OmniSciDB  fe05a0c208
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
anonymous_namespace{ArrowImporter.h} Namespace Reference

Classes

struct  DataBufferBase
 
struct  DataBuffer
 
struct  ArrowValueBase
 
struct  ArrowValue
 
struct  ArrowValue< void * >
 
struct  ArrowValue< bool >
 
struct  ArrowValue< float >
 
struct  ArrowValue< double >
 
struct  ArrowValue< int64_t >
 
struct  ArrowValue< std::string >
 
struct  ArrowValue< arrow::Decimal128 >
 

Typedefs

using VarValue = boost::variant< bool, float, double, int64_t, std::string, void *, arrow::Decimal128 >
 
template<typename T >
using enable_if_integral = typename std::enable_if_t< std::is_integral< T >::value, T >
 
template<typename T >
using enable_if_integral_not_bool = typename std::enable_if_t< std::is_integral< T >::value &&!std::is_same< T, bool >::value, T >
 
template<typename T >
using enable_if_floating = typename std::enable_if_t< std::is_floating_point< T >::value, T >
 

Functions

std::string error_context (const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
 
template<typename SrcType , typename DstType >
VarValue get_numeric_value (const arrow::Array &array, const int64_t idx)
 
template<typename SrcType >
VarValue get_string_value (const arrow::Array &array, const int64_t idx)
 
auto value_getter (const arrow::Array &array, const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
 
void type_conversion_error (const std::string pt, const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
 
template<typename DATA_TYPE , typename VALUE_TYPE >
void data_conversion_error (const VALUE_TYPE v, const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
 
void data_conversion_error (const std::string &v, const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
 
template<typename DATA_TYPE >
auto & operator<< (DataBuffer< DATA_TYPE > &data, const VarValue &var)
 

Variables

constexpr int64_t kMillisecondsInSecond = 1000LL
 
constexpr int64_t kMicrosecondsInSecond = 1000LL * 1000LL
 
constexpr int64_t kNanosecondsinSecond = 1000LL * 1000LL * 1000LL
 
constexpr int32_t kSecondsInDay = 86400
 
static const std::map
< std::pair< int32_t,
arrow::TimeUnit::type >
, std::pair< SQLOps, int64_t > > 
_precision_scale_lookup
 

Typedef Documentation

template<typename T >
using anonymous_namespace{ArrowImporter.h}::enable_if_floating = typedef typename std::enable_if_t<std::is_floating_point<T>::value, T>

Definition at line 72 of file ArrowImporter.h.

template<typename T >
using anonymous_namespace{ArrowImporter.h}::enable_if_integral = typedef typename std::enable_if_t<std::is_integral<T>::value, T>

Definition at line 66 of file ArrowImporter.h.

template<typename T >
using anonymous_namespace{ArrowImporter.h}::enable_if_integral_not_bool = typedef typename std::enable_if_t<std::is_integral<T>::value && !std::is_same<T, bool>::value, T>

Definition at line 70 of file ArrowImporter.h.

using anonymous_namespace{ArrowImporter.h}::VarValue = typedef boost::variant<bool, float, double, int64_t, std::string, void*, arrow::Decimal128>

Definition at line 63 of file ArrowImporter.h.

Function Documentation

template<typename DATA_TYPE , typename VALUE_TYPE >
void anonymous_namespace{ArrowImporter.h}::data_conversion_error ( const VALUE_TYPE  v,
const ColumnDescriptor cd,
import_export::BadRowsTracker *const  bad_rows_tracker 
)
inline

Definition at line 144 of file ArrowImporter.h.

References arrow_throw_if(), error_context(), and to_string().

Referenced by anonymous_namespace{ArrowImporter.h}::ArrowValue< std::string >::operator DATA_TYPE().

146  {
147  arrow_throw_if(true,
148  error_context(cd, bad_rows_tracker) +
149  "Invalid data conversion from parquet value " + std::to_string(v) +
150  " to " + std::to_string(DATA_TYPE(v)));
151 }
std::string to_string(char const *&&v)
std::string error_context(const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
Definition: ArrowImporter.h:76
void arrow_throw_if(const bool cond, const std::string &message)
Definition: ArrowImporter.h:41

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void anonymous_namespace{ArrowImporter.h}::data_conversion_error ( const std::string &  v,
const ColumnDescriptor cd,
import_export::BadRowsTracker *const  bad_rows_tracker 
)
inline

Definition at line 153 of file ArrowImporter.h.

References arrow_throw_if(), ColumnDescriptor::columnType, error_context(), and SQLTypeInfo::get_type_name().

155  {
156  arrow_throw_if(true,
157  error_context(cd, bad_rows_tracker) +
158  "Invalid data conversion from parquet string '" + v + "' to " +
159  cd->columnType.get_type_name() + " column type");
160 }
std::string get_type_name() const
Definition: sqltypes.h:417
SQLTypeInfo columnType
std::string error_context(const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
Definition: ArrowImporter.h:76
void arrow_throw_if(const bool cond, const std::string &message)
Definition: ArrowImporter.h:41

+ Here is the call graph for this function:

std::string anonymous_namespace{ArrowImporter.h}::error_context ( const ColumnDescriptor cd,
import_export::BadRowsTracker *const  bad_rows_tracker 
)
inline

Definition at line 76 of file ArrowImporter.h.

References ColumnDescriptor::columnName, import_export::BadRowsTracker::file_name, import_export::BadRowsTracker::row_group, and to_string().

Referenced by anonymous_namespace{ArrowImporter.h}::ArrowValue< arrow::Decimal128 >::ArrowValue(), import_export::TypedImportBuffer::convert_arrow_val_to_import_buffer(), data_conversion_error(), type_conversion_error(), and value_getter().

77  {
78  return bad_rows_tracker ? "File " + bad_rows_tracker->file_name + ", row-group " +
79  std::to_string(bad_rows_tracker->row_group) +
80  (cd ? ", column " + cd->columnName + ": " : "")
81  : std::string();
82 }
std::string to_string(char const *&&v)
std::string columnName

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename SrcType , typename DstType >
VarValue anonymous_namespace{ArrowImporter.h}::get_numeric_value ( const arrow::Array &  array,
const int64_t  idx 
)
inline

Definition at line 85 of file ArrowImporter.h.

85  {
86  using ArrayType = typename arrow::TypeTraits<SrcType>::ArrayType;
87  return (DstType) static_cast<const ArrayType&>(array).Value(idx);
88 }
template<typename SrcType >
VarValue anonymous_namespace{ArrowImporter.h}::get_string_value ( const arrow::Array &  array,
const int64_t  idx 
)
inline

Definition at line 91 of file ArrowImporter.h.

91  {
92  using ArrayType = typename arrow::TypeTraits<SrcType>::ArrayType;
93  return static_cast<const ArrayType&>(array).GetString(idx);
94 }
template<typename DATA_TYPE >
auto& anonymous_namespace{ArrowImporter.h}::operator<< ( DataBuffer< DATA_TYPE > &  data,
const VarValue &  var 
)
inline

Definition at line 499 of file ArrowImporter.h.

References exprtype.

499  {
500  boost::apply_visitor(
501  [&data](const auto& v) {
502  data.buffer.push_back(DATA_TYPE(ArrowValue<exprtype(v)>(data, v)));
503  },
504  var);
505  return data;
506 }
#define exprtype(expr)
Definition: ArrowImporter.h:74
void anonymous_namespace{ArrowImporter.h}::type_conversion_error ( const std::string  pt,
const ColumnDescriptor cd,
import_export::BadRowsTracker *const  bad_rows_tracker 
)
inline

Definition at line 134 of file ArrowImporter.h.

References arrow_throw_if(), ColumnDescriptor::columnType, error_context(), and SQLTypeInfo::get_type_name().

Referenced by anonymous_namespace{ArrowImporter.h}::ArrowValue< bool >::operator DATA_TYPE().

136  {
137  arrow_throw_if(true,
138  error_context(cd, bad_rows_tracker) +
139  "Invalid type conversion from parquet " + pt + " type to " +
140  cd->columnType.get_type_name());
141 }
std::string get_type_name() const
Definition: sqltypes.h:417
SQLTypeInfo columnType
std::string error_context(const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
Definition: ArrowImporter.h:76
void arrow_throw_if(const bool cond, const std::string &message)
Definition: ArrowImporter.h:41

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

auto anonymous_namespace{ArrowImporter.h}::value_getter ( const arrow::Array &  array,
const ColumnDescriptor cd,
import_export::BadRowsTracker *const  bad_rows_tracker 
)
inline

Definition at line 103 of file ArrowImporter.h.

References arrow_throw_if(), DECIMAL, DOUBLE, error_context(), FLOAT, NUMERIC_CASE, STRING, STRING_CASE, and TIMESTAMP.

Referenced by import_export::TypedImportBuffer::convert_arrow_val_to_import_buffer().

105  {
106  switch (array.type_id()) {
107  NUMERIC_CASE(BOOL, arrow::BooleanType, bool)
108  NUMERIC_CASE(UINT8, arrow::UInt8Type, int64_t)
109  NUMERIC_CASE(UINT16, arrow::UInt16Type, int64_t)
110  NUMERIC_CASE(UINT32, arrow::UInt32Type, int64_t)
111  NUMERIC_CASE(UINT64, arrow::Int64Type, int64_t)
112  NUMERIC_CASE(INT8, arrow::Int8Type, int64_t)
113  NUMERIC_CASE(INT16, arrow::Int16Type, int64_t)
114  NUMERIC_CASE(INT32, arrow::Int32Type, int64_t)
115  NUMERIC_CASE(INT64, arrow::Int64Type, int64_t)
116  NUMERIC_CASE(FLOAT, arrow::FloatType, float)
117  NUMERIC_CASE(DOUBLE, arrow::DoubleType, double)
118  NUMERIC_CASE(DATE32, arrow::Date32Type, int64_t)
119  NUMERIC_CASE(DATE64, arrow::Date64Type, int64_t)
120  NUMERIC_CASE(TIME64, arrow::Time64Type, int64_t)
121  NUMERIC_CASE(TIME32, arrow::Time32Type, int64_t)
122  NUMERIC_CASE(TIMESTAMP, arrow::TimestampType, int64_t)
123  NUMERIC_CASE(DECIMAL, arrow::Decimal128Type, arrow::Decimal128)
124  STRING_CASE(STRING, arrow::StringType)
125  STRING_CASE(BINARY, arrow::BinaryType)
126  default:
128  error_context(cd, bad_rows_tracker) + "Parquet type " +
129  array.type()->name() + " is not supported");
130  throw;
131  }
132 }
#define DOUBLE
string name
Definition: setup.in.py:72
#define NUMERIC_CASE(tid, src_type, var_type)
Definition: ArrowImporter.h:96
bool g_enable_smem_group_by true
#define TIMESTAMP
#define STRING_CASE(tid, src_type)
Definition: ArrowImporter.h:99
#define STRING
#define DECIMAL
std::string error_context(const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
Definition: ArrowImporter.h:76
#define FLOAT
void arrow_throw_if(const bool cond, const std::string &message)
Definition: ArrowImporter.h:41

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Variable Documentation

const std::map<std::pair<int32_t, arrow::TimeUnit::type>, std::pair<SQLOps, int64_t> > anonymous_namespace{ArrowImporter.h}::_precision_scale_lookup
static
Initial value:
{
{{0, arrow::TimeUnit::MILLI}, {kDIVIDE, kMillisecondsInSecond}},
{{0, arrow::TimeUnit::MICRO}, {kDIVIDE, kMicrosecondsInSecond}},
{{0, arrow::TimeUnit::NANO}, {kDIVIDE, kNanosecondsinSecond}},
{{3, arrow::TimeUnit::SECOND}, {kMULTIPLY, kMicrosecondsInSecond}},
{{3, arrow::TimeUnit::MICRO}, {kDIVIDE, kMillisecondsInSecond}},
{{3, arrow::TimeUnit::NANO}, {kDIVIDE, kMicrosecondsInSecond}},
{{6, arrow::TimeUnit::SECOND}, {kMULTIPLY, kMicrosecondsInSecond}},
{{6, arrow::TimeUnit::MILLI}, {kMULTIPLY, kMillisecondsInSecond}},
{{6, arrow::TimeUnit::NANO}, {kDIVIDE, kMillisecondsInSecond}},
{{9, arrow::TimeUnit::SECOND}, {kMULTIPLY, kNanosecondsinSecond}},
{{9, arrow::TimeUnit::MILLI}, {kMULTIPLY, kMicrosecondsInSecond}},
{{9, arrow::TimeUnit::MICRO}, {kMULTIPLY, kMillisecondsInSecond}}}

Definition at line 211 of file ArrowImporter.h.

Referenced by anonymous_namespace{ArrowImporter.h}::ArrowValue< int64_t >::operator DATA_TYPE(), and anonymous_namespace{ArrowImporter.h}::ArrowValueBase< bool >::resolve_time().

constexpr int64_t anonymous_namespace{ArrowImporter.h}::kMicrosecondsInSecond = 1000LL * 1000LL

Definition at line 205 of file ArrowImporter.h.

constexpr int64_t anonymous_namespace{ArrowImporter.h}::kMillisecondsInSecond = 1000LL
constexpr int64_t anonymous_namespace{ArrowImporter.h}::kNanosecondsinSecond = 1000LL * 1000LL * 1000LL

Definition at line 206 of file ArrowImporter.h.

constexpr int32_t anonymous_namespace{ArrowImporter.h}::kSecondsInDay = 86400