OmniSciDB  b24e664e58
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
anonymous_namespace{ArrowImporter.h} Namespace Reference

Classes

struct  DataBufferBase
 
struct  DataBuffer
 
struct  ArrowValueBase
 
struct  ArrowValue
 
struct  ArrowValue< void * >
 
struct  ArrowValue< bool >
 
struct  ArrowValue< float >
 
struct  ArrowValue< double >
 
struct  ArrowValue< int64_t >
 
struct  ArrowValue< std::string >
 
struct  ArrowValue< Decimal128 >
 

Typedefs

using VarValue = boost::variant< bool, float, double, int64_t, std::string, void *, Decimal128 >
 
template<typename T >
using enable_if_integral = typename std::enable_if_t< std::is_integral< T >::value, T >
 
template<typename T >
using enable_if_integral_not_bool = typename std::enable_if_t< std::is_integral< T >::value &&!std::is_same< T, bool >::value, T >
 
template<typename T >
using enable_if_floating = typename std::enable_if_t< std::is_floating_point< T >::value, T >
 

Functions

std::string error_context (const ColumnDescriptor *cd, Importer_NS::BadRowsTracker *const bad_rows_tracker)
 
template<typename SrcType , typename DstType >
VarValue get_numeric_value (const Array &array, const int64_t idx)
 
template<typename SrcType >
VarValue get_string_value (const Array &array, const int64_t idx)
 
auto value_getter (const Array &array, const ColumnDescriptor *cd, Importer_NS::BadRowsTracker *const bad_rows_tracker)
 
void type_conversion_error (const std::string pt, const ColumnDescriptor *cd, Importer_NS::BadRowsTracker *const bad_rows_tracker)
 
template<typename DATA_TYPE , typename VALUE_TYPE >
void data_conversion_error (const VALUE_TYPE v, const ColumnDescriptor *cd, Importer_NS::BadRowsTracker *const bad_rows_tracker)
 
void data_conversion_error (const std::string &v, const ColumnDescriptor *cd, Importer_NS::BadRowsTracker *const bad_rows_tracker)
 
template<typename DATA_TYPE >
auto & operator<< (DataBuffer< DATA_TYPE > &data, const VarValue &var)
 

Variables

constexpr int64_t kMillisecondsInSecond = 1000L
 
constexpr int64_t kMicrosecondsInSecond = 1000L * 1000L
 
constexpr int64_t kNanosecondsinSecond = 1000L * 1000L * 1000L
 
constexpr int32_t kSecondsInDay = 86400
 
static const std::map
< std::pair< int32_t,
arrow::TimeUnit::type >
, std::pair< SQLOps, int64_t > > 
_precision_scale_lookup
 

Typedef Documentation

template<typename T >
using anonymous_namespace{ArrowImporter.h}::enable_if_floating = typedef typename std::enable_if_t<std::is_floating_point<T>::value, T>

Definition at line 71 of file ArrowImporter.h.

template<typename T >
using anonymous_namespace{ArrowImporter.h}::enable_if_integral = typedef typename std::enable_if_t<std::is_integral<T>::value, T>

Definition at line 65 of file ArrowImporter.h.

template<typename T >
using anonymous_namespace{ArrowImporter.h}::enable_if_integral_not_bool = typedef typename std::enable_if_t<std::is_integral<T>::value && !std::is_same<T, bool>::value, T>

Definition at line 69 of file ArrowImporter.h.

using anonymous_namespace{ArrowImporter.h}::VarValue = typedef boost::variant<bool, float, double, int64_t, std::string, void*, Decimal128>

Definition at line 62 of file ArrowImporter.h.

Function Documentation

template<typename DATA_TYPE , typename VALUE_TYPE >
void anonymous_namespace{ArrowImporter.h}::data_conversion_error ( const VALUE_TYPE  v,
const ColumnDescriptor cd,
Importer_NS::BadRowsTracker *const  bad_rows_tracker 
)
inline

Definition at line 143 of file ArrowImporter.h.

References arrow_throw_if(), error_context(), and to_string().

Referenced by anonymous_namespace{ArrowImporter.h}::ArrowValue< std::string >::operator const bool(), and anonymous_namespace{ArrowImporter.h}::ArrowValue< std::string >::operator const DATA_TYPE().

145  {
146  arrow_throw_if(true,
147  error_context(cd, bad_rows_tracker) +
148  "Invalid data conversion from parquet value " + std::to_string(v) +
149  " to " + std::to_string(DATA_TYPE(v)));
150 }
std::string error_context(const ColumnDescriptor *cd, Importer_NS::BadRowsTracker *const bad_rows_tracker)
Definition: ArrowImporter.h:75
std::string to_string(char const *&&v)
void arrow_throw_if(const bool cond, const std::string &message)
Definition: ArrowImporter.h:40

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void anonymous_namespace{ArrowImporter.h}::data_conversion_error ( const std::string &  v,
const ColumnDescriptor cd,
Importer_NS::BadRowsTracker *const  bad_rows_tracker 
)
inline

Definition at line 152 of file ArrowImporter.h.

References arrow_throw_if(), ColumnDescriptor::columnType, error_context(), and SQLTypeInfoCore< TYPE_FACET_PACK >::get_type_name().

154  {
155  arrow_throw_if(true,
156  error_context(cd, bad_rows_tracker) +
157  "Invalid data conversion from parquet string '" + v + "' to " +
158  cd->columnType.get_type_name() + " column type");
159 }
std::string error_context(const ColumnDescriptor *cd, Importer_NS::BadRowsTracker *const bad_rows_tracker)
Definition: ArrowImporter.h:75
std::string get_type_name() const
Definition: sqltypes.h:429
SQLTypeInfo columnType
void arrow_throw_if(const bool cond, const std::string &message)
Definition: ArrowImporter.h:40

+ Here is the call graph for this function:

std::string anonymous_namespace{ArrowImporter.h}::error_context ( const ColumnDescriptor cd,
Importer_NS::BadRowsTracker *const  bad_rows_tracker 
)
inline

Definition at line 75 of file ArrowImporter.h.

References ColumnDescriptor::columnName, Importer_NS::BadRowsTracker::file_name, Importer_NS::BadRowsTracker::row_group, and to_string().

Referenced by anonymous_namespace{ArrowImporter.h}::ArrowValue< Decimal128 >::ArrowValue(), Importer_NS::TypedImportBuffer::convert_arrow_val_to_import_buffer(), data_conversion_error(), type_conversion_error(), and value_getter().

76  {
77  return bad_rows_tracker ? "File " + bad_rows_tracker->file_name + ", row-group " +
78  std::to_string(bad_rows_tracker->row_group) +
79  (cd ? ", column " + cd->columnName + ": " : "")
80  : std::string();
81 }
std::string to_string(char const *&&v)
std::string columnName

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename SrcType , typename DstType >
VarValue anonymous_namespace{ArrowImporter.h}::get_numeric_value ( const Array &  array,
const int64_t  idx 
)
inline

Definition at line 84 of file ArrowImporter.h.

84  {
85  using ArrayType = typename TypeTraits<SrcType>::ArrayType;
86  return (DstType) static_cast<const ArrayType&>(array).Value(idx);
87 }
template<typename SrcType >
VarValue anonymous_namespace{ArrowImporter.h}::get_string_value ( const Array &  array,
const int64_t  idx 
)
inline

Definition at line 90 of file ArrowImporter.h.

90  {
91  using ArrayType = typename TypeTraits<SrcType>::ArrayType;
92  return static_cast<const ArrayType&>(array).GetString(idx);
93 }
template<typename DATA_TYPE >
auto& anonymous_namespace{ArrowImporter.h}::operator<< ( DataBuffer< DATA_TYPE > &  data,
const VarValue &  var 
)
inline

Definition at line 486 of file ArrowImporter.h.

References exprtype.

486  {
487  boost::apply_visitor(
488  [&data](const auto& v) {
489  data.buffer.push_back(DATA_TYPE(ArrowValue<exprtype(v)>(data, v)));
490  },
491  var);
492  return data;
493 }
#define exprtype(expr)
Definition: ArrowImporter.h:73
void anonymous_namespace{ArrowImporter.h}::type_conversion_error ( const std::string  pt,
const ColumnDescriptor cd,
Importer_NS::BadRowsTracker *const  bad_rows_tracker 
)
inline

Definition at line 133 of file ArrowImporter.h.

References arrow_throw_if(), ColumnDescriptor::columnType, error_context(), and SQLTypeInfoCore< TYPE_FACET_PACK >::get_type_name().

Referenced by anonymous_namespace{ArrowImporter.h}::ArrowValue< bool >::operator const DATA_TYPE().

135  {
136  arrow_throw_if(true,
137  error_context(cd, bad_rows_tracker) +
138  "Invalid type conversion from parquet " + pt + " type to " +
139  cd->columnType.get_type_name());
140 }
std::string error_context(const ColumnDescriptor *cd, Importer_NS::BadRowsTracker *const bad_rows_tracker)
Definition: ArrowImporter.h:75
std::string get_type_name() const
Definition: sqltypes.h:429
SQLTypeInfo columnType
void arrow_throw_if(const bool cond, const std::string &message)
Definition: ArrowImporter.h:40

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

auto anonymous_namespace{ArrowImporter.h}::value_getter ( const Array &  array,
const ColumnDescriptor cd,
Importer_NS::BadRowsTracker *const  bad_rows_tracker 
)
inline

Definition at line 102 of file ArrowImporter.h.

References arrow_throw_if(), BINARY, error_context(), NUMERIC_CASE, and STRING_CASE.

Referenced by Importer_NS::TypedImportBuffer::convert_arrow_val_to_import_buffer().

104  {
105  switch (array.type_id()) {
106  NUMERIC_CASE(BOOL, BooleanType, bool)
107  NUMERIC_CASE(UINT8, UInt8Type, int64_t)
108  NUMERIC_CASE(UINT16, UInt16Type, int64_t)
109  NUMERIC_CASE(UINT32, UInt32Type, int64_t)
110  NUMERIC_CASE(UINT64, Int64Type, int64_t)
111  NUMERIC_CASE(INT8, Int8Type, int64_t)
112  NUMERIC_CASE(INT16, Int16Type, int64_t)
113  NUMERIC_CASE(INT32, Int32Type, int64_t)
114  NUMERIC_CASE(INT64, Int64Type, int64_t)
115  NUMERIC_CASE(FLOAT, FloatType, float)
116  NUMERIC_CASE(DOUBLE, DoubleType, double)
117  NUMERIC_CASE(DATE32, Date32Type, int64_t)
118  NUMERIC_CASE(DATE64, Date64Type, int64_t)
119  NUMERIC_CASE(TIME64, Time64Type, int64_t)
120  NUMERIC_CASE(TIME32, Time32Type, int64_t)
121  NUMERIC_CASE(TIMESTAMP, TimestampType, int64_t)
122  NUMERIC_CASE(DECIMAL, Decimal128Type, Decimal128)
123  STRING_CASE(STRING, StringType)
124  STRING_CASE(BINARY, BinaryType)
125  default:
127  error_context(cd, bad_rows_tracker) + "Parquet type " +
128  array.type()->name() + " is not supported");
129  throw;
130  }
131 }
std::string error_context(const ColumnDescriptor *cd, Importer_NS::BadRowsTracker *const bad_rows_tracker)
Definition: ArrowImporter.h:75
#define NUMERIC_CASE(tid, src_type, var_type)
Definition: ArrowImporter.h:95
bool g_enable_smem_group_by true
#define STRING_CASE(tid, src_type)
Definition: ArrowImporter.h:98
void arrow_throw_if(const bool cond, const std::string &message)
Definition: ArrowImporter.h:40

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Variable Documentation

const std::map<std::pair<int32_t, arrow::TimeUnit::type>, std::pair<SQLOps, int64_t> > anonymous_namespace{ArrowImporter.h}::_precision_scale_lookup
static
Initial value:
{{{0, TimeUnit::MILLI}, {kDIVIDE, kMillisecondsInSecond}},
{{0, TimeUnit::MICRO}, {kDIVIDE, kMicrosecondsInSecond}},
{{0, TimeUnit::NANO}, {kDIVIDE, kNanosecondsinSecond}},
{{3, TimeUnit::SECOND}, {kMULTIPLY, kMicrosecondsInSecond}},
{{3, TimeUnit::MICRO}, {kDIVIDE, kMillisecondsInSecond}},
{{3, TimeUnit::NANO}, {kDIVIDE, kMicrosecondsInSecond}},
{{6, TimeUnit::SECOND}, {kMULTIPLY, kMicrosecondsInSecond}},
{{6, TimeUnit::MILLI}, {kMULTIPLY, kMillisecondsInSecond}},
{{6, TimeUnit::NANO}, {kDIVIDE, kMillisecondsInSecond}},
{{9, TimeUnit::SECOND}, {kMULTIPLY, kNanosecondsinSecond}},
{{9, TimeUnit::MILLI}, {kMULTIPLY, kMicrosecondsInSecond}},
{{9, TimeUnit::MICRO}, {kMULTIPLY, kMillisecondsInSecond}}}

Definition at line 209 of file ArrowImporter.h.

Referenced by anonymous_namespace{ArrowImporter.h}::ArrowValue< int64_t >::operator const std::string(), and anonymous_namespace{ArrowImporter.h}::ArrowValueBase< bool >::resolve_time().

constexpr int64_t anonymous_namespace{ArrowImporter.h}::kMicrosecondsInSecond = 1000L * 1000L

Definition at line 203 of file ArrowImporter.h.

constexpr int64_t anonymous_namespace{ArrowImporter.h}::kMillisecondsInSecond = 1000L
constexpr int64_t anonymous_namespace{ArrowImporter.h}::kNanosecondsinSecond = 1000L * 1000L * 1000L

Definition at line 204 of file ArrowImporter.h.

constexpr int32_t anonymous_namespace{ArrowImporter.h}::kSecondsInDay = 86400