OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
anonymous_namespace{ArrowImporter.h} Namespace Reference

Classes

struct  DataBufferBase
 
struct  DataBuffer
 
struct  ArrowValueBase
 
struct  ArrowValue
 
struct  ArrowValue< void * >
 
struct  ArrowValue< bool >
 
struct  ArrowValue< float >
 
struct  ArrowValue< double >
 
struct  ArrowValue< int64_t >
 
struct  ArrowValue< std::string >
 
struct  ArrowValue< arrow::Decimal128 >
 

Typedefs

using VarValue = boost::variant< bool, float, double, int64_t, std::string, void *, arrow::Decimal128 >
 
template<typename T >
using enable_if_integral = typename std::enable_if_t< std::is_integral< T >::value, T >
 
template<typename T >
using enable_if_integral_not_bool = typename std::enable_if_t< std::is_integral< T >::value &&!std::is_same< T, bool >::value, T >
 
template<typename T >
using enable_if_floating = typename std::enable_if_t< std::is_floating_point< T >::value, T >
 

Functions

std::string error_context (const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
 
template<typename SrcType , typename DstType >
VarValue get_numeric_value (const arrow::Array &array, const int64_t idx)
 
template<typename SrcType >
VarValue get_string_value (const arrow::Array &array, const int64_t idx)
 
auto value_getter (const arrow::Array &array, const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
 
void type_conversion_error (const std::string pt, const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
 
template<typename DATA_TYPE , typename VALUE_TYPE >
void data_conversion_error (const VALUE_TYPE v, const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
 
void data_conversion_error (const std::string &v, const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
 
template<typename DATA_TYPE >
auto & operator<< (DataBuffer< DATA_TYPE > &data, const VarValue &var)
 

Variables

constexpr int64_t kMillisecondsInSecond = 1000LL
 
constexpr int64_t kMicrosecondsInSecond = 1000LL * 1000LL
 
constexpr int64_t kNanosecondsinSecond = 1000LL * 1000LL * 1000LL
 
constexpr int32_t kSecondsInDay = 86400
 
static const std::map
< std::pair< int32_t,
arrow::TimeUnit::type >
, std::pair< SQLOps, int64_t > > 
_precision_scale_lookup
 

Typedef Documentation

template<typename T >
using anonymous_namespace{ArrowImporter.h}::enable_if_floating = typedef typename std::enable_if_t<std::is_floating_point<T>::value, T>

Definition at line 73 of file ArrowImporter.h.

template<typename T >
using anonymous_namespace{ArrowImporter.h}::enable_if_integral = typedef typename std::enable_if_t<std::is_integral<T>::value, T>

Definition at line 67 of file ArrowImporter.h.

template<typename T >
using anonymous_namespace{ArrowImporter.h}::enable_if_integral_not_bool = typedef typename std::enable_if_t<std::is_integral<T>::value && !std::is_same<T, bool>::value, T>

Definition at line 71 of file ArrowImporter.h.

using anonymous_namespace{ArrowImporter.h}::VarValue = typedef boost::variant<bool, float, double, int64_t, std::string, void*, arrow::Decimal128>

Definition at line 64 of file ArrowImporter.h.

Function Documentation

template<typename DATA_TYPE , typename VALUE_TYPE >
void anonymous_namespace{ArrowImporter.h}::data_conversion_error ( const VALUE_TYPE  v,
const ColumnDescriptor cd,
import_export::BadRowsTracker *const  bad_rows_tracker 
)
inline

Definition at line 145 of file ArrowImporter.h.

References arrow_throw_if(), error_context(), and to_string().

Referenced by anonymous_namespace{ArrowImporter.h}::ArrowValue< std::string >::operator DATA_TYPE().

147  {
148  arrow_throw_if(true,
149  error_context(cd, bad_rows_tracker) +
150  "Invalid data conversion from parquet value " + std::to_string(v) +
151  " to " + std::to_string(DATA_TYPE(v)));
152 }
std::string to_string(char const *&&v)
std::string error_context(const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
Definition: ArrowImporter.h:77
void arrow_throw_if(const bool cond, const std::string &message)
Definition: ArrowImporter.h:42

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void anonymous_namespace{ArrowImporter.h}::data_conversion_error ( const std::string &  v,
const ColumnDescriptor cd,
import_export::BadRowsTracker *const  bad_rows_tracker 
)
inline

Definition at line 154 of file ArrowImporter.h.

References arrow_throw_if(), ColumnDescriptor::columnType, error_context(), and SQLTypeInfo::get_type_name().

156  {
157  arrow_throw_if(true,
158  error_context(cd, bad_rows_tracker) +
159  "Invalid data conversion from parquet string '" + v + "' to " +
160  cd->columnType.get_type_name() + " column type");
161 }
std::string get_type_name() const
Definition: sqltypes.h:482
SQLTypeInfo columnType
std::string error_context(const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
Definition: ArrowImporter.h:77
void arrow_throw_if(const bool cond, const std::string &message)
Definition: ArrowImporter.h:42

+ Here is the call graph for this function:

std::string anonymous_namespace{ArrowImporter.h}::error_context ( const ColumnDescriptor cd,
import_export::BadRowsTracker *const  bad_rows_tracker 
)
inline

Definition at line 77 of file ArrowImporter.h.

References ColumnDescriptor::columnName, import_export::BadRowsTracker::file_name, import_export::BadRowsTracker::row_group, and to_string().

Referenced by anonymous_namespace{ArrowImporter.h}::ArrowValue< arrow::Decimal128 >::ArrowValue(), import_export::TypedImportBuffer::convert_arrow_val_to_import_buffer(), data_conversion_error(), type_conversion_error(), and value_getter().

78  {
79  return bad_rows_tracker ? "File " + bad_rows_tracker->file_name + ", row-group " +
80  std::to_string(bad_rows_tracker->row_group) +
81  (cd ? ", column " + cd->columnName + ": " : "")
82  : std::string();
83 }
std::string to_string(char const *&&v)
std::string columnName

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename SrcType , typename DstType >
VarValue anonymous_namespace{ArrowImporter.h}::get_numeric_value ( const arrow::Array &  array,
const int64_t  idx 
)
inline

Definition at line 86 of file ArrowImporter.h.

86  {
87  using ArrayType = typename arrow::TypeTraits<SrcType>::ArrayType;
88  return (DstType) static_cast<const ArrayType&>(array).Value(idx);
89 }
template<typename SrcType >
VarValue anonymous_namespace{ArrowImporter.h}::get_string_value ( const arrow::Array &  array,
const int64_t  idx 
)
inline

Definition at line 92 of file ArrowImporter.h.

92  {
93  using ArrayType = typename arrow::TypeTraits<SrcType>::ArrayType;
94  return static_cast<const ArrayType&>(array).GetString(idx);
95 }
template<typename DATA_TYPE >
auto& anonymous_namespace{ArrowImporter.h}::operator<< ( DataBuffer< DATA_TYPE > &  data,
const VarValue &  var 
)
inline

Definition at line 500 of file ArrowImporter.h.

References exprtype.

500  {
501  boost::apply_visitor(
502  [&data](const auto& v) {
503  data.buffer.push_back(DATA_TYPE(ArrowValue<exprtype(v)>(data, v)));
504  },
505  var);
506  return data;
507 }
#define exprtype(expr)
Definition: ArrowImporter.h:75
void anonymous_namespace{ArrowImporter.h}::type_conversion_error ( const std::string  pt,
const ColumnDescriptor cd,
import_export::BadRowsTracker *const  bad_rows_tracker 
)
inline

Definition at line 135 of file ArrowImporter.h.

References arrow_throw_if(), ColumnDescriptor::columnType, error_context(), and SQLTypeInfo::get_type_name().

Referenced by anonymous_namespace{ArrowImporter.h}::ArrowValue< bool >::operator DATA_TYPE().

137  {
138  arrow_throw_if(true,
139  error_context(cd, bad_rows_tracker) +
140  "Invalid type conversion from parquet " + pt + " type to " +
141  cd->columnType.get_type_name());
142 }
std::string get_type_name() const
Definition: sqltypes.h:482
SQLTypeInfo columnType
std::string error_context(const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
Definition: ArrowImporter.h:77
void arrow_throw_if(const bool cond, const std::string &message)
Definition: ArrowImporter.h:42

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

auto anonymous_namespace{ArrowImporter.h}::value_getter ( const arrow::Array &  array,
const ColumnDescriptor cd,
import_export::BadRowsTracker *const  bad_rows_tracker 
)
inline

Definition at line 104 of file ArrowImporter.h.

References arrow_throw_if(), error_context(), NUMERIC_CASE, heavydb.dtypes::STRING, and STRING_CASE.

Referenced by import_export::TypedImportBuffer::convert_arrow_val_to_import_buffer().

106  {
107  switch (array.type_id()) {
108  NUMERIC_CASE(BOOL, arrow::BooleanType, bool)
109  NUMERIC_CASE(UINT8, arrow::UInt8Type, int64_t)
110  NUMERIC_CASE(UINT16, arrow::UInt16Type, int64_t)
111  NUMERIC_CASE(UINT32, arrow::UInt32Type, int64_t)
112  NUMERIC_CASE(UINT64, arrow::Int64Type, int64_t)
113  NUMERIC_CASE(INT8, arrow::Int8Type, int64_t)
114  NUMERIC_CASE(INT16, arrow::Int16Type, int64_t)
115  NUMERIC_CASE(INT32, arrow::Int32Type, int64_t)
116  NUMERIC_CASE(INT64, arrow::Int64Type, int64_t)
117  NUMERIC_CASE(FLOAT, arrow::FloatType, float)
118  NUMERIC_CASE(DOUBLE, arrow::DoubleType, double)
119  NUMERIC_CASE(DATE32, arrow::Date32Type, int64_t)
120  NUMERIC_CASE(DATE64, arrow::Date64Type, int64_t)
121  NUMERIC_CASE(TIME64, arrow::Time64Type, int64_t)
122  NUMERIC_CASE(TIME32, arrow::Time32Type, int64_t)
123  NUMERIC_CASE(TIMESTAMP, arrow::TimestampType, int64_t)
124  NUMERIC_CASE(DECIMAL, arrow::Decimal128Type, arrow::Decimal128)
125  STRING_CASE(STRING, arrow::StringType)
126  STRING_CASE(BINARY, arrow::BinaryType)
127  default:
129  error_context(cd, bad_rows_tracker) + "Parquet type " +
130  array.type()->name() + " is not supported");
131  throw;
132  }
133 }
#define NUMERIC_CASE(tid, src_type, var_type)
Definition: ArrowImporter.h:97
tuple STRING
Definition: dtypes.py:31
bool g_enable_smem_group_by true
#define STRING_CASE(tid, src_type)
std::string error_context(const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
Definition: ArrowImporter.h:77
string name
Definition: setup.in.py:72
void arrow_throw_if(const bool cond, const std::string &message)
Definition: ArrowImporter.h:42

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Variable Documentation

const std::map<std::pair<int32_t, arrow::TimeUnit::type>, std::pair<SQLOps, int64_t> > anonymous_namespace{ArrowImporter.h}::_precision_scale_lookup
static
Initial value:
{
{{0, arrow::TimeUnit::MILLI}, {kDIVIDE, kMillisecondsInSecond}},
{{0, arrow::TimeUnit::MICRO}, {kDIVIDE, kMicrosecondsInSecond}},
{{0, arrow::TimeUnit::NANO}, {kDIVIDE, kNanosecondsinSecond}},
{{3, arrow::TimeUnit::SECOND}, {kMULTIPLY, kMicrosecondsInSecond}},
{{3, arrow::TimeUnit::MICRO}, {kDIVIDE, kMillisecondsInSecond}},
{{3, arrow::TimeUnit::NANO}, {kDIVIDE, kMicrosecondsInSecond}},
{{6, arrow::TimeUnit::SECOND}, {kMULTIPLY, kMicrosecondsInSecond}},
{{6, arrow::TimeUnit::MILLI}, {kMULTIPLY, kMillisecondsInSecond}},
{{6, arrow::TimeUnit::NANO}, {kDIVIDE, kMillisecondsInSecond}},
{{9, arrow::TimeUnit::SECOND}, {kMULTIPLY, kNanosecondsinSecond}},
{{9, arrow::TimeUnit::MILLI}, {kMULTIPLY, kMicrosecondsInSecond}},
{{9, arrow::TimeUnit::MICRO}, {kMULTIPLY, kMillisecondsInSecond}}}

Definition at line 212 of file ArrowImporter.h.

Referenced by anonymous_namespace{ArrowImporter.h}::ArrowValue< int64_t >::operator DATA_TYPE(), and anonymous_namespace{ArrowImporter.h}::ArrowValueBase< bool >::resolve_time().

constexpr int64_t anonymous_namespace{ArrowImporter.h}::kMicrosecondsInSecond = 1000LL * 1000LL

Definition at line 206 of file ArrowImporter.h.

constexpr int64_t anonymous_namespace{ArrowImporter.h}::kMillisecondsInSecond = 1000LL
constexpr int64_t anonymous_namespace{ArrowImporter.h}::kNanosecondsinSecond = 1000LL * 1000LL * 1000LL

Definition at line 207 of file ArrowImporter.h.

constexpr int32_t anonymous_namespace{ArrowImporter.h}::kSecondsInDay = 86400