OmniSciDB  04ee39c94c
ArrowImporter.h
Go to the documentation of this file.
1 /*
2  * Copyright 2019 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 #ifndef ARROW_IMPORTER_H
17 #define ARROW_IMPORTER_H
18 
19 #include <cstdlib>
20 #include <ctime>
21 #include <map>
22 #include <mutex>
23 
24 #include <arrow/api.h>
25 #include <arrow/io/api.h>
26 #include <boost/algorithm/string.hpp>
27 #include <boost/variant.hpp>
28 
29 #include "Shared/SqlTypesLayout.h"
31 #include "Shared/sqltypes.h"
32 
33 using namespace arrow;
34 
35 struct ArrowImporterException : std::runtime_error {
36  using std::runtime_error::runtime_error;
37 };
38 
39 template <typename T = ArrowImporterException>
40 inline void arrow_throw_if(const bool cond, const std::string& message) {
41  if (cond) {
42  // work around race from goooogle log
43  static std::mutex mtx;
44  std::unique_lock<std::mutex> lock(mtx);
45  LOG(ERROR) << message;
46  throw T(message);
47  }
48 }
49 
50 #ifdef ENABLE_IMPORT_PARQUET
51 #include <parquet/api/reader.h>
52 #include <parquet/api/writer.h>
53 #include <parquet/arrow/reader.h>
54 #include <parquet/exception.h>
55 #endif // ENABLE_IMPORT_PARQUET
56 
57 #include "arrow/util/decimal.h"
58 
59 namespace {
60 
61 using VarValue =
62  boost::variant<bool, float, double, int64_t, std::string, void*, Decimal128>;
63 
64 template <typename T>
65 using enable_if_integral = typename std::enable_if_t<std::is_integral<T>::value, T>;
66 template <typename T>
68  typename std::enable_if_t<std::is_integral<T>::value && !std::is_same<T, bool>::value,
69  T>;
70 template <typename T>
71 using enable_if_floating = typename std::enable_if_t<std::is_floating_point<T>::value, T>;
72 
73 #define exprtype(expr) std::decay_t<decltype(expr)>
74 
75 inline std::string error_context(const ColumnDescriptor* cd,
76  Importer_NS::BadRowsTracker* const bad_rows_tracker) {
77  return bad_rows_tracker ? "File " + bad_rows_tracker->file_name + ", row-group " +
78  std::to_string(bad_rows_tracker->row_group) +
79  (cd ? ", column " + cd->columnName + ": " : "")
80  : std::string();
81 }
82 
83 template <typename SrcType, typename DstType>
84 inline VarValue get_numeric_value(const Array& array, const int64_t idx) {
85  using ArrayType = typename TypeTraits<SrcType>::ArrayType;
86  return (DstType) static_cast<const ArrayType&>(array).Value(idx);
87 }
88 
89 template <typename SrcType>
90 inline VarValue get_string_value(const Array& array, const int64_t idx) {
91  using ArrayType = typename TypeTraits<SrcType>::ArrayType;
92  return static_cast<const ArrayType&>(array).GetString(idx);
93 }
94 
95 #define NUMERIC_CASE(tid, src_type, var_type) \
96  case Type::tid: \
97  return get_numeric_value<src_type, var_type>;
98 #define STRING_CASE(tid, src_type) \
99  case Type::tid: \
100  return get_string_value<src_type>;
101 
102 inline auto value_getter(const Array& array,
103  const ColumnDescriptor* cd,
104  Importer_NS::BadRowsTracker* const bad_rows_tracker) {
105  switch (array.type_id()) {
106  NUMERIC_CASE(BOOL, BooleanType, bool)
107  NUMERIC_CASE(UINT8, UInt8Type, int64_t)
108  NUMERIC_CASE(UINT16, UInt16Type, int64_t)
109  NUMERIC_CASE(UINT32, UInt32Type, int64_t)
110  NUMERIC_CASE(UINT64, Int64Type, int64_t)
111  NUMERIC_CASE(INT8, Int8Type, int64_t)
112  NUMERIC_CASE(INT16, Int16Type, int64_t)
113  NUMERIC_CASE(INT32, Int32Type, int64_t)
114  NUMERIC_CASE(INT64, Int64Type, int64_t)
115  NUMERIC_CASE(FLOAT, FloatType, float)
116  NUMERIC_CASE(DOUBLE, DoubleType, double)
117  NUMERIC_CASE(DATE32, Date32Type, int64_t)
118  NUMERIC_CASE(DATE64, Date64Type, int64_t)
119  NUMERIC_CASE(TIME64, Time64Type, int64_t)
120  NUMERIC_CASE(TIME32, Time32Type, int64_t)
121  NUMERIC_CASE(TIMESTAMP, TimestampType, int64_t)
122  NUMERIC_CASE(DECIMAL, Decimal128Type, Decimal128)
123  STRING_CASE(STRING, StringType)
124  STRING_CASE(BINARY, BinaryType)
125  default:
126  arrow_throw_if(true,
127  error_context(cd, bad_rows_tracker) + "Parquet type " +
128  array.type()->name() + " is not supported");
129  throw;
130  }
131 }
132 
133 inline void type_conversion_error(const std::string pt,
134  const ColumnDescriptor* cd,
135  Importer_NS::BadRowsTracker* const bad_rows_tracker) {
136  arrow_throw_if(true,
137  error_context(cd, bad_rows_tracker) +
138  "Invalid type conversion from parquet " + pt + " type to " +
139  cd->columnType.get_type_name());
140 }
141 
142 template <typename DATA_TYPE, typename VALUE_TYPE>
143 inline void data_conversion_error(const VALUE_TYPE v,
144  const ColumnDescriptor* cd,
145  Importer_NS::BadRowsTracker* const bad_rows_tracker) {
146  arrow_throw_if(true,
147  error_context(cd, bad_rows_tracker) +
148  "Invalid data conversion from parquet value " + std::to_string(v) +
149  " to " + std::to_string(DATA_TYPE(v)));
150 }
151 
152 inline void data_conversion_error(const std::string& v,
153  const ColumnDescriptor* cd,
154  Importer_NS::BadRowsTracker* const bad_rows_tracker) {
155  arrow_throw_if(true,
156  error_context(cd, bad_rows_tracker) +
157  "Invalid data conversion from parquet string '" + v + "' to " +
158  cd->columnType.get_type_name() + " column type");
159 }
160 
161 // models the variant data buffers of TypedImportBuffer (LHS)
164  const Array& array;
166  // in case of arrow-decimal to omni-decimal conversion
167  // dont get/set these info on every row of arrow array
168  const DataType& arrow_type;
173  const Array& array,
174  Importer_NS::BadRowsTracker* const bad_rows_tracker)
175  : cd(cd)
176  , array(array)
177  , bad_rows_tracker(bad_rows_tracker)
178  , arrow_type(*array.type())
179  , arrow_decimal_scale(arrow_type.id() == Type::DECIMAL
180  ? static_cast<const Decimal128Type&>(arrow_type).scale()
181  : 0)
182  , old_type(cd->columnType.get_type(),
183  cd->columnType.get_dimension(),
184  arrow_decimal_scale,
185  true)
186  , new_type(cd->columnType.get_type(),
187  cd->columnType.get_dimension(),
188  cd->columnType.get_scale(),
189  true) {}
190 };
191 
192 template <typename DATA_TYPE>
194  std::vector<DATA_TYPE>& buffer;
196  const Array& array,
197  std::vector<DATA_TYPE>& buffer,
198  Importer_NS::BadRowsTracker* const bad_rows_tracker)
199  : DataBufferBase(cd, array, bad_rows_tracker), buffer(buffer) {}
200 };
201 
202 constexpr int64_t kMillisecondsInSecond = 1000L;
203 constexpr int64_t kMicrosecondsInSecond = 1000L * 1000L;
204 constexpr int64_t kNanosecondsinSecond = 1000L * 1000L * 1000L;
205 constexpr int32_t kSecondsInDay = 86400;
206 
207 static const std::map<std::pair<int32_t, arrow::TimeUnit::type>,
208  std::pair<SQLOps, int64_t>>
209  _precision_scale_lookup{{{0, TimeUnit::MILLI}, {kDIVIDE, kMillisecondsInSecond}},
210  {{0, TimeUnit::MICRO}, {kDIVIDE, kMicrosecondsInSecond}},
211  {{0, TimeUnit::NANO}, {kDIVIDE, kNanosecondsinSecond}},
212  {{3, TimeUnit::SECOND}, {kMULTIPLY, kMicrosecondsInSecond}},
213  {{3, TimeUnit::MICRO}, {kDIVIDE, kMillisecondsInSecond}},
214  {{3, TimeUnit::NANO}, {kDIVIDE, kMicrosecondsInSecond}},
215  {{6, TimeUnit::SECOND}, {kMULTIPLY, kMicrosecondsInSecond}},
216  {{6, TimeUnit::MILLI}, {kMULTIPLY, kMillisecondsInSecond}},
217  {{6, TimeUnit::NANO}, {kDIVIDE, kMillisecondsInSecond}},
218  {{9, TimeUnit::SECOND}, {kMULTIPLY, kNanosecondsinSecond}},
219  {{9, TimeUnit::MILLI}, {kMULTIPLY, kMicrosecondsInSecond}},
220  {{9, TimeUnit::MICRO}, {kMULTIPLY, kMillisecondsInSecond}}};
221 
222 // models the variant values of Arrow Array (RHS)
223 template <typename VALUE_TYPE>
226  const VALUE_TYPE v;
227  const int32_t dimension;
228  ArrowValueBase(const DataBufferBase& data, const VALUE_TYPE& v)
229  : data(data)
230  , v(v)
231  , dimension(data.cd->columnType.is_high_precision_timestamp()
232  ? data.cd->columnType.get_dimension()
233  : 0) {}
234  template <bool enabled = std::is_integral<VALUE_TYPE>::value>
235  int64_t resolve_time(const VALUE_TYPE& v, std::enable_if_t<enabled>* = 0) const {
236  const auto& type_id = data.arrow_type.id();
237  if (type_id == Type::DATE32 || type_id == Type::DATE64) {
238  auto& date_type = static_cast<const DateType&>(data.arrow_type);
239  switch (date_type.unit()) {
240  case DateUnit::DAY:
241  return v * kSecondsInDay;
242  case DateUnit::MILLI:
243  return v / kMillisecondsInSecond;
244  }
245  } else if (type_id == Type::TIME32 || type_id == Type::TIME64 ||
246  type_id == Type::TIMESTAMP) {
247  auto& time_type = static_cast<const TimeType&>(data.arrow_type);
248  const auto result =
249  _precision_scale_lookup.find(std::make_pair(dimension, time_type.unit()));
250  if (result != _precision_scale_lookup.end()) {
251  const auto scale = result->second;
252  return scale.first == kMULTIPLY ? v * scale.second : v / scale.second;
253  } else {
254  return v;
255  }
256  }
257  UNREACHABLE() << data.arrow_type << " is not a valid Arrow time or date type";
258  return 0;
259  }
260  template <bool enabled = std::is_integral<VALUE_TYPE>::value>
261  int64_t resolve_time(const VALUE_TYPE& v, std::enable_if_t<!enabled>* = 0) const {
262  static_assert(enabled, "unreachable");
263  return 0;
264  }
265 };
266 
267 template <typename VALUE_TYPE>
268 struct ArrowValue : ArrowValueBase<VALUE_TYPE> {};
269 
270 template <>
272  using VALUE_TYPE = void*;
273  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
274  : ArrowValueBase<VALUE_TYPE>(data, v) {}
275  template <typename DATA_TYPE, typename = enable_if_integral<DATA_TYPE>>
276  explicit operator const DATA_TYPE() const {
277  return inline_fixed_encoding_null_val(data.cd->columnType);
278  }
279  template <typename DATA_TYPE, typename = enable_if_floating<DATA_TYPE>>
280  explicit operator DATA_TYPE() const {
281  return inline_fp_null_val(data.cd->columnType);
282  }
283  explicit operator const std::string() const { return std::string(); }
284 };
285 
286 template <>
287 struct ArrowValue<bool> : ArrowValueBase<bool> {
288  using VALUE_TYPE = bool;
289  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
290  : ArrowValueBase<VALUE_TYPE>(data, v) {}
291  template <typename DATA_TYPE, typename = enable_if_integral<DATA_TYPE>>
292  explicit operator const DATA_TYPE() const {
293  if (!(data.cd->columnType.is_number() || data.cd->columnType.is_boolean())) {
294  type_conversion_error("bool", data.cd, data.bad_rows_tracker);
295  }
296  return v;
297  }
298  template <typename DATA_TYPE, typename = enable_if_floating<DATA_TYPE>>
299  explicit operator DATA_TYPE() const {
300  return v ? 1 : 0;
301  }
302  explicit operator const std::string() const { return v ? "T" : "F"; }
303 };
304 
305 template <>
306 struct ArrowValue<float> : ArrowValueBase<float> {
307  using VALUE_TYPE = float;
308  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
309  : ArrowValueBase<VALUE_TYPE>(data, v) {}
310  template <typename DATA_TYPE, typename = enable_if_integral<DATA_TYPE>>
311  explicit operator const DATA_TYPE() const {
312  const auto ti = data.cd->columnType;
313  DATA_TYPE v = ti.is_decimal() ? this->v * pow(10, ti.get_scale()) : this->v;
314  if (!(std::numeric_limits<DATA_TYPE>::lowest() < v &&
315  v <= std::numeric_limits<DATA_TYPE>::max())) {
316  data_conversion_error<DATA_TYPE>(v, data.cd, data.bad_rows_tracker);
317  }
318  return v;
319  }
320  template <typename DATA_TYPE, typename = enable_if_floating<DATA_TYPE>>
321  explicit operator DATA_TYPE() const {
322  return v;
323  }
324  explicit operator const std::string() const { return std::to_string(v); }
325 };
326 
327 template <>
328 struct ArrowValue<double> : ArrowValueBase<double> {
329  using VALUE_TYPE = double;
330  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
331  : ArrowValueBase<VALUE_TYPE>(data, v) {}
332  template <typename DATA_TYPE, typename = enable_if_integral<DATA_TYPE>>
333  explicit operator const DATA_TYPE() const {
334  const auto ti = data.cd->columnType;
335  DATA_TYPE v = ti.is_decimal() ? this->v * pow(10, ti.get_scale()) : this->v;
336  if (!(std::numeric_limits<DATA_TYPE>::lowest() < v &&
337  v <= std::numeric_limits<DATA_TYPE>::max())) {
338  data_conversion_error<DATA_TYPE>(v, data.cd, data.bad_rows_tracker);
339  }
340  return v;
341  }
342  template <typename DATA_TYPE, typename = enable_if_floating<DATA_TYPE>>
343  explicit operator DATA_TYPE() const {
344  if (std::is_same<DATA_TYPE, float>::value) {
345  if (!(std::numeric_limits<float>::lowest() < v &&
346  v <= std::numeric_limits<float>::max())) {
347  data_conversion_error<float>(v, data.cd, data.bad_rows_tracker);
348  }
349  }
350  return v;
351  }
352  explicit operator const std::string() const { return std::to_string(v); }
353 };
354 
355 template <>
356 struct ArrowValue<int64_t> : ArrowValueBase<int64_t> {
357  using VALUE_TYPE = int64_t;
358  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
359  : ArrowValueBase<VALUE_TYPE>(data, v) {}
360  template <typename DATA_TYPE, typename = enable_if_integral<DATA_TYPE>>
361  explicit operator const DATA_TYPE() const {
362  int64_t v = this->v;
363  if (std::is_same<int64_t, DATA_TYPE>::value) {
364  } else if (std::numeric_limits<DATA_TYPE>::lowest() < v &&
365  v <= std::numeric_limits<DATA_TYPE>::max()) {
366  } else {
367  data_conversion_error<DATA_TYPE>(v, data.cd, data.bad_rows_tracker);
368  }
369  if (data.cd->columnType.is_time()) {
370  v = this->resolve_time(v);
371  }
372  return v;
373  }
374  template <typename DATA_TYPE, typename = enable_if_floating<DATA_TYPE>>
375  explicit operator DATA_TYPE() const {
376  return v;
377  }
378  explicit operator const std::string() const {
379  const auto& type_id = data.arrow_type.id();
380  if (type_id == Type::DATE32 || type_id == Type::DATE64) {
381  auto& date_type = static_cast<const DateType&>(data.arrow_type);
382  SQLTypeInfo ti(kDATE);
383  Datum datum{.bigintval = date_type.unit() == DateUnit::MILLI
385  : v};
386  return DatumToString(datum, ti);
387  } else if (type_id == Type::TIME32 || type_id == Type::TIME64 ||
388  type_id == Type::TIMESTAMP) {
389  auto& time_type = static_cast<const TimeType&>(data.arrow_type);
390  const auto result =
391  _precision_scale_lookup.find(std::make_pair(0, time_type.unit()));
392  int64_t divisor{1};
393  if (result != _precision_scale_lookup.end()) {
394  divisor = result->second.second;
395  }
397  Datum datum{.bigintval = v / divisor};
398  auto time_str = DatumToString(datum, ti);
399  if (divisor != 1 && v % divisor) {
400  time_str += "." + std::to_string(v % divisor);
401  }
402  return time_str;
403  }
404  return std::to_string(v);
405  }
406 };
407 
408 template <>
409 struct ArrowValue<std::string> : ArrowValueBase<std::string> {
410  using VALUE_TYPE = std::string;
411  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
412  : ArrowValueBase<VALUE_TYPE>(data, v) {}
413  explicit operator const bool() const {
414  if (v.size() == 0) {
415  return inline_int_null_value<int8_t>();
416  }
417  try {
418  SQLTypeInfo ti(kBOOLEAN);
419  auto datum = StringToDatum(v, ti);
420  return datum.boolval;
421  } catch (...) {
422  data_conversion_error(v, data.cd, data.bad_rows_tracker);
423  return false;
424  }
425  }
426  template <typename DATA_TYPE, typename = enable_if_integral_not_bool<DATA_TYPE>>
427  explicit operator const DATA_TYPE() const {
428  if (v.size() == 0) {
429  return inline_fixed_encoding_null_val(data.cd->columnType);
430  }
431  try {
432  auto ti = data.cd->columnType;
433  auto datum = StringToDatum(v, ti);
434  return datum.bigintval;
435  } catch (...) {
436  data_conversion_error(v, data.cd, data.bad_rows_tracker);
437  return 0;
438  }
439  }
440  template <typename DATA_TYPE, typename = enable_if_floating<DATA_TYPE>>
441  explicit operator DATA_TYPE() const {
442  return atof(v.data());
443  }
444  explicit operator const std::string() const { return v; }
445 };
446 
447 template <>
448 struct ArrowValue<Decimal128> : ArrowValueBase<Decimal128> {
449  using VALUE_TYPE = Decimal128;
450  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
451  : ArrowValueBase<VALUE_TYPE>(data, v) {
452  // omni decimal has only 64 bits
453  arrow_throw_if(!(v.high_bits() == 0 || v.high_bits() == -1),
454  error_context(data.cd, data.bad_rows_tracker) +
455  "Truncation error on Arrow Decimal128 value");
456  }
457  template <typename DATA_TYPE, typename = enable_if_integral<DATA_TYPE>>
458  explicit operator const DATA_TYPE() const {
459  int64_t v = static_cast<int64_t>(this->v);
460  if (data.cd->columnType.is_decimal()) {
461  return convert_decimal_value_to_scale(v, data.old_type, data.new_type);
462  }
463  if (data.arrow_decimal_scale) {
464  v = std::llround(v / pow(10, data.arrow_decimal_scale));
465  }
466  if (std::is_same<int64_t, DATA_TYPE>::value) {
467  } else if (std::numeric_limits<DATA_TYPE>::lowest() < v &&
468  v <= std::numeric_limits<DATA_TYPE>::max()) {
469  } else {
470  data_conversion_error<DATA_TYPE>(v, data.cd, data.bad_rows_tracker);
471  }
472  return v;
473  }
474  template <typename DATA_TYPE, typename = enable_if_floating<DATA_TYPE>>
475  explicit operator DATA_TYPE() const {
476  int64_t v = static_cast<int64_t>(this->v);
477  return data.arrow_decimal_scale ? v / pow(10, data.arrow_decimal_scale) : v;
478  }
479  explicit operator const std::string() const {
480  return v.ToString(data.arrow_decimal_scale);
481  }
482 };
483 
484 // appends a converted RHS value to LHS data block
485 template <typename DATA_TYPE>
486 inline auto& operator<<(DataBuffer<DATA_TYPE>& data, const VarValue& var) {
487  boost::apply_visitor(
488  [&data](const auto& v) {
489  data.buffer.push_back(DATA_TYPE(ArrowValue<exprtype(v)>(data, v)));
490  },
491  var);
492  return data;
493 }
494 
495 } // namespace
496 #endif // ARROW_IMPORTER_H
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
int64_t resolve_time(const VALUE_TYPE &v, std::enable_if_t<!enabled > *=0) const
std::string DatumToString(Datum d, const SQLTypeInfo &ti)
Definition: Datum.cpp:193
#define exprtype(expr)
Definition: ArrowImporter.h:73
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
std::string error_context(const ColumnDescriptor *cd, Importer_NS::BadRowsTracker *const bad_rows_tracker)
Definition: ArrowImporter.h:75
#define LOG(tag)
Definition: Logger.h:182
#define UNREACHABLE()
Definition: Logger.h:231
Constants for Builtin SQL Types supported by MapD.
auto value_getter(const Array &array, const ColumnDescriptor *cd, Importer_NS::BadRowsTracker *const bad_rows_tracker)
#define NUMERIC_CASE(tid, src_type, var_type)
Definition: ArrowImporter.h:95
double inline_fp_null_val(const SQL_TYPE_INFO &ti)
std::string to_string(char const *&&v)
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
static const std::map< std::pair< int32_t, arrow::TimeUnit::type >, std::pair< SQLOps, int64_t > > _precision_scale_lookup
std::string get_type_name() const
Definition: sqltypes.h:426
T v(const TargetValue &r)
int64_t bigintval
Definition: sqltypes.h:126
void type_conversion_error(const std::string pt, const ColumnDescriptor *cd, Importer_NS::BadRowsTracker *const bad_rows_tracker)
typename std::enable_if_t< std::is_floating_point< T >::value, T > enable_if_floating
Definition: ArrowImporter.h:71
specifies the content in-memory of a row in the column metadata table
VarValue get_string_value(const Array &array, const int64_t idx)
Definition: ArrowImporter.h:90
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
Definition: sqltypes.h:55
Definition: Importer.h:66
Importer_NS::BadRowsTracker *const bad_rows_tracker
int64_t convert_decimal_value_to_scale(const int64_t decimal_value, const SQLTypeInfo &type_info, const SQLTypeInfo &new_type_info)
Definition: Datum.cpp:284
Datum StringToDatum(const std::string &s, SQLTypeInfo &ti)
Definition: Datum.cpp:90
typename std::enable_if_t< std::is_integral< T >::value, T > enable_if_integral
Definition: ArrowImporter.h:65
#define STRING_CASE(tid, src_type)
Definition: ArrowImporter.h:98
void data_conversion_error(const std::string &v, const ColumnDescriptor *cd, Importer_NS::BadRowsTracker *const bad_rows_tracker)
DataBuffer(const ColumnDescriptor *cd, const Array &array, std::vector< DATA_TYPE > &buffer, Importer_NS::BadRowsTracker *const bad_rows_tracker)
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
typename std::enable_if_t< std::is_integral< T >::value &&!std::is_same< T, bool >::value, T > enable_if_integral_not_bool
Definition: ArrowImporter.h:69
SQLTypeInfo columnType
VarValue get_numeric_value(const Array &array, const int64_t idx)
Definition: ArrowImporter.h:84
DataBufferBase(const ColumnDescriptor *cd, const Array &array, Importer_NS::BadRowsTracker *const bad_rows_tracker)
std::string columnName
ArrowValueBase(const DataBufferBase &data, const VALUE_TYPE &v)
void arrow_throw_if(const bool cond, const std::string &message)
Definition: ArrowImporter.h:40
int64_t resolve_time(const VALUE_TYPE &v, std::enable_if_t< enabled > *=0) const
boost::variant< bool, float, double, int64_t, std::string, void *, Decimal128 > VarValue
Definition: ArrowImporter.h:62