OmniSciDB  8a228a1076
ArrowImporter.h
Go to the documentation of this file.
1 /*
2  * Copyright 2019 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 #ifndef ARROW_IMPORTER_H
17 #define ARROW_IMPORTER_H
18 
19 #include <cstdlib>
20 #include <ctime>
21 #include <map>
22 #include <mutex>
23 
24 #include <arrow/api.h>
25 #include <arrow/io/api.h>
26 #include <boost/algorithm/string.hpp>
27 #include <boost/variant.hpp>
28 
29 #include "Shared/SqlTypesLayout.h"
31 #include "Shared/sqltypes.h"
32 
33 using arrow::Array;
34 using arrow::Type;
35 
36 struct ArrowImporterException : std::runtime_error {
37  using std::runtime_error::runtime_error;
38 };
39 
40 template <typename T = ArrowImporterException>
41 inline void arrow_throw_if(const bool cond, const std::string& message) {
42  if (cond) {
43  // work around race from goooogle log
44  static std::mutex mtx;
45  std::unique_lock<std::mutex> lock(mtx);
46  LOG(ERROR) << message;
47  throw T(message);
48  }
49 }
50 
51 #ifdef ENABLE_IMPORT_PARQUET
52 #include <parquet/api/reader.h>
53 #include <parquet/api/writer.h>
54 #include <parquet/arrow/reader.h>
55 #include <parquet/exception.h>
56 #endif // ENABLE_IMPORT_PARQUET
57 
58 #include "arrow/util/decimal.h"
59 
60 namespace {
61 
62 using VarValue =
63  boost::variant<bool, float, double, int64_t, std::string, void*, arrow::Decimal128>;
64 
65 template <typename T>
66 using enable_if_integral = typename std::enable_if_t<std::is_integral<T>::value, T>;
67 template <typename T>
69  typename std::enable_if_t<std::is_integral<T>::value && !std::is_same<T, bool>::value,
70  T>;
71 template <typename T>
72 using enable_if_floating = typename std::enable_if_t<std::is_floating_point<T>::value, T>;
73 
74 #define exprtype(expr) std::decay_t<decltype(expr)>
75 
76 inline std::string error_context(const ColumnDescriptor* cd,
77  import_export::BadRowsTracker* const bad_rows_tracker) {
78  return bad_rows_tracker ? "File " + bad_rows_tracker->file_name + ", row-group " +
79  std::to_string(bad_rows_tracker->row_group) +
80  (cd ? ", column " + cd->columnName + ": " : "")
81  : std::string();
82 }
83 
84 template <typename SrcType, typename DstType>
85 inline VarValue get_numeric_value(const arrow::Array& array, const int64_t idx) {
86  using ArrayType = typename arrow::TypeTraits<SrcType>::ArrayType;
87  return (DstType) static_cast<const ArrayType&>(array).Value(idx);
88 }
89 
90 template <typename SrcType>
91 inline VarValue get_string_value(const arrow::Array& array, const int64_t idx) {
92  using ArrayType = typename arrow::TypeTraits<SrcType>::ArrayType;
93  return static_cast<const ArrayType&>(array).GetString(idx);
94 }
95 
96 #define NUMERIC_CASE(tid, src_type, var_type) \
97  case arrow::Type::tid: \
98  return get_numeric_value<src_type, var_type>;
99 #define STRING_CASE(tid, src_type) \
100  case arrow::Type::tid: \
101  return get_string_value<src_type>;
102 
103 inline auto value_getter(const arrow::Array& array,
104  const ColumnDescriptor* cd,
105  import_export::BadRowsTracker* const bad_rows_tracker) {
106  switch (array.type_id()) {
107  NUMERIC_CASE(BOOL, arrow::BooleanType, bool)
108  NUMERIC_CASE(UINT8, arrow::UInt8Type, int64_t)
109  NUMERIC_CASE(UINT16, arrow::UInt16Type, int64_t)
110  NUMERIC_CASE(UINT32, arrow::UInt32Type, int64_t)
111  NUMERIC_CASE(UINT64, arrow::Int64Type, int64_t)
112  NUMERIC_CASE(INT8, arrow::Int8Type, int64_t)
113  NUMERIC_CASE(INT16, arrow::Int16Type, int64_t)
114  NUMERIC_CASE(INT32, arrow::Int32Type, int64_t)
115  NUMERIC_CASE(INT64, arrow::Int64Type, int64_t)
116  NUMERIC_CASE(FLOAT, arrow::FloatType, float)
117  NUMERIC_CASE(DOUBLE, arrow::DoubleType, double)
118  NUMERIC_CASE(DATE32, arrow::Date32Type, int64_t)
119  NUMERIC_CASE(DATE64, arrow::Date64Type, int64_t)
120  NUMERIC_CASE(TIME64, arrow::Time64Type, int64_t)
121  NUMERIC_CASE(TIME32, arrow::Time32Type, int64_t)
122  NUMERIC_CASE(TIMESTAMP, arrow::TimestampType, int64_t)
123  NUMERIC_CASE(DECIMAL, arrow::Decimal128Type, arrow::Decimal128)
124  STRING_CASE(STRING, arrow::StringType)
125  STRING_CASE(BINARY, arrow::BinaryType)
126  default:
127  arrow_throw_if(true,
128  error_context(cd, bad_rows_tracker) + "Parquet type " +
129  array.type()->name() + " is not supported");
130  throw;
131  }
132 }
133 
134 inline void type_conversion_error(const std::string pt,
135  const ColumnDescriptor* cd,
136  import_export::BadRowsTracker* const bad_rows_tracker) {
137  arrow_throw_if(true,
138  error_context(cd, bad_rows_tracker) +
139  "Invalid type conversion from parquet " + pt + " type to " +
140  cd->columnType.get_type_name());
141 }
142 
143 template <typename DATA_TYPE, typename VALUE_TYPE>
144 inline void data_conversion_error(const VALUE_TYPE v,
145  const ColumnDescriptor* cd,
146  import_export::BadRowsTracker* const bad_rows_tracker) {
147  arrow_throw_if(true,
148  error_context(cd, bad_rows_tracker) +
149  "Invalid data conversion from parquet value " + std::to_string(v) +
150  " to " + std::to_string(DATA_TYPE(v)));
151 }
152 
153 inline void data_conversion_error(const std::string& v,
154  const ColumnDescriptor* cd,
155  import_export::BadRowsTracker* const bad_rows_tracker) {
156  arrow_throw_if(true,
157  error_context(cd, bad_rows_tracker) +
158  "Invalid data conversion from parquet string '" + v + "' to " +
159  cd->columnType.get_type_name() + " column type");
160 }
161 
162 // models the variant data buffers of TypedImportBuffer (LHS)
165  const arrow::Array& array;
167  // in case of arrow-decimal to omni-decimal conversion
168  // dont get/set these info on every row of arrow array
169  const arrow::DataType& arrow_type;
174  const arrow::Array& array,
175  import_export::BadRowsTracker* const bad_rows_tracker)
176  : cd(cd)
177  , array(array)
178  , bad_rows_tracker(bad_rows_tracker)
179  , arrow_type(*array.type())
180  , arrow_decimal_scale(
181  arrow_type.id() == arrow::Type::DECIMAL
182  ? static_cast<const arrow::Decimal128Type&>(arrow_type).scale()
183  : 0)
184  , old_type(cd->columnType.get_type(),
185  cd->columnType.get_dimension(),
186  arrow_decimal_scale,
187  true)
188  , new_type(cd->columnType.get_type(),
189  cd->columnType.get_dimension(),
190  cd->columnType.get_scale(),
191  true) {}
192 };
193 
194 template <typename DATA_TYPE>
196  std::vector<DATA_TYPE>& buffer;
198  const arrow::Array& array,
199  std::vector<DATA_TYPE>& buffer,
200  import_export::BadRowsTracker* const bad_rows_tracker)
201  : DataBufferBase(cd, array, bad_rows_tracker), buffer(buffer) {}
202 };
203 
204 constexpr int64_t kMillisecondsInSecond = 1000L;
205 constexpr int64_t kMicrosecondsInSecond = 1000L * 1000L;
206 constexpr int64_t kNanosecondsinSecond = 1000L * 1000L * 1000L;
207 constexpr int32_t kSecondsInDay = 86400;
208 
209 static const std::map<std::pair<int32_t, arrow::TimeUnit::type>,
210  std::pair<SQLOps, int64_t>>
212  {{0, arrow::TimeUnit::MILLI}, {kDIVIDE, kMillisecondsInSecond}},
213  {{0, arrow::TimeUnit::MICRO}, {kDIVIDE, kMicrosecondsInSecond}},
214  {{0, arrow::TimeUnit::NANO}, {kDIVIDE, kNanosecondsinSecond}},
215  {{3, arrow::TimeUnit::SECOND}, {kMULTIPLY, kMicrosecondsInSecond}},
216  {{3, arrow::TimeUnit::MICRO}, {kDIVIDE, kMillisecondsInSecond}},
217  {{3, arrow::TimeUnit::NANO}, {kDIVIDE, kMicrosecondsInSecond}},
218  {{6, arrow::TimeUnit::SECOND}, {kMULTIPLY, kMicrosecondsInSecond}},
219  {{6, arrow::TimeUnit::MILLI}, {kMULTIPLY, kMillisecondsInSecond}},
220  {{6, arrow::TimeUnit::NANO}, {kDIVIDE, kMillisecondsInSecond}},
221  {{9, arrow::TimeUnit::SECOND}, {kMULTIPLY, kNanosecondsinSecond}},
222  {{9, arrow::TimeUnit::MILLI}, {kMULTIPLY, kMicrosecondsInSecond}},
223  {{9, arrow::TimeUnit::MICRO}, {kMULTIPLY, kMillisecondsInSecond}}};
224 
225 // models the variant values of Arrow Array (RHS)
226 template <typename VALUE_TYPE>
229  const VALUE_TYPE v;
230  const int32_t dimension;
231  ArrowValueBase(const DataBufferBase& data, const VALUE_TYPE& v)
232  : data(data)
233  , v(v)
234  , dimension(data.cd->columnType.is_high_precision_timestamp()
235  ? data.cd->columnType.get_dimension()
236  : 0) {}
237  template <bool enabled = std::is_integral<VALUE_TYPE>::value>
238  int64_t resolve_time(const VALUE_TYPE& v, std::enable_if_t<enabled>* = 0) const {
239  const auto& type_id = data.arrow_type.id();
240  if (type_id == arrow::Type::DATE32 || type_id == arrow::Type::DATE64) {
241  auto& date_type = static_cast<const arrow::DateType&>(data.arrow_type);
242  switch (date_type.unit()) {
243  case arrow::DateUnit::DAY:
244  return v * kSecondsInDay;
245  case arrow::DateUnit::MILLI:
246  return v / kMillisecondsInSecond;
247  }
248  } else if (type_id == arrow::Type::TIME32 || type_id == arrow::Type::TIME64 ||
249  type_id == arrow::Type::TIMESTAMP) {
250  auto& time_type = static_cast<const arrow::TimeType&>(data.arrow_type);
251  const auto result =
252  _precision_scale_lookup.find(std::make_pair(dimension, time_type.unit()));
253  if (result != _precision_scale_lookup.end()) {
254  const auto scale = result->second;
255  return scale.first == kMULTIPLY ? v * scale.second : v / scale.second;
256  } else {
257  return v;
258  }
259  }
260  UNREACHABLE() << data.arrow_type << " is not a valid Arrow time or date type";
261  return 0;
262  }
263  template <bool enabled = std::is_integral<VALUE_TYPE>::value>
264  int64_t resolve_time(const VALUE_TYPE& v, std::enable_if_t<!enabled>* = 0) const {
265  static_assert(enabled, "unreachable");
266  return 0;
267  }
268 };
269 
270 template <typename VALUE_TYPE>
271 struct ArrowValue : ArrowValueBase<VALUE_TYPE> {};
272 
273 template <>
275  using VALUE_TYPE = void*;
276  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
277  : ArrowValueBase<VALUE_TYPE>(data, v) {}
278  template <typename DATA_TYPE, typename = enable_if_integral<DATA_TYPE>>
279  explicit operator const DATA_TYPE() const {
280  return inline_fixed_encoding_null_val(data.cd->columnType);
281  }
282  template <typename DATA_TYPE, typename = enable_if_floating<DATA_TYPE>>
283  explicit operator DATA_TYPE() const {
284  return inline_fp_null_val(data.cd->columnType);
285  }
286  explicit operator const std::string() const { return std::string(); }
287 };
288 
289 template <>
290 struct ArrowValue<bool> : ArrowValueBase<bool> {
291  using VALUE_TYPE = bool;
292  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
293  : ArrowValueBase<VALUE_TYPE>(data, v) {}
294  template <typename DATA_TYPE, typename = enable_if_integral<DATA_TYPE>>
295  explicit operator const DATA_TYPE() const {
296  if (!(data.cd->columnType.is_number() || data.cd->columnType.is_boolean())) {
297  type_conversion_error("bool", data.cd, data.bad_rows_tracker);
298  }
299  return v;
300  }
301  template <typename DATA_TYPE, typename = enable_if_floating<DATA_TYPE>>
302  explicit operator DATA_TYPE() const {
303  return v ? 1 : 0;
304  }
305  explicit operator const std::string() const { return v ? "T" : "F"; }
306 };
307 
308 template <>
309 struct ArrowValue<float> : ArrowValueBase<float> {
310  using VALUE_TYPE = float;
311  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
312  : ArrowValueBase<VALUE_TYPE>(data, v) {}
313  template <typename DATA_TYPE, typename = enable_if_integral<DATA_TYPE>>
314  explicit operator const DATA_TYPE() const {
315  const auto ti = data.cd->columnType;
316  DATA_TYPE v = ti.is_decimal() ? this->v * pow(10, ti.get_scale()) : this->v;
317  if (!(std::numeric_limits<DATA_TYPE>::lowest() < v &&
318  v <= std::numeric_limits<DATA_TYPE>::max())) {
319  data_conversion_error<DATA_TYPE>(v, data.cd, data.bad_rows_tracker);
320  }
321  return v;
322  }
323  template <typename DATA_TYPE, typename = enable_if_floating<DATA_TYPE>>
324  explicit operator DATA_TYPE() const {
325  return v;
326  }
327  explicit operator const std::string() const { return std::to_string(v); }
328 };
329 
330 template <>
331 struct ArrowValue<double> : ArrowValueBase<double> {
332  using VALUE_TYPE = double;
333  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
334  : ArrowValueBase<VALUE_TYPE>(data, v) {}
335  template <typename DATA_TYPE, typename = enable_if_integral<DATA_TYPE>>
336  explicit operator const DATA_TYPE() const {
337  const auto ti = data.cd->columnType;
338  DATA_TYPE v = ti.is_decimal() ? this->v * pow(10, ti.get_scale()) : this->v;
339  if (!(std::numeric_limits<DATA_TYPE>::lowest() < v &&
340  v <= std::numeric_limits<DATA_TYPE>::max())) {
341  data_conversion_error<DATA_TYPE>(v, data.cd, data.bad_rows_tracker);
342  }
343  return v;
344  }
345  template <typename DATA_TYPE, typename = enable_if_floating<DATA_TYPE>>
346  explicit operator DATA_TYPE() const {
347  if (std::is_same<DATA_TYPE, float>::value) {
348  if (!(std::numeric_limits<float>::lowest() < v &&
349  v <= std::numeric_limits<float>::max())) {
350  data_conversion_error<float>(v, data.cd, data.bad_rows_tracker);
351  }
352  }
353  return v;
354  }
355  explicit operator const std::string() const { return std::to_string(v); }
356 };
357 
358 template <>
359 struct ArrowValue<int64_t> : ArrowValueBase<int64_t> {
360  using VALUE_TYPE = int64_t;
361  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
362  : ArrowValueBase<VALUE_TYPE>(data, v) {}
363  template <typename DATA_TYPE, typename = enable_if_integral<DATA_TYPE>>
364  explicit operator const DATA_TYPE() const {
365  int64_t v = this->v;
366  if (std::is_same<int64_t, DATA_TYPE>::value) {
367  } else if (std::numeric_limits<DATA_TYPE>::lowest() < v &&
368  v <= std::numeric_limits<DATA_TYPE>::max()) {
369  } else {
370  data_conversion_error<DATA_TYPE>(v, data.cd, data.bad_rows_tracker);
371  }
372  if (data.cd->columnType.is_time()) {
373  v = this->resolve_time(v);
374  }
375  return v;
376  }
377  template <typename DATA_TYPE, typename = enable_if_floating<DATA_TYPE>>
378  explicit operator DATA_TYPE() const {
379  return v;
380  }
381  explicit operator const std::string() const {
382  const auto& type_id = data.arrow_type.id();
383  if (type_id == arrow::Type::DATE32 || type_id == arrow::Type::DATE64) {
384  auto& date_type = static_cast<const arrow::DateType&>(data.arrow_type);
385  SQLTypeInfo ti(kDATE);
386  Datum datum{.bigintval = date_type.unit() == arrow::DateUnit::MILLI
388  : v};
389  return DatumToString(datum, ti);
390  } else if (type_id == arrow::Type::TIME32 || type_id == arrow::Type::TIME64 ||
391  type_id == arrow::Type::TIMESTAMP) {
392  auto& time_type = static_cast<const arrow::TimeType&>(data.arrow_type);
393  const auto result =
394  _precision_scale_lookup.find(std::make_pair(0, time_type.unit()));
395  int64_t divisor{1};
396  if (result != _precision_scale_lookup.end()) {
397  divisor = result->second.second;
398  }
400  Datum datum{.bigintval = v / divisor};
401  auto time_str = DatumToString(datum, ti);
402  if (divisor != 1 && v % divisor) {
403  time_str += "." + std::to_string(v % divisor);
404  }
405  return time_str;
406  }
407  return std::to_string(v);
408  }
409 };
410 
411 template <>
412 struct ArrowValue<std::string> : ArrowValueBase<std::string> {
413  using VALUE_TYPE = std::string;
414  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
415  : ArrowValueBase<VALUE_TYPE>(data, v) {}
416  explicit operator const bool() const {
417  if (v.size() == 0) {
418  return inline_int_null_value<int8_t>();
419  }
420  try {
421  SQLTypeInfo ti(kBOOLEAN);
422  auto datum = StringToDatum(v, ti);
423  return datum.boolval;
424  } catch (...) {
425  data_conversion_error(v, data.cd, data.bad_rows_tracker);
426  return false;
427  }
428  }
429  template <typename DATA_TYPE, typename = enable_if_integral_not_bool<DATA_TYPE>>
430  explicit operator const DATA_TYPE() const {
431  if (v.size() == 0) {
432  return inline_fixed_encoding_null_val(data.cd->columnType);
433  }
434  try {
435  auto ti = data.cd->columnType;
436  auto datum = StringToDatum(v, ti);
437  return datum.bigintval;
438  } catch (...) {
439  data_conversion_error(v, data.cd, data.bad_rows_tracker);
440  return 0;
441  }
442  }
443  template <typename DATA_TYPE, typename = enable_if_floating<DATA_TYPE>>
444  explicit operator DATA_TYPE() const {
445  return atof(v.data());
446  }
447  explicit operator const std::string() const { return v; }
448 };
449 
450 template <>
451 struct ArrowValue<arrow::Decimal128> : ArrowValueBase<arrow::Decimal128> {
452  using VALUE_TYPE = arrow::Decimal128;
453  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
454  : ArrowValueBase<VALUE_TYPE>(data, v) {
455  // omni decimal has only 64 bits
456  arrow_throw_if(!(v.high_bits() == 0 || v.high_bits() == -1),
457  error_context(data.cd, data.bad_rows_tracker) +
458  "Truncation error on Arrow Decimal128 value");
459  }
460  template <typename DATA_TYPE, typename = enable_if_integral<DATA_TYPE>>
461  explicit operator const DATA_TYPE() const {
462  int64_t v = static_cast<int64_t>(this->v);
463  if (data.cd->columnType.is_decimal()) {
464  return convert_decimal_value_to_scale(v, data.old_type, data.new_type);
465  }
466  if (data.arrow_decimal_scale) {
467  v = std::llround(v / pow(10, data.arrow_decimal_scale));
468  }
469  if (std::is_same<int64_t, DATA_TYPE>::value) {
470  } else if (std::numeric_limits<DATA_TYPE>::lowest() < v &&
471  v <= std::numeric_limits<DATA_TYPE>::max()) {
472  } else {
473  data_conversion_error<DATA_TYPE>(v, data.cd, data.bad_rows_tracker);
474  }
475  return v;
476  }
477  template <typename DATA_TYPE, typename = enable_if_floating<DATA_TYPE>>
478  explicit operator DATA_TYPE() const {
479  int64_t v = static_cast<int64_t>(this->v);
480  return data.arrow_decimal_scale ? v / pow(10, data.arrow_decimal_scale) : v;
481  }
482  explicit operator const std::string() const {
483  return v.ToString(data.arrow_decimal_scale);
484  }
485 };
486 
487 // appends a converted RHS value to LHS data block
488 template <typename DATA_TYPE>
489 inline auto& operator<<(DataBuffer<DATA_TYPE>& data, const VarValue& var) {
490  boost::apply_visitor(
491  [&data](const auto& v) {
492  data.buffer.push_back(DATA_TYPE(ArrowValue<exprtype(v)>(data, v)));
493  },
494  var);
495  return data;
496 }
497 
498 } // namespace
499 #endif // ARROW_IMPORTER_H
boost::variant< bool, float, double, int64_t, std::string, void *, arrow::Decimal128 > VarValue
Definition: ArrowImporter.h:63
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
int64_t resolve_time(const VALUE_TYPE &v, std::enable_if_t<!enabled > *=0) const
std::string DatumToString(Datum d, const SQLTypeInfo &ti)
Definition: Datum.cpp:236
#define exprtype(expr)
Definition: ArrowImporter.h:74
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
#define LOG(tag)
Definition: Logger.h:188
DataBufferBase(const ColumnDescriptor *cd, const arrow::Array &array, import_export::BadRowsTracker *const bad_rows_tracker)
auto value_getter(const arrow::Array &array, const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
#define UNREACHABLE()
Definition: Logger.h:241
Constants for Builtin SQL Types supported by OmniSci.
#define NUMERIC_CASE(tid, src_type, var_type)
Definition: ArrowImporter.h:96
double inline_fp_null_val(const SQL_TYPE_INFO &ti)
std::string to_string(char const *&&v)
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
static const std::map< std::pair< int32_t, arrow::TimeUnit::type >, std::pair< SQLOps, int64_t > > _precision_scale_lookup
void data_conversion_error(const std::string &v, const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
int64_t bigintval
Definition: sqltypes.h:138
typename std::enable_if_t< std::is_floating_point< T >::value, T > enable_if_floating
Definition: ArrowImporter.h:72
void type_conversion_error(const std::string pt, const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
import_export::BadRowsTracker *const bad_rows_tracker
DataBuffer(const ColumnDescriptor *cd, const arrow::Array &array, std::vector< DATA_TYPE > &buffer, import_export::BadRowsTracker *const bad_rows_tracker)
Datum StringToDatum(std::string_view s, SQLTypeInfo &ti)
Definition: Datum.cpp:124
specifies the content in-memory of a row in the column metadata table
VarValue get_string_value(const arrow::Array &array, const int64_t idx)
Definition: ArrowImporter.h:91
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
Definition: sqltypes.h:55
Definition: Importer.h:62
std::string get_type_name() const
Definition: sqltypes.h:362
int64_t convert_decimal_value_to_scale(const int64_t decimal_value, const SQLTypeInfo &type_info, const SQLTypeInfo &new_type_info)
Definition: Datum.cpp:315
typename std::enable_if_t< std::is_integral< T >::value, T > enable_if_integral
Definition: ArrowImporter.h:66
#define STRING_CASE(tid, src_type)
Definition: ArrowImporter.h:99
VarValue get_numeric_value(const arrow::Array &array, const int64_t idx)
Definition: ArrowImporter.h:85
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
typename std::enable_if_t< std::is_integral< T >::value &&!std::is_same< T, bool >::value, T > enable_if_integral_not_bool
Definition: ArrowImporter.h:70
SQLTypeInfo columnType
std::string error_context(const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
Definition: ArrowImporter.h:76
std::string columnName
ArrowValueBase(const DataBufferBase &data, const VALUE_TYPE &v)
void arrow_throw_if(const bool cond, const std::string &message)
Definition: ArrowImporter.h:41
int64_t resolve_time(const VALUE_TYPE &v, std::enable_if_t< enabled > *=0) const