OmniSciDB  c0231cc57d
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
ArrowImporter.h
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef ARROW_IMPORTER_H
18 #define ARROW_IMPORTER_H
19 
20 #include <cstdlib>
21 #include <ctime>
22 #include <map>
23 #include <mutex>
24 
25 #include <arrow/api.h>
26 #include <arrow/io/api.h>
27 #include <boost/algorithm/string.hpp>
28 #include <boost/variant.hpp>
29 
30 #include "Shared/SqlTypesLayout.h"
32 #include "Shared/sqltypes.h"
33 
34 using arrow::Array;
35 using arrow::Type;
36 
37 struct ArrowImporterException : std::runtime_error {
38  using std::runtime_error::runtime_error;
39 };
40 
41 template <typename T = ArrowImporterException>
42 inline void arrow_throw_if(const bool cond, const std::string& message) {
43  if (cond) {
44  // work around race from goooogle log
45  static std::mutex mtx;
46  std::unique_lock<std::mutex> lock(mtx);
47  LOG(ERROR) << message;
48  throw T(message);
49  }
50 }
51 
52 #ifdef ENABLE_IMPORT_PARQUET
53 #include <parquet/api/reader.h>
54 #include <parquet/api/writer.h>
55 #include <parquet/arrow/reader.h>
56 #include <parquet/exception.h>
57 #endif // ENABLE_IMPORT_PARQUET
58 
59 #include "arrow/util/decimal.h"
60 
61 namespace {
62 
63 using VarValue =
64  boost::variant<bool, float, double, int64_t, std::string, void*, arrow::Decimal128>;
65 
66 template <typename T>
67 using enable_if_integral = typename std::enable_if_t<std::is_integral<T>::value, T>;
68 template <typename T>
70  typename std::enable_if_t<std::is_integral<T>::value && !std::is_same<T, bool>::value,
71  T>;
72 template <typename T>
73 using enable_if_floating = typename std::enable_if_t<std::is_floating_point<T>::value, T>;
74 
75 #define exprtype(expr) std::decay_t<decltype(expr)>
76 
77 inline std::string error_context(const ColumnDescriptor* cd,
78  import_export::BadRowsTracker* const bad_rows_tracker) {
79  return bad_rows_tracker ? "File " + bad_rows_tracker->file_name + ", row-group " +
80  std::to_string(bad_rows_tracker->row_group) +
81  (cd ? ", column " + cd->columnName + ": " : "")
82  : std::string();
83 }
84 
85 template <typename SrcType, typename DstType>
86 inline VarValue get_numeric_value(const arrow::Array& array, const int64_t idx) {
87  using ArrayType = typename arrow::TypeTraits<SrcType>::ArrayType;
88  return (DstType) static_cast<const ArrayType&>(array).Value(idx);
89 }
90 
91 template <typename SrcType>
92 inline VarValue get_string_value(const arrow::Array& array, const int64_t idx) {
93  using ArrayType = typename arrow::TypeTraits<SrcType>::ArrayType;
94  return static_cast<const ArrayType&>(array).GetString(idx);
95 }
96 
97 #define NUMERIC_CASE(tid, src_type, var_type) \
98  case arrow::Type::tid: \
99  return get_numeric_value<src_type, var_type>;
100 #define STRING_CASE(tid, src_type) \
101  case arrow::Type::tid: \
102  return get_string_value<src_type>;
103 
104 inline auto value_getter(const arrow::Array& array,
105  const ColumnDescriptor* cd,
106  import_export::BadRowsTracker* const bad_rows_tracker) {
107  switch (array.type_id()) {
108  NUMERIC_CASE(BOOL, arrow::BooleanType, bool)
109  NUMERIC_CASE(UINT8, arrow::UInt8Type, int64_t)
110  NUMERIC_CASE(UINT16, arrow::UInt16Type, int64_t)
111  NUMERIC_CASE(UINT32, arrow::UInt32Type, int64_t)
112  NUMERIC_CASE(UINT64, arrow::Int64Type, int64_t)
113  NUMERIC_CASE(INT8, arrow::Int8Type, int64_t)
114  NUMERIC_CASE(INT16, arrow::Int16Type, int64_t)
115  NUMERIC_CASE(INT32, arrow::Int32Type, int64_t)
116  NUMERIC_CASE(INT64, arrow::Int64Type, int64_t)
117  NUMERIC_CASE(FLOAT, arrow::FloatType, float)
118  NUMERIC_CASE(DOUBLE, arrow::DoubleType, double)
119  NUMERIC_CASE(DATE32, arrow::Date32Type, int64_t)
120  NUMERIC_CASE(DATE64, arrow::Date64Type, int64_t)
121  NUMERIC_CASE(TIME64, arrow::Time64Type, int64_t)
122  NUMERIC_CASE(TIME32, arrow::Time32Type, int64_t)
123  NUMERIC_CASE(TIMESTAMP, arrow::TimestampType, int64_t)
124  NUMERIC_CASE(DECIMAL, arrow::Decimal128Type, arrow::Decimal128)
125  STRING_CASE(STRING, arrow::StringType)
126  STRING_CASE(BINARY, arrow::BinaryType)
127  default:
128  arrow_throw_if(true,
129  error_context(cd, bad_rows_tracker) + "Parquet type " +
130  array.type()->name() + " is not supported");
131  throw;
132  }
133 }
134 
135 inline void type_conversion_error(const std::string pt,
136  const ColumnDescriptor* cd,
137  import_export::BadRowsTracker* const bad_rows_tracker) {
138  arrow_throw_if(true,
139  error_context(cd, bad_rows_tracker) +
140  "Invalid type conversion from parquet " + pt + " type to " +
141  cd->columnType.get_type_name());
142 }
143 
144 template <typename DATA_TYPE, typename VALUE_TYPE>
145 inline void data_conversion_error(const VALUE_TYPE v,
146  const ColumnDescriptor* cd,
147  import_export::BadRowsTracker* const bad_rows_tracker) {
148  arrow_throw_if(true,
149  error_context(cd, bad_rows_tracker) +
150  "Invalid data conversion from parquet value " + std::to_string(v) +
151  " to " + std::to_string(DATA_TYPE(v)));
152 }
153 
154 inline void data_conversion_error(const std::string& v,
155  const ColumnDescriptor* cd,
156  import_export::BadRowsTracker* const bad_rows_tracker) {
157  arrow_throw_if(true,
158  error_context(cd, bad_rows_tracker) +
159  "Invalid data conversion from parquet string '" + v + "' to " +
160  cd->columnType.get_type_name() + " column type");
161 }
162 
163 // models the variant data buffers of TypedImportBuffer (LHS)
166  const arrow::Array& array;
168  // in case of arrow-decimal to omni-decimal conversion
169  // dont get/set these info on every row of arrow array
170  const arrow::DataType& arrow_type;
175  const arrow::Array& array,
176  import_export::BadRowsTracker* const bad_rows_tracker)
177  : cd(cd)
178  , array(array)
179  , bad_rows_tracker(bad_rows_tracker)
180  , arrow_type(*array.type())
181  , arrow_decimal_scale(
182  arrow_type.id() == arrow::Type::DECIMAL
183  ? static_cast<const arrow::Decimal128Type&>(arrow_type).scale()
184  : 0)
185  , old_type(cd->columnType.get_type(),
186  cd->columnType.get_dimension(),
187  arrow_decimal_scale,
188  true)
189  , new_type(cd->columnType.get_type(),
190  cd->columnType.get_dimension(),
191  cd->columnType.get_scale(),
192  true) {}
193 };
194 
195 template <typename DATA_TYPE>
197  std::vector<DATA_TYPE>& buffer;
199  const arrow::Array& array,
200  std::vector<DATA_TYPE>& buffer,
201  import_export::BadRowsTracker* const bad_rows_tracker)
202  : DataBufferBase(cd, array, bad_rows_tracker), buffer(buffer) {}
203 };
204 
205 constexpr int64_t kMillisecondsInSecond = 1000LL;
206 constexpr int64_t kMicrosecondsInSecond = 1000LL * 1000LL;
207 constexpr int64_t kNanosecondsinSecond = 1000LL * 1000LL * 1000LL;
208 constexpr int32_t kSecondsInDay = 86400;
209 
210 static const std::map<std::pair<int32_t, arrow::TimeUnit::type>,
211  std::pair<SQLOps, int64_t>>
213  {{0, arrow::TimeUnit::MILLI}, {kDIVIDE, kMillisecondsInSecond}},
214  {{0, arrow::TimeUnit::MICRO}, {kDIVIDE, kMicrosecondsInSecond}},
215  {{0, arrow::TimeUnit::NANO}, {kDIVIDE, kNanosecondsinSecond}},
216  {{3, arrow::TimeUnit::SECOND}, {kMULTIPLY, kMicrosecondsInSecond}},
217  {{3, arrow::TimeUnit::MICRO}, {kDIVIDE, kMillisecondsInSecond}},
218  {{3, arrow::TimeUnit::NANO}, {kDIVIDE, kMicrosecondsInSecond}},
219  {{6, arrow::TimeUnit::SECOND}, {kMULTIPLY, kMicrosecondsInSecond}},
220  {{6, arrow::TimeUnit::MILLI}, {kMULTIPLY, kMillisecondsInSecond}},
221  {{6, arrow::TimeUnit::NANO}, {kDIVIDE, kMillisecondsInSecond}},
222  {{9, arrow::TimeUnit::SECOND}, {kMULTIPLY, kNanosecondsinSecond}},
223  {{9, arrow::TimeUnit::MILLI}, {kMULTIPLY, kMicrosecondsInSecond}},
224  {{9, arrow::TimeUnit::MICRO}, {kMULTIPLY, kMillisecondsInSecond}}};
225 
226 // models the variant values of Arrow Array (RHS)
227 template <typename VALUE_TYPE>
230  const VALUE_TYPE v;
231  const int32_t dimension;
232  ArrowValueBase(const DataBufferBase& data, const VALUE_TYPE& v)
233  : data(data)
234  , v(v)
235  , dimension(data.cd->columnType.is_high_precision_timestamp()
236  ? data.cd->columnType.get_dimension()
237  : 0) {}
238  template <bool enabled = std::is_integral<VALUE_TYPE>::value>
239  int64_t resolve_time(const VALUE_TYPE& v, std::enable_if_t<enabled>* = 0) const {
240  const auto& type_id = data.arrow_type.id();
241  if (type_id == arrow::Type::DATE32 || type_id == arrow::Type::DATE64) {
242  auto& date_type = static_cast<const arrow::DateType&>(data.arrow_type);
243  switch (date_type.unit()) {
244  case arrow::DateUnit::DAY:
245  return v * kSecondsInDay;
246  case arrow::DateUnit::MILLI:
247  return v / kMillisecondsInSecond;
248  }
249  } else if (type_id == arrow::Type::TIME32 || type_id == arrow::Type::TIME64 ||
250  type_id == arrow::Type::TIMESTAMP) {
251  auto& time_type = static_cast<const arrow::TimeType&>(data.arrow_type);
252  const auto result =
253  _precision_scale_lookup.find(std::make_pair(dimension, time_type.unit()));
254  if (result != _precision_scale_lookup.end()) {
255  const auto scale = result->second;
256  return scale.first == kMULTIPLY ? v * scale.second : v / scale.second;
257  } else {
258  return v;
259  }
260  }
261  UNREACHABLE() << data.arrow_type << " is not a valid Arrow time or date type";
262  return 0;
263  }
264  template <bool enabled = std::is_integral<VALUE_TYPE>::value>
265  int64_t resolve_time(const VALUE_TYPE& v, std::enable_if_t<!enabled>* = 0) const {
266  static_assert(enabled, "unreachable");
267  return 0;
268  }
269 };
270 
271 template <typename VALUE_TYPE>
272 struct ArrowValue : ArrowValueBase<VALUE_TYPE> {};
273 
274 template <>
276  using VALUE_TYPE = void*;
277  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
278  : ArrowValueBase<VALUE_TYPE>(data, v) {}
279 
280  template <typename DATA_TYPE>
281  explicit operator DATA_TYPE() const {
282  if constexpr (std::is_integral<DATA_TYPE>::value) { // NOLINT
283  return inline_fixed_encoding_null_val(data.cd->columnType);
284  } else if constexpr (std::is_floating_point<DATA_TYPE>::value) { // NOLINT
285  return inline_fp_null_val(data.cd->columnType);
286  } else if constexpr (std::is_same<DATA_TYPE, std::string>::value) { // NOLINT
287  return std::string();
288  }
289  }
290 };
291 
292 template <>
293 struct ArrowValue<bool> : ArrowValueBase<bool> {
294  using VALUE_TYPE = bool;
295  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
296  : ArrowValueBase<VALUE_TYPE>(data, v) {}
297 
298  template <typename DATA_TYPE>
299  explicit operator DATA_TYPE() const {
300  if constexpr (std::is_integral<DATA_TYPE>::value) { // NOLINT
301  if (!(data.cd->columnType.is_number() || data.cd->columnType.is_boolean())) {
302  type_conversion_error("bool", data.cd, data.bad_rows_tracker);
303  }
304  return v;
305  } else if constexpr (std::is_floating_point<DATA_TYPE>::value) { // NOLINT
306  return v ? 1 : 0;
307  } else if constexpr (std::is_same<DATA_TYPE, std::string>::value) { // NOLINT
308  return v ? "T" : "F";
309  }
310  }
311 };
312 
313 template <>
314 struct ArrowValue<float> : ArrowValueBase<float> {
315  using VALUE_TYPE = float;
316  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
317  : ArrowValueBase<VALUE_TYPE>(data, v) {}
318 
319  template <typename DATA_TYPE>
320  explicit operator DATA_TYPE() const {
321  if constexpr (std::is_integral<DATA_TYPE>::value) { // NOLINT
322  const auto ti = data.cd->columnType;
323  DATA_TYPE v = ti.is_decimal() ? this->v * pow(10, ti.get_scale()) : this->v;
324  if (!(std::numeric_limits<DATA_TYPE>::lowest() < v &&
325  v <= std::numeric_limits<DATA_TYPE>::max())) {
326  data_conversion_error<DATA_TYPE>(v, data.cd, data.bad_rows_tracker);
327  }
328  return v;
329  } else if constexpr (std::is_floating_point<DATA_TYPE>::value) { // NOLINT
330  return v;
331  } else if constexpr (std::is_same<DATA_TYPE, std::string>::value) { // NOLINT
332  return std::to_string(v);
333  }
334  }
335 };
336 
337 template <>
338 struct ArrowValue<double> : ArrowValueBase<double> {
339  using VALUE_TYPE = double;
340  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
341  : ArrowValueBase<VALUE_TYPE>(data, v) {}
342 
343  template <typename DATA_TYPE>
344  explicit operator DATA_TYPE() const {
345  if constexpr (std::is_integral<DATA_TYPE>::value) { // NOLINT
346  const auto ti = data.cd->columnType;
347  DATA_TYPE v = ti.is_decimal() ? this->v * pow(10, ti.get_scale()) : this->v;
348  if (!(std::numeric_limits<DATA_TYPE>::lowest() < v &&
349  v <= std::numeric_limits<DATA_TYPE>::max())) {
350  data_conversion_error<DATA_TYPE>(v, data.cd, data.bad_rows_tracker);
351  }
352  return v;
353  } else if constexpr (std::is_floating_point<DATA_TYPE>::value) { // NOLINT
354  if (std::is_same<DATA_TYPE, float>::value) {
355  if (!(std::numeric_limits<float>::lowest() < v &&
356  v <= std::numeric_limits<float>::max())) {
357  data_conversion_error<float>(v, data.cd, data.bad_rows_tracker);
358  }
359  }
360  return v;
361  } else if constexpr (std::is_same<DATA_TYPE, std::string>::value) { // NOLINT
362  return std::to_string(v);
363  }
364  }
365 };
366 
367 template <>
368 struct ArrowValue<int64_t> : ArrowValueBase<int64_t> {
369  using VALUE_TYPE = int64_t;
370  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
371  : ArrowValueBase<VALUE_TYPE>(data, v) {}
372 
373  template <typename DATA_TYPE>
374  explicit operator DATA_TYPE() const {
375  if constexpr (std::is_integral<DATA_TYPE>::value) { // NOLINT
376  int64_t v = this->v;
377  if (std::is_same<int64_t, DATA_TYPE>::value) {
378  } else if (std::numeric_limits<DATA_TYPE>::lowest() < v &&
379  v <= std::numeric_limits<DATA_TYPE>::max()) {
380  } else {
381  data_conversion_error<DATA_TYPE>(v, data.cd, data.bad_rows_tracker);
382  }
383  if (data.cd->columnType.is_time()) {
384  v = this->resolve_time(v);
385  }
386  return v;
387  } else if constexpr (std::is_floating_point<DATA_TYPE>::value) { // NOLINT
388  return v;
389  } else if constexpr (std::is_same<DATA_TYPE, std::string>::value) { // NOLINT
390  const auto& type_id = data.arrow_type.id();
391  if (type_id == arrow::Type::DATE32 || type_id == arrow::Type::DATE64) {
392  auto& date_type = static_cast<const arrow::DateType&>(data.arrow_type);
393  SQLTypeInfo ti(kDATE);
394  Datum datum;
395  datum.bigintval =
396  date_type.unit() == arrow::DateUnit::MILLI ? v / kMicrosecondsInSecond : v;
397  return DatumToString(datum, ti);
398  } else if (type_id == arrow::Type::TIME32 || type_id == arrow::Type::TIME64 ||
399  type_id == arrow::Type::TIMESTAMP) {
400  auto& time_type = static_cast<const arrow::TimeType&>(data.arrow_type);
401  const auto result =
402  _precision_scale_lookup.find(std::make_pair(0, time_type.unit()));
403  int64_t divisor{1};
404  if (result != _precision_scale_lookup.end()) {
405  divisor = result->second.second;
406  }
408  Datum datum;
409  datum.bigintval = v / divisor;
410  auto time_str = DatumToString(datum, ti);
411  if (divisor != 1 && v % divisor) {
412  time_str += "." + std::to_string(v % divisor);
413  }
414  return time_str;
415  }
416  return std::to_string(v);
417  }
418  }
419 };
420 
421 template <>
422 struct ArrowValue<std::string> : ArrowValueBase<std::string> {
423  using VALUE_TYPE = std::string;
424  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
425  : ArrowValueBase<VALUE_TYPE>(data, v) {}
426 
427  template <typename DATA_TYPE>
428  explicit operator DATA_TYPE() const {
429  if constexpr (std::is_same<DATA_TYPE, bool>::value) { // NOLINT
430  if (v.size() == 0) {
431  return inline_int_null_value<int8_t>();
432  }
433  try {
434  SQLTypeInfo ti(kBOOLEAN);
435  auto datum = StringToDatum(v, ti);
436  return datum.boolval;
437  } catch (...) {
438  data_conversion_error(v, data.cd, data.bad_rows_tracker);
439  return false;
440  }
441  } else if constexpr (std::is_integral<DATA_TYPE>::value) { // NOLINT
442  if (v.size() == 0) {
443  return inline_fixed_encoding_null_val(data.cd->columnType);
444  }
445  try {
446  auto ti = data.cd->columnType;
447  auto datum = StringToDatum(v, ti);
448  return datum.bigintval;
449  } catch (...) {
450  data_conversion_error(v, data.cd, data.bad_rows_tracker);
451  return 0;
452  }
453  } else if constexpr (std::is_floating_point<DATA_TYPE>::value) { // NOLINT
454  return atof(v.data());
455  } else if constexpr (std::is_same<DATA_TYPE, std::string>::value) { // NOLINT
456  return v;
457  }
458  }
459 };
460 
461 template <>
462 struct ArrowValue<arrow::Decimal128> : ArrowValueBase<arrow::Decimal128> {
463  using VALUE_TYPE = arrow::Decimal128;
464  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
465  : ArrowValueBase<VALUE_TYPE>(data, v) {
466  // omni decimal has only 64 bits
467  arrow_throw_if(!(v.high_bits() == 0 || v.high_bits() == -1),
468  error_context(data.cd, data.bad_rows_tracker) +
469  "Truncation error on Arrow Decimal128 value");
470  }
471 
472  template <typename DATA_TYPE>
473  explicit operator DATA_TYPE() const {
474  if constexpr (std::is_integral<DATA_TYPE>::value) { // NOLINT
475  int64_t v = static_cast<int64_t>(this->v);
476  if (data.cd->columnType.is_decimal()) {
477  return convert_decimal_value_to_scale(v, data.old_type, data.new_type);
478  }
479  if (data.arrow_decimal_scale) {
480  v = std::llround(v / pow(10, data.arrow_decimal_scale));
481  }
482  if (std::is_same<int64_t, DATA_TYPE>::value) {
483  } else if (std::numeric_limits<DATA_TYPE>::lowest() < v &&
484  v <= std::numeric_limits<DATA_TYPE>::max()) {
485  } else {
486  data_conversion_error<DATA_TYPE>(v, data.cd, data.bad_rows_tracker);
487  }
488  return v;
489  } else if constexpr (std::is_floating_point<DATA_TYPE>::value) { // NOLINT
490  int64_t v = static_cast<int64_t>(this->v);
491  return data.arrow_decimal_scale ? v / pow(10, data.arrow_decimal_scale) : v;
492  } else if constexpr (std::is_same<DATA_TYPE, std::string>::value) { // NOLINT
493  return v.ToString(data.arrow_decimal_scale);
494  }
495  }
496 };
497 
498 // appends a converted RHS value to LHS data block
499 template <typename DATA_TYPE>
500 inline auto& operator<<(DataBuffer<DATA_TYPE>& data, const VarValue& var) {
501  boost::apply_visitor(
502  [&data](const auto& v) {
503  data.buffer.push_back(DATA_TYPE(ArrowValue<exprtype(v)>(data, v)));
504  },
505  var);
506  return data;
507 }
508 
509 } // namespace
510 #endif // ARROW_IMPORTER_H
void data_conversion_error(const VALUE_TYPE v, const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
boost::variant< bool, float, double, int64_t, std::string, void *, arrow::Decimal128 > VarValue
Definition: ArrowImporter.h:64
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
std::string DatumToString(Datum d, const SQLTypeInfo &ti)
Definition: Datum.cpp:398
#define exprtype(expr)
Definition: ArrowImporter.h:75
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
#define LOG(tag)
Definition: Logger.h:216
DataBufferBase(const ColumnDescriptor *cd, const arrow::Array &array, import_export::BadRowsTracker *const bad_rows_tracker)
auto value_getter(const arrow::Array &array, const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
DEVICE double get_scale(const double domain_min, const double domain_max, const int32_t num_bins)
#define UNREACHABLE()
Definition: Logger.h:266
Constants for Builtin SQL Types supported by HEAVY.AI.
#define NUMERIC_CASE(tid, src_type, var_type)
Definition: ArrowImporter.h:97
double inline_fp_null_val(const SQL_TYPE_INFO &ti)
tuple STRING
Definition: dtypes.py:31
std::string to_string(char const *&&v)
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
static const std::map< std::pair< int32_t, arrow::TimeUnit::type >, std::pair< SQLOps, int64_t > > _precision_scale_lookup
int64_t bigintval
Definition: sqltypes.h:235
typename std::enable_if_t< std::is_floating_point< T >::value, T > enable_if_floating
Definition: ArrowImporter.h:73
void type_conversion_error(const std::string pt, const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
import_export::BadRowsTracker *const bad_rows_tracker
DataBuffer(const ColumnDescriptor *cd, const arrow::Array &array, std::vector< DATA_TYPE > &buffer, import_export::BadRowsTracker *const bad_rows_tracker)
Datum StringToDatum(std::string_view s, SQLTypeInfo &ti)
Definition: Datum.cpp:277
specifies the content in-memory of a row in the column metadata table
bool g_enable_smem_group_by true
VarValue get_string_value(const arrow::Array &array, const int64_t idx)
Definition: ArrowImporter.h:92
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
Definition: sqltypes.h:67
int64_t convert_decimal_value_to_scale(const int64_t decimal_value, const SQLTypeInfo &type_info, const SQLTypeInfo &new_type_info)
Definition: Datum.cpp:562
int64_t resolve_time(const VALUE_TYPE &v, std::enable_if_t< enabled > *=0) const
std::string get_type_name() const
Definition: sqltypes.h:528
typename std::enable_if_t< std::is_integral< T >::value, T > enable_if_integral
Definition: ArrowImporter.h:67
#define STRING_CASE(tid, src_type)
int64_t resolve_time(const VALUE_TYPE &v, std::enable_if_t<!enabled > *=0) const
VarValue get_numeric_value(const arrow::Array &array, const int64_t idx)
Definition: ArrowImporter.h:86
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
typename std::enable_if_t< std::is_integral< T >::value &&!std::is_same< T, bool >::value, T > enable_if_integral_not_bool
Definition: ArrowImporter.h:71
SQLTypeInfo columnType
std::string error_context(const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
Definition: ArrowImporter.h:77
std::string columnName
ArrowValueBase(const DataBufferBase &data, const VALUE_TYPE &v)
void arrow_throw_if(const bool cond, const std::string &message)
Definition: ArrowImporter.h:42