OmniSciDB  6686921089
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
ArrowImporter.h
Go to the documentation of this file.
1 /*
2  * Copyright 2019 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 #ifndef ARROW_IMPORTER_H
17 #define ARROW_IMPORTER_H
18 
19 #include <cstdlib>
20 #include <ctime>
21 #include <map>
22 #include <mutex>
23 
24 #include <arrow/api.h>
25 #include <arrow/io/api.h>
26 #include <boost/algorithm/string.hpp>
27 #include <boost/variant.hpp>
28 
29 #include "Shared/SqlTypesLayout.h"
31 #include "Shared/sqltypes.h"
32 
33 using arrow::Array;
34 using arrow::Type;
35 
36 struct ArrowImporterException : std::runtime_error {
37  using std::runtime_error::runtime_error;
38 };
39 
40 template <typename T = ArrowImporterException>
41 inline void arrow_throw_if(const bool cond, const std::string& message) {
42  if (cond) {
43  // work around race from goooogle log
44  static std::mutex mtx;
45  std::unique_lock<std::mutex> lock(mtx);
46  LOG(ERROR) << message;
47  throw T(message);
48  }
49 }
50 
51 #ifdef ENABLE_IMPORT_PARQUET
52 #include <parquet/api/reader.h>
53 #include <parquet/api/writer.h>
54 #include <parquet/arrow/reader.h>
55 #include <parquet/exception.h>
56 #endif // ENABLE_IMPORT_PARQUET
57 
58 #include "arrow/util/decimal.h"
59 
60 namespace {
61 
62 using VarValue =
63  boost::variant<bool, float, double, int64_t, std::string, void*, arrow::Decimal128>;
64 
65 template <typename T>
66 using enable_if_integral = typename std::enable_if_t<std::is_integral<T>::value, T>;
67 template <typename T>
69  typename std::enable_if_t<std::is_integral<T>::value && !std::is_same<T, bool>::value,
70  T>;
71 template <typename T>
72 using enable_if_floating = typename std::enable_if_t<std::is_floating_point<T>::value, T>;
73 
74 #define exprtype(expr) std::decay_t<decltype(expr)>
75 
76 inline std::string error_context(const ColumnDescriptor* cd,
77  import_export::BadRowsTracker* const bad_rows_tracker) {
78  return bad_rows_tracker ? "File " + bad_rows_tracker->file_name + ", row-group " +
79  std::to_string(bad_rows_tracker->row_group) +
80  (cd ? ", column " + cd->columnName + ": " : "")
81  : std::string();
82 }
83 
84 template <typename SrcType, typename DstType>
85 inline VarValue get_numeric_value(const arrow::Array& array, const int64_t idx) {
86  using ArrayType = typename arrow::TypeTraits<SrcType>::ArrayType;
87  return (DstType) static_cast<const ArrayType&>(array).Value(idx);
88 }
89 
90 template <typename SrcType>
91 inline VarValue get_string_value(const arrow::Array& array, const int64_t idx) {
92  using ArrayType = typename arrow::TypeTraits<SrcType>::ArrayType;
93  return static_cast<const ArrayType&>(array).GetString(idx);
94 }
95 
96 #define NUMERIC_CASE(tid, src_type, var_type) \
97  case arrow::Type::tid: \
98  return get_numeric_value<src_type, var_type>;
99 #define STRING_CASE(tid, src_type) \
100  case arrow::Type::tid: \
101  return get_string_value<src_type>;
102 
103 inline auto value_getter(const arrow::Array& array,
104  const ColumnDescriptor* cd,
105  import_export::BadRowsTracker* const bad_rows_tracker) {
106  switch (array.type_id()) {
107  NUMERIC_CASE(BOOL, arrow::BooleanType, bool)
108  NUMERIC_CASE(UINT8, arrow::UInt8Type, int64_t)
109  NUMERIC_CASE(UINT16, arrow::UInt16Type, int64_t)
110  NUMERIC_CASE(UINT32, arrow::UInt32Type, int64_t)
111  NUMERIC_CASE(UINT64, arrow::Int64Type, int64_t)
112  NUMERIC_CASE(INT8, arrow::Int8Type, int64_t)
113  NUMERIC_CASE(INT16, arrow::Int16Type, int64_t)
114  NUMERIC_CASE(INT32, arrow::Int32Type, int64_t)
115  NUMERIC_CASE(INT64, arrow::Int64Type, int64_t)
116  NUMERIC_CASE(FLOAT, arrow::FloatType, float)
117  NUMERIC_CASE(DOUBLE, arrow::DoubleType, double)
118  NUMERIC_CASE(DATE32, arrow::Date32Type, int64_t)
119  NUMERIC_CASE(DATE64, arrow::Date64Type, int64_t)
120  NUMERIC_CASE(TIME64, arrow::Time64Type, int64_t)
121  NUMERIC_CASE(TIME32, arrow::Time32Type, int64_t)
122  NUMERIC_CASE(TIMESTAMP, arrow::TimestampType, int64_t)
123  NUMERIC_CASE(DECIMAL, arrow::Decimal128Type, arrow::Decimal128)
124  STRING_CASE(STRING, arrow::StringType)
125  STRING_CASE(BINARY, arrow::BinaryType)
126  default:
127  arrow_throw_if(true,
128  error_context(cd, bad_rows_tracker) + "Parquet type " +
129  array.type()->name() + " is not supported");
130  throw;
131  }
132 }
133 
134 inline void type_conversion_error(const std::string pt,
135  const ColumnDescriptor* cd,
136  import_export::BadRowsTracker* const bad_rows_tracker) {
137  arrow_throw_if(true,
138  error_context(cd, bad_rows_tracker) +
139  "Invalid type conversion from parquet " + pt + " type to " +
140  cd->columnType.get_type_name());
141 }
142 
143 template <typename DATA_TYPE, typename VALUE_TYPE>
144 inline void data_conversion_error(const VALUE_TYPE v,
145  const ColumnDescriptor* cd,
146  import_export::BadRowsTracker* const bad_rows_tracker) {
147  arrow_throw_if(true,
148  error_context(cd, bad_rows_tracker) +
149  "Invalid data conversion from parquet value " + std::to_string(v) +
150  " to " + std::to_string(DATA_TYPE(v)));
151 }
152 
153 inline void data_conversion_error(const std::string& v,
154  const ColumnDescriptor* cd,
155  import_export::BadRowsTracker* const bad_rows_tracker) {
156  arrow_throw_if(true,
157  error_context(cd, bad_rows_tracker) +
158  "Invalid data conversion from parquet string '" + v + "' to " +
159  cd->columnType.get_type_name() + " column type");
160 }
161 
162 // models the variant data buffers of TypedImportBuffer (LHS)
165  const arrow::Array& array;
167  // in case of arrow-decimal to omni-decimal conversion
168  // dont get/set these info on every row of arrow array
169  const arrow::DataType& arrow_type;
174  const arrow::Array& array,
175  import_export::BadRowsTracker* const bad_rows_tracker)
176  : cd(cd)
177  , array(array)
178  , bad_rows_tracker(bad_rows_tracker)
179  , arrow_type(*array.type())
180  , arrow_decimal_scale(
181  arrow_type.id() == arrow::Type::DECIMAL
182  ? static_cast<const arrow::Decimal128Type&>(arrow_type).scale()
183  : 0)
184  , old_type(cd->columnType.get_type(),
185  cd->columnType.get_dimension(),
186  arrow_decimal_scale,
187  true)
188  , new_type(cd->columnType.get_type(),
189  cd->columnType.get_dimension(),
190  cd->columnType.get_scale(),
191  true) {}
192 };
193 
194 template <typename DATA_TYPE>
196  std::vector<DATA_TYPE>& buffer;
198  const arrow::Array& array,
199  std::vector<DATA_TYPE>& buffer,
200  import_export::BadRowsTracker* const bad_rows_tracker)
201  : DataBufferBase(cd, array, bad_rows_tracker), buffer(buffer) {}
202 };
203 
204 constexpr int64_t kMillisecondsInSecond = 1000LL;
205 constexpr int64_t kMicrosecondsInSecond = 1000LL * 1000LL;
206 constexpr int64_t kNanosecondsinSecond = 1000LL * 1000LL * 1000LL;
207 constexpr int32_t kSecondsInDay = 86400;
208 
209 static const std::map<std::pair<int32_t, arrow::TimeUnit::type>,
210  std::pair<SQLOps, int64_t>>
212  {{0, arrow::TimeUnit::MILLI}, {kDIVIDE, kMillisecondsInSecond}},
213  {{0, arrow::TimeUnit::MICRO}, {kDIVIDE, kMicrosecondsInSecond}},
214  {{0, arrow::TimeUnit::NANO}, {kDIVIDE, kNanosecondsinSecond}},
215  {{3, arrow::TimeUnit::SECOND}, {kMULTIPLY, kMicrosecondsInSecond}},
216  {{3, arrow::TimeUnit::MICRO}, {kDIVIDE, kMillisecondsInSecond}},
217  {{3, arrow::TimeUnit::NANO}, {kDIVIDE, kMicrosecondsInSecond}},
218  {{6, arrow::TimeUnit::SECOND}, {kMULTIPLY, kMicrosecondsInSecond}},
219  {{6, arrow::TimeUnit::MILLI}, {kMULTIPLY, kMillisecondsInSecond}},
220  {{6, arrow::TimeUnit::NANO}, {kDIVIDE, kMillisecondsInSecond}},
221  {{9, arrow::TimeUnit::SECOND}, {kMULTIPLY, kNanosecondsinSecond}},
222  {{9, arrow::TimeUnit::MILLI}, {kMULTIPLY, kMicrosecondsInSecond}},
223  {{9, arrow::TimeUnit::MICRO}, {kMULTIPLY, kMillisecondsInSecond}}};
224 
225 // models the variant values of Arrow Array (RHS)
226 template <typename VALUE_TYPE>
229  const VALUE_TYPE v;
230  const int32_t dimension;
231  ArrowValueBase(const DataBufferBase& data, const VALUE_TYPE& v)
232  : data(data)
233  , v(v)
234  , dimension(data.cd->columnType.is_high_precision_timestamp()
235  ? data.cd->columnType.get_dimension()
236  : 0) {}
237  template <bool enabled = std::is_integral<VALUE_TYPE>::value>
238  int64_t resolve_time(const VALUE_TYPE& v, std::enable_if_t<enabled>* = 0) const {
239  const auto& type_id = data.arrow_type.id();
240  if (type_id == arrow::Type::DATE32 || type_id == arrow::Type::DATE64) {
241  auto& date_type = static_cast<const arrow::DateType&>(data.arrow_type);
242  switch (date_type.unit()) {
243  case arrow::DateUnit::DAY:
244  return v * kSecondsInDay;
245  case arrow::DateUnit::MILLI:
246  return v / kMillisecondsInSecond;
247  }
248  } else if (type_id == arrow::Type::TIME32 || type_id == arrow::Type::TIME64 ||
249  type_id == arrow::Type::TIMESTAMP) {
250  auto& time_type = static_cast<const arrow::TimeType&>(data.arrow_type);
251  const auto result =
252  _precision_scale_lookup.find(std::make_pair(dimension, time_type.unit()));
253  if (result != _precision_scale_lookup.end()) {
254  const auto scale = result->second;
255  return scale.first == kMULTIPLY ? v * scale.second : v / scale.second;
256  } else {
257  return v;
258  }
259  }
260  UNREACHABLE() << data.arrow_type << " is not a valid Arrow time or date type";
261  return 0;
262  }
263  template <bool enabled = std::is_integral<VALUE_TYPE>::value>
264  int64_t resolve_time(const VALUE_TYPE& v, std::enable_if_t<!enabled>* = 0) const {
265  static_assert(enabled, "unreachable");
266  return 0;
267  }
268 };
269 
270 template <typename VALUE_TYPE>
271 struct ArrowValue : ArrowValueBase<VALUE_TYPE> {};
272 
273 template <>
275  using VALUE_TYPE = void*;
276  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
277  : ArrowValueBase<VALUE_TYPE>(data, v) {}
278 
279  template <typename DATA_TYPE>
280  explicit operator DATA_TYPE() const {
281  if constexpr (std::is_integral<DATA_TYPE>::value) { // NOLINT
282  return inline_fixed_encoding_null_val(data.cd->columnType);
283  } else if constexpr (std::is_floating_point<DATA_TYPE>::value) { // NOLINT
284  return inline_fp_null_val(data.cd->columnType);
285  } else if constexpr (std::is_same<DATA_TYPE, std::string>::value) { // NOLINT
286  return std::string();
287  }
288  }
289 };
290 
291 template <>
292 struct ArrowValue<bool> : ArrowValueBase<bool> {
293  using VALUE_TYPE = bool;
294  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
295  : ArrowValueBase<VALUE_TYPE>(data, v) {}
296 
297  template <typename DATA_TYPE>
298  explicit operator DATA_TYPE() const {
299  if constexpr (std::is_integral<DATA_TYPE>::value) { // NOLINT
300  if (!(data.cd->columnType.is_number() || data.cd->columnType.is_boolean())) {
301  type_conversion_error("bool", data.cd, data.bad_rows_tracker);
302  }
303  return v;
304  } else if constexpr (std::is_floating_point<DATA_TYPE>::value) { // NOLINT
305  return v ? 1 : 0;
306  } else if constexpr (std::is_same<DATA_TYPE, std::string>::value) { // NOLINT
307  return v ? "T" : "F";
308  }
309  }
310 };
311 
312 template <>
313 struct ArrowValue<float> : ArrowValueBase<float> {
314  using VALUE_TYPE = float;
315  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
316  : ArrowValueBase<VALUE_TYPE>(data, v) {}
317 
318  template <typename DATA_TYPE>
319  explicit operator DATA_TYPE() const {
320  if constexpr (std::is_integral<DATA_TYPE>::value) { // NOLINT
321  const auto ti = data.cd->columnType;
322  DATA_TYPE v = ti.is_decimal() ? this->v * pow(10, ti.get_scale()) : this->v;
323  if (!(std::numeric_limits<DATA_TYPE>::lowest() < v &&
324  v <= std::numeric_limits<DATA_TYPE>::max())) {
325  data_conversion_error<DATA_TYPE>(v, data.cd, data.bad_rows_tracker);
326  }
327  return v;
328  } else if constexpr (std::is_floating_point<DATA_TYPE>::value) { // NOLINT
329  return v;
330  } else if constexpr (std::is_same<DATA_TYPE, std::string>::value) { // NOLINT
331  return std::to_string(v);
332  }
333  }
334 };
335 
336 template <>
337 struct ArrowValue<double> : ArrowValueBase<double> {
338  using VALUE_TYPE = double;
339  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
340  : ArrowValueBase<VALUE_TYPE>(data, v) {}
341 
342  template <typename DATA_TYPE>
343  explicit operator DATA_TYPE() const {
344  if constexpr (std::is_integral<DATA_TYPE>::value) { // NOLINT
345  const auto ti = data.cd->columnType;
346  DATA_TYPE v = ti.is_decimal() ? this->v * pow(10, ti.get_scale()) : this->v;
347  if (!(std::numeric_limits<DATA_TYPE>::lowest() < v &&
348  v <= std::numeric_limits<DATA_TYPE>::max())) {
349  data_conversion_error<DATA_TYPE>(v, data.cd, data.bad_rows_tracker);
350  }
351  return v;
352  } else if constexpr (std::is_floating_point<DATA_TYPE>::value) { // NOLINT
353  if (std::is_same<DATA_TYPE, float>::value) {
354  if (!(std::numeric_limits<float>::lowest() < v &&
355  v <= std::numeric_limits<float>::max())) {
356  data_conversion_error<float>(v, data.cd, data.bad_rows_tracker);
357  }
358  }
359  return v;
360  } else if constexpr (std::is_same<DATA_TYPE, std::string>::value) { // NOLINT
361  return std::to_string(v);
362  }
363  }
364 };
365 
366 template <>
367 struct ArrowValue<int64_t> : ArrowValueBase<int64_t> {
368  using VALUE_TYPE = int64_t;
369  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
370  : ArrowValueBase<VALUE_TYPE>(data, v) {}
371 
372  template <typename DATA_TYPE>
373  explicit operator DATA_TYPE() const {
374  if constexpr (std::is_integral<DATA_TYPE>::value) { // NOLINT
375  int64_t v = this->v;
376  if (std::is_same<int64_t, DATA_TYPE>::value) {
377  } else if (std::numeric_limits<DATA_TYPE>::lowest() < v &&
378  v <= std::numeric_limits<DATA_TYPE>::max()) {
379  } else {
380  data_conversion_error<DATA_TYPE>(v, data.cd, data.bad_rows_tracker);
381  }
382  if (data.cd->columnType.is_time()) {
383  v = this->resolve_time(v);
384  }
385  return v;
386  } else if constexpr (std::is_floating_point<DATA_TYPE>::value) { // NOLINT
387  return v;
388  } else if constexpr (std::is_same<DATA_TYPE, std::string>::value) { // NOLINT
389  const auto& type_id = data.arrow_type.id();
390  if (type_id == arrow::Type::DATE32 || type_id == arrow::Type::DATE64) {
391  auto& date_type = static_cast<const arrow::DateType&>(data.arrow_type);
392  SQLTypeInfo ti(kDATE);
393  Datum datum;
394  datum.bigintval =
395  date_type.unit() == arrow::DateUnit::MILLI ? v / kMicrosecondsInSecond : v;
396  return DatumToString(datum, ti);
397  } else if (type_id == arrow::Type::TIME32 || type_id == arrow::Type::TIME64 ||
398  type_id == arrow::Type::TIMESTAMP) {
399  auto& time_type = static_cast<const arrow::TimeType&>(data.arrow_type);
400  const auto result =
401  _precision_scale_lookup.find(std::make_pair(0, time_type.unit()));
402  int64_t divisor{1};
403  if (result != _precision_scale_lookup.end()) {
404  divisor = result->second.second;
405  }
407  Datum datum;
408  datum.bigintval = v / divisor;
409  auto time_str = DatumToString(datum, ti);
410  if (divisor != 1 && v % divisor) {
411  time_str += "." + std::to_string(v % divisor);
412  }
413  return time_str;
414  }
415  return std::to_string(v);
416  }
417  }
418 };
419 
420 template <>
421 struct ArrowValue<std::string> : ArrowValueBase<std::string> {
422  using VALUE_TYPE = std::string;
423  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
424  : ArrowValueBase<VALUE_TYPE>(data, v) {}
425 
426  template <typename DATA_TYPE>
427  explicit operator DATA_TYPE() const {
428  if constexpr (std::is_same<DATA_TYPE, bool>::value) { // NOLINT
429  if (v.size() == 0) {
430  return inline_int_null_value<int8_t>();
431  }
432  try {
433  SQLTypeInfo ti(kBOOLEAN);
434  auto datum = StringToDatum(v, ti);
435  return datum.boolval;
436  } catch (...) {
437  data_conversion_error(v, data.cd, data.bad_rows_tracker);
438  return false;
439  }
440  } else if constexpr (std::is_integral<DATA_TYPE>::value) { // NOLINT
441  if (v.size() == 0) {
442  return inline_fixed_encoding_null_val(data.cd->columnType);
443  }
444  try {
445  auto ti = data.cd->columnType;
446  auto datum = StringToDatum(v, ti);
447  return datum.bigintval;
448  } catch (...) {
449  data_conversion_error(v, data.cd, data.bad_rows_tracker);
450  return 0;
451  }
452  } else if constexpr (std::is_floating_point<DATA_TYPE>::value) { // NOLINT
453  return atof(v.data());
454  } else if constexpr (std::is_same<DATA_TYPE, std::string>::value) { // NOLINT
455  return v;
456  }
457  }
458 };
459 
460 template <>
461 struct ArrowValue<arrow::Decimal128> : ArrowValueBase<arrow::Decimal128> {
462  using VALUE_TYPE = arrow::Decimal128;
463  ArrowValue(const DataBufferBase& data, const VALUE_TYPE& v)
464  : ArrowValueBase<VALUE_TYPE>(data, v) {
465  // omni decimal has only 64 bits
466  arrow_throw_if(!(v.high_bits() == 0 || v.high_bits() == -1),
467  error_context(data.cd, data.bad_rows_tracker) +
468  "Truncation error on Arrow Decimal128 value");
469  }
470 
471  template <typename DATA_TYPE>
472  explicit operator DATA_TYPE() const {
473  if constexpr (std::is_integral<DATA_TYPE>::value) { // NOLINT
474  int64_t v = static_cast<int64_t>(this->v);
475  if (data.cd->columnType.is_decimal()) {
476  return convert_decimal_value_to_scale(v, data.old_type, data.new_type);
477  }
478  if (data.arrow_decimal_scale) {
479  v = std::llround(v / pow(10, data.arrow_decimal_scale));
480  }
481  if (std::is_same<int64_t, DATA_TYPE>::value) {
482  } else if (std::numeric_limits<DATA_TYPE>::lowest() < v &&
483  v <= std::numeric_limits<DATA_TYPE>::max()) {
484  } else {
485  data_conversion_error<DATA_TYPE>(v, data.cd, data.bad_rows_tracker);
486  }
487  return v;
488  } else if constexpr (std::is_floating_point<DATA_TYPE>::value) { // NOLINT
489  int64_t v = static_cast<int64_t>(this->v);
490  return data.arrow_decimal_scale ? v / pow(10, data.arrow_decimal_scale) : v;
491  } else if constexpr (std::is_same<DATA_TYPE, std::string>::value) { // NOLINT
492  return v.ToString(data.arrow_decimal_scale);
493  }
494  }
495 };
496 
497 // appends a converted RHS value to LHS data block
498 template <typename DATA_TYPE>
499 inline auto& operator<<(DataBuffer<DATA_TYPE>& data, const VarValue& var) {
500  boost::apply_visitor(
501  [&data](const auto& v) {
502  data.buffer.push_back(DATA_TYPE(ArrowValue<exprtype(v)>(data, v)));
503  },
504  var);
505  return data;
506 }
507 
508 } // namespace
509 #endif // ARROW_IMPORTER_H
void data_conversion_error(const VALUE_TYPE v, const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
boost::variant< bool, float, double, int64_t, std::string, void *, arrow::Decimal128 > VarValue
Definition: ArrowImporter.h:63
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
std::string DatumToString(Datum d, const SQLTypeInfo &ti)
Definition: Datum.cpp:392
#define exprtype(expr)
Definition: ArrowImporter.h:74
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
#define const
#define LOG(tag)
Definition: Logger.h:203
DataBufferBase(const ColumnDescriptor *cd, const arrow::Array &array, import_export::BadRowsTracker *const bad_rows_tracker)
auto value_getter(const arrow::Array &array, const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
#define DOUBLE
#define UNREACHABLE()
Definition: Logger.h:253
Constants for Builtin SQL Types supported by OmniSci.
#define NUMERIC_CASE(tid, src_type, var_type)
Definition: ArrowImporter.h:96
double inline_fp_null_val(const SQL_TYPE_INFO &ti)
std::string to_string(char const *&&v)
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
static const std::map< std::pair< int32_t, arrow::TimeUnit::type >, std::pair< SQLOps, int64_t > > _precision_scale_lookup
int64_t bigintval
Definition: sqltypes.h:215
typename std::enable_if_t< std::is_floating_point< T >::value, T > enable_if_floating
Definition: ArrowImporter.h:72
void type_conversion_error(const std::string pt, const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
import_export::BadRowsTracker *const bad_rows_tracker
DataBuffer(const ColumnDescriptor *cd, const arrow::Array &array, std::vector< DATA_TYPE > &buffer, import_export::BadRowsTracker *const bad_rows_tracker)
Datum StringToDatum(std::string_view s, SQLTypeInfo &ti)
Definition: Datum.cpp:275
specifies the content in-memory of a row in the column metadata table
bool g_enable_smem_group_by true
VarValue get_string_value(const arrow::Array &array, const int64_t idx)
Definition: ArrowImporter.h:91
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
ArrowValue(const DataBufferBase &data, const VALUE_TYPE &v)
Definition: sqltypes.h:53
int64_t convert_decimal_value_to_scale(const int64_t decimal_value, const SQLTypeInfo &type_info, const SQLTypeInfo &new_type_info)
Definition: Datum.cpp:473
int64_t resolve_time(const VALUE_TYPE &v, std::enable_if_t< enabled > *=0) const
#define TIMESTAMP
std::string get_type_name() const
Definition: sqltypes.h:432
typename std::enable_if_t< std::is_integral< T >::value, T > enable_if_integral
Definition: ArrowImporter.h:66
#define STRING_CASE(tid, src_type)
Definition: ArrowImporter.h:99
int64_t resolve_time(const VALUE_TYPE &v, std::enable_if_t<!enabled > *=0) const
#define STRING
VarValue get_numeric_value(const arrow::Array &array, const int64_t idx)
Definition: ArrowImporter.h:85
#define DECIMAL
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
typename std::enable_if_t< std::is_integral< T >::value &&!std::is_same< T, bool >::value, T > enable_if_integral_not_bool
Definition: ArrowImporter.h:70
SQLTypeInfo columnType
std::string error_context(const ColumnDescriptor *cd, import_export::BadRowsTracker *const bad_rows_tracker)
Definition: ArrowImporter.h:76
#define FLOAT
std::string columnName
ArrowValueBase(const DataBufferBase &data, const VALUE_TYPE &v)
void arrow_throw_if(const bool cond, const std::string &message)
Definition: ArrowImporter.h:41