OmniSciDB  04ee39c94c
Importer_NS::TypedImportBuffer Class Reference

#include <Importer.h>

+ Inheritance diagram for Importer_NS::TypedImportBuffer:
+ Collaboration diagram for Importer_NS::TypedImportBuffer:

Public Member Functions

 TypedImportBuffer (const ColumnDescriptor *col_desc, StringDictionary *string_dict)
 
 ~TypedImportBuffer ()
 
void addBoolean (const int8_t v)
 
void addTinyint (const int8_t v)
 
void addSmallint (const int16_t v)
 
void addInt (const int32_t v)
 
void addBigint (const int64_t v)
 
void addFloat (const float v)
 
void addDouble (const double v)
 
void addString (const std::string &v)
 
void addGeoString (const std::string &v)
 
void addArray (const ArrayDatum &v)
 
std::vector< std::string > & addStringArray ()
 
void addStringArray (const std::vector< std::string > &arr)
 
void addDictEncodedString (const std::vector< std::string > &string_vec)
 
void addDictEncodedStringArray (const std::vector< std::vector< std::string >> &string_array_vec)
 
const SQLTypeInfogetTypeInfo () const
 
const ColumnDescriptorgetColumnDesc () const
 
StringDictionarygetStringDictionary () const
 
int8_t * getAsBytes () const
 
size_t getElementSize () const
 
std::vector< std::string > * getStringBuffer () const
 
std::vector< std::string > * getGeoStringBuffer () const
 
std::vector< ArrayDatum > * getArrayBuffer () const
 
std::vector< std::vector< std::string > > * getStringArrayBuffer () const
 
std::vector< ArrayDatum > * getStringArrayDictBuffer () const
 
int8_t * getStringDictBuffer () const
 
bool stringDictCheckpoint ()
 
void clear ()
 
size_t add_values (const ColumnDescriptor *cd, const TColumn &data)
 
size_t add_arrow_values (const ColumnDescriptor *cd, const arrow::Array &data, const bool exact_type_match, const ArraySliceRange &slice_range, BadRowsTracker *bad_rows_tracker)
 
void add_value (const ColumnDescriptor *cd, const std::string &val, const bool is_null, const CopyParams &copy_params, const int64_t replicate_count=0)
 
void add_value (const ColumnDescriptor *cd, const TDatum &val, const bool is_null, const int64_t replicate_count=0)
 
void pop_value ()
 
int64_t get_replicate_count () const
 
void set_replicate_count (const int64_t replicate_count)
 
template<typename DATA_TYPE >
size_t convert_arrow_val_to_import_buffer (const ColumnDescriptor *cd, const arrow::Array &array, std::vector< DATA_TYPE > &buffer, const ArraySliceRange &slice_range, BadRowsTracker *const bad_rows_tracker)
 
template<typename DATA_TYPE >
auto del_values (std::vector< DATA_TYPE > &buffer, BadRowsTracker *const bad_rows_tracker)
 
auto del_values (const SQLTypes type, BadRowsTracker *const bad_rows_tracker)
 
template<typename DATA_TYPE >
size_t convert_arrow_val_to_import_buffer (const ColumnDescriptor *cd, const Array &array, std::vector< DATA_TYPE > &buffer, const ArraySliceRange &slice_range, Importer_NS::BadRowsTracker *const bad_rows_tracker)
 

Public Attributes

std::vector< std::unique_ptr< TypedImportBuffer > > * import_buffers
 
size_t col_idx
 
std::vector< int8_t > * bool_buffer_
 
std::vector< int8_t > * tinyint_buffer_
 
std::vector< int16_t > * smallint_buffer_
 
std::vector< int32_t > * int_buffer_
 
std::vector< int64_t > * bigint_buffer_
 
std::vector< float > * float_buffer_
 
std::vector< double > * double_buffer_
 
std::vector< std::string > * string_buffer_
 
std::vector< std::string > * geo_string_buffer_
 
std::vector< ArrayDatum > * array_buffer_
 
std::vector< std::vector< std::string > > * string_array_buffer_
 
std::vector< uint8_t > * string_dict_i8_buffer_
 
std::vector< uint16_t > * string_dict_i16_buffer_
 
std::vector< int32_t > * string_dict_i32_buffer_
 
std::vector< ArrayDatum > * string_array_dict_buffer_
 

Private Attributes

union {
   std::vector< int8_t > *   bool_buffer_
 
   std::vector< int8_t > *   tinyint_buffer_
 
   std::vector< int16_t > *   smallint_buffer_
 
   std::vector< int32_t > *   int_buffer_
 
   std::vector< int64_t > *   bigint_buffer_
 
   std::vector< float > *   float_buffer_
 
   std::vector< double > *   double_buffer_
 
   std::vector< std::string > *   string_buffer_
 
   std::vector< std::string > *   geo_string_buffer_
 
   std::vector< ArrayDatum > *   array_buffer_
 
   std::vector< std::vector< std::string > > *   string_array_buffer_
 
}; 
 
union {
   std::vector< uint8_t > *   string_dict_i8_buffer_
 
   std::vector< uint16_t > *   string_dict_i16_buffer_
 
   std::vector< int32_t > *   string_dict_i32_buffer_
 
   std::vector< ArrayDatum > *   string_array_dict_buffer_
 
}; 
 
const ColumnDescriptorcolumn_desc_
 
StringDictionarystring_dict_
 
size_t replicate_count_ = 0
 

Detailed Description

Definition at line 184 of file Importer.h.

Constructor & Destructor Documentation

◆ TypedImportBuffer()

Importer_NS::TypedImportBuffer::TypedImportBuffer ( const ColumnDescriptor col_desc,
StringDictionary string_dict 
)
inline

Definition at line 186 of file Importer.h.

References CHECK, ColumnDescriptor::columnType, SQLTypeInfoCore< TYPE_FACET_PACK >::get_compression(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_size(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_subtype(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kENCODING_DICT, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, and kVARCHAR.

187  : column_desc_(col_desc), string_dict_(string_dict) {
188  switch (col_desc->columnType.get_type()) {
189  case kBOOLEAN:
190  bool_buffer_ = new std::vector<int8_t>();
191  break;
192  case kTINYINT:
193  tinyint_buffer_ = new std::vector<int8_t>();
194  break;
195  case kSMALLINT:
196  smallint_buffer_ = new std::vector<int16_t>();
197  break;
198  case kINT:
199  int_buffer_ = new std::vector<int32_t>();
200  break;
201  case kBIGINT:
202  case kNUMERIC:
203  case kDECIMAL:
204  bigint_buffer_ = new std::vector<int64_t>();
205  break;
206  case kFLOAT:
207  float_buffer_ = new std::vector<float>();
208  break;
209  case kDOUBLE:
210  double_buffer_ = new std::vector<double>();
211  break;
212  case kTEXT:
213  case kVARCHAR:
214  case kCHAR:
215  string_buffer_ = new std::vector<std::string>();
216  if (col_desc->columnType.get_compression() == kENCODING_DICT) {
217  switch (col_desc->columnType.get_size()) {
218  case 1:
219  string_dict_i8_buffer_ = new std::vector<uint8_t>();
220  break;
221  case 2:
222  string_dict_i16_buffer_ = new std::vector<uint16_t>();
223  break;
224  case 4:
225  string_dict_i32_buffer_ = new std::vector<int32_t>();
226  break;
227  default:
228  CHECK(false);
229  }
230  }
231  break;
232  case kDATE:
233  case kTIME:
234  case kTIMESTAMP:
235  bigint_buffer_ = new std::vector<int64_t>();
236  break;
237  case kARRAY:
238  if (IS_STRING(col_desc->columnType.get_subtype())) {
240  string_array_buffer_ = new std::vector<std::vector<std::string>>();
241  string_array_dict_buffer_ = new std::vector<ArrayDatum>();
242  } else {
243  array_buffer_ = new std::vector<ArrayDatum>();
244  }
245  break;
246  case kPOINT:
247  case kLINESTRING:
248  case kPOLYGON:
249  case kMULTIPOLYGON:
250  geo_string_buffer_ = new std::vector<std::string>();
251  break;
252  default:
253  CHECK(false);
254  }
255  }
std::vector< int16_t > * smallint_buffer_
Definition: Importer.h:615
HOST DEVICE int get_size() const
Definition: sqltypes.h:333
std::vector< int64_t > * bigint_buffer_
Definition: Importer.h:617
Definition: sqltypes.h:51
std::vector< float > * float_buffer_
Definition: Importer.h:618
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:323
std::vector< uint16_t > * string_dict_i16_buffer_
Definition: Importer.h:627
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:331
std::vector< int8_t > * tinyint_buffer_
Definition: Importer.h:614
std::vector< std::string > * string_buffer_
Definition: Importer.h:620
StringDictionary * string_dict_
Definition: Importer.h:632
std::vector< int32_t > * string_dict_i32_buffer_
Definition: Importer.h:628
std::vector< std::vector< std::string > > * string_array_buffer_
Definition: Importer.h:623
std::vector< int32_t > * int_buffer_
Definition: Importer.h:616
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:324
std::vector< int8_t > * bool_buffer_
Definition: Importer.h:613
Definition: sqltypes.h:54
Definition: sqltypes.h:55
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:621
std::vector< ArrayDatum > * string_array_dict_buffer_
Definition: Importer.h:629
std::vector< double > * double_buffer_
Definition: Importer.h:619
Definition: sqltypes.h:43
const ColumnDescriptor * column_desc_
Definition: Importer.h:631
#define IS_STRING(T)
Definition: sqltypes.h:163
#define CHECK(condition)
Definition: Logger.h:187
Definition: sqltypes.h:47
SQLTypeInfo columnType
std::vector< uint8_t > * string_dict_i8_buffer_
Definition: Importer.h:626
std::vector< ArrayDatum > * array_buffer_
Definition: Importer.h:622
+ Here is the call graph for this function:

◆ ~TypedImportBuffer()

Importer_NS::TypedImportBuffer::~TypedImportBuffer ( )
inline

Definition at line 257 of file Importer.h.

References CHECK, IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kENCODING_DICT, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, and kVARCHAR.

257  {
258  switch (column_desc_->columnType.get_type()) {
259  case kBOOLEAN:
260  delete bool_buffer_;
261  break;
262  case kTINYINT:
263  delete tinyint_buffer_;
264  break;
265  case kSMALLINT:
266  delete smallint_buffer_;
267  break;
268  case kINT:
269  delete int_buffer_;
270  break;
271  case kBIGINT:
272  case kNUMERIC:
273  case kDECIMAL:
274  delete bigint_buffer_;
275  break;
276  case kFLOAT:
277  delete float_buffer_;
278  break;
279  case kDOUBLE:
280  delete double_buffer_;
281  break;
282  case kTEXT:
283  case kVARCHAR:
284  case kCHAR:
285  delete string_buffer_;
287  switch (column_desc_->columnType.get_size()) {
288  case 1:
289  delete string_dict_i8_buffer_;
290  break;
291  case 2:
293  break;
294  case 4:
296  break;
297  }
298  }
299  break;
300  case kDATE:
301  case kTIME:
302  case kTIMESTAMP:
303  delete bigint_buffer_;
304  break;
305  case kARRAY:
307  delete string_array_buffer_;
309  } else {
310  delete array_buffer_;
311  }
312  break;
313  case kPOINT:
314  case kLINESTRING:
315  case kPOLYGON:
316  case kMULTIPOLYGON:
317  delete geo_string_buffer_;
318  break;
319  default:
320  CHECK(false);
321  }
322  }
std::vector< int16_t > * smallint_buffer_
Definition: Importer.h:615
HOST DEVICE int get_size() const
Definition: sqltypes.h:333
std::vector< int64_t > * bigint_buffer_
Definition: Importer.h:617
Definition: sqltypes.h:51
std::vector< float > * float_buffer_
Definition: Importer.h:618
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:323
std::vector< uint16_t > * string_dict_i16_buffer_
Definition: Importer.h:627
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:331
std::vector< int8_t > * tinyint_buffer_
Definition: Importer.h:614
std::vector< std::string > * string_buffer_
Definition: Importer.h:620
std::vector< int32_t > * string_dict_i32_buffer_
Definition: Importer.h:628
std::vector< std::vector< std::string > > * string_array_buffer_
Definition: Importer.h:623
std::vector< int32_t > * int_buffer_
Definition: Importer.h:616
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:324
std::vector< int8_t > * bool_buffer_
Definition: Importer.h:613
Definition: sqltypes.h:54
Definition: sqltypes.h:55
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:621
std::vector< ArrayDatum > * string_array_dict_buffer_
Definition: Importer.h:629
std::vector< double > * double_buffer_
Definition: Importer.h:619
Definition: sqltypes.h:43
const ColumnDescriptor * column_desc_
Definition: Importer.h:631
#define IS_STRING(T)
Definition: sqltypes.h:163
#define CHECK(condition)
Definition: Logger.h:187
Definition: sqltypes.h:47
SQLTypeInfo columnType
std::vector< uint8_t > * string_dict_i8_buffer_
Definition: Importer.h:626
std::vector< ArrayDatum > * array_buffer_
Definition: Importer.h:622

Member Function Documentation

◆ add_arrow_values()

size_t Importer_NS::TypedImportBuffer::add_arrow_values ( const ColumnDescriptor cd,
const arrow::Array &  data,
const bool  exact_type_match,
const ArraySliceRange slice_range,
BadRowsTracker bad_rows_tracker 
)

Definition at line 959 of file Importer.cpp.

References arrow_throw_if(), ColumnDescriptor::columnName, ColumnDescriptor::columnType, decimal_to_int_type(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_notnull(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), SQLTypeInfoCore< TYPE_FACET_PACK >::is_decimal(), kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, and run-benchmark-import::type.

963  {
964  const auto type = cd->columnType.is_decimal() ? decimal_to_int_type(cd->columnType)
965  : cd->columnType.get_type();
966  if (cd->columnType.get_notnull()) {
967  // We can't have any null values for this column; to have them is an error
968  arrow_throw_if(col.null_count() > 0, "NULL not allowed for column " + cd->columnName);
969  }
970 
971  switch (type) {
972  case kBOOLEAN:
973  if (exact_type_match) {
974  arrow_throw_if(col.type_id() != Type::BOOL, "Expected boolean type");
975  }
977  cd, col, *bool_buffer_, slice_range, bad_rows_tracker);
978  case kTINYINT:
979  if (exact_type_match) {
980  arrow_throw_if(col.type_id() != Type::INT8, "Expected int8 type");
981  }
983  cd, col, *tinyint_buffer_, slice_range, bad_rows_tracker);
984  case kSMALLINT:
985  if (exact_type_match) {
986  arrow_throw_if(col.type_id() != Type::INT16, "Expected int16 type");
987  }
989  cd, col, *smallint_buffer_, slice_range, bad_rows_tracker);
990  case kINT:
991  if (exact_type_match) {
992  arrow_throw_if(col.type_id() != Type::INT32, "Expected int32 type");
993  }
995  cd, col, *int_buffer_, slice_range, bad_rows_tracker);
996  case kBIGINT:
997  if (exact_type_match) {
998  arrow_throw_if(col.type_id() != Type::INT64, "Expected int64 type");
999  }
1001  cd, col, *bigint_buffer_, slice_range, bad_rows_tracker);
1002  case kFLOAT:
1003  if (exact_type_match) {
1004  arrow_throw_if(col.type_id() != Type::FLOAT, "Expected float type");
1005  }
1007  cd, col, *float_buffer_, slice_range, bad_rows_tracker);
1008  case kDOUBLE:
1009  if (exact_type_match) {
1010  arrow_throw_if(col.type_id() != Type::DOUBLE, "Expected double type");
1011  }
1013  cd, col, *double_buffer_, slice_range, bad_rows_tracker);
1014  case kTEXT:
1015  case kVARCHAR:
1016  case kCHAR:
1017  if (exact_type_match) {
1018  arrow_throw_if(col.type_id() != Type::BINARY && col.type_id() != Type::STRING,
1019  "Expected string type");
1020  }
1022  cd, col, *string_buffer_, slice_range, bad_rows_tracker);
1023  case kTIME:
1024  if (exact_type_match) {
1025  arrow_throw_if(col.type_id() != Type::TIME32 && col.type_id() != Type::TIME64,
1026  "Expected time32 or time64 type");
1027  }
1029  cd, col, *bigint_buffer_, slice_range, bad_rows_tracker);
1030  case kTIMESTAMP:
1031  if (exact_type_match) {
1032  arrow_throw_if(col.type_id() != Type::TIMESTAMP, "Expected timestamp type");
1033  }
1035  cd, col, *bigint_buffer_, slice_range, bad_rows_tracker);
1036  case kDATE:
1037  if (exact_type_match) {
1038  arrow_throw_if(col.type_id() != Type::DATE32 && col.type_id() != Type::DATE64,
1039  "Expected date32 or date64 type");
1040  }
1042  cd, col, *bigint_buffer_, slice_range, bad_rows_tracker);
1043  case kPOINT:
1044  case kLINESTRING:
1045  case kPOLYGON:
1046  case kMULTIPOLYGON:
1047  arrow_throw_if(col.type_id() != Type::BINARY && col.type_id() != Type::STRING,
1048  "Expected string type");
1050  cd, col, *geo_string_buffer_, slice_range, bad_rows_tracker);
1051  case kARRAY:
1052  throw std::runtime_error("Arrow array appends not yet supported");
1053  default:
1054  throw std::runtime_error("Invalid Type");
1055  }
1056 }
std::vector< int16_t > * smallint_buffer_
Definition: Importer.h:615
std::vector< int64_t > * bigint_buffer_
Definition: Importer.h:617
Definition: sqltypes.h:51
std::vector< float > * float_buffer_
Definition: Importer.h:618
size_t convert_arrow_val_to_import_buffer(const ColumnDescriptor *cd, const arrow::Array &array, std::vector< DATA_TYPE > &buffer, const ArraySliceRange &slice_range, BadRowsTracker *const bad_rows_tracker)
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:330
std::vector< int8_t > * tinyint_buffer_
Definition: Importer.h:614
std::vector< std::string > * string_buffer_
Definition: Importer.h:620
std::vector< int32_t > * int_buffer_
Definition: Importer.h:616
bool is_decimal() const
Definition: sqltypes.h:453
std::vector< int8_t > * bool_buffer_
Definition: Importer.h:613
SQLTypes decimal_to_int_type(const SQLTypeInfo &ti)
Definition: Datum.cpp:268
Definition: sqltypes.h:54
Definition: sqltypes.h:55
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:621
std::vector< double > * double_buffer_
Definition: Importer.h:619
Definition: sqltypes.h:43
Definition: sqltypes.h:47
SQLTypeInfo columnType
std::string columnName
void arrow_throw_if(const bool cond, const std::string &message)
Definition: ArrowImporter.h:40
+ Here is the call graph for this function:

◆ add_value() [1/2]

void Importer_NS::TypedImportBuffer::add_value ( const ColumnDescriptor cd,
const std::string &  val,
const bool  is_null,
const CopyParams copy_params,
const int64_t  replicate_count = 0 
)

Definition at line 615 of file Importer.cpp.

References Datum::bigintval, Datum::boolval, CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, convert_decimal_value_to_scale(), anonymous_namespace{ImportTest.cpp}::d(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_elem_type(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_notnull(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_size(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_subtype(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), inline_fixed_encoding_null_val(), Datum::intval, IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, StringDictionary::MAX_STRLEN, NULL_DOUBLE, NULL_FLOAT, Importer_NS::NullArray(), Importer_NS::ImporterUtils::parseStringArray(), Datum::smallintval, Importer_NS::StringToArray(), StringToDatum(), Datum::tinyintval, to_string(), and run-benchmark-import::type.

619  {
620  set_replicate_count(replicate_count);
621  const auto type = cd->columnType.get_type();
622  switch (type) {
623  case kBOOLEAN: {
624  if (is_null) {
625  if (cd->columnType.get_notnull()) {
626  throw std::runtime_error("NULL for column " + cd->columnName);
627  }
629  } else {
630  SQLTypeInfo ti = cd->columnType;
631  Datum d = StringToDatum(val, ti);
632  addBoolean((int8_t)d.boolval);
633  }
634  break;
635  }
636  case kTINYINT: {
637  if (!is_null && (isdigit(val[0]) || val[0] == '-')) {
638  SQLTypeInfo ti = cd->columnType;
639  Datum d = StringToDatum(val, ti);
641  } else {
642  if (cd->columnType.get_notnull()) {
643  throw std::runtime_error("NULL for column " + cd->columnName);
644  }
646  }
647  break;
648  }
649  case kSMALLINT: {
650  if (!is_null && (isdigit(val[0]) || val[0] == '-')) {
651  SQLTypeInfo ti = cd->columnType;
652  Datum d = StringToDatum(val, ti);
654  } else {
655  if (cd->columnType.get_notnull()) {
656  throw std::runtime_error("NULL for column " + cd->columnName);
657  }
659  }
660  break;
661  }
662  case kINT: {
663  if (!is_null && (isdigit(val[0]) || val[0] == '-')) {
664  SQLTypeInfo ti = cd->columnType;
665  Datum d = StringToDatum(val, ti);
666  addInt(d.intval);
667  } else {
668  if (cd->columnType.get_notnull()) {
669  throw std::runtime_error("NULL for column " + cd->columnName);
670  }
672  }
673  break;
674  }
675  case kBIGINT: {
676  if (!is_null && (isdigit(val[0]) || val[0] == '-')) {
677  SQLTypeInfo ti = cd->columnType;
678  Datum d = StringToDatum(val, ti);
679  addBigint(d.bigintval);
680  } else {
681  if (cd->columnType.get_notnull()) {
682  throw std::runtime_error("NULL for column " + cd->columnName);
683  }
685  }
686  break;
687  }
688  case kDECIMAL:
689  case kNUMERIC: {
690  if (!is_null) {
691  SQLTypeInfo ti(kNUMERIC, 0, 0, false);
692  Datum d = StringToDatum(val, ti);
693  const auto converted_decimal_value =
695  addBigint(converted_decimal_value);
696  } else {
697  if (cd->columnType.get_notnull()) {
698  throw std::runtime_error("NULL for column " + cd->columnName);
699  }
701  }
702  break;
703  }
704  case kFLOAT:
705  if (!is_null && (val[0] == '.' || isdigit(val[0]) || val[0] == '-')) {
706  addFloat((float)std::atof(val.c_str()));
707  } else {
708  if (cd->columnType.get_notnull()) {
709  throw std::runtime_error("NULL for column " + cd->columnName);
710  }
712  }
713  break;
714  case kDOUBLE:
715  if (!is_null && (val[0] == '.' || isdigit(val[0]) || val[0] == '-')) {
716  addDouble(std::atof(val.c_str()));
717  } else {
718  if (cd->columnType.get_notnull()) {
719  throw std::runtime_error("NULL for column " + cd->columnName);
720  }
722  }
723  break;
724  case kTEXT:
725  case kVARCHAR:
726  case kCHAR: {
727  // @TODO(wei) for now, use empty string for nulls
728  if (is_null) {
729  if (cd->columnType.get_notnull()) {
730  throw std::runtime_error("NULL for column " + cd->columnName);
731  }
732  addString(std::string());
733  } else {
734  if (val.length() > StringDictionary::MAX_STRLEN) {
735  throw std::runtime_error("String too long for column " + cd->columnName +
736  " was " + std::to_string(val.length()) + " max is " +
738  }
739  addString(val);
740  }
741  break;
742  }
743  case kTIME:
744  case kTIMESTAMP:
745  case kDATE:
746  if (!is_null && (isdigit(val[0]) || val[0] == '-')) {
747  SQLTypeInfo ti = cd->columnType;
748  Datum d = StringToDatum(val, ti);
749  addBigint(d.bigintval);
750  } else {
751  if (cd->columnType.get_notnull()) {
752  throw std::runtime_error("NULL for column " + cd->columnName);
753  }
755  }
756  break;
757  case kARRAY: {
758  if (is_null && cd->columnType.get_notnull()) {
759  throw std::runtime_error("NULL for column " + cd->columnName);
760  }
761  SQLTypeInfo ti = cd->columnType;
762  if (IS_STRING(ti.get_subtype())) {
763  std::vector<std::string> string_vec;
764  // Just parse string array, don't push it to buffer yet as we might throw
765  ImporterUtils::parseStringArray(val, copy_params, string_vec);
766  if (!is_null) {
767  // TODO: add support for NULL string arrays
768  if (ti.get_size() > 0) {
769  auto sti = ti.get_elem_type();
770  size_t expected_size = ti.get_size() / sti.get_size();
771  size_t actual_size = string_vec.size();
772  if (actual_size != expected_size) {
773  throw std::runtime_error("Fixed length array column " + cd->columnName +
774  " expects " + std::to_string(expected_size) +
775  " values, received " +
776  std::to_string(actual_size));
777  }
778  }
779  addStringArray(string_vec);
780  } else {
781  if (ti.get_size() > 0) {
782  // TODO: remove once NULL fixlen arrays are allowed
783  throw std::runtime_error("Fixed length array column " + cd->columnName +
784  " currently cannot accept NULL arrays");
785  }
786  // TODO: add support for NULL string arrays, replace with addStringArray(),
787  // for now add whatever parseStringArray() outputs for NULLs ("NULL")
788  addStringArray(string_vec);
789  }
790  } else {
791  if (!is_null) {
792  ArrayDatum d = StringToArray(val, ti, copy_params);
793  if (d.is_null) { // val could be "NULL"
794  addArray(NullArray(ti));
795  } else {
796  if (ti.get_size() > 0 && static_cast<size_t>(ti.get_size()) != d.length) {
797  throw std::runtime_error("Fixed length array for column " + cd->columnName +
798  " has incorrect length: " + val);
799  }
800  addArray(d);
801  }
802  } else {
803  addArray(NullArray(ti));
804  }
805  }
806  break;
807  }
808  case kPOINT:
809  case kLINESTRING:
810  case kPOLYGON:
811  case kMULTIPOLYGON:
812  addGeoString(val);
813  break;
814  default:
815  CHECK(false) << "TypedImportBuffer::add_value() does not support type " << type;
816  }
817 }
int8_t tinyintval
Definition: sqltypes.h:123
void addSmallint(const int16_t v)
Definition: Importer.h:328
#define NULL_DOUBLE
Definition: sqltypes.h:176
void d(const SQLTypes expected_type, const std::string &str)
Definition: ImportTest.cpp:268
HOST DEVICE int get_size() const
Definition: sqltypes.h:333
void addGeoString(const std::string &v)
Definition: Importer.h:340
Definition: sqltypes.h:51
void addTinyint(const int8_t v)
Definition: Importer.h:326
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:330
bool boolval
Definition: sqltypes.h:122
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:323
void addInt(const int32_t v)
Definition: Importer.h:330
int32_t intval
Definition: sqltypes.h:125
ArrayDatum StringToArray(const std::string &s, const SQLTypeInfo &ti, const CopyParams &copy_params)
Definition: Importer.cpp:441
std::string to_string(char const *&&v)
void set_replicate_count(const int64_t replicate_count)
Definition: Importer.h:596
void addBoolean(const int8_t v)
Definition: Importer.h:324
void addFloat(const float v)
Definition: Importer.h:334
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:324
int64_t bigintval
Definition: sqltypes.h:126
#define NULL_FLOAT
Definition: sqltypes.h:175
int16_t smallintval
Definition: sqltypes.h:124
SQLTypeInfoCore get_elem_type() const
Definition: sqltypes.h:632
void addBigint(const int64_t v)
Definition: Importer.h:332
static bool parseStringArray(const std::string &s, const CopyParams &copy_params, std::vector< std::string > &string_vec)
Definition: Importer.h:814
Definition: sqltypes.h:54
Definition: sqltypes.h:55
bool is_null(const T &v, const SQLTypeInfo &t)
int64_t convert_decimal_value_to_scale(const int64_t decimal_value, const SQLTypeInfo &type_info, const SQLTypeInfo &new_type_info)
Definition: Datum.cpp:284
void addArray(const ArrayDatum &v)
Definition: Importer.h:342
Datum StringToDatum(const std::string &s, SQLTypeInfo &ti)
Definition: Datum.cpp:90
Definition: sqltypes.h:43
ArrayDatum NullArray(const SQLTypeInfo &ti)
Definition: Importer.cpp:494
#define IS_STRING(T)
Definition: sqltypes.h:163
void addString(const std::string &v)
Definition: Importer.h:338
#define CHECK(condition)
Definition: Logger.h:187
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
static constexpr size_t MAX_STRLEN
Definition: sqltypes.h:47
SQLTypeInfo columnType
std::vector< std::string > & addStringArray()
Definition: Importer.h:344
std::string columnName
std::conditional_t< isCudaCC(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:119
void addDouble(const double v)
Definition: Importer.h:336
+ Here is the call graph for this function:

◆ add_value() [2/2]

void Importer_NS::TypedImportBuffer::add_value ( const ColumnDescriptor cd,
const TDatum &  val,
const bool  is_null,
const int64_t  replicate_count = 0 
)

Definition at line 1380 of file Importer.cpp.

References Importer_NS::addBinaryStringArray(), CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, decimal_to_int_type(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_notnull(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_subtype(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), inline_fixed_encoding_null_val(), SQLTypeInfoCore< TYPE_FACET_PACK >::is_decimal(), IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, NULL_DOUBLE, NULL_FLOAT, Importer_NS::NullArray(), Importer_NS::TDatumToArrayDatum(), and run-benchmark-import::type.

1383  {
1384  set_replicate_count(replicate_count);
1385  const auto type = cd->columnType.is_decimal() ? decimal_to_int_type(cd->columnType)
1386  : cd->columnType.get_type();
1387  switch (type) {
1388  case kBOOLEAN: {
1389  if (is_null) {
1390  if (cd->columnType.get_notnull()) {
1391  throw std::runtime_error("NULL for column " + cd->columnName);
1392  }
1394  } else {
1395  addBoolean((int8_t)datum.val.int_val);
1396  }
1397  break;
1398  }
1399  case kTINYINT:
1400  if (!is_null) {
1401  addTinyint((int8_t)datum.val.int_val);
1402  } else {
1403  if (cd->columnType.get_notnull()) {
1404  throw std::runtime_error("NULL for column " + cd->columnName);
1405  }
1407  }
1408  break;
1409  case kSMALLINT:
1410  if (!is_null) {
1411  addSmallint((int16_t)datum.val.int_val);
1412  } else {
1413  if (cd->columnType.get_notnull()) {
1414  throw std::runtime_error("NULL for column " + cd->columnName);
1415  }
1417  }
1418  break;
1419  case kINT:
1420  if (!is_null) {
1421  addInt((int32_t)datum.val.int_val);
1422  } else {
1423  if (cd->columnType.get_notnull()) {
1424  throw std::runtime_error("NULL for column " + cd->columnName);
1425  }
1427  }
1428  break;
1429  case kBIGINT:
1430  if (!is_null) {
1431  addBigint(datum.val.int_val);
1432  } else {
1433  if (cd->columnType.get_notnull()) {
1434  throw std::runtime_error("NULL for column " + cd->columnName);
1435  }
1437  }
1438  break;
1439  case kFLOAT:
1440  if (!is_null) {
1441  addFloat((float)datum.val.real_val);
1442  } else {
1443  if (cd->columnType.get_notnull()) {
1444  throw std::runtime_error("NULL for column " + cd->columnName);
1445  }
1447  }
1448  break;
1449  case kDOUBLE:
1450  if (!is_null) {
1451  addDouble(datum.val.real_val);
1452  } else {
1453  if (cd->columnType.get_notnull()) {
1454  throw std::runtime_error("NULL for column " + cd->columnName);
1455  }
1457  }
1458  break;
1459  case kTEXT:
1460  case kVARCHAR:
1461  case kCHAR: {
1462  // @TODO(wei) for now, use empty string for nulls
1463  if (is_null) {
1464  if (cd->columnType.get_notnull()) {
1465  throw std::runtime_error("NULL for column " + cd->columnName);
1466  }
1467  addString(std::string());
1468  } else {
1469  addString(datum.val.str_val);
1470  }
1471  break;
1472  }
1473  case kTIME:
1474  case kTIMESTAMP:
1475  case kDATE: {
1476  if (!is_null) {
1477  addBigint(datum.val.int_val);
1478  } else {
1479  if (cd->columnType.get_notnull()) {
1480  throw std::runtime_error("NULL for column " + cd->columnName);
1481  }
1483  }
1484  break;
1485  }
1486  case kARRAY:
1487  if (is_null && cd->columnType.get_notnull()) {
1488  throw std::runtime_error("NULL for column " + cd->columnName);
1489  }
1490  if (IS_STRING(cd->columnType.get_subtype())) {
1491  std::vector<std::string>& string_vec = addStringArray();
1492  addBinaryStringArray(datum, string_vec);
1493  } else {
1494  if (!is_null) {
1495  addArray(TDatumToArrayDatum(datum, cd->columnType));
1496  } else {
1498  }
1499  }
1500  break;
1501  case kPOINT:
1502  case kLINESTRING:
1503  case kPOLYGON:
1504  case kMULTIPOLYGON:
1505  if (is_null) {
1506  if (cd->columnType.get_notnull()) {
1507  throw std::runtime_error("NULL for column " + cd->columnName);
1508  }
1509  addGeoString(std::string());
1510  } else {
1511  addGeoString(datum.val.str_val);
1512  }
1513  break;
1514  default:
1515  CHECK(false) << "TypedImportBuffer::add_value() does not support type " << type;
1516  }
1517 }
void addSmallint(const int16_t v)
Definition: Importer.h:328
#define NULL_DOUBLE
Definition: sqltypes.h:176
void addGeoString(const std::string &v)
Definition: Importer.h:340
Definition: sqltypes.h:51
void addTinyint(const int8_t v)
Definition: Importer.h:326
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:330
void addInt(const int32_t v)
Definition: Importer.h:330
void set_replicate_count(const int64_t replicate_count)
Definition: Importer.h:596
void addBoolean(const int8_t v)
Definition: Importer.h:324
void addFloat(const float v)
Definition: Importer.h:334
bool is_decimal() const
Definition: sqltypes.h:453
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:324
#define NULL_FLOAT
Definition: sqltypes.h:175
void addBigint(const int64_t v)
Definition: Importer.h:332
SQLTypes decimal_to_int_type(const SQLTypeInfo &ti)
Definition: Datum.cpp:268
Definition: sqltypes.h:54
Definition: sqltypes.h:55
bool is_null(const T &v, const SQLTypeInfo &t)
void addArray(const ArrayDatum &v)
Definition: Importer.h:342
ArrayDatum TDatumToArrayDatum(const TDatum &datum, const SQLTypeInfo &ti)
Definition: Importer.cpp:578
Definition: sqltypes.h:43
ArrayDatum NullArray(const SQLTypeInfo &ti)
Definition: Importer.cpp:494
#define IS_STRING(T)
Definition: sqltypes.h:163
void addString(const std::string &v)
Definition: Importer.h:338
#define CHECK(condition)
Definition: Logger.h:187
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
Definition: sqltypes.h:47
SQLTypeInfo columnType
std::vector< std::string > & addStringArray()
Definition: Importer.h:344
std::string columnName
void addBinaryStringArray(const TDatum &datum, std::vector< std::string > &string_vec)
Definition: Importer.cpp:526
void addDouble(const double v)
Definition: Importer.h:336
+ Here is the call graph for this function:

◆ add_values()

size_t Importer_NS::TypedImportBuffer::add_values ( const ColumnDescriptor cd,
const TColumn &  data 
)

Definition at line 1059 of file Importer.cpp.

References checked_malloc(), ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfoCore< TYPE_FACET_PACK >::get_notnull(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_subtype(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), inline_fixed_encoding_null_val(), IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, NULL_DOUBLE, NULL_FLOAT, and Importer_NS::NullArray().

1059  {
1060  size_t dataSize = 0;
1061  if (cd->columnType.get_notnull()) {
1062  // We can't have any null values for this column; to have them is an error
1063  if (std::any_of(col.nulls.begin(), col.nulls.end(), [](int i) { return i != 0; })) {
1064  throw std::runtime_error("NULL for column " + cd->columnName);
1065  }
1066  }
1067 
1068  switch (cd->columnType.get_type()) {
1069  case kBOOLEAN: {
1070  dataSize = col.data.int_col.size();
1071  bool_buffer_->reserve(dataSize);
1072  for (size_t i = 0; i < dataSize; i++) {
1073  if (col.nulls[i]) {
1075  } else {
1076  bool_buffer_->push_back((int8_t)col.data.int_col[i]);
1077  }
1078  }
1079  break;
1080  }
1081  case kTINYINT: {
1082  dataSize = col.data.int_col.size();
1083  tinyint_buffer_->reserve(dataSize);
1084  for (size_t i = 0; i < dataSize; i++) {
1085  if (col.nulls[i]) {
1087  } else {
1088  tinyint_buffer_->push_back((int8_t)col.data.int_col[i]);
1089  }
1090  }
1091  break;
1092  }
1093  case kSMALLINT: {
1094  dataSize = col.data.int_col.size();
1095  smallint_buffer_->reserve(dataSize);
1096  for (size_t i = 0; i < dataSize; i++) {
1097  if (col.nulls[i]) {
1099  } else {
1100  smallint_buffer_->push_back((int16_t)col.data.int_col[i]);
1101  }
1102  }
1103  break;
1104  }
1105  case kINT: {
1106  dataSize = col.data.int_col.size();
1107  int_buffer_->reserve(dataSize);
1108  for (size_t i = 0; i < dataSize; i++) {
1109  if (col.nulls[i]) {
1111  } else {
1112  int_buffer_->push_back((int32_t)col.data.int_col[i]);
1113  }
1114  }
1115  break;
1116  }
1117  case kBIGINT:
1118  case kNUMERIC:
1119  case kDECIMAL: {
1120  dataSize = col.data.int_col.size();
1121  bigint_buffer_->reserve(dataSize);
1122  for (size_t i = 0; i < dataSize; i++) {
1123  if (col.nulls[i]) {
1125  } else {
1126  bigint_buffer_->push_back((int64_t)col.data.int_col[i]);
1127  }
1128  }
1129  break;
1130  }
1131  case kFLOAT: {
1132  dataSize = col.data.real_col.size();
1133  float_buffer_->reserve(dataSize);
1134  for (size_t i = 0; i < dataSize; i++) {
1135  if (col.nulls[i]) {
1136  float_buffer_->push_back(NULL_FLOAT);
1137  } else {
1138  float_buffer_->push_back((float)col.data.real_col[i]);
1139  }
1140  }
1141  break;
1142  }
1143  case kDOUBLE: {
1144  dataSize = col.data.real_col.size();
1145  double_buffer_->reserve(dataSize);
1146  for (size_t i = 0; i < dataSize; i++) {
1147  if (col.nulls[i]) {
1148  double_buffer_->push_back(NULL_DOUBLE);
1149  } else {
1150  double_buffer_->push_back((double)col.data.real_col[i]);
1151  }
1152  }
1153  break;
1154  }
1155  case kTEXT:
1156  case kVARCHAR:
1157  case kCHAR: {
1158  // TODO: for now, use empty string for nulls
1159  dataSize = col.data.str_col.size();
1160  string_buffer_->reserve(dataSize);
1161  for (size_t i = 0; i < dataSize; i++) {
1162  if (col.nulls[i]) {
1163  string_buffer_->push_back(std::string());
1164  } else {
1165  string_buffer_->push_back(col.data.str_col[i]);
1166  }
1167  }
1168  break;
1169  }
1170  case kTIME:
1171  case kTIMESTAMP:
1172  case kDATE: {
1173  dataSize = col.data.int_col.size();
1174  bigint_buffer_->reserve(dataSize);
1175  for (size_t i = 0; i < dataSize; i++) {
1176  if (col.nulls[i]) {
1178  } else {
1179  bigint_buffer_->push_back(static_cast<int64_t>(col.data.int_col[i]));
1180  }
1181  }
1182  break;
1183  }
1184  case kPOINT:
1185  case kLINESTRING:
1186  case kPOLYGON:
1187  case kMULTIPOLYGON: {
1188  dataSize = col.data.str_col.size();
1189  geo_string_buffer_->reserve(dataSize);
1190  for (size_t i = 0; i < dataSize; i++) {
1191  if (col.nulls[i]) {
1192  // TODO: add support for NULL geo
1193  geo_string_buffer_->push_back(std::string());
1194  } else {
1195  geo_string_buffer_->push_back(col.data.str_col[i]);
1196  }
1197  }
1198  break;
1199  }
1200  case kARRAY: {
1201  dataSize = col.data.arr_col.size();
1202  if (IS_STRING(cd->columnType.get_subtype())) {
1203  for (size_t i = 0; i < dataSize; i++) {
1204  std::vector<std::string>& string_vec = addStringArray();
1205  if (!col.nulls[i]) {
1206  size_t stringArrSize = col.data.arr_col[i].data.str_col.size();
1207  for (size_t str_idx = 0; str_idx != stringArrSize; ++str_idx) {
1208  string_vec.push_back(col.data.arr_col[i].data.str_col[str_idx]);
1209  }
1210  }
1211  }
1212  } else {
1213  auto elem_ti = cd->columnType.get_subtype();
1214  switch (elem_ti) {
1215  case kBOOLEAN: {
1216  for (size_t i = 0; i < dataSize; i++) {
1217  if (col.nulls[i]) {
1219  } else {
1220  size_t len = col.data.arr_col[i].data.int_col.size();
1221  size_t byteSize = len * sizeof(int8_t);
1222  int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
1223  int8_t* p = buf;
1224  for (size_t j = 0; j < len; ++j) {
1225  *(bool*)p = static_cast<bool>(col.data.arr_col[i].data.int_col[j]);
1226  p += sizeof(bool);
1227  }
1228  addArray(ArrayDatum(byteSize, buf, false));
1229  }
1230  }
1231  break;
1232  }
1233  case kTINYINT: {
1234  for (size_t i = 0; i < dataSize; i++) {
1235  if (col.nulls[i]) {
1237  } else {
1238  size_t len = col.data.arr_col[i].data.int_col.size();
1239  size_t byteSize = len * sizeof(int8_t);
1240  int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
1241  int8_t* p = buf;
1242  for (size_t j = 0; j < len; ++j) {
1243  *(int8_t*)p = static_cast<int8_t>(col.data.arr_col[i].data.int_col[j]);
1244  p += sizeof(int8_t);
1245  }
1246  addArray(ArrayDatum(byteSize, buf, false));
1247  }
1248  }
1249  break;
1250  }
1251  case kSMALLINT: {
1252  for (size_t i = 0; i < dataSize; i++) {
1253  if (col.nulls[i]) {
1255  } else {
1256  size_t len = col.data.arr_col[i].data.int_col.size();
1257  size_t byteSize = len * sizeof(int16_t);
1258  int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
1259  int8_t* p = buf;
1260  for (size_t j = 0; j < len; ++j) {
1261  *(int16_t*)p =
1262  static_cast<int16_t>(col.data.arr_col[i].data.int_col[j]);
1263  p += sizeof(int16_t);
1264  }
1265  addArray(ArrayDatum(byteSize, buf, false));
1266  }
1267  }
1268  break;
1269  }
1270  case kINT: {
1271  for (size_t i = 0; i < dataSize; i++) {
1272  if (col.nulls[i]) {
1274  } else {
1275  size_t len = col.data.arr_col[i].data.int_col.size();
1276  size_t byteSize = len * sizeof(int32_t);
1277  int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
1278  int8_t* p = buf;
1279  for (size_t j = 0; j < len; ++j) {
1280  *(int32_t*)p =
1281  static_cast<int32_t>(col.data.arr_col[i].data.int_col[j]);
1282  p += sizeof(int32_t);
1283  }
1284  addArray(ArrayDatum(byteSize, buf, false));
1285  }
1286  }
1287  break;
1288  }
1289  case kBIGINT:
1290  case kNUMERIC:
1291  case kDECIMAL: {
1292  for (size_t i = 0; i < dataSize; i++) {
1293  if (col.nulls[i]) {
1295  } else {
1296  size_t len = col.data.arr_col[i].data.int_col.size();
1297  size_t byteSize = len * sizeof(int64_t);
1298  int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
1299  int8_t* p = buf;
1300  for (size_t j = 0; j < len; ++j) {
1301  *(int64_t*)p =
1302  static_cast<int64_t>(col.data.arr_col[j].data.int_col[j]);
1303  p += sizeof(int64_t);
1304  }
1305  addArray(ArrayDatum(byteSize, buf, false));
1306  }
1307  }
1308  break;
1309  }
1310  case kFLOAT: {
1311  for (size_t i = 0; i < dataSize; i++) {
1312  if (col.nulls[i]) {
1314  } else {
1315  size_t len = col.data.arr_col[i].data.real_col.size();
1316  size_t byteSize = len * sizeof(float);
1317  int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
1318  int8_t* p = buf;
1319  for (size_t j = 0; j < len; ++j) {
1320  *(float*)p = static_cast<float>(col.data.arr_col[i].data.real_col[j]);
1321  p += sizeof(float);
1322  }
1323  addArray(ArrayDatum(byteSize, buf, false));
1324  }
1325  }
1326  break;
1327  }
1328  case kDOUBLE: {
1329  for (size_t i = 0; i < dataSize; i++) {
1330  if (col.nulls[i]) {
1332  } else {
1333  size_t len = col.data.arr_col[i].data.real_col.size();
1334  size_t byteSize = len * sizeof(double);
1335  int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
1336  int8_t* p = buf;
1337  for (size_t j = 0; j < len; ++j) {
1338  *(double*)p = static_cast<double>(col.data.arr_col[i].data.real_col[j]);
1339  p += sizeof(double);
1340  }
1341  addArray(ArrayDatum(byteSize, buf, false));
1342  }
1343  }
1344  break;
1345  }
1346  case kTIME:
1347  case kTIMESTAMP:
1348  case kDATE: {
1349  for (size_t i = 0; i < dataSize; i++) {
1350  if (col.nulls[i]) {
1352  } else {
1353  size_t len = col.data.arr_col[i].data.int_col.size();
1354  size_t byteWidth = sizeof(int64_t);
1355  size_t byteSize = len * byteWidth;
1356  int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
1357  int8_t* p = buf;
1358  for (size_t j = 0; j < len; ++j) {
1359  *reinterpret_cast<int64_t*>(p) =
1360  static_cast<int64_t>(col.data.arr_col[i].data.int_col[j]);
1361  p += sizeof(int64_t);
1362  }
1363  addArray(ArrayDatum(byteSize, buf, false));
1364  }
1365  }
1366  break;
1367  }
1368  default:
1369  throw std::runtime_error("Invalid Array Type");
1370  }
1371  }
1372  break;
1373  }
1374  default:
1375  throw std::runtime_error("Invalid Type");
1376  }
1377  return dataSize;
1378 }
std::vector< int16_t > * smallint_buffer_
Definition: Importer.h:615
#define NULL_DOUBLE
Definition: sqltypes.h:176
std::vector< int64_t > * bigint_buffer_
Definition: Importer.h:617
Definition: sqltypes.h:51
std::vector< float > * float_buffer_
Definition: Importer.h:618
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:330
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:323
std::vector< int8_t > * tinyint_buffer_
Definition: Importer.h:614
std::vector< std::string > * string_buffer_
Definition: Importer.h:620
std::vector< int32_t > * int_buffer_
Definition: Importer.h:616
void * checked_malloc(const size_t size)
Definition: checked_alloc.h:40
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:324
#define NULL_FLOAT
Definition: sqltypes.h:175
std::vector< int8_t > * bool_buffer_
Definition: Importer.h:613
Definition: sqltypes.h:54
Definition: sqltypes.h:55
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:621
void addArray(const ArrayDatum &v)
Definition: Importer.h:342
std::vector< double > * double_buffer_
Definition: Importer.h:619
Definition: sqltypes.h:43
ArrayDatum NullArray(const SQLTypeInfo &ti)
Definition: Importer.cpp:494
#define IS_STRING(T)
Definition: sqltypes.h:163
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
Definition: sqltypes.h:47
SQLTypeInfo columnType
std::vector< std::string > & addStringArray()
Definition: Importer.h:344
std::string columnName
std::conditional_t< isCudaCC(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:119
+ Here is the call graph for this function:

◆ addArray()

void Importer_NS::TypedImportBuffer::addArray ( const ArrayDatum v)
inline

Definition at line 342 of file Importer.h.

342 { array_buffer_->push_back(v); }
T v(const TargetValue &r)
std::vector< ArrayDatum > * array_buffer_
Definition: Importer.h:622

◆ addBigint()

void Importer_NS::TypedImportBuffer::addBigint ( const int64_t  v)
inline

Definition at line 332 of file Importer.h.

332 { bigint_buffer_->push_back(v); }
std::vector< int64_t > * bigint_buffer_
Definition: Importer.h:617
T v(const TargetValue &r)

◆ addBoolean()

void Importer_NS::TypedImportBuffer::addBoolean ( const int8_t  v)
inline

Definition at line 324 of file Importer.h.

324 { bool_buffer_->push_back(v); }
T v(const TargetValue &r)
std::vector< int8_t > * bool_buffer_
Definition: Importer.h:613

◆ addDictEncodedString()

void Importer_NS::TypedImportBuffer::addDictEncodedString ( const std::vector< std::string > &  string_vec)
inline

Definition at line 353 of file Importer.h.

References CHECK, and StringDictionary::MAX_STRLEN.

353  {
355  for (const auto& str : string_vec) {
356  if (str.size() > StringDictionary::MAX_STRLEN) {
357  throw std::runtime_error("String too long for dictionary encoding.");
358  }
359  }
360  switch (column_desc_->columnType.get_size()) {
361  case 1:
362  string_dict_i8_buffer_->resize(string_vec.size());
363  string_dict_->getOrAddBulk(string_vec, string_dict_i8_buffer_->data());
364  break;
365  case 2:
366  string_dict_i16_buffer_->resize(string_vec.size());
367  string_dict_->getOrAddBulk(string_vec, string_dict_i16_buffer_->data());
368  break;
369  case 4:
370  string_dict_i32_buffer_->resize(string_vec.size());
371  string_dict_->getOrAddBulk(string_vec, string_dict_i32_buffer_->data());
372  break;
373  default:
374  CHECK(false);
375  }
376  }
HOST DEVICE int get_size() const
Definition: sqltypes.h:333
std::vector< uint16_t > * string_dict_i16_buffer_
Definition: Importer.h:627
StringDictionary * string_dict_
Definition: Importer.h:632
std::vector< int32_t > * string_dict_i32_buffer_
Definition: Importer.h:628
void getOrAddBulk(const std::vector< std::string > &string_vec, T *encoded_vec)
const ColumnDescriptor * column_desc_
Definition: Importer.h:631
#define CHECK(condition)
Definition: Logger.h:187
static constexpr size_t MAX_STRLEN
SQLTypeInfo columnType
std::vector< uint8_t > * string_dict_i8_buffer_
Definition: Importer.h:626

◆ addDictEncodedStringArray()

void Importer_NS::TypedImportBuffer::addDictEncodedStringArray ( const std::vector< std::vector< std::string >> &  string_array_vec)
inline

Definition at line 378 of file Importer.h.

References CHECK, checked_malloc(), and StringDictionary::MAX_STRLEN.

379  {
381 
382  // first check data is ok
383  for (auto& p : string_array_vec) {
384  for (const auto& str : p) {
385  if (str.size() > StringDictionary::MAX_STRLEN) {
386  throw std::runtime_error("String too long for dictionary encoding.");
387  }
388  }
389  }
390 
391  std::vector<std::vector<int32_t>> ids_array(0);
392  string_dict_->getOrAddBulkArray(string_array_vec, ids_array);
393 
394  for (auto& p : ids_array) {
395  size_t len = p.size() * sizeof(int32_t);
396  auto a = static_cast<int32_t*>(checked_malloc(len));
397  memcpy(a, &p[0], len);
398  // TODO: distinguish between empty and NULL
399  string_array_dict_buffer_->push_back(
400  ArrayDatum(len, reinterpret_cast<int8_t*>(a), len == 0));
401  }
402  }
StringDictionary * string_dict_
Definition: Importer.h:632
void * checked_malloc(const size_t size)
Definition: checked_alloc.h:40
void getOrAddBulkArray(const std::vector< std::vector< std::string >> &string_array_vec, std::vector< std::vector< int32_t >> &ids_array_vec)
std::vector< ArrayDatum > * string_array_dict_buffer_
Definition: Importer.h:629
#define CHECK(condition)
Definition: Logger.h:187
static constexpr size_t MAX_STRLEN
std::conditional_t< isCudaCC(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:119
+ Here is the call graph for this function:

◆ addDouble()

void Importer_NS::TypedImportBuffer::addDouble ( const double  v)
inline

Definition at line 336 of file Importer.h.

336 { double_buffer_->push_back(v); }
T v(const TargetValue &r)
std::vector< double > * double_buffer_
Definition: Importer.h:619

◆ addFloat()

void Importer_NS::TypedImportBuffer::addFloat ( const float  v)
inline

Definition at line 334 of file Importer.h.

334 { float_buffer_->push_back(v); }
std::vector< float > * float_buffer_
Definition: Importer.h:618
T v(const TargetValue &r)

◆ addGeoString()

void Importer_NS::TypedImportBuffer::addGeoString ( const std::string &  v)
inline

Definition at line 340 of file Importer.h.

340 { geo_string_buffer_->push_back(v); }
T v(const TargetValue &r)
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:621

◆ addInt()

void Importer_NS::TypedImportBuffer::addInt ( const int32_t  v)
inline

Definition at line 330 of file Importer.h.

330 { int_buffer_->push_back(v); }
T v(const TargetValue &r)
std::vector< int32_t > * int_buffer_
Definition: Importer.h:616

◆ addSmallint()

void Importer_NS::TypedImportBuffer::addSmallint ( const int16_t  v)
inline

Definition at line 328 of file Importer.h.

328 { smallint_buffer_->push_back(v); }
std::vector< int16_t > * smallint_buffer_
Definition: Importer.h:615
T v(const TargetValue &r)

◆ addString()

void Importer_NS::TypedImportBuffer::addString ( const std::string &  v)
inline

Definition at line 338 of file Importer.h.

338 { string_buffer_->push_back(v); }
T v(const TargetValue &r)
std::vector< std::string > * string_buffer_
Definition: Importer.h:620

◆ addStringArray() [1/2]

std::vector<std::string>& Importer_NS::TypedImportBuffer::addStringArray ( )
inline

Definition at line 344 of file Importer.h.

344  {
345  string_array_buffer_->push_back(std::vector<std::string>());
346  return string_array_buffer_->back();
347  }
std::vector< std::vector< std::string > > * string_array_buffer_
Definition: Importer.h:623

◆ addStringArray() [2/2]

void Importer_NS::TypedImportBuffer::addStringArray ( const std::vector< std::string > &  arr)
inline

Definition at line 349 of file Importer.h.

349  {
350  string_array_buffer_->push_back(arr);
351  }
std::vector< std::vector< std::string > > * string_array_buffer_
Definition: Importer.h:623

◆ addTinyint()

void Importer_NS::TypedImportBuffer::addTinyint ( const int8_t  v)
inline

Definition at line 326 of file Importer.h.

326 { tinyint_buffer_->push_back(v); }
std::vector< int8_t > * tinyint_buffer_
Definition: Importer.h:614
T v(const TargetValue &r)

◆ clear()

void Importer_NS::TypedImportBuffer::clear ( )
inline

Definition at line 498 of file Importer.h.

References Experimental::Array, CHECK, anonymous_namespace{TypedDataAccessors.h}::is_null(), IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kENCODING_DICT, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, and kVARCHAR.

498  {
499  switch (column_desc_->columnType.get_type()) {
500  case kBOOLEAN: {
501  bool_buffer_->clear();
502  break;
503  }
504  case kTINYINT: {
505  tinyint_buffer_->clear();
506  break;
507  }
508  case kSMALLINT: {
509  smallint_buffer_->clear();
510  break;
511  }
512  case kINT: {
513  int_buffer_->clear();
514  break;
515  }
516  case kBIGINT:
517  case kNUMERIC:
518  case kDECIMAL: {
519  bigint_buffer_->clear();
520  break;
521  }
522  case kFLOAT: {
523  float_buffer_->clear();
524  break;
525  }
526  case kDOUBLE: {
527  double_buffer_->clear();
528  break;
529  }
530  case kTEXT:
531  case kVARCHAR:
532  case kCHAR: {
533  string_buffer_->clear();
535  switch (column_desc_->columnType.get_size()) {
536  case 1:
537  string_dict_i8_buffer_->clear();
538  break;
539  case 2:
540  string_dict_i16_buffer_->clear();
541  break;
542  case 4:
543  string_dict_i32_buffer_->clear();
544  break;
545  default:
546  CHECK(false);
547  }
548  }
549  break;
550  }
551  case kDATE:
552  case kTIME:
553  case kTIMESTAMP:
554  bigint_buffer_->clear();
555  break;
556  case kARRAY: {
558  string_array_buffer_->clear();
559  string_array_dict_buffer_->clear();
560  } else {
561  array_buffer_->clear();
562  }
563  break;
564  }
565  case kPOINT:
566  case kLINESTRING:
567  case kPOLYGON:
568  case kMULTIPOLYGON:
569  geo_string_buffer_->clear();
570  break;
571  default:
572  CHECK(false);
573  }
574  }
std::vector< int16_t > * smallint_buffer_
Definition: Importer.h:615
HOST DEVICE int get_size() const
Definition: sqltypes.h:333
std::vector< int64_t > * bigint_buffer_
Definition: Importer.h:617
Definition: sqltypes.h:51
std::vector< float > * float_buffer_
Definition: Importer.h:618
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:323
std::vector< uint16_t > * string_dict_i16_buffer_
Definition: Importer.h:627
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:331
std::vector< int8_t > * tinyint_buffer_
Definition: Importer.h:614
std::vector< std::string > * string_buffer_
Definition: Importer.h:620
std::vector< int32_t > * string_dict_i32_buffer_
Definition: Importer.h:628
std::vector< std::vector< std::string > > * string_array_buffer_
Definition: Importer.h:623
std::vector< int32_t > * int_buffer_
Definition: Importer.h:616
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:324
std::vector< int8_t > * bool_buffer_
Definition: Importer.h:613
Definition: sqltypes.h:54
Definition: sqltypes.h:55
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:621
std::vector< ArrayDatum > * string_array_dict_buffer_
Definition: Importer.h:629
std::vector< double > * double_buffer_
Definition: Importer.h:619
Definition: sqltypes.h:43
const ColumnDescriptor * column_desc_
Definition: Importer.h:631
#define IS_STRING(T)
Definition: sqltypes.h:163
#define CHECK(condition)
Definition: Logger.h:187
Definition: sqltypes.h:47
SQLTypeInfo columnType
std::vector< uint8_t > * string_dict_i8_buffer_
Definition: Importer.h:626
std::vector< ArrayDatum > * array_buffer_
Definition: Importer.h:622
+ Here is the call graph for this function:

◆ convert_arrow_val_to_import_buffer() [1/2]

template<typename DATA_TYPE >
size_t Importer_NS::TypedImportBuffer::convert_arrow_val_to_import_buffer ( const ColumnDescriptor cd,
const arrow::Array &  array,
std::vector< DATA_TYPE > &  buffer,
const ArraySliceRange slice_range,
BadRowsTracker *const  bad_rows_tracker 
)

◆ convert_arrow_val_to_import_buffer() [2/2]

template<typename DATA_TYPE >
size_t Importer_NS::TypedImportBuffer::convert_arrow_val_to_import_buffer ( const ColumnDescriptor cd,
const Array &  array,
std::vector< DATA_TYPE > &  buffer,
const ArraySliceRange slice_range,
Importer_NS::BadRowsTracker *const  bad_rows_tracker 
)

Definition at line 879 of file Importer.cpp.

References Importer_NS::Importer::buffer, anonymous_namespace{ArrowImporter.h}::error_context(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), Importer_NS::Importer::getCatalog(), Geo_namespace::GeoTypesFactory::getGeoColumns(), Importer_NS::BadRowsTracker::importer, Importer_NS::BadRowsTracker::mutex, Importer_NS::BadRowsTracker::rows, Importer_NS::Importer::set_geo_physical_import_buffer(), and anonymous_namespace{ArrowImporter.h}::value_getter().

884  {
885  auto data =
886  std::make_unique<DataBuffer<DATA_TYPE>>(cd, array, buffer, bad_rows_tracker);
887  auto f_value_getter = value_getter(array, cd, bad_rows_tracker);
888  std::function<void(const int64_t)> f_add_geo_phy_cols = [&](const int64_t row) {};
889  if (bad_rows_tracker && cd->columnType.is_geometry()) {
890  f_add_geo_phy_cols = [&](const int64_t row) {
891  // Populate physical columns (ref. MapDHandler::load_table)
892  std::vector<double> coords, bounds;
893  std::vector<int> ring_sizes, poly_rings;
894  int render_group = 0;
895  SQLTypeInfo ti;
896  // replace any unexpected exception from getGeoColumns or other
897  // on this path with a GeoImportException so that we wont over
898  // push a null to the logical column...
899  try {
900  arrow_throw_if<GeoImportException>(
901  array.IsNull(row) ||
903  ti,
904  coords,
905  bounds,
906  ring_sizes,
907  poly_rings,
908  false),
909  error_context(cd, bad_rows_tracker) + "Invalid geometry");
910  arrow_throw_if<GeoImportException>(
911  cd->columnType.get_type() != ti.get_type(),
912  error_context(cd, bad_rows_tracker) + "Geometry type mismatch");
913  auto col_idx_workpad = col_idx; // what a pitfall!!
915  bad_rows_tracker->importer->getCatalog(),
916  cd,
918  col_idx_workpad,
919  coords,
920  bounds,
921  ring_sizes,
922  poly_rings,
923  render_group);
924  } catch (GeoImportException&) {
925  throw;
926  } catch (std::runtime_error& e) {
927  throw GeoImportException(e.what());
928  } catch (const std::exception& e) {
929  throw GeoImportException(e.what());
930  } catch (...) {
931  throw GeoImportException("unknown exception");
932  }
933  };
934  }
935  auto f_mark_a_bad_row = [&](const auto row) {
936  std::unique_lock<std::mutex> lck(bad_rows_tracker->mutex);
937  bad_rows_tracker->rows.insert(row - slice_range.first);
938  };
939  buffer.reserve(slice_range.second - slice_range.first);
940  for (size_t row = slice_range.first; row < slice_range.second; ++row) {
941  try {
942  *data << (array.IsNull(row) ? nullptr : f_value_getter(array, row));
943  f_add_geo_phy_cols(row);
944  } catch (GeoImportException&) {
945  f_mark_a_bad_row(row);
946  } catch (ArrowImporterException&) {
947  // trace bad rows of each column; otherwise rethrow.
948  if (bad_rows_tracker) {
949  *data << nullptr;
950  f_mark_a_bad_row(row);
951  } else {
952  throw;
953  }
954  }
955  }
956  return buffer.size();
957 }
std::string error_context(const ColumnDescriptor *cd, Importer_NS::BadRowsTracker *const bad_rows_tracker)
Definition: ArrowImporter.h:75
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:323
auto value_getter(const Array &array, const ColumnDescriptor *cd, Importer_NS::BadRowsTracker *const bad_rows_tracker)
Catalog_Namespace::Catalog & getCatalog()
Definition: Importer.h:932
static void set_geo_physical_import_buffer(const Catalog_Namespace::Catalog &catalog, const ColumnDescriptor *cd, std::vector< std::unique_ptr< TypedImportBuffer >> &import_buffers, size_t &col_idx, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, int render_group, const int64_t replicate_count=0)
Definition: Importer.cpp:1581
std::vector< std::unique_ptr< TypedImportBuffer > > * import_buffers
Definition: Importer.h:608
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:621
static bool getGeoColumns(const std::string &wkt, SQLTypeInfo &ti, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, const bool promote_poly_to_mpoly=false)
Definition: geo_types.cpp:459
std::set< int64_t > rows
Definition: Importer.h:80
+ Here is the call graph for this function:

◆ del_values() [1/2]

template<typename DATA_TYPE >
auto Importer_NS::TypedImportBuffer::del_values ( std::vector< DATA_TYPE > &  buffer,
BadRowsTracker *const  bad_rows_tracker 
)

Referenced by Importer_NS::DataStreamSink::archivePlumber().

+ Here is the caller graph for this function:

◆ del_values() [2/2]

auto Importer_NS::TypedImportBuffer::del_values ( const SQLTypes  type,
BadRowsTracker *const  bad_rows_tracker 
)

◆ get_replicate_count()

int64_t Importer_NS::TypedImportBuffer::get_replicate_count ( ) const
inline

Definition at line 595 of file Importer.h.

595 { return replicate_count_; }

◆ getArrayBuffer()

std::vector<ArrayDatum>* Importer_NS::TypedImportBuffer::getArrayBuffer ( ) const
inline

Definition at line 468 of file Importer.h.

468 { return array_buffer_; }
std::vector< ArrayDatum > * array_buffer_
Definition: Importer.h:622

◆ getAsBytes()

int8_t* Importer_NS::TypedImportBuffer::getAsBytes ( ) const
inline

Definition at line 410 of file Importer.h.

References kBIGINT, kBOOLEAN, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kNUMERIC, kSMALLINT, kTIME, kTIMESTAMP, and kTINYINT.

Referenced by Importer_NS::anonymous_namespace{Importer.cpp}::double_value_at(), Importer_NS::anonymous_namespace{Importer.cpp}::float_value_at(), and Importer_NS::anonymous_namespace{Importer.cpp}::int_value_at().

410  {
411  switch (column_desc_->columnType.get_type()) {
412  case kBOOLEAN:
413  return reinterpret_cast<int8_t*>(&((*bool_buffer_)[0]));
414  case kTINYINT:
415  return reinterpret_cast<int8_t*>(&((*tinyint_buffer_)[0]));
416  case kSMALLINT:
417  return reinterpret_cast<int8_t*>(&((*smallint_buffer_)[0]));
418  case kINT:
419  return reinterpret_cast<int8_t*>(&((*int_buffer_)[0]));
420  case kBIGINT:
421  case kNUMERIC:
422  case kDECIMAL:
423  return reinterpret_cast<int8_t*>(&((*bigint_buffer_)[0]));
424  case kFLOAT:
425  return reinterpret_cast<int8_t*>(&((*float_buffer_)[0]));
426  case kDOUBLE:
427  return reinterpret_cast<int8_t*>(&((*double_buffer_)[0]));
428  case kDATE:
429  case kTIME:
430  case kTIMESTAMP:
431  return reinterpret_cast<int8_t*>(&((*bigint_buffer_)[0]));
432  default:
433  abort();
434  }
435  }
Definition: sqltypes.h:51
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:323
Definition: sqltypes.h:55
const ColumnDescriptor * column_desc_
Definition: Importer.h:631
Definition: sqltypes.h:47
SQLTypeInfo columnType
+ Here is the caller graph for this function:

◆ getColumnDesc()

const ColumnDescriptor* Importer_NS::TypedImportBuffer::getColumnDesc ( ) const
inline

Definition at line 406 of file Importer.h.

406 { return column_desc_; }
const ColumnDescriptor * column_desc_
Definition: Importer.h:631

◆ getElementSize()

size_t Importer_NS::TypedImportBuffer::getElementSize ( ) const
inline

Definition at line 437 of file Importer.h.

References kBIGINT, kBOOLEAN, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kNUMERIC, kSMALLINT, kTIME, kTIMESTAMP, and kTINYINT.

437  {
438  switch (column_desc_->columnType.get_type()) {
439  case kBOOLEAN:
440  return sizeof((*bool_buffer_)[0]);
441  case kTINYINT:
442  return sizeof((*tinyint_buffer_)[0]);
443  case kSMALLINT:
444  return sizeof((*smallint_buffer_)[0]);
445  case kINT:
446  return sizeof((*int_buffer_)[0]);
447  case kBIGINT:
448  case kNUMERIC:
449  case kDECIMAL:
450  return sizeof((*bigint_buffer_)[0]);
451  case kFLOAT:
452  return sizeof((*float_buffer_)[0]);
453  case kDOUBLE:
454  return sizeof((*double_buffer_)[0]);
455  case kDATE:
456  case kTIME:
457  case kTIMESTAMP:
458  return sizeof((*bigint_buffer_)[0]);
459  default:
460  abort();
461  }
462  }
Definition: sqltypes.h:51
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:323
Definition: sqltypes.h:55
const ColumnDescriptor * column_desc_
Definition: Importer.h:631
Definition: sqltypes.h:47
SQLTypeInfo columnType

◆ getGeoStringBuffer()

std::vector<std::string>* Importer_NS::TypedImportBuffer::getGeoStringBuffer ( ) const
inline

Definition at line 466 of file Importer.h.

466 { return geo_string_buffer_; }
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:621

◆ getStringArrayBuffer()

std::vector<std::vector<std::string> >* Importer_NS::TypedImportBuffer::getStringArrayBuffer ( ) const
inline

Definition at line 470 of file Importer.h.

470  {
471  return string_array_buffer_;
472  }
std::vector< std::vector< std::string > > * string_array_buffer_
Definition: Importer.h:623

◆ getStringArrayDictBuffer()

std::vector<ArrayDatum>* Importer_NS::TypedImportBuffer::getStringArrayDictBuffer ( ) const
inline

Definition at line 474 of file Importer.h.

474  {
476  }
std::vector< ArrayDatum > * string_array_dict_buffer_
Definition: Importer.h:629

◆ getStringBuffer()

std::vector<std::string>* Importer_NS::TypedImportBuffer::getStringBuffer ( ) const
inline

Definition at line 464 of file Importer.h.

464 { return string_buffer_; }
std::vector< std::string > * string_buffer_
Definition: Importer.h:620

◆ getStringDictBuffer()

int8_t* Importer_NS::TypedImportBuffer::getStringDictBuffer ( ) const
inline

Definition at line 478 of file Importer.h.

Referenced by Importer_NS::anonymous_namespace{Importer.cpp}::int_value_at().

478  {
479  switch (column_desc_->columnType.get_size()) {
480  case 1:
481  return reinterpret_cast<int8_t*>(&((*string_dict_i8_buffer_)[0]));
482  case 2:
483  return reinterpret_cast<int8_t*>(&((*string_dict_i16_buffer_)[0]));
484  case 4:
485  return reinterpret_cast<int8_t*>(&((*string_dict_i32_buffer_)[0]));
486  default:
487  abort();
488  }
489  }
HOST DEVICE int get_size() const
Definition: sqltypes.h:333
const ColumnDescriptor * column_desc_
Definition: Importer.h:631
SQLTypeInfo columnType
+ Here is the caller graph for this function:

◆ getStringDictionary()

StringDictionary* Importer_NS::TypedImportBuffer::getStringDictionary ( ) const
inline

Definition at line 408 of file Importer.h.

408 { return string_dict_; }
StringDictionary * string_dict_
Definition: Importer.h:632

◆ getTypeInfo()

const SQLTypeInfo& Importer_NS::TypedImportBuffer::getTypeInfo ( ) const
inline

Definition at line 404 of file Importer.h.

Referenced by Importer_NS::anonymous_namespace{Importer.cpp}::double_value_at(), Importer_NS::anonymous_namespace{Importer.cpp}::float_value_at(), and Importer_NS::anonymous_namespace{Importer.cpp}::int_value_at().

404 { return column_desc_->columnType; }
const ColumnDescriptor * column_desc_
Definition: Importer.h:631
SQLTypeInfo columnType
+ Here is the caller graph for this function:

◆ pop_value()

void Importer_NS::TypedImportBuffer::pop_value ( )

Definition at line 819 of file Importer.cpp.

References CHECK, decimal_to_int_type(), IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, and run-benchmark-import::type.

819  {
820  const auto type = column_desc_->columnType.is_decimal()
822  : column_desc_->columnType.get_type();
823  switch (type) {
824  case kBOOLEAN:
825  bool_buffer_->pop_back();
826  break;
827  case kTINYINT:
828  tinyint_buffer_->pop_back();
829  break;
830  case kSMALLINT:
831  smallint_buffer_->pop_back();
832  break;
833  case kINT:
834  int_buffer_->pop_back();
835  break;
836  case kBIGINT:
837  bigint_buffer_->pop_back();
838  break;
839  case kFLOAT:
840  float_buffer_->pop_back();
841  break;
842  case kDOUBLE:
843  double_buffer_->pop_back();
844  break;
845  case kTEXT:
846  case kVARCHAR:
847  case kCHAR:
848  string_buffer_->pop_back();
849  break;
850  case kDATE:
851  case kTIME:
852  case kTIMESTAMP:
853  bigint_buffer_->pop_back();
854  break;
855  case kARRAY:
857  string_array_buffer_->pop_back();
858  } else {
859  array_buffer_->pop_back();
860  }
861  break;
862  case kPOINT:
863  case kLINESTRING:
864  case kPOLYGON:
865  case kMULTIPOLYGON:
866  geo_string_buffer_->pop_back();
867  break;
868  default:
869  CHECK(false) << "TypedImportBuffer::pop_value() does not support type " << type;
870  }
871 }
std::vector< int16_t > * smallint_buffer_
Definition: Importer.h:615
std::vector< int64_t > * bigint_buffer_
Definition: Importer.h:617
Definition: sqltypes.h:51
std::vector< float > * float_buffer_
Definition: Importer.h:618
std::vector< int8_t > * tinyint_buffer_
Definition: Importer.h:614
std::vector< std::string > * string_buffer_
Definition: Importer.h:620
std::vector< std::vector< std::string > > * string_array_buffer_
Definition: Importer.h:623
std::vector< int32_t > * int_buffer_
Definition: Importer.h:616
bool is_decimal() const
Definition: sqltypes.h:453
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:324
std::vector< int8_t > * bool_buffer_
Definition: Importer.h:613
SQLTypes decimal_to_int_type(const SQLTypeInfo &ti)
Definition: Datum.cpp:268
Definition: sqltypes.h:54
Definition: sqltypes.h:55
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:621
std::vector< double > * double_buffer_
Definition: Importer.h:619
Definition: sqltypes.h:43
const ColumnDescriptor * column_desc_
Definition: Importer.h:631
#define IS_STRING(T)
Definition: sqltypes.h:163
#define CHECK(condition)
Definition: Logger.h:187
Definition: sqltypes.h:47
SQLTypeInfo columnType
std::vector< ArrayDatum > * array_buffer_
Definition: Importer.h:622
+ Here is the call graph for this function:

◆ set_replicate_count()

void Importer_NS::TypedImportBuffer::set_replicate_count ( const int64_t  replicate_count)
inline

Definition at line 596 of file Importer.h.

References Experimental::Array, and run-benchmark-import::type.

596  {
597  replicate_count_ = replicate_count;
598  }

◆ stringDictCheckpoint()

bool Importer_NS::TypedImportBuffer::stringDictCheckpoint ( )
inline

Definition at line 491 of file Importer.h.

491  {
492  if (string_dict_ == nullptr) {
493  return true;
494  }
495  return string_dict_->checkpoint();
496  }
StringDictionary * string_dict_
Definition: Importer.h:632
bool checkpoint() noexcept

Member Data Documentation

◆ @1

union { ... }

◆ @3

union { ... }

◆ array_buffer_

std::vector<ArrayDatum>* Importer_NS::TypedImportBuffer::array_buffer_

Definition at line 622 of file Importer.h.

◆ bigint_buffer_

std::vector<int64_t>* Importer_NS::TypedImportBuffer::bigint_buffer_

Definition at line 617 of file Importer.h.

◆ bool_buffer_

std::vector<int8_t>* Importer_NS::TypedImportBuffer::bool_buffer_

Definition at line 613 of file Importer.h.

◆ col_idx

size_t Importer_NS::TypedImportBuffer::col_idx

Definition at line 609 of file Importer.h.

◆ column_desc_

const ColumnDescriptor* Importer_NS::TypedImportBuffer::column_desc_
private

Definition at line 631 of file Importer.h.

◆ double_buffer_

std::vector<double>* Importer_NS::TypedImportBuffer::double_buffer_

Definition at line 619 of file Importer.h.

◆ float_buffer_

std::vector<float>* Importer_NS::TypedImportBuffer::float_buffer_

Definition at line 618 of file Importer.h.

◆ geo_string_buffer_

std::vector<std::string>* Importer_NS::TypedImportBuffer::geo_string_buffer_

Definition at line 621 of file Importer.h.

◆ import_buffers

std::vector<std::unique_ptr<TypedImportBuffer> >* Importer_NS::TypedImportBuffer::import_buffers

Definition at line 608 of file Importer.h.

◆ int_buffer_

std::vector<int32_t>* Importer_NS::TypedImportBuffer::int_buffer_

Definition at line 616 of file Importer.h.

◆ replicate_count_

size_t Importer_NS::TypedImportBuffer::replicate_count_ = 0
private

Definition at line 633 of file Importer.h.

◆ smallint_buffer_

std::vector<int16_t>* Importer_NS::TypedImportBuffer::smallint_buffer_

Definition at line 615 of file Importer.h.

◆ string_array_buffer_

std::vector<std::vector<std::string> >* Importer_NS::TypedImportBuffer::string_array_buffer_

Definition at line 623 of file Importer.h.

◆ string_array_dict_buffer_

std::vector<ArrayDatum>* Importer_NS::TypedImportBuffer::string_array_dict_buffer_

Definition at line 629 of file Importer.h.

◆ string_buffer_

std::vector<std::string>* Importer_NS::TypedImportBuffer::string_buffer_

Definition at line 620 of file Importer.h.

◆ string_dict_

StringDictionary* Importer_NS::TypedImportBuffer::string_dict_
private

Definition at line 632 of file Importer.h.

◆ string_dict_i16_buffer_

std::vector<uint16_t>* Importer_NS::TypedImportBuffer::string_dict_i16_buffer_

Definition at line 627 of file Importer.h.

◆ string_dict_i32_buffer_

std::vector<int32_t>* Importer_NS::TypedImportBuffer::string_dict_i32_buffer_

Definition at line 628 of file Importer.h.

◆ string_dict_i8_buffer_

std::vector<uint8_t>* Importer_NS::TypedImportBuffer::string_dict_i8_buffer_

Definition at line 626 of file Importer.h.

◆ tinyint_buffer_

std::vector<int8_t>* Importer_NS::TypedImportBuffer::tinyint_buffer_

Definition at line 614 of file Importer.h.


The documentation for this class was generated from the following files: