OmniSciDB  7bf56492aa
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Importer_NS::TypedImportBuffer Class Reference

#include <Importer.h>

+ Inheritance diagram for Importer_NS::TypedImportBuffer:
+ Collaboration diagram for Importer_NS::TypedImportBuffer:

Public Member Functions

 TypedImportBuffer (const ColumnDescriptor *col_desc, StringDictionary *string_dict)
 
 ~TypedImportBuffer ()
 
void addBoolean (const int8_t v)
 
void addTinyint (const int8_t v)
 
void addSmallint (const int16_t v)
 
void addInt (const int32_t v)
 
void addBigint (const int64_t v)
 
void addFloat (const float v)
 
void addDouble (const double v)
 
void addString (const std::string &v)
 
void addGeoString (const std::string &v)
 
void addArray (const ArrayDatum &v)
 
std::vector< std::string > & addStringArray ()
 
void addStringArray (const std::vector< std::string > &arr)
 
void addDictEncodedString (const std::vector< std::string > &string_vec)
 
void addDictEncodedStringArray (const std::vector< std::vector< std::string >> &string_array_vec)
 
const SQLTypeInfogetTypeInfo () const
 
const ColumnDescriptorgetColumnDesc () const
 
StringDictionarygetStringDictionary () const
 
int8_t * getAsBytes () const
 
size_t getElementSize () const
 
std::vector< std::string > * getStringBuffer () const
 
std::vector< std::string > * getGeoStringBuffer () const
 
std::vector< ArrayDatum > * getArrayBuffer () const
 
std::vector< std::vector
< std::string > > * 
getStringArrayBuffer () const
 
std::vector< ArrayDatum > * getStringArrayDictBuffer () const
 
int8_t * getStringDictBuffer () const
 
bool stringDictCheckpoint ()
 
void clear ()
 
size_t add_values (const ColumnDescriptor *cd, const TColumn &data)
 
size_t add_arrow_values (const ColumnDescriptor *cd, const arrow::Array &data, const bool exact_type_match, const ArraySliceRange &slice_range, BadRowsTracker *bad_rows_tracker)
 
void add_value (const ColumnDescriptor *cd, const std::string &val, const bool is_null, const CopyParams &copy_params, const int64_t replicate_count=0)
 
void add_value (const ColumnDescriptor *cd, const TDatum &val, const bool is_null, const int64_t replicate_count=0)
 
void pop_value ()
 
int64_t get_replicate_count () const
 
void set_replicate_count (const int64_t replicate_count)
 
template<typename DATA_TYPE >
size_t convert_arrow_val_to_import_buffer (const ColumnDescriptor *cd, const arrow::Array &array, std::vector< DATA_TYPE > &buffer, const ArraySliceRange &slice_range, BadRowsTracker *const bad_rows_tracker)
 
template<typename DATA_TYPE >
auto del_values (std::vector< DATA_TYPE > &buffer, BadRowsTracker *const bad_rows_tracker)
 
auto del_values (const SQLTypes type, BadRowsTracker *const bad_rows_tracker)
 
template<typename DATA_TYPE >
size_t convert_arrow_val_to_import_buffer (const ColumnDescriptor *cd, const Array &array, std::vector< DATA_TYPE > &buffer, const ArraySliceRange &slice_range, Importer_NS::BadRowsTracker *const bad_rows_tracker)
 

Public Attributes

std::vector< std::unique_ptr
< TypedImportBuffer > > * 
import_buffers
 
size_t col_idx
 
std::vector< int8_t > * bool_buffer_
 
std::vector< int8_t > * tinyint_buffer_
 
std::vector< int16_t > * smallint_buffer_
 
std::vector< int32_t > * int_buffer_
 
std::vector< int64_t > * bigint_buffer_
 
std::vector< float > * float_buffer_
 
std::vector< double > * double_buffer_
 
std::vector< std::string > * string_buffer_
 
std::vector< std::string > * geo_string_buffer_
 
std::vector< ArrayDatum > * array_buffer_
 
std::vector< std::vector
< std::string > > * 
string_array_buffer_
 
std::vector< uint8_t > * string_dict_i8_buffer_
 
std::vector< uint16_t > * string_dict_i16_buffer_
 
std::vector< int32_t > * string_dict_i32_buffer_
 
std::vector< ArrayDatum > * string_array_dict_buffer_
 

Private Attributes

union {
   std::vector< int8_t > *   bool_buffer_
 
   std::vector< int8_t > *   tinyint_buffer_
 
   std::vector< int16_t > *   smallint_buffer_
 
   std::vector< int32_t > *   int_buffer_
 
   std::vector< int64_t > *   bigint_buffer_
 
   std::vector< float > *   float_buffer_
 
   std::vector< double > *   double_buffer_
 
   std::vector< std::string > *   string_buffer_
 
   std::vector< std::string > *   geo_string_buffer_
 
   std::vector< ArrayDatum > *   array_buffer_
 
   std::vector< std::vector
< std::string > > *   string_array_buffer_
 
}; 
 
union {
   std::vector< uint8_t > *   string_dict_i8_buffer_
 
   std::vector< uint16_t > *   string_dict_i16_buffer_
 
   std::vector< int32_t > *   string_dict_i32_buffer_
 
   std::vector< ArrayDatum > *   string_array_dict_buffer_
 
}; 
 
const ColumnDescriptorcolumn_desc_
 
StringDictionarystring_dict_
 
size_t replicate_count_ = 0
 

Detailed Description

Definition at line 82 of file Importer.h.

Constructor & Destructor Documentation

Importer_NS::TypedImportBuffer::TypedImportBuffer ( const ColumnDescriptor col_desc,
StringDictionary string_dict 
)
inline

Definition at line 84 of file Importer.h.

References array_buffer_, bigint_buffer_, bool_buffer_, CHECK(), ColumnDescriptor::columnType, double_buffer_, float_buffer_, geo_string_buffer_, SQLTypeInfo::get_compression(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), int_buffer_, IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kENCODING_DICT, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, smallint_buffer_, string_array_buffer_, string_array_dict_buffer_, string_buffer_, string_dict_i16_buffer_, string_dict_i32_buffer_, string_dict_i8_buffer_, and tinyint_buffer_.

85  : column_desc_(col_desc), string_dict_(string_dict) {
86  switch (col_desc->columnType.get_type()) {
87  case kBOOLEAN:
88  bool_buffer_ = new std::vector<int8_t>();
89  break;
90  case kTINYINT:
91  tinyint_buffer_ = new std::vector<int8_t>();
92  break;
93  case kSMALLINT:
94  smallint_buffer_ = new std::vector<int16_t>();
95  break;
96  case kINT:
97  int_buffer_ = new std::vector<int32_t>();
98  break;
99  case kBIGINT:
100  case kNUMERIC:
101  case kDECIMAL:
102  bigint_buffer_ = new std::vector<int64_t>();
103  break;
104  case kFLOAT:
105  float_buffer_ = new std::vector<float>();
106  break;
107  case kDOUBLE:
108  double_buffer_ = new std::vector<double>();
109  break;
110  case kTEXT:
111  case kVARCHAR:
112  case kCHAR:
113  string_buffer_ = new std::vector<std::string>();
114  if (col_desc->columnType.get_compression() == kENCODING_DICT) {
115  switch (col_desc->columnType.get_size()) {
116  case 1:
117  string_dict_i8_buffer_ = new std::vector<uint8_t>();
118  break;
119  case 2:
120  string_dict_i16_buffer_ = new std::vector<uint16_t>();
121  break;
122  case 4:
123  string_dict_i32_buffer_ = new std::vector<int32_t>();
124  break;
125  default:
126  CHECK(false);
127  }
128  }
129  break;
130  case kDATE:
131  case kTIME:
132  case kTIMESTAMP:
133  bigint_buffer_ = new std::vector<int64_t>();
134  break;
135  case kARRAY:
136  if (IS_STRING(col_desc->columnType.get_subtype())) {
138  string_array_buffer_ = new std::vector<std::vector<std::string>>();
139  string_array_dict_buffer_ = new std::vector<ArrayDatum>();
140  } else {
141  array_buffer_ = new std::vector<ArrayDatum>();
142  }
143  break;
144  case kPOINT:
145  case kLINESTRING:
146  case kPOLYGON:
147  case kMULTIPOLYGON:
148  geo_string_buffer_ = new std::vector<std::string>();
149  break;
150  default:
151  CHECK(false);
152  }
153  }
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:249
std::vector< int16_t > * smallint_buffer_
Definition: Importer.h:490
HOST DEVICE int get_size() const
Definition: sqltypes.h:258
std::vector< int64_t > * bigint_buffer_
Definition: Importer.h:492
Definition: sqltypes.h:50
std::vector< float > * float_buffer_
Definition: Importer.h:493
std::vector< uint16_t > * string_dict_i16_buffer_
Definition: Importer.h:502
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:248
std::vector< int8_t > * tinyint_buffer_
Definition: Importer.h:489
CHECK(cgen_state)
std::vector< std::string > * string_buffer_
Definition: Importer.h:495
StringDictionary * string_dict_
Definition: Importer.h:507
std::vector< int32_t > * string_dict_i32_buffer_
Definition: Importer.h:503
std::vector< std::vector< std::string > > * string_array_buffer_
Definition: Importer.h:498
std::vector< int32_t > * int_buffer_
Definition: Importer.h:491
std::vector< int8_t > * bool_buffer_
Definition: Importer.h:488
Definition: sqltypes.h:53
Definition: sqltypes.h:54
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:256
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:496
std::vector< ArrayDatum > * string_array_dict_buffer_
Definition: Importer.h:504
std::vector< double > * double_buffer_
Definition: Importer.h:494
Definition: sqltypes.h:42
const ColumnDescriptor * column_desc_
Definition: Importer.h:506
#define IS_STRING(T)
Definition: sqltypes.h:162
Definition: sqltypes.h:46
SQLTypeInfo columnType
std::vector< uint8_t > * string_dict_i8_buffer_
Definition: Importer.h:501
std::vector< ArrayDatum > * array_buffer_
Definition: Importer.h:497

+ Here is the call graph for this function:

Importer_NS::TypedImportBuffer::~TypedImportBuffer ( )
inline

Definition at line 155 of file Importer.h.

References array_buffer_, bigint_buffer_, bool_buffer_, CHECK(), column_desc_, ColumnDescriptor::columnType, double_buffer_, float_buffer_, geo_string_buffer_, SQLTypeInfo::get_compression(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), int_buffer_, IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kENCODING_DICT, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, smallint_buffer_, string_array_buffer_, string_array_dict_buffer_, string_buffer_, string_dict_i16_buffer_, string_dict_i32_buffer_, string_dict_i8_buffer_, and tinyint_buffer_.

155  {
156  switch (column_desc_->columnType.get_type()) {
157  case kBOOLEAN:
158  delete bool_buffer_;
159  break;
160  case kTINYINT:
161  delete tinyint_buffer_;
162  break;
163  case kSMALLINT:
164  delete smallint_buffer_;
165  break;
166  case kINT:
167  delete int_buffer_;
168  break;
169  case kBIGINT:
170  case kNUMERIC:
171  case kDECIMAL:
172  delete bigint_buffer_;
173  break;
174  case kFLOAT:
175  delete float_buffer_;
176  break;
177  case kDOUBLE:
178  delete double_buffer_;
179  break;
180  case kTEXT:
181  case kVARCHAR:
182  case kCHAR:
183  delete string_buffer_;
185  switch (column_desc_->columnType.get_size()) {
186  case 1:
187  delete string_dict_i8_buffer_;
188  break;
189  case 2:
191  break;
192  case 4:
194  break;
195  }
196  }
197  break;
198  case kDATE:
199  case kTIME:
200  case kTIMESTAMP:
201  delete bigint_buffer_;
202  break;
203  case kARRAY:
205  delete string_array_buffer_;
207  } else {
208  delete array_buffer_;
209  }
210  break;
211  case kPOINT:
212  case kLINESTRING:
213  case kPOLYGON:
214  case kMULTIPOLYGON:
215  delete geo_string_buffer_;
216  break;
217  default:
218  CHECK(false);
219  }
220  }
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:249
std::vector< int16_t > * smallint_buffer_
Definition: Importer.h:490
HOST DEVICE int get_size() const
Definition: sqltypes.h:258
std::vector< int64_t > * bigint_buffer_
Definition: Importer.h:492
Definition: sqltypes.h:50
std::vector< float > * float_buffer_
Definition: Importer.h:493
std::vector< uint16_t > * string_dict_i16_buffer_
Definition: Importer.h:502
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:248
std::vector< int8_t > * tinyint_buffer_
Definition: Importer.h:489
CHECK(cgen_state)
std::vector< std::string > * string_buffer_
Definition: Importer.h:495
std::vector< int32_t > * string_dict_i32_buffer_
Definition: Importer.h:503
std::vector< std::vector< std::string > > * string_array_buffer_
Definition: Importer.h:498
std::vector< int32_t > * int_buffer_
Definition: Importer.h:491
std::vector< int8_t > * bool_buffer_
Definition: Importer.h:488
Definition: sqltypes.h:53
Definition: sqltypes.h:54
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:256
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:496
std::vector< ArrayDatum > * string_array_dict_buffer_
Definition: Importer.h:504
std::vector< double > * double_buffer_
Definition: Importer.h:494
Definition: sqltypes.h:42
const ColumnDescriptor * column_desc_
Definition: Importer.h:506
#define IS_STRING(T)
Definition: sqltypes.h:162
Definition: sqltypes.h:46
SQLTypeInfo columnType
std::vector< uint8_t > * string_dict_i8_buffer_
Definition: Importer.h:501
std::vector< ArrayDatum > * array_buffer_
Definition: Importer.h:497

+ Here is the call graph for this function:

Member Function Documentation

size_t Importer_NS::TypedImportBuffer::add_arrow_values ( const ColumnDescriptor cd,
const arrow::Array &  data,
const bool  exact_type_match,
const ArraySliceRange slice_range,
BadRowsTracker bad_rows_tracker 
)

Definition at line 839 of file Importer.cpp.

References arrow_throw_if(), bigint_buffer_, bool_buffer_, ColumnDescriptor::columnName, ColumnDescriptor::columnType, convert_arrow_val_to_import_buffer(), decimal_to_int_type(), double_buffer_, float_buffer_, geo_string_buffer_, SQLTypeInfo::get_notnull(), SQLTypeInfo::get_type(), int_buffer_, SQLTypeInfo::is_decimal(), kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, smallint_buffer_, string_buffer_, tinyint_buffer_, and run_benchmark_import::type.

843  {
844  const auto type = cd->columnType.is_decimal() ? decimal_to_int_type(cd->columnType)
845  : cd->columnType.get_type();
846  if (cd->columnType.get_notnull()) {
847  // We can't have any null values for this column; to have them is an error
848  arrow_throw_if(col.null_count() > 0, "NULL not allowed for column " + cd->columnName);
849  }
850 
851  switch (type) {
852  case kBOOLEAN:
853  if (exact_type_match) {
854  arrow_throw_if(col.type_id() != Type::BOOL, "Expected boolean type");
855  }
857  cd, col, *bool_buffer_, slice_range, bad_rows_tracker);
858  case kTINYINT:
859  if (exact_type_match) {
860  arrow_throw_if(col.type_id() != Type::INT8, "Expected int8 type");
861  }
863  cd, col, *tinyint_buffer_, slice_range, bad_rows_tracker);
864  case kSMALLINT:
865  if (exact_type_match) {
866  arrow_throw_if(col.type_id() != Type::INT16, "Expected int16 type");
867  }
869  cd, col, *smallint_buffer_, slice_range, bad_rows_tracker);
870  case kINT:
871  if (exact_type_match) {
872  arrow_throw_if(col.type_id() != Type::INT32, "Expected int32 type");
873  }
875  cd, col, *int_buffer_, slice_range, bad_rows_tracker);
876  case kBIGINT:
877  if (exact_type_match) {
878  arrow_throw_if(col.type_id() != Type::INT64, "Expected int64 type");
879  }
881  cd, col, *bigint_buffer_, slice_range, bad_rows_tracker);
882  case kFLOAT:
883  if (exact_type_match) {
884  arrow_throw_if(col.type_id() != Type::FLOAT, "Expected float type");
885  }
887  cd, col, *float_buffer_, slice_range, bad_rows_tracker);
888  case kDOUBLE:
889  if (exact_type_match) {
890  arrow_throw_if(col.type_id() != Type::DOUBLE, "Expected double type");
891  }
893  cd, col, *double_buffer_, slice_range, bad_rows_tracker);
894  case kTEXT:
895  case kVARCHAR:
896  case kCHAR:
897  if (exact_type_match) {
898  arrow_throw_if(col.type_id() != Type::BINARY && col.type_id() != Type::STRING,
899  "Expected string type");
900  }
902  cd, col, *string_buffer_, slice_range, bad_rows_tracker);
903  case kTIME:
904  if (exact_type_match) {
905  arrow_throw_if(col.type_id() != Type::TIME32 && col.type_id() != Type::TIME64,
906  "Expected time32 or time64 type");
907  }
909  cd, col, *bigint_buffer_, slice_range, bad_rows_tracker);
910  case kTIMESTAMP:
911  if (exact_type_match) {
912  arrow_throw_if(col.type_id() != Type::TIMESTAMP, "Expected timestamp type");
913  }
915  cd, col, *bigint_buffer_, slice_range, bad_rows_tracker);
916  case kDATE:
917  if (exact_type_match) {
918  arrow_throw_if(col.type_id() != Type::DATE32 && col.type_id() != Type::DATE64,
919  "Expected date32 or date64 type");
920  }
922  cd, col, *bigint_buffer_, slice_range, bad_rows_tracker);
923  case kPOINT:
924  case kLINESTRING:
925  case kPOLYGON:
926  case kMULTIPOLYGON:
927  arrow_throw_if(col.type_id() != Type::BINARY && col.type_id() != Type::STRING,
928  "Expected string type");
930  cd, col, *geo_string_buffer_, slice_range, bad_rows_tracker);
931  case kARRAY:
932  throw std::runtime_error("Arrow array appends not yet supported");
933  default:
934  throw std::runtime_error("Invalid Type");
935  }
936 }
std::vector< int16_t > * smallint_buffer_
Definition: Importer.h:490
std::vector< int64_t > * bigint_buffer_
Definition: Importer.h:492
Definition: sqltypes.h:50
std::vector< float > * float_buffer_
Definition: Importer.h:493
size_t convert_arrow_val_to_import_buffer(const ColumnDescriptor *cd, const arrow::Array &array, std::vector< DATA_TYPE > &buffer, const ArraySliceRange &slice_range, BadRowsTracker *const bad_rows_tracker)
std::vector< int8_t > * tinyint_buffer_
Definition: Importer.h:489
std::vector< std::string > * string_buffer_
Definition: Importer.h:495
std::vector< int32_t > * int_buffer_
Definition: Importer.h:491
std::vector< int8_t > * bool_buffer_
Definition: Importer.h:488
SQLTypes decimal_to_int_type(const SQLTypeInfo &ti)
Definition: Datum.cpp:299
Definition: sqltypes.h:53
Definition: sqltypes.h:54
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:496
std::vector< double > * double_buffer_
Definition: Importer.h:494
Definition: sqltypes.h:42
Definition: sqltypes.h:46
SQLTypeInfo columnType
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:255
bool is_decimal() const
Definition: sqltypes.h:402
std::string columnName
void arrow_throw_if(const bool cond, const std::string &message)
Definition: ArrowImporter.h:40

+ Here is the call graph for this function:

void Importer_NS::TypedImportBuffer::add_value ( const ColumnDescriptor cd,
const std::string &  val,
const bool  is_null,
const CopyParams copy_params,
const int64_t  replicate_count = 0 
)

Definition at line 498 of file Importer.cpp.

References addArray(), addBigint(), addBoolean(), addDouble(), addFloat(), addGeoString(), addInt(), addSmallint(), addString(), addStringArray(), addTinyint(), Datum::bigintval, Datum::boolval, CHECK(), ColumnDescriptor::columnName, ColumnDescriptor::columnType, convert_decimal_value_to_scale(), SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), inline_fixed_encoding_null_val(), IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, StringDictionary::MAX_STRLEN, NULL_DOUBLE, NULL_FLOAT, Importer_NS::NullArray(), Importer_NS::DelimitedParserUtils::parseStringArray(), set_replicate_count(), Importer_NS::StringToArray(), StringToDatum(), to_string(), and run_benchmark_import::type.

502  {
503  set_replicate_count(replicate_count);
504  const auto type = cd->columnType.get_type();
505  switch (type) {
506  case kBOOLEAN: {
507  if (is_null) {
508  if (cd->columnType.get_notnull()) {
509  throw std::runtime_error("NULL for column " + cd->columnName);
510  }
512  } else {
513  SQLTypeInfo ti = cd->columnType;
514  Datum d = StringToDatum(val, ti);
515  addBoolean((int8_t)d.boolval);
516  }
517  break;
518  }
519  case kTINYINT: {
520  if (!is_null && (isdigit(val[0]) || val[0] == '-')) {
521  addTinyint(std::stoi(val));
522  } else {
523  if (cd->columnType.get_notnull()) {
524  throw std::runtime_error("NULL for column " + cd->columnName);
525  }
527  }
528  break;
529  }
530  case kSMALLINT: {
531  if (!is_null && (isdigit(val[0]) || val[0] == '-')) {
532  addSmallint(std::stoi(val));
533  } else {
534  if (cd->columnType.get_notnull()) {
535  throw std::runtime_error("NULL for column " + cd->columnName);
536  }
538  }
539  break;
540  }
541  case kINT: {
542  if (!is_null && (isdigit(val[0]) || val[0] == '-')) {
543  addInt(std::stoi(val));
544  } else {
545  if (cd->columnType.get_notnull()) {
546  throw std::runtime_error("NULL for column " + cd->columnName);
547  }
549  }
550  break;
551  }
552  case kBIGINT: {
553  if (!is_null && (isdigit(val[0]) || val[0] == '-')) {
554  addBigint(std::stoll(val));
555  } else {
556  if (cd->columnType.get_notnull()) {
557  throw std::runtime_error("NULL for column " + cd->columnName);
558  }
560  }
561  break;
562  }
563  case kDECIMAL:
564  case kNUMERIC: {
565  if (!is_null) {
566  SQLTypeInfo ti(kNUMERIC, 0, 0, false);
567  Datum d = StringToDatum(val, ti);
568  const auto converted_decimal_value =
570  addBigint(converted_decimal_value);
571  } else {
572  if (cd->columnType.get_notnull()) {
573  throw std::runtime_error("NULL for column " + cd->columnName);
574  }
576  }
577  break;
578  }
579  case kFLOAT:
580  if (!is_null && (val[0] == '.' || isdigit(val[0]) || val[0] == '-')) {
581  addFloat((float)std::atof(val.c_str()));
582  } else {
583  if (cd->columnType.get_notnull()) {
584  throw std::runtime_error("NULL for column " + cd->columnName);
585  }
587  }
588  break;
589  case kDOUBLE:
590  if (!is_null && (val[0] == '.' || isdigit(val[0]) || val[0] == '-')) {
591  addDouble(std::atof(val.c_str()));
592  } else {
593  if (cd->columnType.get_notnull()) {
594  throw std::runtime_error("NULL for column " + cd->columnName);
595  }
597  }
598  break;
599  case kTEXT:
600  case kVARCHAR:
601  case kCHAR: {
602  // @TODO(wei) for now, use empty string for nulls
603  if (is_null) {
604  if (cd->columnType.get_notnull()) {
605  throw std::runtime_error("NULL for column " + cd->columnName);
606  }
607  addString(std::string());
608  } else {
609  if (val.length() > StringDictionary::MAX_STRLEN) {
610  throw std::runtime_error("String too long for column " + cd->columnName +
611  " was " + std::to_string(val.length()) + " max is " +
613  }
614  addString(val);
615  }
616  break;
617  }
618  case kTIME:
619  case kTIMESTAMP:
620  case kDATE:
621  if (!is_null && (isdigit(val[0]) || val[0] == '-')) {
622  SQLTypeInfo ti = cd->columnType;
623  Datum d = StringToDatum(val, ti);
624  addBigint(d.bigintval);
625  } else {
626  if (cd->columnType.get_notnull()) {
627  throw std::runtime_error("NULL for column " + cd->columnName);
628  }
630  }
631  break;
632  case kARRAY: {
633  if (is_null && cd->columnType.get_notnull()) {
634  throw std::runtime_error("NULL for column " + cd->columnName);
635  }
636  SQLTypeInfo ti = cd->columnType;
637  if (IS_STRING(ti.get_subtype())) {
638  std::vector<std::string> string_vec;
639  // Just parse string array, don't push it to buffer yet as we might throw
640  Importer_NS::DelimitedParserUtils::parseStringArray(val, copy_params, string_vec);
641  if (!is_null) {
642  // TODO: add support for NULL string arrays
643  if (ti.get_size() > 0) {
644  auto sti = ti.get_elem_type();
645  size_t expected_size = ti.get_size() / sti.get_size();
646  size_t actual_size = string_vec.size();
647  if (actual_size != expected_size) {
648  throw std::runtime_error("Fixed length array column " + cd->columnName +
649  " expects " + std::to_string(expected_size) +
650  " values, received " +
651  std::to_string(actual_size));
652  }
653  }
654  addStringArray(string_vec);
655  } else {
656  if (ti.get_size() > 0) {
657  // TODO: remove once NULL fixlen arrays are allowed
658  throw std::runtime_error("Fixed length array column " + cd->columnName +
659  " currently cannot accept NULL arrays");
660  }
661  // TODO: add support for NULL string arrays, replace with addStringArray(),
662  // for now add whatever parseStringArray() outputs for NULLs ("NULL")
663  addStringArray(string_vec);
664  }
665  } else {
666  if (!is_null) {
667  ArrayDatum d = StringToArray(val, ti, copy_params);
668  if (d.is_null) { // val could be "NULL"
669  addArray(NullArray(ti));
670  } else {
671  if (ti.get_size() > 0 && static_cast<size_t>(ti.get_size()) != d.length) {
672  throw std::runtime_error("Fixed length array for column " + cd->columnName +
673  " has incorrect length: " + val);
674  }
675  addArray(d);
676  }
677  } else {
678  addArray(NullArray(ti));
679  }
680  }
681  break;
682  }
683  case kPOINT:
684  case kLINESTRING:
685  case kPOLYGON:
686  case kMULTIPOLYGON:
687  addGeoString(val);
688  break;
689  default:
690  CHECK(false) << "TypedImportBuffer::add_value() does not support type " << type;
691  }
692 }
void addSmallint(const int16_t v)
Definition: Importer.h:226
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:249
#define NULL_DOUBLE
Definition: sqltypes.h:175
HOST DEVICE int get_size() const
Definition: sqltypes.h:258
void addGeoString(const std::string &v)
Definition: Importer.h:238
Definition: sqltypes.h:50
void addTinyint(const int8_t v)
Definition: Importer.h:224
bool boolval
Definition: sqltypes.h:123
void addInt(const int32_t v)
Definition: Importer.h:228
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:248
ArrayDatum StringToArray(const std::string &s, const SQLTypeInfo &ti, const CopyParams &copy_params)
Definition: Importer.cpp:314
std::string to_string(char const *&&v)
void set_replicate_count(const int64_t replicate_count)
Definition: Importer.h:471
void addBoolean(const int8_t v)
Definition: Importer.h:222
void addFloat(const float v)
Definition: Importer.h:232
CHECK(cgen_state)
int64_t bigintval
Definition: sqltypes.h:127
#define NULL_FLOAT
Definition: sqltypes.h:174
void addBigint(const int64_t v)
Definition: Importer.h:230
Definition: sqltypes.h:53
Definition: sqltypes.h:54
bool is_null(const T &v, const SQLTypeInfo &t)
int64_t convert_decimal_value_to_scale(const int64_t decimal_value, const SQLTypeInfo &type_info, const SQLTypeInfo &new_type_info)
Definition: Datum.cpp:315
void addArray(const ArrayDatum &v)
Definition: Importer.h:240
Datum StringToDatum(const std::string &s, SQLTypeInfo &ti)
Definition: Datum.cpp:121
Definition: sqltypes.h:42
ArrayDatum NullArray(const SQLTypeInfo &ti)
Definition: Importer.cpp:367
#define IS_STRING(T)
Definition: sqltypes.h:162
void addString(const std::string &v)
Definition: Importer.h:236
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
static constexpr size_t MAX_STRLEN
Definition: sqltypes.h:46
SQLTypeInfo columnType
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:255
SQLTypeInfo get_elem_type() const
Definition: sqltypes.h:603
std::vector< std::string > & addStringArray()
Definition: Importer.h:242
static void parseStringArray(const std::string &s, const Importer_NS::CopyParams &copy_params, std::vector< std::string > &string_vec)
Parses given string array and inserts into given vector of strings.
std::string columnName
std::conditional_t< isCudaCC(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:120
void addDouble(const double v)
Definition: Importer.h:234

+ Here is the call graph for this function:

void Importer_NS::TypedImportBuffer::add_value ( const ColumnDescriptor cd,
const TDatum &  val,
const bool  is_null,
const int64_t  replicate_count = 0 
)

Definition at line 1260 of file Importer.cpp.

References addArray(), addBigint(), Importer_NS::addBinaryStringArray(), addBoolean(), addDouble(), addFloat(), addGeoString(), addInt(), addSmallint(), addString(), addStringArray(), addTinyint(), CHECK(), ColumnDescriptor::columnName, ColumnDescriptor::columnType, decimal_to_int_type(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), inline_fixed_encoding_null_val(), SQLTypeInfo::is_decimal(), IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, NULL_DOUBLE, NULL_FLOAT, Importer_NS::NullArray(), set_replicate_count(), Importer_NS::TDatumToArrayDatum(), and run_benchmark_import::type.

1263  {
1264  set_replicate_count(replicate_count);
1265  const auto type = cd->columnType.is_decimal() ? decimal_to_int_type(cd->columnType)
1266  : cd->columnType.get_type();
1267  switch (type) {
1268  case kBOOLEAN: {
1269  if (is_null) {
1270  if (cd->columnType.get_notnull()) {
1271  throw std::runtime_error("NULL for column " + cd->columnName);
1272  }
1274  } else {
1275  addBoolean((int8_t)datum.val.int_val);
1276  }
1277  break;
1278  }
1279  case kTINYINT:
1280  if (!is_null) {
1281  addTinyint((int8_t)datum.val.int_val);
1282  } else {
1283  if (cd->columnType.get_notnull()) {
1284  throw std::runtime_error("NULL for column " + cd->columnName);
1285  }
1287  }
1288  break;
1289  case kSMALLINT:
1290  if (!is_null) {
1291  addSmallint((int16_t)datum.val.int_val);
1292  } else {
1293  if (cd->columnType.get_notnull()) {
1294  throw std::runtime_error("NULL for column " + cd->columnName);
1295  }
1297  }
1298  break;
1299  case kINT:
1300  if (!is_null) {
1301  addInt((int32_t)datum.val.int_val);
1302  } else {
1303  if (cd->columnType.get_notnull()) {
1304  throw std::runtime_error("NULL for column " + cd->columnName);
1305  }
1307  }
1308  break;
1309  case kBIGINT:
1310  if (!is_null) {
1311  addBigint(datum.val.int_val);
1312  } else {
1313  if (cd->columnType.get_notnull()) {
1314  throw std::runtime_error("NULL for column " + cd->columnName);
1315  }
1317  }
1318  break;
1319  case kFLOAT:
1320  if (!is_null) {
1321  addFloat((float)datum.val.real_val);
1322  } else {
1323  if (cd->columnType.get_notnull()) {
1324  throw std::runtime_error("NULL for column " + cd->columnName);
1325  }
1327  }
1328  break;
1329  case kDOUBLE:
1330  if (!is_null) {
1331  addDouble(datum.val.real_val);
1332  } else {
1333  if (cd->columnType.get_notnull()) {
1334  throw std::runtime_error("NULL for column " + cd->columnName);
1335  }
1337  }
1338  break;
1339  case kTEXT:
1340  case kVARCHAR:
1341  case kCHAR: {
1342  // @TODO(wei) for now, use empty string for nulls
1343  if (is_null) {
1344  if (cd->columnType.get_notnull()) {
1345  throw std::runtime_error("NULL for column " + cd->columnName);
1346  }
1347  addString(std::string());
1348  } else {
1349  addString(datum.val.str_val);
1350  }
1351  break;
1352  }
1353  case kTIME:
1354  case kTIMESTAMP:
1355  case kDATE: {
1356  if (!is_null) {
1357  addBigint(datum.val.int_val);
1358  } else {
1359  if (cd->columnType.get_notnull()) {
1360  throw std::runtime_error("NULL for column " + cd->columnName);
1361  }
1363  }
1364  break;
1365  }
1366  case kARRAY:
1367  if (is_null && cd->columnType.get_notnull()) {
1368  throw std::runtime_error("NULL for column " + cd->columnName);
1369  }
1370  if (IS_STRING(cd->columnType.get_subtype())) {
1371  std::vector<std::string>& string_vec = addStringArray();
1372  addBinaryStringArray(datum, string_vec);
1373  } else {
1374  if (!is_null) {
1375  addArray(TDatumToArrayDatum(datum, cd->columnType));
1376  } else {
1378  }
1379  }
1380  break;
1381  case kPOINT:
1382  case kLINESTRING:
1383  case kPOLYGON:
1384  case kMULTIPOLYGON:
1385  if (is_null) {
1386  if (cd->columnType.get_notnull()) {
1387  throw std::runtime_error("NULL for column " + cd->columnName);
1388  }
1389  addGeoString(std::string());
1390  } else {
1391  addGeoString(datum.val.str_val);
1392  }
1393  break;
1394  default:
1395  CHECK(false) << "TypedImportBuffer::add_value() does not support type " << type;
1396  }
1397 }
void addSmallint(const int16_t v)
Definition: Importer.h:226
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:249
#define NULL_DOUBLE
Definition: sqltypes.h:175
void addGeoString(const std::string &v)
Definition: Importer.h:238
Definition: sqltypes.h:50
void addTinyint(const int8_t v)
Definition: Importer.h:224
void addInt(const int32_t v)
Definition: Importer.h:228
void set_replicate_count(const int64_t replicate_count)
Definition: Importer.h:471
void addBoolean(const int8_t v)
Definition: Importer.h:222
void addFloat(const float v)
Definition: Importer.h:232
CHECK(cgen_state)
#define NULL_FLOAT
Definition: sqltypes.h:174
void addBigint(const int64_t v)
Definition: Importer.h:230
SQLTypes decimal_to_int_type(const SQLTypeInfo &ti)
Definition: Datum.cpp:299
Definition: sqltypes.h:53
Definition: sqltypes.h:54
bool is_null(const T &v, const SQLTypeInfo &t)
void addArray(const ArrayDatum &v)
Definition: Importer.h:240
ArrayDatum TDatumToArrayDatum(const TDatum &datum, const SQLTypeInfo &ti)
Definition: Importer.cpp:451
Definition: sqltypes.h:42
ArrayDatum NullArray(const SQLTypeInfo &ti)
Definition: Importer.cpp:367
#define IS_STRING(T)
Definition: sqltypes.h:162
void addString(const std::string &v)
Definition: Importer.h:236
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
Definition: sqltypes.h:46
SQLTypeInfo columnType
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:255
bool is_decimal() const
Definition: sqltypes.h:402
std::vector< std::string > & addStringArray()
Definition: Importer.h:242
std::string columnName
void addBinaryStringArray(const TDatum &datum, std::vector< std::string > &string_vec)
Definition: Importer.cpp:399
void addDouble(const double v)
Definition: Importer.h:234

+ Here is the call graph for this function:

size_t Importer_NS::TypedImportBuffer::add_values ( const ColumnDescriptor cd,
const TColumn &  data 
)

Definition at line 939 of file Importer.cpp.

References addArray(), addStringArray(), bigint_buffer_, bool_buffer_, checked_malloc(), ColumnDescriptor::columnName, ColumnDescriptor::columnType, double_buffer_, float_buffer_, geo_string_buffer_, SQLTypeInfo::get_notnull(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), inline_fixed_encoding_null_val(), int_buffer_, IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, NULL_DOUBLE, NULL_FLOAT, Importer_NS::NullArray(), smallint_buffer_, string_buffer_, and tinyint_buffer_.

939  {
940  size_t dataSize = 0;
941  if (cd->columnType.get_notnull()) {
942  // We can't have any null values for this column; to have them is an error
943  if (std::any_of(col.nulls.begin(), col.nulls.end(), [](int i) { return i != 0; })) {
944  throw std::runtime_error("NULL for column " + cd->columnName);
945  }
946  }
947 
948  switch (cd->columnType.get_type()) {
949  case kBOOLEAN: {
950  dataSize = col.data.int_col.size();
951  bool_buffer_->reserve(dataSize);
952  for (size_t i = 0; i < dataSize; i++) {
953  if (col.nulls[i]) {
955  } else {
956  bool_buffer_->push_back((int8_t)col.data.int_col[i]);
957  }
958  }
959  break;
960  }
961  case kTINYINT: {
962  dataSize = col.data.int_col.size();
963  tinyint_buffer_->reserve(dataSize);
964  for (size_t i = 0; i < dataSize; i++) {
965  if (col.nulls[i]) {
967  } else {
968  tinyint_buffer_->push_back((int8_t)col.data.int_col[i]);
969  }
970  }
971  break;
972  }
973  case kSMALLINT: {
974  dataSize = col.data.int_col.size();
975  smallint_buffer_->reserve(dataSize);
976  for (size_t i = 0; i < dataSize; i++) {
977  if (col.nulls[i]) {
979  } else {
980  smallint_buffer_->push_back((int16_t)col.data.int_col[i]);
981  }
982  }
983  break;
984  }
985  case kINT: {
986  dataSize = col.data.int_col.size();
987  int_buffer_->reserve(dataSize);
988  for (size_t i = 0; i < dataSize; i++) {
989  if (col.nulls[i]) {
991  } else {
992  int_buffer_->push_back((int32_t)col.data.int_col[i]);
993  }
994  }
995  break;
996  }
997  case kBIGINT:
998  case kNUMERIC:
999  case kDECIMAL: {
1000  dataSize = col.data.int_col.size();
1001  bigint_buffer_->reserve(dataSize);
1002  for (size_t i = 0; i < dataSize; i++) {
1003  if (col.nulls[i]) {
1005  } else {
1006  bigint_buffer_->push_back((int64_t)col.data.int_col[i]);
1007  }
1008  }
1009  break;
1010  }
1011  case kFLOAT: {
1012  dataSize = col.data.real_col.size();
1013  float_buffer_->reserve(dataSize);
1014  for (size_t i = 0; i < dataSize; i++) {
1015  if (col.nulls[i]) {
1016  float_buffer_->push_back(NULL_FLOAT);
1017  } else {
1018  float_buffer_->push_back((float)col.data.real_col[i]);
1019  }
1020  }
1021  break;
1022  }
1023  case kDOUBLE: {
1024  dataSize = col.data.real_col.size();
1025  double_buffer_->reserve(dataSize);
1026  for (size_t i = 0; i < dataSize; i++) {
1027  if (col.nulls[i]) {
1028  double_buffer_->push_back(NULL_DOUBLE);
1029  } else {
1030  double_buffer_->push_back((double)col.data.real_col[i]);
1031  }
1032  }
1033  break;
1034  }
1035  case kTEXT:
1036  case kVARCHAR:
1037  case kCHAR: {
1038  // TODO: for now, use empty string for nulls
1039  dataSize = col.data.str_col.size();
1040  string_buffer_->reserve(dataSize);
1041  for (size_t i = 0; i < dataSize; i++) {
1042  if (col.nulls[i]) {
1043  string_buffer_->push_back(std::string());
1044  } else {
1045  string_buffer_->push_back(col.data.str_col[i]);
1046  }
1047  }
1048  break;
1049  }
1050  case kTIME:
1051  case kTIMESTAMP:
1052  case kDATE: {
1053  dataSize = col.data.int_col.size();
1054  bigint_buffer_->reserve(dataSize);
1055  for (size_t i = 0; i < dataSize; i++) {
1056  if (col.nulls[i]) {
1058  } else {
1059  bigint_buffer_->push_back(static_cast<int64_t>(col.data.int_col[i]));
1060  }
1061  }
1062  break;
1063  }
1064  case kPOINT:
1065  case kLINESTRING:
1066  case kPOLYGON:
1067  case kMULTIPOLYGON: {
1068  dataSize = col.data.str_col.size();
1069  geo_string_buffer_->reserve(dataSize);
1070  for (size_t i = 0; i < dataSize; i++) {
1071  if (col.nulls[i]) {
1072  // TODO: add support for NULL geo
1073  geo_string_buffer_->push_back(std::string());
1074  } else {
1075  geo_string_buffer_->push_back(col.data.str_col[i]);
1076  }
1077  }
1078  break;
1079  }
1080  case kARRAY: {
1081  dataSize = col.data.arr_col.size();
1082  if (IS_STRING(cd->columnType.get_subtype())) {
1083  for (size_t i = 0; i < dataSize; i++) {
1084  std::vector<std::string>& string_vec = addStringArray();
1085  if (!col.nulls[i]) {
1086  size_t stringArrSize = col.data.arr_col[i].data.str_col.size();
1087  for (size_t str_idx = 0; str_idx != stringArrSize; ++str_idx) {
1088  string_vec.push_back(col.data.arr_col[i].data.str_col[str_idx]);
1089  }
1090  }
1091  }
1092  } else {
1093  auto elem_ti = cd->columnType.get_subtype();
1094  switch (elem_ti) {
1095  case kBOOLEAN: {
1096  for (size_t i = 0; i < dataSize; i++) {
1097  if (col.nulls[i]) {
1099  } else {
1100  size_t len = col.data.arr_col[i].data.int_col.size();
1101  size_t byteSize = len * sizeof(int8_t);
1102  int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
1103  int8_t* p = buf;
1104  for (size_t j = 0; j < len; ++j) {
1105  *(bool*)p = static_cast<bool>(col.data.arr_col[i].data.int_col[j]);
1106  p += sizeof(bool);
1107  }
1108  addArray(ArrayDatum(byteSize, buf, false));
1109  }
1110  }
1111  break;
1112  }
1113  case kTINYINT: {
1114  for (size_t i = 0; i < dataSize; i++) {
1115  if (col.nulls[i]) {
1117  } else {
1118  size_t len = col.data.arr_col[i].data.int_col.size();
1119  size_t byteSize = len * sizeof(int8_t);
1120  int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
1121  int8_t* p = buf;
1122  for (size_t j = 0; j < len; ++j) {
1123  *(int8_t*)p = static_cast<int8_t>(col.data.arr_col[i].data.int_col[j]);
1124  p += sizeof(int8_t);
1125  }
1126  addArray(ArrayDatum(byteSize, buf, false));
1127  }
1128  }
1129  break;
1130  }
1131  case kSMALLINT: {
1132  for (size_t i = 0; i < dataSize; i++) {
1133  if (col.nulls[i]) {
1135  } else {
1136  size_t len = col.data.arr_col[i].data.int_col.size();
1137  size_t byteSize = len * sizeof(int16_t);
1138  int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
1139  int8_t* p = buf;
1140  for (size_t j = 0; j < len; ++j) {
1141  *(int16_t*)p =
1142  static_cast<int16_t>(col.data.arr_col[i].data.int_col[j]);
1143  p += sizeof(int16_t);
1144  }
1145  addArray(ArrayDatum(byteSize, buf, false));
1146  }
1147  }
1148  break;
1149  }
1150  case kINT: {
1151  for (size_t i = 0; i < dataSize; i++) {
1152  if (col.nulls[i]) {
1154  } else {
1155  size_t len = col.data.arr_col[i].data.int_col.size();
1156  size_t byteSize = len * sizeof(int32_t);
1157  int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
1158  int8_t* p = buf;
1159  for (size_t j = 0; j < len; ++j) {
1160  *(int32_t*)p =
1161  static_cast<int32_t>(col.data.arr_col[i].data.int_col[j]);
1162  p += sizeof(int32_t);
1163  }
1164  addArray(ArrayDatum(byteSize, buf, false));
1165  }
1166  }
1167  break;
1168  }
1169  case kBIGINT:
1170  case kNUMERIC:
1171  case kDECIMAL: {
1172  for (size_t i = 0; i < dataSize; i++) {
1173  if (col.nulls[i]) {
1175  } else {
1176  size_t len = col.data.arr_col[i].data.int_col.size();
1177  size_t byteSize = len * sizeof(int64_t);
1178  int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
1179  int8_t* p = buf;
1180  for (size_t j = 0; j < len; ++j) {
1181  *(int64_t*)p =
1182  static_cast<int64_t>(col.data.arr_col[j].data.int_col[j]);
1183  p += sizeof(int64_t);
1184  }
1185  addArray(ArrayDatum(byteSize, buf, false));
1186  }
1187  }
1188  break;
1189  }
1190  case kFLOAT: {
1191  for (size_t i = 0; i < dataSize; i++) {
1192  if (col.nulls[i]) {
1194  } else {
1195  size_t len = col.data.arr_col[i].data.real_col.size();
1196  size_t byteSize = len * sizeof(float);
1197  int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
1198  int8_t* p = buf;
1199  for (size_t j = 0; j < len; ++j) {
1200  *(float*)p = static_cast<float>(col.data.arr_col[i].data.real_col[j]);
1201  p += sizeof(float);
1202  }
1203  addArray(ArrayDatum(byteSize, buf, false));
1204  }
1205  }
1206  break;
1207  }
1208  case kDOUBLE: {
1209  for (size_t i = 0; i < dataSize; i++) {
1210  if (col.nulls[i]) {
1212  } else {
1213  size_t len = col.data.arr_col[i].data.real_col.size();
1214  size_t byteSize = len * sizeof(double);
1215  int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
1216  int8_t* p = buf;
1217  for (size_t j = 0; j < len; ++j) {
1218  *(double*)p = static_cast<double>(col.data.arr_col[i].data.real_col[j]);
1219  p += sizeof(double);
1220  }
1221  addArray(ArrayDatum(byteSize, buf, false));
1222  }
1223  }
1224  break;
1225  }
1226  case kTIME:
1227  case kTIMESTAMP:
1228  case kDATE: {
1229  for (size_t i = 0; i < dataSize; i++) {
1230  if (col.nulls[i]) {
1232  } else {
1233  size_t len = col.data.arr_col[i].data.int_col.size();
1234  size_t byteWidth = sizeof(int64_t);
1235  size_t byteSize = len * byteWidth;
1236  int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
1237  int8_t* p = buf;
1238  for (size_t j = 0; j < len; ++j) {
1239  *reinterpret_cast<int64_t*>(p) =
1240  static_cast<int64_t>(col.data.arr_col[i].data.int_col[j]);
1241  p += sizeof(int64_t);
1242  }
1243  addArray(ArrayDatum(byteSize, buf, false));
1244  }
1245  }
1246  break;
1247  }
1248  default:
1249  throw std::runtime_error("Invalid Array Type");
1250  }
1251  }
1252  break;
1253  }
1254  default:
1255  throw std::runtime_error("Invalid Type");
1256  }
1257  return dataSize;
1258 }
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:249
std::vector< int16_t > * smallint_buffer_
Definition: Importer.h:490
#define NULL_DOUBLE
Definition: sqltypes.h:175
std::vector< int64_t > * bigint_buffer_
Definition: Importer.h:492
Definition: sqltypes.h:50
std::vector< float > * float_buffer_
Definition: Importer.h:493
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:248
std::vector< int8_t > * tinyint_buffer_
Definition: Importer.h:489
std::vector< std::string > * string_buffer_
Definition: Importer.h:495
std::vector< int32_t > * int_buffer_
Definition: Importer.h:491
void * checked_malloc(const size_t size)
Definition: checked_alloc.h:40
#define NULL_FLOAT
Definition: sqltypes.h:174
std::vector< int8_t > * bool_buffer_
Definition: Importer.h:488
Definition: sqltypes.h:53
Definition: sqltypes.h:54
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:496
void addArray(const ArrayDatum &v)
Definition: Importer.h:240
std::vector< double > * double_buffer_
Definition: Importer.h:494
Definition: sqltypes.h:42
ArrayDatum NullArray(const SQLTypeInfo &ti)
Definition: Importer.cpp:367
#define IS_STRING(T)
Definition: sqltypes.h:162
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
Definition: sqltypes.h:46
SQLTypeInfo columnType
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:255
std::vector< std::string > & addStringArray()
Definition: Importer.h:242
std::string columnName
std::conditional_t< isCudaCC(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:120

+ Here is the call graph for this function:

void Importer_NS::TypedImportBuffer::addArray ( const ArrayDatum v)
inline

Definition at line 240 of file Importer.h.

References array_buffer_.

Referenced by add_value(), and add_values().

240 { array_buffer_->push_back(v); }
std::vector< ArrayDatum > * array_buffer_
Definition: Importer.h:497

+ Here is the caller graph for this function:

void Importer_NS::TypedImportBuffer::addBigint ( const int64_t  v)
inline

Definition at line 230 of file Importer.h.

References bigint_buffer_.

Referenced by add_value().

230 { bigint_buffer_->push_back(v); }
std::vector< int64_t > * bigint_buffer_
Definition: Importer.h:492

+ Here is the caller graph for this function:

void Importer_NS::TypedImportBuffer::addBoolean ( const int8_t  v)
inline

Definition at line 222 of file Importer.h.

References bool_buffer_.

Referenced by add_value().

222 { bool_buffer_->push_back(v); }
std::vector< int8_t > * bool_buffer_
Definition: Importer.h:488

+ Here is the caller graph for this function:

void Importer_NS::TypedImportBuffer::addDictEncodedString ( const std::vector< std::string > &  string_vec)

Definition at line 470 of file Importer.cpp.

References CHECK(), column_desc_, ColumnDescriptor::columnType, SQLTypeInfo::get_size(), StringDictionary::getOrAddBulk(), StringDictionary::MAX_STRLEN, string_dict_, string_dict_i16_buffer_, string_dict_i32_buffer_, and string_dict_i8_buffer_.

470  {
472  std::vector<std::string_view> string_view_vec;
473  string_view_vec.reserve(string_vec.size());
474  for (const auto& str : string_vec) {
475  if (str.size() > StringDictionary::MAX_STRLEN) {
476  throw std::runtime_error("String too long for dictionary encoding.");
477  }
478  string_view_vec.push_back(str);
479  }
480  switch (column_desc_->columnType.get_size()) {
481  case 1:
482  string_dict_i8_buffer_->resize(string_view_vec.size());
483  string_dict_->getOrAddBulk(string_view_vec, string_dict_i8_buffer_->data());
484  break;
485  case 2:
486  string_dict_i16_buffer_->resize(string_view_vec.size());
487  string_dict_->getOrAddBulk(string_view_vec, string_dict_i16_buffer_->data());
488  break;
489  case 4:
490  string_dict_i32_buffer_->resize(string_view_vec.size());
491  string_dict_->getOrAddBulk(string_view_vec, string_dict_i32_buffer_->data());
492  break;
493  default:
494  CHECK(false);
495  }
496 }
HOST DEVICE int get_size() const
Definition: sqltypes.h:258
std::vector< uint16_t > * string_dict_i16_buffer_
Definition: Importer.h:502
CHECK(cgen_state)
StringDictionary * string_dict_
Definition: Importer.h:507
std::vector< int32_t > * string_dict_i32_buffer_
Definition: Importer.h:503
void getOrAddBulk(const std::vector< String > &string_vec, T *encoded_vec)
const ColumnDescriptor * column_desc_
Definition: Importer.h:506
static constexpr size_t MAX_STRLEN
SQLTypeInfo columnType
std::vector< uint8_t > * string_dict_i8_buffer_
Definition: Importer.h:501

+ Here is the call graph for this function:

void Importer_NS::TypedImportBuffer::addDictEncodedStringArray ( const std::vector< std::vector< std::string >> &  string_array_vec)
inline

Definition at line 253 of file Importer.h.

References CHECK(), checked_malloc(), StringDictionary::getOrAddBulkArray(), StringDictionary::MAX_STRLEN, string_array_dict_buffer_, and string_dict_.

254  {
256 
257  // first check data is ok
258  for (auto& p : string_array_vec) {
259  for (const auto& str : p) {
260  if (str.size() > StringDictionary::MAX_STRLEN) {
261  throw std::runtime_error("String too long for dictionary encoding.");
262  }
263  }
264  }
265 
266  std::vector<std::vector<int32_t>> ids_array(0);
267  string_dict_->getOrAddBulkArray(string_array_vec, ids_array);
268 
269  for (auto& p : ids_array) {
270  size_t len = p.size() * sizeof(int32_t);
271  auto a = static_cast<int32_t*>(checked_malloc(len));
272  memcpy(a, &p[0], len);
273  // TODO: distinguish between empty and NULL
274  string_array_dict_buffer_->push_back(
275  ArrayDatum(len, reinterpret_cast<int8_t*>(a), len == 0));
276  }
277  }
CHECK(cgen_state)
StringDictionary * string_dict_
Definition: Importer.h:507
void * checked_malloc(const size_t size)
Definition: checked_alloc.h:40
void getOrAddBulkArray(const std::vector< std::vector< String >> &string_array_vec, std::vector< std::vector< int32_t >> &ids_array_vec)
std::vector< ArrayDatum > * string_array_dict_buffer_
Definition: Importer.h:504
static constexpr size_t MAX_STRLEN
std::conditional_t< isCudaCC(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:120

+ Here is the call graph for this function:

void Importer_NS::TypedImportBuffer::addDouble ( const double  v)
inline

Definition at line 234 of file Importer.h.

References double_buffer_.

Referenced by add_value().

234 { double_buffer_->push_back(v); }
std::vector< double > * double_buffer_
Definition: Importer.h:494

+ Here is the caller graph for this function:

void Importer_NS::TypedImportBuffer::addFloat ( const float  v)
inline

Definition at line 232 of file Importer.h.

References float_buffer_.

Referenced by add_value().

232 { float_buffer_->push_back(v); }
std::vector< float > * float_buffer_
Definition: Importer.h:493

+ Here is the caller graph for this function:

void Importer_NS::TypedImportBuffer::addGeoString ( const std::string &  v)
inline

Definition at line 238 of file Importer.h.

References geo_string_buffer_.

Referenced by add_value().

238 { geo_string_buffer_->push_back(v); }
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:496

+ Here is the caller graph for this function:

void Importer_NS::TypedImportBuffer::addInt ( const int32_t  v)
inline

Definition at line 228 of file Importer.h.

References int_buffer_.

Referenced by add_value().

228 { int_buffer_->push_back(v); }
std::vector< int32_t > * int_buffer_
Definition: Importer.h:491

+ Here is the caller graph for this function:

void Importer_NS::TypedImportBuffer::addSmallint ( const int16_t  v)
inline

Definition at line 226 of file Importer.h.

References smallint_buffer_.

Referenced by add_value().

226 { smallint_buffer_->push_back(v); }
std::vector< int16_t > * smallint_buffer_
Definition: Importer.h:490

+ Here is the caller graph for this function:

void Importer_NS::TypedImportBuffer::addString ( const std::string &  v)
inline

Definition at line 236 of file Importer.h.

References string_buffer_.

Referenced by add_value().

236 { string_buffer_->push_back(v); }
std::vector< std::string > * string_buffer_
Definition: Importer.h:495

+ Here is the caller graph for this function:

std::vector<std::string>& Importer_NS::TypedImportBuffer::addStringArray ( )
inline

Definition at line 242 of file Importer.h.

References string_array_buffer_.

Referenced by add_value(), and add_values().

242  {
243  string_array_buffer_->push_back(std::vector<std::string>());
244  return string_array_buffer_->back();
245  }
std::vector< std::vector< std::string > > * string_array_buffer_
Definition: Importer.h:498

+ Here is the caller graph for this function:

void Importer_NS::TypedImportBuffer::addStringArray ( const std::vector< std::string > &  arr)
inline

Definition at line 247 of file Importer.h.

References string_array_buffer_.

247  {
248  string_array_buffer_->push_back(arr);
249  }
std::vector< std::vector< std::string > > * string_array_buffer_
Definition: Importer.h:498
void Importer_NS::TypedImportBuffer::addTinyint ( const int8_t  v)
inline

Definition at line 224 of file Importer.h.

References tinyint_buffer_.

Referenced by add_value().

224 { tinyint_buffer_->push_back(v); }
std::vector< int8_t > * tinyint_buffer_
Definition: Importer.h:489

+ Here is the caller graph for this function:

void Importer_NS::TypedImportBuffer::clear ( )
inline

Definition at line 373 of file Importer.h.

References array_buffer_, bigint_buffer_, bool_buffer_, CHECK(), column_desc_, ColumnDescriptor::columnType, double_buffer_, float_buffer_, geo_string_buffer_, SQLTypeInfo::get_compression(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), int_buffer_, IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kENCODING_DICT, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, smallint_buffer_, string_array_buffer_, string_array_dict_buffer_, string_buffer_, string_dict_i16_buffer_, string_dict_i32_buffer_, string_dict_i8_buffer_, and tinyint_buffer_.

373  {
374  switch (column_desc_->columnType.get_type()) {
375  case kBOOLEAN: {
376  bool_buffer_->clear();
377  break;
378  }
379  case kTINYINT: {
380  tinyint_buffer_->clear();
381  break;
382  }
383  case kSMALLINT: {
384  smallint_buffer_->clear();
385  break;
386  }
387  case kINT: {
388  int_buffer_->clear();
389  break;
390  }
391  case kBIGINT:
392  case kNUMERIC:
393  case kDECIMAL: {
394  bigint_buffer_->clear();
395  break;
396  }
397  case kFLOAT: {
398  float_buffer_->clear();
399  break;
400  }
401  case kDOUBLE: {
402  double_buffer_->clear();
403  break;
404  }
405  case kTEXT:
406  case kVARCHAR:
407  case kCHAR: {
408  string_buffer_->clear();
410  switch (column_desc_->columnType.get_size()) {
411  case 1:
412  string_dict_i8_buffer_->clear();
413  break;
414  case 2:
415  string_dict_i16_buffer_->clear();
416  break;
417  case 4:
418  string_dict_i32_buffer_->clear();
419  break;
420  default:
421  CHECK(false);
422  }
423  }
424  break;
425  }
426  case kDATE:
427  case kTIME:
428  case kTIMESTAMP:
429  bigint_buffer_->clear();
430  break;
431  case kARRAY: {
433  string_array_buffer_->clear();
434  string_array_dict_buffer_->clear();
435  } else {
436  array_buffer_->clear();
437  }
438  break;
439  }
440  case kPOINT:
441  case kLINESTRING:
442  case kPOLYGON:
443  case kMULTIPOLYGON:
444  geo_string_buffer_->clear();
445  break;
446  default:
447  CHECK(false);
448  }
449  }
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:249
std::vector< int16_t > * smallint_buffer_
Definition: Importer.h:490
HOST DEVICE int get_size() const
Definition: sqltypes.h:258
std::vector< int64_t > * bigint_buffer_
Definition: Importer.h:492
Definition: sqltypes.h:50
std::vector< float > * float_buffer_
Definition: Importer.h:493
std::vector< uint16_t > * string_dict_i16_buffer_
Definition: Importer.h:502
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:248
std::vector< int8_t > * tinyint_buffer_
Definition: Importer.h:489
CHECK(cgen_state)
std::vector< std::string > * string_buffer_
Definition: Importer.h:495
std::vector< int32_t > * string_dict_i32_buffer_
Definition: Importer.h:503
std::vector< std::vector< std::string > > * string_array_buffer_
Definition: Importer.h:498
std::vector< int32_t > * int_buffer_
Definition: Importer.h:491
std::vector< int8_t > * bool_buffer_
Definition: Importer.h:488
Definition: sqltypes.h:53
Definition: sqltypes.h:54
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:256
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:496
std::vector< ArrayDatum > * string_array_dict_buffer_
Definition: Importer.h:504
std::vector< double > * double_buffer_
Definition: Importer.h:494
Definition: sqltypes.h:42
const ColumnDescriptor * column_desc_
Definition: Importer.h:506
#define IS_STRING(T)
Definition: sqltypes.h:162
Definition: sqltypes.h:46
SQLTypeInfo columnType
std::vector< uint8_t > * string_dict_i8_buffer_
Definition: Importer.h:501
std::vector< ArrayDatum > * array_buffer_
Definition: Importer.h:497

+ Here is the call graph for this function:

template<typename DATA_TYPE >
size_t Importer_NS::TypedImportBuffer::convert_arrow_val_to_import_buffer ( const ColumnDescriptor cd,
const arrow::Array &  array,
std::vector< DATA_TYPE > &  buffer,
const ArraySliceRange slice_range,
BadRowsTracker *const  bad_rows_tracker 
)

Referenced by add_arrow_values().

+ Here is the caller graph for this function:

template<typename DATA_TYPE >
size_t Importer_NS::TypedImportBuffer::convert_arrow_val_to_import_buffer ( const ColumnDescriptor cd,
const Array array,
std::vector< DATA_TYPE > &  buffer,
const ArraySliceRange slice_range,
Importer_NS::BadRowsTracker *const  bad_rows_tracker 
)

Definition at line 754 of file Importer.cpp.

References col_idx, anonymous_namespace{ArrowImporter.h}::error_context(), geo_string_buffer_, SQLTypeInfo::get_type(), Importer_NS::Importer::getCatalog(), Geo_namespace::GeoTypesFactory::getGeoColumns(), Geo_namespace::GeoTypesFactory::getNullGeoColumns(), import_buffers, Importer_NS::BadRowsTracker::importer, Importer_NS::BadRowsTracker::mutex, Importer_NS::BadRowsTracker::rows, Importer_NS::Importer::set_geo_physical_import_buffer(), and anonymous_namespace{ArrowImporter.h}::value_getter().

759  {
760  auto data =
761  std::make_unique<DataBuffer<DATA_TYPE>>(cd, array, buffer, bad_rows_tracker);
762  auto f_value_getter = value_getter(array, cd, bad_rows_tracker);
763  std::function<void(const int64_t)> f_add_geo_phy_cols = [&](const int64_t row) {};
764  if (bad_rows_tracker && cd->columnType.is_geometry()) {
765  f_add_geo_phy_cols = [&](const int64_t row) {
766  // Populate physical columns (ref. MapDHandler::load_table)
767  std::vector<double> coords, bounds;
768  std::vector<int> ring_sizes, poly_rings;
769  int render_group = 0;
770  SQLTypeInfo ti;
771  // replace any unexpected exception from getGeoColumns or other
772  // on this path with a GeoImportException so that we wont over
773  // push a null to the logical column...
774  try {
775  SQLTypeInfo import_ti{ti};
776  if (array.IsNull(row)) {
778  import_ti, coords, bounds, ring_sizes, poly_rings, false);
779  } else {
780  arrow_throw_if<GeoImportException>(
782  ti,
783  coords,
784  bounds,
785  ring_sizes,
786  poly_rings,
787  false),
788  error_context(cd, bad_rows_tracker) + "Invalid geometry");
789  arrow_throw_if<GeoImportException>(
790  cd->columnType.get_type() != ti.get_type(),
791  error_context(cd, bad_rows_tracker) + "Geometry type mismatch");
792  }
793  auto col_idx_workpad = col_idx; // what a pitfall!!
795  bad_rows_tracker->importer->getCatalog(),
796  cd,
798  col_idx_workpad,
799  coords,
800  bounds,
801  ring_sizes,
802  poly_rings,
803  render_group);
804  } catch (GeoImportException&) {
805  throw;
806  } catch (std::runtime_error& e) {
807  throw GeoImportException(e.what());
808  } catch (const std::exception& e) {
809  throw GeoImportException(e.what());
810  } catch (...) {
811  throw GeoImportException("unknown exception");
812  }
813  };
814  }
815  auto f_mark_a_bad_row = [&](const auto row) {
816  std::unique_lock<std::mutex> lck(bad_rows_tracker->mutex);
817  bad_rows_tracker->rows.insert(row - slice_range.first);
818  };
819  buffer.reserve(slice_range.second - slice_range.first);
820  for (size_t row = slice_range.first; row < slice_range.second; ++row) {
821  try {
822  *data << (array.IsNull(row) ? nullptr : f_value_getter(array, row));
823  f_add_geo_phy_cols(row);
824  } catch (GeoImportException&) {
825  f_mark_a_bad_row(row);
826  } catch (ArrowImporterException&) {
827  // trace bad rows of each column; otherwise rethrow.
828  if (bad_rows_tracker) {
829  *data << nullptr;
830  f_mark_a_bad_row(row);
831  } else {
832  throw;
833  }
834  }
835  }
836  return buffer.size();
837 }
std::string error_context(const ColumnDescriptor *cd, Importer_NS::BadRowsTracker *const bad_rows_tracker)
Definition: ArrowImporter.h:75
auto value_getter(const Array &array, const ColumnDescriptor *cd, Importer_NS::BadRowsTracker *const bad_rows_tracker)
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:248
Catalog_Namespace::Catalog & getCatalog()
Definition: Importer.h:769
static void set_geo_physical_import_buffer(const Catalog_Namespace::Catalog &catalog, const ColumnDescriptor *cd, std::vector< std::unique_ptr< TypedImportBuffer >> &import_buffers, size_t &col_idx, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, int render_group, const int64_t replicate_count=0)
Definition: Importer.cpp:1411
std::vector< std::unique_ptr< TypedImportBuffer > > * import_buffers
Definition: Importer.h:483
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:496
static void getNullGeoColumns(SQLTypeInfo &ti, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, const bool promote_poly_to_mpoly=false)
Definition: geo_types.cpp:626
static bool getGeoColumns(const std::string &wkt, SQLTypeInfo &ti, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, const bool promote_poly_to_mpoly=false)
Definition: geo_types.cpp:459
std::set< int64_t > rows
Definition: Importer.h:75

+ Here is the call graph for this function:

template<typename DATA_TYPE >
auto Importer_NS::TypedImportBuffer::del_values ( std::vector< DATA_TYPE > &  buffer,
BadRowsTracker *const  bad_rows_tracker 
)
auto Importer_NS::TypedImportBuffer::del_values ( const SQLTypes  type,
BadRowsTracker *const  bad_rows_tracker 
)
int64_t Importer_NS::TypedImportBuffer::get_replicate_count ( ) const
inline

Definition at line 470 of file Importer.h.

References replicate_count_.

470 { return replicate_count_; }
std::vector<ArrayDatum>* Importer_NS::TypedImportBuffer::getArrayBuffer ( ) const
inline

Definition at line 343 of file Importer.h.

References array_buffer_.

343 { return array_buffer_; }
std::vector< ArrayDatum > * array_buffer_
Definition: Importer.h:497
int8_t* Importer_NS::TypedImportBuffer::getAsBytes ( ) const
inline

Definition at line 285 of file Importer.h.

References column_desc_, ColumnDescriptor::columnType, SQLTypeInfo::get_type(), kBIGINT, kBOOLEAN, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kNUMERIC, kSMALLINT, kTIME, kTIMESTAMP, and kTINYINT.

Referenced by Importer_NS::anonymous_namespace{Importer.cpp}::double_value_at(), Importer_NS::anonymous_namespace{Importer.cpp}::float_value_at(), and Importer_NS::anonymous_namespace{Importer.cpp}::int_value_at().

285  {
286  switch (column_desc_->columnType.get_type()) {
287  case kBOOLEAN:
288  return reinterpret_cast<int8_t*>(&((*bool_buffer_)[0]));
289  case kTINYINT:
290  return reinterpret_cast<int8_t*>(&((*tinyint_buffer_)[0]));
291  case kSMALLINT:
292  return reinterpret_cast<int8_t*>(&((*smallint_buffer_)[0]));
293  case kINT:
294  return reinterpret_cast<int8_t*>(&((*int_buffer_)[0]));
295  case kBIGINT:
296  case kNUMERIC:
297  case kDECIMAL:
298  return reinterpret_cast<int8_t*>(&((*bigint_buffer_)[0]));
299  case kFLOAT:
300  return reinterpret_cast<int8_t*>(&((*float_buffer_)[0]));
301  case kDOUBLE:
302  return reinterpret_cast<int8_t*>(&((*double_buffer_)[0]));
303  case kDATE:
304  case kTIME:
305  case kTIMESTAMP:
306  return reinterpret_cast<int8_t*>(&((*bigint_buffer_)[0]));
307  default:
308  abort();
309  }
310  }
Definition: sqltypes.h:50
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:248
Definition: sqltypes.h:54
const ColumnDescriptor * column_desc_
Definition: Importer.h:506
Definition: sqltypes.h:46
SQLTypeInfo columnType

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const ColumnDescriptor* Importer_NS::TypedImportBuffer::getColumnDesc ( ) const
inline

Definition at line 281 of file Importer.h.

References column_desc_.

281 { return column_desc_; }
const ColumnDescriptor * column_desc_
Definition: Importer.h:506
size_t Importer_NS::TypedImportBuffer::getElementSize ( ) const
inline

Definition at line 312 of file Importer.h.

References column_desc_, ColumnDescriptor::columnType, SQLTypeInfo::get_type(), kBIGINT, kBOOLEAN, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kNUMERIC, kSMALLINT, kTIME, kTIMESTAMP, and kTINYINT.

312  {
313  switch (column_desc_->columnType.get_type()) {
314  case kBOOLEAN:
315  return sizeof((*bool_buffer_)[0]);
316  case kTINYINT:
317  return sizeof((*tinyint_buffer_)[0]);
318  case kSMALLINT:
319  return sizeof((*smallint_buffer_)[0]);
320  case kINT:
321  return sizeof((*int_buffer_)[0]);
322  case kBIGINT:
323  case kNUMERIC:
324  case kDECIMAL:
325  return sizeof((*bigint_buffer_)[0]);
326  case kFLOAT:
327  return sizeof((*float_buffer_)[0]);
328  case kDOUBLE:
329  return sizeof((*double_buffer_)[0]);
330  case kDATE:
331  case kTIME:
332  case kTIMESTAMP:
333  return sizeof((*bigint_buffer_)[0]);
334  default:
335  abort();
336  }
337  }
Definition: sqltypes.h:50
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:248
Definition: sqltypes.h:54
const ColumnDescriptor * column_desc_
Definition: Importer.h:506
Definition: sqltypes.h:46
SQLTypeInfo columnType

+ Here is the call graph for this function:

std::vector<std::string>* Importer_NS::TypedImportBuffer::getGeoStringBuffer ( ) const
inline

Definition at line 341 of file Importer.h.

References geo_string_buffer_.

341 { return geo_string_buffer_; }
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:496
std::vector<std::vector<std::string> >* Importer_NS::TypedImportBuffer::getStringArrayBuffer ( ) const
inline

Definition at line 345 of file Importer.h.

References string_array_buffer_.

345  {
346  return string_array_buffer_;
347  }
std::vector< std::vector< std::string > > * string_array_buffer_
Definition: Importer.h:498
std::vector<ArrayDatum>* Importer_NS::TypedImportBuffer::getStringArrayDictBuffer ( ) const
inline

Definition at line 349 of file Importer.h.

References string_array_dict_buffer_.

349  {
351  }
std::vector< ArrayDatum > * string_array_dict_buffer_
Definition: Importer.h:504
std::vector<std::string>* Importer_NS::TypedImportBuffer::getStringBuffer ( ) const
inline

Definition at line 339 of file Importer.h.

References string_buffer_.

339 { return string_buffer_; }
std::vector< std::string > * string_buffer_
Definition: Importer.h:495
int8_t* Importer_NS::TypedImportBuffer::getStringDictBuffer ( ) const
inline

Definition at line 353 of file Importer.h.

References column_desc_, ColumnDescriptor::columnType, and SQLTypeInfo::get_size().

Referenced by Importer_NS::anonymous_namespace{Importer.cpp}::int_value_at().

353  {
354  switch (column_desc_->columnType.get_size()) {
355  case 1:
356  return reinterpret_cast<int8_t*>(&((*string_dict_i8_buffer_)[0]));
357  case 2:
358  return reinterpret_cast<int8_t*>(&((*string_dict_i16_buffer_)[0]));
359  case 4:
360  return reinterpret_cast<int8_t*>(&((*string_dict_i32_buffer_)[0]));
361  default:
362  abort();
363  }
364  }
HOST DEVICE int get_size() const
Definition: sqltypes.h:258
const ColumnDescriptor * column_desc_
Definition: Importer.h:506
SQLTypeInfo columnType

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

StringDictionary* Importer_NS::TypedImportBuffer::getStringDictionary ( ) const
inline

Definition at line 283 of file Importer.h.

References string_dict_.

283 { return string_dict_; }
StringDictionary * string_dict_
Definition: Importer.h:507
const SQLTypeInfo& Importer_NS::TypedImportBuffer::getTypeInfo ( ) const
inline

Definition at line 279 of file Importer.h.

References column_desc_, and ColumnDescriptor::columnType.

Referenced by Importer_NS::anonymous_namespace{Importer.cpp}::double_value_at(), Importer_NS::anonymous_namespace{Importer.cpp}::float_value_at(), and Importer_NS::anonymous_namespace{Importer.cpp}::int_value_at().

279 { return column_desc_->columnType; }
const ColumnDescriptor * column_desc_
Definition: Importer.h:506
SQLTypeInfo columnType

+ Here is the caller graph for this function:

void Importer_NS::TypedImportBuffer::pop_value ( )

Definition at line 694 of file Importer.cpp.

References array_buffer_, bigint_buffer_, bool_buffer_, CHECK(), column_desc_, ColumnDescriptor::columnType, decimal_to_int_type(), double_buffer_, float_buffer_, geo_string_buffer_, SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), int_buffer_, SQLTypeInfo::is_decimal(), IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, smallint_buffer_, string_array_buffer_, string_buffer_, tinyint_buffer_, and run_benchmark_import::type.

694  {
695  const auto type = column_desc_->columnType.is_decimal()
697  : column_desc_->columnType.get_type();
698  switch (type) {
699  case kBOOLEAN:
700  bool_buffer_->pop_back();
701  break;
702  case kTINYINT:
703  tinyint_buffer_->pop_back();
704  break;
705  case kSMALLINT:
706  smallint_buffer_->pop_back();
707  break;
708  case kINT:
709  int_buffer_->pop_back();
710  break;
711  case kBIGINT:
712  bigint_buffer_->pop_back();
713  break;
714  case kFLOAT:
715  float_buffer_->pop_back();
716  break;
717  case kDOUBLE:
718  double_buffer_->pop_back();
719  break;
720  case kTEXT:
721  case kVARCHAR:
722  case kCHAR:
723  string_buffer_->pop_back();
724  break;
725  case kDATE:
726  case kTIME:
727  case kTIMESTAMP:
728  bigint_buffer_->pop_back();
729  break;
730  case kARRAY:
732  string_array_buffer_->pop_back();
733  } else {
734  array_buffer_->pop_back();
735  }
736  break;
737  case kPOINT:
738  case kLINESTRING:
739  case kPOLYGON:
740  case kMULTIPOLYGON:
741  geo_string_buffer_->pop_back();
742  break;
743  default:
744  CHECK(false) << "TypedImportBuffer::pop_value() does not support type " << type;
745  }
746 }
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:249
std::vector< int16_t > * smallint_buffer_
Definition: Importer.h:490
std::vector< int64_t > * bigint_buffer_
Definition: Importer.h:492
Definition: sqltypes.h:50
std::vector< float > * float_buffer_
Definition: Importer.h:493
std::vector< int8_t > * tinyint_buffer_
Definition: Importer.h:489
CHECK(cgen_state)
std::vector< std::string > * string_buffer_
Definition: Importer.h:495
std::vector< std::vector< std::string > > * string_array_buffer_
Definition: Importer.h:498
std::vector< int32_t > * int_buffer_
Definition: Importer.h:491
std::vector< int8_t > * bool_buffer_
Definition: Importer.h:488
SQLTypes decimal_to_int_type(const SQLTypeInfo &ti)
Definition: Datum.cpp:299
Definition: sqltypes.h:53
Definition: sqltypes.h:54
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:496
std::vector< double > * double_buffer_
Definition: Importer.h:494
Definition: sqltypes.h:42
const ColumnDescriptor * column_desc_
Definition: Importer.h:506
#define IS_STRING(T)
Definition: sqltypes.h:162
Definition: sqltypes.h:46
SQLTypeInfo columnType
bool is_decimal() const
Definition: sqltypes.h:402
std::vector< ArrayDatum > * array_buffer_
Definition: Importer.h:497

+ Here is the call graph for this function:

void Importer_NS::TypedImportBuffer::set_replicate_count ( const int64_t  replicate_count)
inline

Definition at line 471 of file Importer.h.

References replicate_count_.

Referenced by add_value().

471  {
472  replicate_count_ = replicate_count;
473  }

+ Here is the caller graph for this function:

bool Importer_NS::TypedImportBuffer::stringDictCheckpoint ( )
inline

Definition at line 366 of file Importer.h.

References StringDictionary::checkpoint(), and string_dict_.

366  {
367  if (string_dict_ == nullptr) {
368  return true;
369  }
370  return string_dict_->checkpoint();
371  }
StringDictionary * string_dict_
Definition: Importer.h:507
bool checkpoint() noexcept

+ Here is the call graph for this function:

Member Data Documentation

union { ... }
union { ... }
std::vector<ArrayDatum>* Importer_NS::TypedImportBuffer::array_buffer_
std::vector<int64_t>* Importer_NS::TypedImportBuffer::bigint_buffer_
std::vector<int8_t>* Importer_NS::TypedImportBuffer::bool_buffer_
size_t Importer_NS::TypedImportBuffer::col_idx

Definition at line 484 of file Importer.h.

Referenced by convert_arrow_val_to_import_buffer().

const ColumnDescriptor* Importer_NS::TypedImportBuffer::column_desc_
private
std::vector<double>* Importer_NS::TypedImportBuffer::double_buffer_
std::vector<float>* Importer_NS::TypedImportBuffer::float_buffer_
std::vector<std::string>* Importer_NS::TypedImportBuffer::geo_string_buffer_
std::vector<std::unique_ptr<TypedImportBuffer> >* Importer_NS::TypedImportBuffer::import_buffers

Definition at line 483 of file Importer.h.

Referenced by convert_arrow_val_to_import_buffer().

std::vector<int32_t>* Importer_NS::TypedImportBuffer::int_buffer_
size_t Importer_NS::TypedImportBuffer::replicate_count_ = 0
private

Definition at line 508 of file Importer.h.

Referenced by get_replicate_count(), and set_replicate_count().

std::vector<int16_t>* Importer_NS::TypedImportBuffer::smallint_buffer_
std::vector<std::vector<std::string> >* Importer_NS::TypedImportBuffer::string_array_buffer_
std::vector<ArrayDatum>* Importer_NS::TypedImportBuffer::string_array_dict_buffer_
std::vector<std::string>* Importer_NS::TypedImportBuffer::string_buffer_
StringDictionary* Importer_NS::TypedImportBuffer::string_dict_
private
std::vector<uint16_t>* Importer_NS::TypedImportBuffer::string_dict_i16_buffer_

Definition at line 502 of file Importer.h.

Referenced by addDictEncodedString(), clear(), TypedImportBuffer(), and ~TypedImportBuffer().

std::vector<int32_t>* Importer_NS::TypedImportBuffer::string_dict_i32_buffer_

Definition at line 503 of file Importer.h.

Referenced by addDictEncodedString(), clear(), TypedImportBuffer(), and ~TypedImportBuffer().

std::vector<uint8_t>* Importer_NS::TypedImportBuffer::string_dict_i8_buffer_

Definition at line 501 of file Importer.h.

Referenced by addDictEncodedString(), clear(), TypedImportBuffer(), and ~TypedImportBuffer().

std::vector<int8_t>* Importer_NS::TypedImportBuffer::tinyint_buffer_

The documentation for this class was generated from the following files: