OmniSciDB  1dac507f6e
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Importer_NS::TypedImportBuffer Class Reference

#include <Importer.h>

+ Inheritance diagram for Importer_NS::TypedImportBuffer:
+ Collaboration diagram for Importer_NS::TypedImportBuffer:

Public Member Functions

 TypedImportBuffer (const ColumnDescriptor *col_desc, StringDictionary *string_dict)
 
 ~TypedImportBuffer ()
 
void addBoolean (const int8_t v)
 
void addTinyint (const int8_t v)
 
void addSmallint (const int16_t v)
 
void addInt (const int32_t v)
 
void addBigint (const int64_t v)
 
void addFloat (const float v)
 
void addDouble (const double v)
 
void addString (const std::string &v)
 
void addGeoString (const std::string &v)
 
void addArray (const ArrayDatum &v)
 
std::vector< std::string > & addStringArray ()
 
void addStringArray (const std::vector< std::string > &arr)
 
void addDictEncodedString (const std::vector< std::string > &string_vec)
 
void addDictEncodedStringArray (const std::vector< std::vector< std::string >> &string_array_vec)
 
const SQLTypeInfogetTypeInfo () const
 
const ColumnDescriptorgetColumnDesc () const
 
StringDictionarygetStringDictionary () const
 
int8_t * getAsBytes () const
 
size_t getElementSize () const
 
std::vector< std::string > * getStringBuffer () const
 
std::vector< std::string > * getGeoStringBuffer () const
 
std::vector< ArrayDatum > * getArrayBuffer () const
 
std::vector< std::vector
< std::string > > * 
getStringArrayBuffer () const
 
std::vector< ArrayDatum > * getStringArrayDictBuffer () const
 
int8_t * getStringDictBuffer () const
 
bool stringDictCheckpoint ()
 
void clear ()
 
size_t add_values (const ColumnDescriptor *cd, const TColumn &data)
 
size_t add_arrow_values (const ColumnDescriptor *cd, const arrow::Array &data, const bool exact_type_match, const ArraySliceRange &slice_range, BadRowsTracker *bad_rows_tracker)
 
void add_value (const ColumnDescriptor *cd, const std::string &val, const bool is_null, const CopyParams &copy_params, const int64_t replicate_count=0)
 
void add_value (const ColumnDescriptor *cd, const TDatum &val, const bool is_null, const int64_t replicate_count=0)
 
void pop_value ()
 
int64_t get_replicate_count () const
 
void set_replicate_count (const int64_t replicate_count)
 
template<typename DATA_TYPE >
size_t convert_arrow_val_to_import_buffer (const ColumnDescriptor *cd, const arrow::Array &array, std::vector< DATA_TYPE > &buffer, const ArraySliceRange &slice_range, BadRowsTracker *const bad_rows_tracker)
 
template<typename DATA_TYPE >
auto del_values (std::vector< DATA_TYPE > &buffer, BadRowsTracker *const bad_rows_tracker)
 
auto del_values (const SQLTypes type, BadRowsTracker *const bad_rows_tracker)
 
template<typename DATA_TYPE >
size_t convert_arrow_val_to_import_buffer (const ColumnDescriptor *cd, const Array &array, std::vector< DATA_TYPE > &buffer, const ArraySliceRange &slice_range, Importer_NS::BadRowsTracker *const bad_rows_tracker)
 

Public Attributes

std::vector< std::unique_ptr
< TypedImportBuffer > > * 
import_buffers
 
size_t col_idx
 
std::vector< int8_t > * bool_buffer_
 
std::vector< int8_t > * tinyint_buffer_
 
std::vector< int16_t > * smallint_buffer_
 
std::vector< int32_t > * int_buffer_
 
std::vector< int64_t > * bigint_buffer_
 
std::vector< float > * float_buffer_
 
std::vector< double > * double_buffer_
 
std::vector< std::string > * string_buffer_
 
std::vector< std::string > * geo_string_buffer_
 
std::vector< ArrayDatum > * array_buffer_
 
std::vector< std::vector
< std::string > > * 
string_array_buffer_
 
std::vector< uint8_t > * string_dict_i8_buffer_
 
std::vector< uint16_t > * string_dict_i16_buffer_
 
std::vector< int32_t > * string_dict_i32_buffer_
 
std::vector< ArrayDatum > * string_array_dict_buffer_
 

Private Attributes

union {
   std::vector< int8_t > *   bool_buffer_
 
   std::vector< int8_t > *   tinyint_buffer_
 
   std::vector< int16_t > *   smallint_buffer_
 
   std::vector< int32_t > *   int_buffer_
 
   std::vector< int64_t > *   bigint_buffer_
 
   std::vector< float > *   float_buffer_
 
   std::vector< double > *   double_buffer_
 
   std::vector< std::string > *   string_buffer_
 
   std::vector< std::string > *   geo_string_buffer_
 
   std::vector< ArrayDatum > *   array_buffer_
 
   std::vector< std::vector
< std::string > > *   string_array_buffer_
 
}; 
 
union {
   std::vector< uint8_t > *   string_dict_i8_buffer_
 
   std::vector< uint16_t > *   string_dict_i16_buffer_
 
   std::vector< int32_t > *   string_dict_i32_buffer_
 
   std::vector< ArrayDatum > *   string_array_dict_buffer_
 
}; 
 
const ColumnDescriptorcolumn_desc_
 
StringDictionarystring_dict_
 
size_t replicate_count_ = 0
 

Detailed Description

Definition at line 85 of file Importer.h.

Constructor & Destructor Documentation

Importer_NS::TypedImportBuffer::TypedImportBuffer ( const ColumnDescriptor col_desc,
StringDictionary string_dict 
)
inline

Definition at line 87 of file Importer.h.

References array_buffer_, bigint_buffer_, bool_buffer_, CHECK(), ColumnDescriptor::columnType, double_buffer_, float_buffer_, geo_string_buffer_, SQLTypeInfoCore< TYPE_FACET_PACK >::get_compression(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_size(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_subtype(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), int_buffer_, IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kENCODING_DICT, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, smallint_buffer_, string_array_buffer_, string_array_dict_buffer_, string_buffer_, string_dict_i16_buffer_, string_dict_i32_buffer_, string_dict_i8_buffer_, and tinyint_buffer_.

88  : column_desc_(col_desc), string_dict_(string_dict) {
89  switch (col_desc->columnType.get_type()) {
90  case kBOOLEAN:
91  bool_buffer_ = new std::vector<int8_t>();
92  break;
93  case kTINYINT:
94  tinyint_buffer_ = new std::vector<int8_t>();
95  break;
96  case kSMALLINT:
97  smallint_buffer_ = new std::vector<int16_t>();
98  break;
99  case kINT:
100  int_buffer_ = new std::vector<int32_t>();
101  break;
102  case kBIGINT:
103  case kNUMERIC:
104  case kDECIMAL:
105  bigint_buffer_ = new std::vector<int64_t>();
106  break;
107  case kFLOAT:
108  float_buffer_ = new std::vector<float>();
109  break;
110  case kDOUBLE:
111  double_buffer_ = new std::vector<double>();
112  break;
113  case kTEXT:
114  case kVARCHAR:
115  case kCHAR:
116  string_buffer_ = new std::vector<std::string>();
117  if (col_desc->columnType.get_compression() == kENCODING_DICT) {
118  switch (col_desc->columnType.get_size()) {
119  case 1:
120  string_dict_i8_buffer_ = new std::vector<uint8_t>();
121  break;
122  case 2:
123  string_dict_i16_buffer_ = new std::vector<uint16_t>();
124  break;
125  case 4:
126  string_dict_i32_buffer_ = new std::vector<int32_t>();
127  break;
128  default:
129  CHECK(false);
130  }
131  }
132  break;
133  case kDATE:
134  case kTIME:
135  case kTIMESTAMP:
136  bigint_buffer_ = new std::vector<int64_t>();
137  break;
138  case kARRAY:
139  if (IS_STRING(col_desc->columnType.get_subtype())) {
141  string_array_buffer_ = new std::vector<std::vector<std::string>>();
142  string_array_dict_buffer_ = new std::vector<ArrayDatum>();
143  } else {
144  array_buffer_ = new std::vector<ArrayDatum>();
145  }
146  break;
147  case kPOINT:
148  case kLINESTRING:
149  case kPOLYGON:
150  case kMULTIPOLYGON:
151  geo_string_buffer_ = new std::vector<std::string>();
152  break;
153  default:
154  CHECK(false);
155  }
156  }
std::vector< int16_t > * smallint_buffer_
Definition: Importer.h:516
std::vector< int64_t > * bigint_buffer_
Definition: Importer.h:518
Definition: sqltypes.h:52
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:334
std::vector< float > * float_buffer_
Definition: Importer.h:519
HOST DEVICE int get_size() const
Definition: sqltypes.h:336
std::vector< uint16_t > * string_dict_i16_buffer_
Definition: Importer.h:528
std::vector< int8_t > * tinyint_buffer_
Definition: Importer.h:515
CHECK(cgen_state)
std::vector< std::string > * string_buffer_
Definition: Importer.h:521
StringDictionary * string_dict_
Definition: Importer.h:533
std::vector< int32_t > * string_dict_i32_buffer_
Definition: Importer.h:529
std::vector< std::vector< std::string > > * string_array_buffer_
Definition: Importer.h:524
std::vector< int32_t > * int_buffer_
Definition: Importer.h:517
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:326
std::vector< int8_t > * bool_buffer_
Definition: Importer.h:514
Definition: sqltypes.h:55
Definition: sqltypes.h:56
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:522
std::vector< ArrayDatum > * string_array_dict_buffer_
Definition: Importer.h:530
std::vector< double > * double_buffer_
Definition: Importer.h:520
Definition: sqltypes.h:44
const ColumnDescriptor * column_desc_
Definition: Importer.h:532
#define IS_STRING(T)
Definition: sqltypes.h:166
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:327
Definition: sqltypes.h:48
SQLTypeInfo columnType
std::vector< uint8_t > * string_dict_i8_buffer_
Definition: Importer.h:527
std::vector< ArrayDatum > * array_buffer_
Definition: Importer.h:523

+ Here is the call graph for this function:

Importer_NS::TypedImportBuffer::~TypedImportBuffer ( )
inline

Definition at line 158 of file Importer.h.

References array_buffer_, bigint_buffer_, bool_buffer_, CHECK(), column_desc_, ColumnDescriptor::columnType, double_buffer_, float_buffer_, geo_string_buffer_, SQLTypeInfoCore< TYPE_FACET_PACK >::get_compression(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_size(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_subtype(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), int_buffer_, IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kENCODING_DICT, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, smallint_buffer_, string_array_buffer_, string_array_dict_buffer_, string_buffer_, string_dict_i16_buffer_, string_dict_i32_buffer_, string_dict_i8_buffer_, and tinyint_buffer_.

158  {
159  switch (column_desc_->columnType.get_type()) {
160  case kBOOLEAN:
161  delete bool_buffer_;
162  break;
163  case kTINYINT:
164  delete tinyint_buffer_;
165  break;
166  case kSMALLINT:
167  delete smallint_buffer_;
168  break;
169  case kINT:
170  delete int_buffer_;
171  break;
172  case kBIGINT:
173  case kNUMERIC:
174  case kDECIMAL:
175  delete bigint_buffer_;
176  break;
177  case kFLOAT:
178  delete float_buffer_;
179  break;
180  case kDOUBLE:
181  delete double_buffer_;
182  break;
183  case kTEXT:
184  case kVARCHAR:
185  case kCHAR:
186  delete string_buffer_;
188  switch (column_desc_->columnType.get_size()) {
189  case 1:
190  delete string_dict_i8_buffer_;
191  break;
192  case 2:
194  break;
195  case 4:
197  break;
198  }
199  }
200  break;
201  case kDATE:
202  case kTIME:
203  case kTIMESTAMP:
204  delete bigint_buffer_;
205  break;
206  case kARRAY:
208  delete string_array_buffer_;
210  } else {
211  delete array_buffer_;
212  }
213  break;
214  case kPOINT:
215  case kLINESTRING:
216  case kPOLYGON:
217  case kMULTIPOLYGON:
218  delete geo_string_buffer_;
219  break;
220  default:
221  CHECK(false);
222  }
223  }
std::vector< int16_t > * smallint_buffer_
Definition: Importer.h:516
std::vector< int64_t > * bigint_buffer_
Definition: Importer.h:518
Definition: sqltypes.h:52
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:334
std::vector< float > * float_buffer_
Definition: Importer.h:519
HOST DEVICE int get_size() const
Definition: sqltypes.h:336
std::vector< uint16_t > * string_dict_i16_buffer_
Definition: Importer.h:528
std::vector< int8_t > * tinyint_buffer_
Definition: Importer.h:515
CHECK(cgen_state)
std::vector< std::string > * string_buffer_
Definition: Importer.h:521
std::vector< int32_t > * string_dict_i32_buffer_
Definition: Importer.h:529
std::vector< std::vector< std::string > > * string_array_buffer_
Definition: Importer.h:524
std::vector< int32_t > * int_buffer_
Definition: Importer.h:517
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:326
std::vector< int8_t > * bool_buffer_
Definition: Importer.h:514
Definition: sqltypes.h:55
Definition: sqltypes.h:56
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:522
std::vector< ArrayDatum > * string_array_dict_buffer_
Definition: Importer.h:530
std::vector< double > * double_buffer_
Definition: Importer.h:520
Definition: sqltypes.h:44
const ColumnDescriptor * column_desc_
Definition: Importer.h:532
#define IS_STRING(T)
Definition: sqltypes.h:166
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:327
Definition: sqltypes.h:48
SQLTypeInfo columnType
std::vector< uint8_t > * string_dict_i8_buffer_
Definition: Importer.h:527
std::vector< ArrayDatum > * array_buffer_
Definition: Importer.h:523

+ Here is the call graph for this function:

Member Function Documentation

size_t Importer_NS::TypedImportBuffer::add_arrow_values ( const ColumnDescriptor cd,
const arrow::Array &  data,
const bool  exact_type_match,
const ArraySliceRange slice_range,
BadRowsTracker bad_rows_tracker 
)

Definition at line 842 of file Importer.cpp.

References arrow_throw_if(), bigint_buffer_, bool_buffer_, ColumnDescriptor::columnName, ColumnDescriptor::columnType, convert_arrow_val_to_import_buffer(), decimal_to_int_type(), double_buffer_, float_buffer_, geo_string_buffer_, SQLTypeInfoCore< TYPE_FACET_PACK >::get_notnull(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), int_buffer_, SQLTypeInfoCore< TYPE_FACET_PACK >::is_decimal(), kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, smallint_buffer_, string_buffer_, tinyint_buffer_, and run_benchmark_import::type.

846  {
847  const auto type = cd->columnType.is_decimal() ? decimal_to_int_type(cd->columnType)
848  : cd->columnType.get_type();
849  if (cd->columnType.get_notnull()) {
850  // We can't have any null values for this column; to have them is an error
851  arrow_throw_if(col.null_count() > 0, "NULL not allowed for column " + cd->columnName);
852  }
853 
854  switch (type) {
855  case kBOOLEAN:
856  if (exact_type_match) {
857  arrow_throw_if(col.type_id() != Type::BOOL, "Expected boolean type");
858  }
860  cd, col, *bool_buffer_, slice_range, bad_rows_tracker);
861  case kTINYINT:
862  if (exact_type_match) {
863  arrow_throw_if(col.type_id() != Type::INT8, "Expected int8 type");
864  }
866  cd, col, *tinyint_buffer_, slice_range, bad_rows_tracker);
867  case kSMALLINT:
868  if (exact_type_match) {
869  arrow_throw_if(col.type_id() != Type::INT16, "Expected int16 type");
870  }
872  cd, col, *smallint_buffer_, slice_range, bad_rows_tracker);
873  case kINT:
874  if (exact_type_match) {
875  arrow_throw_if(col.type_id() != Type::INT32, "Expected int32 type");
876  }
878  cd, col, *int_buffer_, slice_range, bad_rows_tracker);
879  case kBIGINT:
880  if (exact_type_match) {
881  arrow_throw_if(col.type_id() != Type::INT64, "Expected int64 type");
882  }
884  cd, col, *bigint_buffer_, slice_range, bad_rows_tracker);
885  case kFLOAT:
886  if (exact_type_match) {
887  arrow_throw_if(col.type_id() != Type::FLOAT, "Expected float type");
888  }
890  cd, col, *float_buffer_, slice_range, bad_rows_tracker);
891  case kDOUBLE:
892  if (exact_type_match) {
893  arrow_throw_if(col.type_id() != Type::DOUBLE, "Expected double type");
894  }
896  cd, col, *double_buffer_, slice_range, bad_rows_tracker);
897  case kTEXT:
898  case kVARCHAR:
899  case kCHAR:
900  if (exact_type_match) {
901  arrow_throw_if(col.type_id() != Type::BINARY && col.type_id() != Type::STRING,
902  "Expected string type");
903  }
905  cd, col, *string_buffer_, slice_range, bad_rows_tracker);
906  case kTIME:
907  if (exact_type_match) {
908  arrow_throw_if(col.type_id() != Type::TIME32 && col.type_id() != Type::TIME64,
909  "Expected time32 or time64 type");
910  }
912  cd, col, *bigint_buffer_, slice_range, bad_rows_tracker);
913  case kTIMESTAMP:
914  if (exact_type_match) {
915  arrow_throw_if(col.type_id() != Type::TIMESTAMP, "Expected timestamp type");
916  }
918  cd, col, *bigint_buffer_, slice_range, bad_rows_tracker);
919  case kDATE:
920  if (exact_type_match) {
921  arrow_throw_if(col.type_id() != Type::DATE32 && col.type_id() != Type::DATE64,
922  "Expected date32 or date64 type");
923  }
925  cd, col, *bigint_buffer_, slice_range, bad_rows_tracker);
926  case kPOINT:
927  case kLINESTRING:
928  case kPOLYGON:
929  case kMULTIPOLYGON:
930  arrow_throw_if(col.type_id() != Type::BINARY && col.type_id() != Type::STRING,
931  "Expected string type");
933  cd, col, *geo_string_buffer_, slice_range, bad_rows_tracker);
934  case kARRAY:
935  throw std::runtime_error("Arrow array appends not yet supported");
936  default:
937  throw std::runtime_error("Invalid Type");
938  }
939 }
std::vector< int16_t > * smallint_buffer_
Definition: Importer.h:516
std::vector< int64_t > * bigint_buffer_
Definition: Importer.h:518
Definition: sqltypes.h:52
std::vector< float > * float_buffer_
Definition: Importer.h:519
size_t convert_arrow_val_to_import_buffer(const ColumnDescriptor *cd, const arrow::Array &array, std::vector< DATA_TYPE > &buffer, const ArraySliceRange &slice_range, BadRowsTracker *const bad_rows_tracker)
std::vector< int8_t > * tinyint_buffer_
Definition: Importer.h:515
std::vector< std::string > * string_buffer_
Definition: Importer.h:521
std::vector< int32_t > * int_buffer_
Definition: Importer.h:517
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:333
std::vector< int8_t > * bool_buffer_
Definition: Importer.h:514
SQLTypes decimal_to_int_type(const SQLTypeInfo &ti)
Definition: Datum.cpp:268
Definition: sqltypes.h:55
Definition: sqltypes.h:56
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:522
std::vector< double > * double_buffer_
Definition: Importer.h:520
Definition: sqltypes.h:44
Definition: sqltypes.h:48
SQLTypeInfo columnType
bool is_decimal() const
Definition: sqltypes.h:480
std::string columnName
void arrow_throw_if(const bool cond, const std::string &message)
Definition: ArrowImporter.h:40

+ Here is the call graph for this function:

void Importer_NS::TypedImportBuffer::add_value ( const ColumnDescriptor cd,
const std::string &  val,
const bool  is_null,
const CopyParams copy_params,
const int64_t  replicate_count = 0 
)

Definition at line 506 of file Importer.cpp.

References addArray(), addBigint(), addBoolean(), addDouble(), addFloat(), addGeoString(), addInt(), addSmallint(), addString(), addStringArray(), addTinyint(), Datum::bigintval, Datum::boolval, CHECK(), ColumnDescriptor::columnName, ColumnDescriptor::columnType, convert_decimal_value_to_scale(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_elem_type(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_notnull(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_size(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_subtype(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), inline_fixed_encoding_null_val(), IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, StringDictionary::MAX_STRLEN, NULL_DOUBLE, NULL_FLOAT, Importer_NS::NullArray(), Importer_NS::DelimitedParserUtils::parseStringArray(), set_replicate_count(), Importer_NS::StringToArray(), StringToDatum(), to_string(), and run_benchmark_import::type.

510  {
511  set_replicate_count(replicate_count);
512  const auto type = cd->columnType.get_type();
513  switch (type) {
514  case kBOOLEAN: {
515  if (is_null) {
516  if (cd->columnType.get_notnull()) {
517  throw std::runtime_error("NULL for column " + cd->columnName);
518  }
520  } else {
521  SQLTypeInfo ti = cd->columnType;
522  Datum d = StringToDatum(val, ti);
523  addBoolean((int8_t)d.boolval);
524  }
525  break;
526  }
527  case kTINYINT: {
528  if (!is_null && (isdigit(val[0]) || val[0] == '-')) {
529  addTinyint(std::stoi(val));
530  } else {
531  if (cd->columnType.get_notnull()) {
532  throw std::runtime_error("NULL for column " + cd->columnName);
533  }
535  }
536  break;
537  }
538  case kSMALLINT: {
539  if (!is_null && (isdigit(val[0]) || val[0] == '-')) {
540  addSmallint(std::stoi(val));
541  } else {
542  if (cd->columnType.get_notnull()) {
543  throw std::runtime_error("NULL for column " + cd->columnName);
544  }
546  }
547  break;
548  }
549  case kINT: {
550  if (!is_null && (isdigit(val[0]) || val[0] == '-')) {
551  addInt(std::stoi(val));
552  } else {
553  if (cd->columnType.get_notnull()) {
554  throw std::runtime_error("NULL for column " + cd->columnName);
555  }
557  }
558  break;
559  }
560  case kBIGINT: {
561  if (!is_null && (isdigit(val[0]) || val[0] == '-')) {
562  addBigint(std::stoll(val));
563  } else {
564  if (cd->columnType.get_notnull()) {
565  throw std::runtime_error("NULL for column " + cd->columnName);
566  }
568  }
569  break;
570  }
571  case kDECIMAL:
572  case kNUMERIC: {
573  if (!is_null) {
574  SQLTypeInfo ti(kNUMERIC, 0, 0, false);
575  Datum d = StringToDatum(val, ti);
576  const auto converted_decimal_value =
578  addBigint(converted_decimal_value);
579  } else {
580  if (cd->columnType.get_notnull()) {
581  throw std::runtime_error("NULL for column " + cd->columnName);
582  }
584  }
585  break;
586  }
587  case kFLOAT:
588  if (!is_null && (val[0] == '.' || isdigit(val[0]) || val[0] == '-')) {
589  addFloat((float)std::atof(val.c_str()));
590  } else {
591  if (cd->columnType.get_notnull()) {
592  throw std::runtime_error("NULL for column " + cd->columnName);
593  }
595  }
596  break;
597  case kDOUBLE:
598  if (!is_null && (val[0] == '.' || isdigit(val[0]) || val[0] == '-')) {
599  addDouble(std::atof(val.c_str()));
600  } else {
601  if (cd->columnType.get_notnull()) {
602  throw std::runtime_error("NULL for column " + cd->columnName);
603  }
605  }
606  break;
607  case kTEXT:
608  case kVARCHAR:
609  case kCHAR: {
610  // @TODO(wei) for now, use empty string for nulls
611  if (is_null) {
612  if (cd->columnType.get_notnull()) {
613  throw std::runtime_error("NULL for column " + cd->columnName);
614  }
615  addString(std::string());
616  } else {
617  if (val.length() > StringDictionary::MAX_STRLEN) {
618  throw std::runtime_error("String too long for column " + cd->columnName +
619  " was " + std::to_string(val.length()) + " max is " +
621  }
622  addString(val);
623  }
624  break;
625  }
626  case kTIME:
627  case kTIMESTAMP:
628  case kDATE:
629  if (!is_null && (isdigit(val[0]) || val[0] == '-')) {
630  SQLTypeInfo ti = cd->columnType;
631  Datum d = StringToDatum(val, ti);
632  addBigint(d.bigintval);
633  } else {
634  if (cd->columnType.get_notnull()) {
635  throw std::runtime_error("NULL for column " + cd->columnName);
636  }
638  }
639  break;
640  case kARRAY: {
641  if (is_null && cd->columnType.get_notnull()) {
642  throw std::runtime_error("NULL for column " + cd->columnName);
643  }
644  SQLTypeInfo ti = cd->columnType;
645  if (IS_STRING(ti.get_subtype())) {
646  std::vector<std::string> string_vec;
647  // Just parse string array, don't push it to buffer yet as we might throw
648  Importer_NS::DelimitedParserUtils::parseStringArray(val, copy_params, string_vec);
649  if (!is_null) {
650  // TODO: add support for NULL string arrays
651  if (ti.get_size() > 0) {
652  auto sti = ti.get_elem_type();
653  size_t expected_size = ti.get_size() / sti.get_size();
654  size_t actual_size = string_vec.size();
655  if (actual_size != expected_size) {
656  throw std::runtime_error("Fixed length array column " + cd->columnName +
657  " expects " + std::to_string(expected_size) +
658  " values, received " +
659  std::to_string(actual_size));
660  }
661  }
662  addStringArray(string_vec);
663  } else {
664  if (ti.get_size() > 0) {
665  // TODO: remove once NULL fixlen arrays are allowed
666  throw std::runtime_error("Fixed length array column " + cd->columnName +
667  " currently cannot accept NULL arrays");
668  }
669  // TODO: add support for NULL string arrays, replace with addStringArray(),
670  // for now add whatever parseStringArray() outputs for NULLs ("NULL")
671  addStringArray(string_vec);
672  }
673  } else {
674  if (!is_null) {
675  ArrayDatum d = StringToArray(val, ti, copy_params);
676  if (d.is_null) { // val could be "NULL"
677  addArray(NullArray(ti));
678  } else {
679  if (ti.get_size() > 0 && static_cast<size_t>(ti.get_size()) != d.length) {
680  throw std::runtime_error("Fixed length array for column " + cd->columnName +
681  " has incorrect length: " + val);
682  }
683  addArray(d);
684  }
685  } else {
686  addArray(NullArray(ti));
687  }
688  }
689  break;
690  }
691  case kPOINT:
692  case kLINESTRING:
693  case kPOLYGON:
694  case kMULTIPOLYGON:
695  addGeoString(val);
696  break;
697  default:
698  CHECK(false) << "TypedImportBuffer::add_value() does not support type " << type;
699  }
700 }
void addSmallint(const int16_t v)
Definition: Importer.h:229
#define NULL_DOUBLE
Definition: sqltypes.h:179
void addGeoString(const std::string &v)
Definition: Importer.h:241
Definition: sqltypes.h:52
void addTinyint(const int8_t v)
Definition: Importer.h:227
bool boolval
Definition: sqltypes.h:125
HOST DEVICE int get_size() const
Definition: sqltypes.h:336
void addInt(const int32_t v)
Definition: Importer.h:231
ArrayDatum StringToArray(const std::string &s, const SQLTypeInfo &ti, const CopyParams &copy_params)
Definition: Importer.cpp:350
std::string to_string(char const *&&v)
void set_replicate_count(const int64_t replicate_count)
Definition: Importer.h:497
void addBoolean(const int8_t v)
Definition: Importer.h:225
void addFloat(const float v)
Definition: Importer.h:235
CHECK(cgen_state)
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:326
int64_t bigintval
Definition: sqltypes.h:129
#define NULL_FLOAT
Definition: sqltypes.h:178
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:333
void addBigint(const int64_t v)
Definition: Importer.h:233
Definition: sqltypes.h:55
Definition: sqltypes.h:56
bool is_null(const T &v, const SQLTypeInfo &t)
int64_t convert_decimal_value_to_scale(const int64_t decimal_value, const SQLTypeInfo &type_info, const SQLTypeInfo &new_type_info)
Definition: Datum.cpp:284
void addArray(const ArrayDatum &v)
Definition: Importer.h:243
Datum StringToDatum(const std::string &s, SQLTypeInfo &ti)
Definition: Datum.cpp:90
Definition: sqltypes.h:44
ArrayDatum NullArray(const SQLTypeInfo &ti)
Definition: Importer.cpp:403
#define IS_STRING(T)
Definition: sqltypes.h:166
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:327
void addString(const std::string &v)
Definition: Importer.h:239
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
static constexpr size_t MAX_STRLEN
Definition: sqltypes.h:48
SQLTypeInfoCore get_elem_type() const
Definition: sqltypes.h:659
SQLTypeInfo columnType
std::vector< std::string > & addStringArray()
Definition: Importer.h:245
static void parseStringArray(const std::string &s, const Importer_NS::CopyParams &copy_params, std::vector< std::string > &string_vec)
Parses given string array and inserts into given vector of strings.
std::string columnName
std::conditional_t< isCudaCC(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:122
void addDouble(const double v)
Definition: Importer.h:237

+ Here is the call graph for this function:

void Importer_NS::TypedImportBuffer::add_value ( const ColumnDescriptor cd,
const TDatum &  val,
const bool  is_null,
const int64_t  replicate_count = 0 
)

Definition at line 1263 of file Importer.cpp.

References addArray(), addBigint(), Importer_NS::addBinaryStringArray(), addBoolean(), addDouble(), addFloat(), addGeoString(), addInt(), addSmallint(), addString(), addStringArray(), addTinyint(), CHECK(), ColumnDescriptor::columnName, ColumnDescriptor::columnType, decimal_to_int_type(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_notnull(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_subtype(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), inline_fixed_encoding_null_val(), SQLTypeInfoCore< TYPE_FACET_PACK >::is_decimal(), IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, NULL_DOUBLE, NULL_FLOAT, Importer_NS::NullArray(), set_replicate_count(), Importer_NS::TDatumToArrayDatum(), and run_benchmark_import::type.

1266  {
1267  set_replicate_count(replicate_count);
1268  const auto type = cd->columnType.is_decimal() ? decimal_to_int_type(cd->columnType)
1269  : cd->columnType.get_type();
1270  switch (type) {
1271  case kBOOLEAN: {
1272  if (is_null) {
1273  if (cd->columnType.get_notnull()) {
1274  throw std::runtime_error("NULL for column " + cd->columnName);
1275  }
1277  } else {
1278  addBoolean((int8_t)datum.val.int_val);
1279  }
1280  break;
1281  }
1282  case kTINYINT:
1283  if (!is_null) {
1284  addTinyint((int8_t)datum.val.int_val);
1285  } else {
1286  if (cd->columnType.get_notnull()) {
1287  throw std::runtime_error("NULL for column " + cd->columnName);
1288  }
1290  }
1291  break;
1292  case kSMALLINT:
1293  if (!is_null) {
1294  addSmallint((int16_t)datum.val.int_val);
1295  } else {
1296  if (cd->columnType.get_notnull()) {
1297  throw std::runtime_error("NULL for column " + cd->columnName);
1298  }
1300  }
1301  break;
1302  case kINT:
1303  if (!is_null) {
1304  addInt((int32_t)datum.val.int_val);
1305  } else {
1306  if (cd->columnType.get_notnull()) {
1307  throw std::runtime_error("NULL for column " + cd->columnName);
1308  }
1310  }
1311  break;
1312  case kBIGINT:
1313  if (!is_null) {
1314  addBigint(datum.val.int_val);
1315  } else {
1316  if (cd->columnType.get_notnull()) {
1317  throw std::runtime_error("NULL for column " + cd->columnName);
1318  }
1320  }
1321  break;
1322  case kFLOAT:
1323  if (!is_null) {
1324  addFloat((float)datum.val.real_val);
1325  } else {
1326  if (cd->columnType.get_notnull()) {
1327  throw std::runtime_error("NULL for column " + cd->columnName);
1328  }
1330  }
1331  break;
1332  case kDOUBLE:
1333  if (!is_null) {
1334  addDouble(datum.val.real_val);
1335  } else {
1336  if (cd->columnType.get_notnull()) {
1337  throw std::runtime_error("NULL for column " + cd->columnName);
1338  }
1340  }
1341  break;
1342  case kTEXT:
1343  case kVARCHAR:
1344  case kCHAR: {
1345  // @TODO(wei) for now, use empty string for nulls
1346  if (is_null) {
1347  if (cd->columnType.get_notnull()) {
1348  throw std::runtime_error("NULL for column " + cd->columnName);
1349  }
1350  addString(std::string());
1351  } else {
1352  addString(datum.val.str_val);
1353  }
1354  break;
1355  }
1356  case kTIME:
1357  case kTIMESTAMP:
1358  case kDATE: {
1359  if (!is_null) {
1360  addBigint(datum.val.int_val);
1361  } else {
1362  if (cd->columnType.get_notnull()) {
1363  throw std::runtime_error("NULL for column " + cd->columnName);
1364  }
1366  }
1367  break;
1368  }
1369  case kARRAY:
1370  if (is_null && cd->columnType.get_notnull()) {
1371  throw std::runtime_error("NULL for column " + cd->columnName);
1372  }
1373  if (IS_STRING(cd->columnType.get_subtype())) {
1374  std::vector<std::string>& string_vec = addStringArray();
1375  addBinaryStringArray(datum, string_vec);
1376  } else {
1377  if (!is_null) {
1378  addArray(TDatumToArrayDatum(datum, cd->columnType));
1379  } else {
1381  }
1382  }
1383  break;
1384  case kPOINT:
1385  case kLINESTRING:
1386  case kPOLYGON:
1387  case kMULTIPOLYGON:
1388  if (is_null) {
1389  if (cd->columnType.get_notnull()) {
1390  throw std::runtime_error("NULL for column " + cd->columnName);
1391  }
1392  addGeoString(std::string());
1393  } else {
1394  addGeoString(datum.val.str_val);
1395  }
1396  break;
1397  default:
1398  CHECK(false) << "TypedImportBuffer::add_value() does not support type " << type;
1399  }
1400 }
void addSmallint(const int16_t v)
Definition: Importer.h:229
#define NULL_DOUBLE
Definition: sqltypes.h:179
void addGeoString(const std::string &v)
Definition: Importer.h:241
Definition: sqltypes.h:52
void addTinyint(const int8_t v)
Definition: Importer.h:227
void addInt(const int32_t v)
Definition: Importer.h:231
void set_replicate_count(const int64_t replicate_count)
Definition: Importer.h:497
void addBoolean(const int8_t v)
Definition: Importer.h:225
void addFloat(const float v)
Definition: Importer.h:235
CHECK(cgen_state)
#define NULL_FLOAT
Definition: sqltypes.h:178
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:333
void addBigint(const int64_t v)
Definition: Importer.h:233
SQLTypes decimal_to_int_type(const SQLTypeInfo &ti)
Definition: Datum.cpp:268
Definition: sqltypes.h:55
Definition: sqltypes.h:56
bool is_null(const T &v, const SQLTypeInfo &t)
void addArray(const ArrayDatum &v)
Definition: Importer.h:243
ArrayDatum TDatumToArrayDatum(const TDatum &datum, const SQLTypeInfo &ti)
Definition: Importer.cpp:487
Definition: sqltypes.h:44
ArrayDatum NullArray(const SQLTypeInfo &ti)
Definition: Importer.cpp:403
#define IS_STRING(T)
Definition: sqltypes.h:166
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:327
void addString(const std::string &v)
Definition: Importer.h:239
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
Definition: sqltypes.h:48
SQLTypeInfo columnType
bool is_decimal() const
Definition: sqltypes.h:480
std::vector< std::string > & addStringArray()
Definition: Importer.h:245
std::string columnName
void addBinaryStringArray(const TDatum &datum, std::vector< std::string > &string_vec)
Definition: Importer.cpp:435
void addDouble(const double v)
Definition: Importer.h:237

+ Here is the call graph for this function:

size_t Importer_NS::TypedImportBuffer::add_values ( const ColumnDescriptor cd,
const TColumn &  data 
)

Definition at line 942 of file Importer.cpp.

References addArray(), addStringArray(), bigint_buffer_, bool_buffer_, checked_malloc(), ColumnDescriptor::columnName, ColumnDescriptor::columnType, double_buffer_, float_buffer_, geo_string_buffer_, SQLTypeInfoCore< TYPE_FACET_PACK >::get_notnull(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_subtype(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), inline_fixed_encoding_null_val(), int_buffer_, IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, NULL_DOUBLE, NULL_FLOAT, Importer_NS::NullArray(), smallint_buffer_, string_buffer_, and tinyint_buffer_.

942  {
943  size_t dataSize = 0;
944  if (cd->columnType.get_notnull()) {
945  // We can't have any null values for this column; to have them is an error
946  if (std::any_of(col.nulls.begin(), col.nulls.end(), [](int i) { return i != 0; })) {
947  throw std::runtime_error("NULL for column " + cd->columnName);
948  }
949  }
950 
951  switch (cd->columnType.get_type()) {
952  case kBOOLEAN: {
953  dataSize = col.data.int_col.size();
954  bool_buffer_->reserve(dataSize);
955  for (size_t i = 0; i < dataSize; i++) {
956  if (col.nulls[i]) {
958  } else {
959  bool_buffer_->push_back((int8_t)col.data.int_col[i]);
960  }
961  }
962  break;
963  }
964  case kTINYINT: {
965  dataSize = col.data.int_col.size();
966  tinyint_buffer_->reserve(dataSize);
967  for (size_t i = 0; i < dataSize; i++) {
968  if (col.nulls[i]) {
970  } else {
971  tinyint_buffer_->push_back((int8_t)col.data.int_col[i]);
972  }
973  }
974  break;
975  }
976  case kSMALLINT: {
977  dataSize = col.data.int_col.size();
978  smallint_buffer_->reserve(dataSize);
979  for (size_t i = 0; i < dataSize; i++) {
980  if (col.nulls[i]) {
982  } else {
983  smallint_buffer_->push_back((int16_t)col.data.int_col[i]);
984  }
985  }
986  break;
987  }
988  case kINT: {
989  dataSize = col.data.int_col.size();
990  int_buffer_->reserve(dataSize);
991  for (size_t i = 0; i < dataSize; i++) {
992  if (col.nulls[i]) {
994  } else {
995  int_buffer_->push_back((int32_t)col.data.int_col[i]);
996  }
997  }
998  break;
999  }
1000  case kBIGINT:
1001  case kNUMERIC:
1002  case kDECIMAL: {
1003  dataSize = col.data.int_col.size();
1004  bigint_buffer_->reserve(dataSize);
1005  for (size_t i = 0; i < dataSize; i++) {
1006  if (col.nulls[i]) {
1008  } else {
1009  bigint_buffer_->push_back((int64_t)col.data.int_col[i]);
1010  }
1011  }
1012  break;
1013  }
1014  case kFLOAT: {
1015  dataSize = col.data.real_col.size();
1016  float_buffer_->reserve(dataSize);
1017  for (size_t i = 0; i < dataSize; i++) {
1018  if (col.nulls[i]) {
1019  float_buffer_->push_back(NULL_FLOAT);
1020  } else {
1021  float_buffer_->push_back((float)col.data.real_col[i]);
1022  }
1023  }
1024  break;
1025  }
1026  case kDOUBLE: {
1027  dataSize = col.data.real_col.size();
1028  double_buffer_->reserve(dataSize);
1029  for (size_t i = 0; i < dataSize; i++) {
1030  if (col.nulls[i]) {
1031  double_buffer_->push_back(NULL_DOUBLE);
1032  } else {
1033  double_buffer_->push_back((double)col.data.real_col[i]);
1034  }
1035  }
1036  break;
1037  }
1038  case kTEXT:
1039  case kVARCHAR:
1040  case kCHAR: {
1041  // TODO: for now, use empty string for nulls
1042  dataSize = col.data.str_col.size();
1043  string_buffer_->reserve(dataSize);
1044  for (size_t i = 0; i < dataSize; i++) {
1045  if (col.nulls[i]) {
1046  string_buffer_->push_back(std::string());
1047  } else {
1048  string_buffer_->push_back(col.data.str_col[i]);
1049  }
1050  }
1051  break;
1052  }
1053  case kTIME:
1054  case kTIMESTAMP:
1055  case kDATE: {
1056  dataSize = col.data.int_col.size();
1057  bigint_buffer_->reserve(dataSize);
1058  for (size_t i = 0; i < dataSize; i++) {
1059  if (col.nulls[i]) {
1061  } else {
1062  bigint_buffer_->push_back(static_cast<int64_t>(col.data.int_col[i]));
1063  }
1064  }
1065  break;
1066  }
1067  case kPOINT:
1068  case kLINESTRING:
1069  case kPOLYGON:
1070  case kMULTIPOLYGON: {
1071  dataSize = col.data.str_col.size();
1072  geo_string_buffer_->reserve(dataSize);
1073  for (size_t i = 0; i < dataSize; i++) {
1074  if (col.nulls[i]) {
1075  // TODO: add support for NULL geo
1076  geo_string_buffer_->push_back(std::string());
1077  } else {
1078  geo_string_buffer_->push_back(col.data.str_col[i]);
1079  }
1080  }
1081  break;
1082  }
1083  case kARRAY: {
1084  dataSize = col.data.arr_col.size();
1085  if (IS_STRING(cd->columnType.get_subtype())) {
1086  for (size_t i = 0; i < dataSize; i++) {
1087  std::vector<std::string>& string_vec = addStringArray();
1088  if (!col.nulls[i]) {
1089  size_t stringArrSize = col.data.arr_col[i].data.str_col.size();
1090  for (size_t str_idx = 0; str_idx != stringArrSize; ++str_idx) {
1091  string_vec.push_back(col.data.arr_col[i].data.str_col[str_idx]);
1092  }
1093  }
1094  }
1095  } else {
1096  auto elem_ti = cd->columnType.get_subtype();
1097  switch (elem_ti) {
1098  case kBOOLEAN: {
1099  for (size_t i = 0; i < dataSize; i++) {
1100  if (col.nulls[i]) {
1102  } else {
1103  size_t len = col.data.arr_col[i].data.int_col.size();
1104  size_t byteSize = len * sizeof(int8_t);
1105  int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
1106  int8_t* p = buf;
1107  for (size_t j = 0; j < len; ++j) {
1108  *(bool*)p = static_cast<bool>(col.data.arr_col[i].data.int_col[j]);
1109  p += sizeof(bool);
1110  }
1111  addArray(ArrayDatum(byteSize, buf, false));
1112  }
1113  }
1114  break;
1115  }
1116  case kTINYINT: {
1117  for (size_t i = 0; i < dataSize; i++) {
1118  if (col.nulls[i]) {
1120  } else {
1121  size_t len = col.data.arr_col[i].data.int_col.size();
1122  size_t byteSize = len * sizeof(int8_t);
1123  int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
1124  int8_t* p = buf;
1125  for (size_t j = 0; j < len; ++j) {
1126  *(int8_t*)p = static_cast<int8_t>(col.data.arr_col[i].data.int_col[j]);
1127  p += sizeof(int8_t);
1128  }
1129  addArray(ArrayDatum(byteSize, buf, false));
1130  }
1131  }
1132  break;
1133  }
1134  case kSMALLINT: {
1135  for (size_t i = 0; i < dataSize; i++) {
1136  if (col.nulls[i]) {
1138  } else {
1139  size_t len = col.data.arr_col[i].data.int_col.size();
1140  size_t byteSize = len * sizeof(int16_t);
1141  int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
1142  int8_t* p = buf;
1143  for (size_t j = 0; j < len; ++j) {
1144  *(int16_t*)p =
1145  static_cast<int16_t>(col.data.arr_col[i].data.int_col[j]);
1146  p += sizeof(int16_t);
1147  }
1148  addArray(ArrayDatum(byteSize, buf, false));
1149  }
1150  }
1151  break;
1152  }
1153  case kINT: {
1154  for (size_t i = 0; i < dataSize; i++) {
1155  if (col.nulls[i]) {
1157  } else {
1158  size_t len = col.data.arr_col[i].data.int_col.size();
1159  size_t byteSize = len * sizeof(int32_t);
1160  int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
1161  int8_t* p = buf;
1162  for (size_t j = 0; j < len; ++j) {
1163  *(int32_t*)p =
1164  static_cast<int32_t>(col.data.arr_col[i].data.int_col[j]);
1165  p += sizeof(int32_t);
1166  }
1167  addArray(ArrayDatum(byteSize, buf, false));
1168  }
1169  }
1170  break;
1171  }
1172  case kBIGINT:
1173  case kNUMERIC:
1174  case kDECIMAL: {
1175  for (size_t i = 0; i < dataSize; i++) {
1176  if (col.nulls[i]) {
1178  } else {
1179  size_t len = col.data.arr_col[i].data.int_col.size();
1180  size_t byteSize = len * sizeof(int64_t);
1181  int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
1182  int8_t* p = buf;
1183  for (size_t j = 0; j < len; ++j) {
1184  *(int64_t*)p =
1185  static_cast<int64_t>(col.data.arr_col[j].data.int_col[j]);
1186  p += sizeof(int64_t);
1187  }
1188  addArray(ArrayDatum(byteSize, buf, false));
1189  }
1190  }
1191  break;
1192  }
1193  case kFLOAT: {
1194  for (size_t i = 0; i < dataSize; i++) {
1195  if (col.nulls[i]) {
1197  } else {
1198  size_t len = col.data.arr_col[i].data.real_col.size();
1199  size_t byteSize = len * sizeof(float);
1200  int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
1201  int8_t* p = buf;
1202  for (size_t j = 0; j < len; ++j) {
1203  *(float*)p = static_cast<float>(col.data.arr_col[i].data.real_col[j]);
1204  p += sizeof(float);
1205  }
1206  addArray(ArrayDatum(byteSize, buf, false));
1207  }
1208  }
1209  break;
1210  }
1211  case kDOUBLE: {
1212  for (size_t i = 0; i < dataSize; i++) {
1213  if (col.nulls[i]) {
1215  } else {
1216  size_t len = col.data.arr_col[i].data.real_col.size();
1217  size_t byteSize = len * sizeof(double);
1218  int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
1219  int8_t* p = buf;
1220  for (size_t j = 0; j < len; ++j) {
1221  *(double*)p = static_cast<double>(col.data.arr_col[i].data.real_col[j]);
1222  p += sizeof(double);
1223  }
1224  addArray(ArrayDatum(byteSize, buf, false));
1225  }
1226  }
1227  break;
1228  }
1229  case kTIME:
1230  case kTIMESTAMP:
1231  case kDATE: {
1232  for (size_t i = 0; i < dataSize; i++) {
1233  if (col.nulls[i]) {
1235  } else {
1236  size_t len = col.data.arr_col[i].data.int_col.size();
1237  size_t byteWidth = sizeof(int64_t);
1238  size_t byteSize = len * byteWidth;
1239  int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
1240  int8_t* p = buf;
1241  for (size_t j = 0; j < len; ++j) {
1242  *reinterpret_cast<int64_t*>(p) =
1243  static_cast<int64_t>(col.data.arr_col[i].data.int_col[j]);
1244  p += sizeof(int64_t);
1245  }
1246  addArray(ArrayDatum(byteSize, buf, false));
1247  }
1248  }
1249  break;
1250  }
1251  default:
1252  throw std::runtime_error("Invalid Array Type");
1253  }
1254  }
1255  break;
1256  }
1257  default:
1258  throw std::runtime_error("Invalid Type");
1259  }
1260  return dataSize;
1261 }
std::vector< int16_t > * smallint_buffer_
Definition: Importer.h:516
#define NULL_DOUBLE
Definition: sqltypes.h:179
std::vector< int64_t > * bigint_buffer_
Definition: Importer.h:518
Definition: sqltypes.h:52
std::vector< float > * float_buffer_
Definition: Importer.h:519
std::vector< int8_t > * tinyint_buffer_
Definition: Importer.h:515
std::vector< std::string > * string_buffer_
Definition: Importer.h:521
std::vector< int32_t > * int_buffer_
Definition: Importer.h:517
void * checked_malloc(const size_t size)
Definition: checked_alloc.h:40
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:326
#define NULL_FLOAT
Definition: sqltypes.h:178
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:333
std::vector< int8_t > * bool_buffer_
Definition: Importer.h:514
Definition: sqltypes.h:55
Definition: sqltypes.h:56
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:522
void addArray(const ArrayDatum &v)
Definition: Importer.h:243
std::vector< double > * double_buffer_
Definition: Importer.h:520
Definition: sqltypes.h:44
ArrayDatum NullArray(const SQLTypeInfo &ti)
Definition: Importer.cpp:403
#define IS_STRING(T)
Definition: sqltypes.h:166
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:327
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
Definition: sqltypes.h:48
SQLTypeInfo columnType
std::vector< std::string > & addStringArray()
Definition: Importer.h:245
std::string columnName
std::conditional_t< isCudaCC(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:122

+ Here is the call graph for this function:

void Importer_NS::TypedImportBuffer::addArray ( const ArrayDatum v)
inline

Definition at line 243 of file Importer.h.

References array_buffer_.

Referenced by add_value(), and add_values().

243 { array_buffer_->push_back(v); }
std::vector< ArrayDatum > * array_buffer_
Definition: Importer.h:523

+ Here is the caller graph for this function:

void Importer_NS::TypedImportBuffer::addBigint ( const int64_t  v)
inline

Definition at line 233 of file Importer.h.

References bigint_buffer_.

Referenced by add_value().

233 { bigint_buffer_->push_back(v); }
std::vector< int64_t > * bigint_buffer_
Definition: Importer.h:518

+ Here is the caller graph for this function:

void Importer_NS::TypedImportBuffer::addBoolean ( const int8_t  v)
inline

Definition at line 225 of file Importer.h.

References bool_buffer_.

Referenced by add_value().

225 { bool_buffer_->push_back(v); }
std::vector< int8_t > * bool_buffer_
Definition: Importer.h:514

+ Here is the caller graph for this function:

void Importer_NS::TypedImportBuffer::addDictEncodedString ( const std::vector< std::string > &  string_vec)
inline

Definition at line 254 of file Importer.h.

References CHECK(), column_desc_, ColumnDescriptor::columnType, SQLTypeInfoCore< TYPE_FACET_PACK >::get_size(), StringDictionary::getOrAddBulk(), StringDictionary::MAX_STRLEN, string_dict_, string_dict_i16_buffer_, string_dict_i32_buffer_, and string_dict_i8_buffer_.

254  {
256  for (const auto& str : string_vec) {
257  if (str.size() > StringDictionary::MAX_STRLEN) {
258  throw std::runtime_error("String too long for dictionary encoding.");
259  }
260  }
261  switch (column_desc_->columnType.get_size()) {
262  case 1:
263  string_dict_i8_buffer_->resize(string_vec.size());
264  string_dict_->getOrAddBulk(string_vec, string_dict_i8_buffer_->data());
265  break;
266  case 2:
267  string_dict_i16_buffer_->resize(string_vec.size());
268  string_dict_->getOrAddBulk(string_vec, string_dict_i16_buffer_->data());
269  break;
270  case 4:
271  string_dict_i32_buffer_->resize(string_vec.size());
272  string_dict_->getOrAddBulk(string_vec, string_dict_i32_buffer_->data());
273  break;
274  default:
275  CHECK(false);
276  }
277  }
HOST DEVICE int get_size() const
Definition: sqltypes.h:336
std::vector< uint16_t > * string_dict_i16_buffer_
Definition: Importer.h:528
CHECK(cgen_state)
StringDictionary * string_dict_
Definition: Importer.h:533
std::vector< int32_t > * string_dict_i32_buffer_
Definition: Importer.h:529
void getOrAddBulk(const std::vector< std::string > &string_vec, T *encoded_vec)
const ColumnDescriptor * column_desc_
Definition: Importer.h:532
static constexpr size_t MAX_STRLEN
SQLTypeInfo columnType
std::vector< uint8_t > * string_dict_i8_buffer_
Definition: Importer.h:527

+ Here is the call graph for this function:

void Importer_NS::TypedImportBuffer::addDictEncodedStringArray ( const std::vector< std::vector< std::string >> &  string_array_vec)
inline

Definition at line 279 of file Importer.h.

References CHECK(), checked_malloc(), StringDictionary::getOrAddBulkArray(), StringDictionary::MAX_STRLEN, string_array_dict_buffer_, and string_dict_.

280  {
282 
283  // first check data is ok
284  for (auto& p : string_array_vec) {
285  for (const auto& str : p) {
286  if (str.size() > StringDictionary::MAX_STRLEN) {
287  throw std::runtime_error("String too long for dictionary encoding.");
288  }
289  }
290  }
291 
292  std::vector<std::vector<int32_t>> ids_array(0);
293  string_dict_->getOrAddBulkArray(string_array_vec, ids_array);
294 
295  for (auto& p : ids_array) {
296  size_t len = p.size() * sizeof(int32_t);
297  auto a = static_cast<int32_t*>(checked_malloc(len));
298  memcpy(a, &p[0], len);
299  // TODO: distinguish between empty and NULL
300  string_array_dict_buffer_->push_back(
301  ArrayDatum(len, reinterpret_cast<int8_t*>(a), len == 0));
302  }
303  }
CHECK(cgen_state)
StringDictionary * string_dict_
Definition: Importer.h:533
void * checked_malloc(const size_t size)
Definition: checked_alloc.h:40
void getOrAddBulkArray(const std::vector< std::vector< std::string >> &string_array_vec, std::vector< std::vector< int32_t >> &ids_array_vec)
std::vector< ArrayDatum > * string_array_dict_buffer_
Definition: Importer.h:530
static constexpr size_t MAX_STRLEN
std::conditional_t< isCudaCC(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:122

+ Here is the call graph for this function:

void Importer_NS::TypedImportBuffer::addDouble ( const double  v)
inline

Definition at line 237 of file Importer.h.

References double_buffer_.

Referenced by add_value().

237 { double_buffer_->push_back(v); }
std::vector< double > * double_buffer_
Definition: Importer.h:520

+ Here is the caller graph for this function:

void Importer_NS::TypedImportBuffer::addFloat ( const float  v)
inline

Definition at line 235 of file Importer.h.

References float_buffer_.

Referenced by add_value().

235 { float_buffer_->push_back(v); }
std::vector< float > * float_buffer_
Definition: Importer.h:519

+ Here is the caller graph for this function:

void Importer_NS::TypedImportBuffer::addGeoString ( const std::string &  v)
inline

Definition at line 241 of file Importer.h.

References geo_string_buffer_.

Referenced by add_value().

241 { geo_string_buffer_->push_back(v); }
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:522

+ Here is the caller graph for this function:

void Importer_NS::TypedImportBuffer::addInt ( const int32_t  v)
inline

Definition at line 231 of file Importer.h.

References int_buffer_.

Referenced by add_value().

231 { int_buffer_->push_back(v); }
std::vector< int32_t > * int_buffer_
Definition: Importer.h:517

+ Here is the caller graph for this function:

void Importer_NS::TypedImportBuffer::addSmallint ( const int16_t  v)
inline

Definition at line 229 of file Importer.h.

References smallint_buffer_.

Referenced by add_value().

229 { smallint_buffer_->push_back(v); }
std::vector< int16_t > * smallint_buffer_
Definition: Importer.h:516

+ Here is the caller graph for this function:

void Importer_NS::TypedImportBuffer::addString ( const std::string &  v)
inline

Definition at line 239 of file Importer.h.

References string_buffer_.

Referenced by add_value().

239 { string_buffer_->push_back(v); }
std::vector< std::string > * string_buffer_
Definition: Importer.h:521

+ Here is the caller graph for this function:

std::vector<std::string>& Importer_NS::TypedImportBuffer::addStringArray ( )
inline

Definition at line 245 of file Importer.h.

References string_array_buffer_.

Referenced by add_value(), and add_values().

245  {
246  string_array_buffer_->push_back(std::vector<std::string>());
247  return string_array_buffer_->back();
248  }
std::vector< std::vector< std::string > > * string_array_buffer_
Definition: Importer.h:524

+ Here is the caller graph for this function:

void Importer_NS::TypedImportBuffer::addStringArray ( const std::vector< std::string > &  arr)
inline

Definition at line 250 of file Importer.h.

References string_array_buffer_.

250  {
251  string_array_buffer_->push_back(arr);
252  }
std::vector< std::vector< std::string > > * string_array_buffer_
Definition: Importer.h:524
void Importer_NS::TypedImportBuffer::addTinyint ( const int8_t  v)
inline

Definition at line 227 of file Importer.h.

References tinyint_buffer_.

Referenced by add_value().

227 { tinyint_buffer_->push_back(v); }
std::vector< int8_t > * tinyint_buffer_
Definition: Importer.h:515

+ Here is the caller graph for this function:

void Importer_NS::TypedImportBuffer::clear ( )
inline

Definition at line 399 of file Importer.h.

References array_buffer_, bigint_buffer_, bool_buffer_, CHECK(), column_desc_, ColumnDescriptor::columnType, double_buffer_, float_buffer_, geo_string_buffer_, SQLTypeInfoCore< TYPE_FACET_PACK >::get_compression(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_size(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_subtype(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), int_buffer_, IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kENCODING_DICT, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, smallint_buffer_, string_array_buffer_, string_array_dict_buffer_, string_buffer_, string_dict_i16_buffer_, string_dict_i32_buffer_, string_dict_i8_buffer_, and tinyint_buffer_.

399  {
400  switch (column_desc_->columnType.get_type()) {
401  case kBOOLEAN: {
402  bool_buffer_->clear();
403  break;
404  }
405  case kTINYINT: {
406  tinyint_buffer_->clear();
407  break;
408  }
409  case kSMALLINT: {
410  smallint_buffer_->clear();
411  break;
412  }
413  case kINT: {
414  int_buffer_->clear();
415  break;
416  }
417  case kBIGINT:
418  case kNUMERIC:
419  case kDECIMAL: {
420  bigint_buffer_->clear();
421  break;
422  }
423  case kFLOAT: {
424  float_buffer_->clear();
425  break;
426  }
427  case kDOUBLE: {
428  double_buffer_->clear();
429  break;
430  }
431  case kTEXT:
432  case kVARCHAR:
433  case kCHAR: {
434  string_buffer_->clear();
436  switch (column_desc_->columnType.get_size()) {
437  case 1:
438  string_dict_i8_buffer_->clear();
439  break;
440  case 2:
441  string_dict_i16_buffer_->clear();
442  break;
443  case 4:
444  string_dict_i32_buffer_->clear();
445  break;
446  default:
447  CHECK(false);
448  }
449  }
450  break;
451  }
452  case kDATE:
453  case kTIME:
454  case kTIMESTAMP:
455  bigint_buffer_->clear();
456  break;
457  case kARRAY: {
459  string_array_buffer_->clear();
460  string_array_dict_buffer_->clear();
461  } else {
462  array_buffer_->clear();
463  }
464  break;
465  }
466  case kPOINT:
467  case kLINESTRING:
468  case kPOLYGON:
469  case kMULTIPOLYGON:
470  geo_string_buffer_->clear();
471  break;
472  default:
473  CHECK(false);
474  }
475  }
std::vector< int16_t > * smallint_buffer_
Definition: Importer.h:516
std::vector< int64_t > * bigint_buffer_
Definition: Importer.h:518
Definition: sqltypes.h:52
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:334
std::vector< float > * float_buffer_
Definition: Importer.h:519
HOST DEVICE int get_size() const
Definition: sqltypes.h:336
std::vector< uint16_t > * string_dict_i16_buffer_
Definition: Importer.h:528
std::vector< int8_t > * tinyint_buffer_
Definition: Importer.h:515
CHECK(cgen_state)
std::vector< std::string > * string_buffer_
Definition: Importer.h:521
std::vector< int32_t > * string_dict_i32_buffer_
Definition: Importer.h:529
std::vector< std::vector< std::string > > * string_array_buffer_
Definition: Importer.h:524
std::vector< int32_t > * int_buffer_
Definition: Importer.h:517
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:326
std::vector< int8_t > * bool_buffer_
Definition: Importer.h:514
Definition: sqltypes.h:55
Definition: sqltypes.h:56
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:522
std::vector< ArrayDatum > * string_array_dict_buffer_
Definition: Importer.h:530
std::vector< double > * double_buffer_
Definition: Importer.h:520
Definition: sqltypes.h:44
const ColumnDescriptor * column_desc_
Definition: Importer.h:532
#define IS_STRING(T)
Definition: sqltypes.h:166
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:327
Definition: sqltypes.h:48
SQLTypeInfo columnType
std::vector< uint8_t > * string_dict_i8_buffer_
Definition: Importer.h:527
std::vector< ArrayDatum > * array_buffer_
Definition: Importer.h:523

+ Here is the call graph for this function:

template<typename DATA_TYPE >
size_t Importer_NS::TypedImportBuffer::convert_arrow_val_to_import_buffer ( const ColumnDescriptor cd,
const arrow::Array &  array,
std::vector< DATA_TYPE > &  buffer,
const ArraySliceRange slice_range,
BadRowsTracker *const  bad_rows_tracker 
)

Referenced by add_arrow_values().

+ Here is the caller graph for this function:

template<typename DATA_TYPE >
size_t Importer_NS::TypedImportBuffer::convert_arrow_val_to_import_buffer ( const ColumnDescriptor cd,
const Array &  array,
std::vector< DATA_TYPE > &  buffer,
const ArraySliceRange slice_range,
Importer_NS::BadRowsTracker *const  bad_rows_tracker 
)

Definition at line 762 of file Importer.cpp.

References col_idx, anonymous_namespace{ArrowImporter.h}::error_context(), geo_string_buffer_, SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), Importer_NS::Importer::getCatalog(), Geo_namespace::GeoTypesFactory::getGeoColumns(), import_buffers, Importer_NS::BadRowsTracker::importer, Importer_NS::BadRowsTracker::mutex, Importer_NS::BadRowsTracker::rows, Importer_NS::Importer::set_geo_physical_import_buffer(), and anonymous_namespace{ArrowImporter.h}::value_getter().

767  {
768  auto data =
769  std::make_unique<DataBuffer<DATA_TYPE>>(cd, array, buffer, bad_rows_tracker);
770  auto f_value_getter = value_getter(array, cd, bad_rows_tracker);
771  std::function<void(const int64_t)> f_add_geo_phy_cols = [&](const int64_t row) {};
772  if (bad_rows_tracker && cd->columnType.is_geometry()) {
773  f_add_geo_phy_cols = [&](const int64_t row) {
774  // Populate physical columns (ref. MapDHandler::load_table)
775  std::vector<double> coords, bounds;
776  std::vector<int> ring_sizes, poly_rings;
777  int render_group = 0;
778  SQLTypeInfo ti;
779  // replace any unexpected exception from getGeoColumns or other
780  // on this path with a GeoImportException so that we wont over
781  // push a null to the logical column...
782  try {
783  arrow_throw_if<GeoImportException>(
784  array.IsNull(row) ||
786  ti,
787  coords,
788  bounds,
789  ring_sizes,
790  poly_rings,
791  false),
792  error_context(cd, bad_rows_tracker) + "Invalid geometry");
793  arrow_throw_if<GeoImportException>(
794  cd->columnType.get_type() != ti.get_type(),
795  error_context(cd, bad_rows_tracker) + "Geometry type mismatch");
796  auto col_idx_workpad = col_idx; // what a pitfall!!
798  bad_rows_tracker->importer->getCatalog(),
799  cd,
801  col_idx_workpad,
802  coords,
803  bounds,
804  ring_sizes,
805  poly_rings,
806  render_group);
807  } catch (GeoImportException&) {
808  throw;
809  } catch (std::runtime_error& e) {
810  throw GeoImportException(e.what());
811  } catch (const std::exception& e) {
812  throw GeoImportException(e.what());
813  } catch (...) {
814  throw GeoImportException("unknown exception");
815  }
816  };
817  }
818  auto f_mark_a_bad_row = [&](const auto row) {
819  std::unique_lock<std::mutex> lck(bad_rows_tracker->mutex);
820  bad_rows_tracker->rows.insert(row - slice_range.first);
821  };
822  buffer.reserve(slice_range.second - slice_range.first);
823  for (size_t row = slice_range.first; row < slice_range.second; ++row) {
824  try {
825  *data << (array.IsNull(row) ? nullptr : f_value_getter(array, row));
826  f_add_geo_phy_cols(row);
827  } catch (GeoImportException&) {
828  f_mark_a_bad_row(row);
829  } catch (ArrowImporterException&) {
830  // trace bad rows of each column; otherwise rethrow.
831  if (bad_rows_tracker) {
832  *data << nullptr;
833  f_mark_a_bad_row(row);
834  } else {
835  throw;
836  }
837  }
838  }
839  return buffer.size();
840 }
std::string error_context(const ColumnDescriptor *cd, Importer_NS::BadRowsTracker *const bad_rows_tracker)
Definition: ArrowImporter.h:75
auto value_getter(const Array &array, const ColumnDescriptor *cd, Importer_NS::BadRowsTracker *const bad_rows_tracker)
Catalog_Namespace::Catalog & getCatalog()
Definition: Importer.h:794
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:326
static void set_geo_physical_import_buffer(const Catalog_Namespace::Catalog &catalog, const ColumnDescriptor *cd, std::vector< std::unique_ptr< TypedImportBuffer >> &import_buffers, size_t &col_idx, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, int render_group, const int64_t replicate_count=0)
Definition: Importer.cpp:1457
std::vector< std::unique_ptr< TypedImportBuffer > > * import_buffers
Definition: Importer.h:509
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:522
static bool getGeoColumns(const std::string &wkt, SQLTypeInfo &ti, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, const bool promote_poly_to_mpoly=false)
Definition: geo_types.cpp:459
std::set< int64_t > rows
Definition: Importer.h:78

+ Here is the call graph for this function:

template<typename DATA_TYPE >
auto Importer_NS::TypedImportBuffer::del_values ( std::vector< DATA_TYPE > &  buffer,
BadRowsTracker *const  bad_rows_tracker 
)
auto Importer_NS::TypedImportBuffer::del_values ( const SQLTypes  type,
BadRowsTracker *const  bad_rows_tracker 
)
int64_t Importer_NS::TypedImportBuffer::get_replicate_count ( ) const
inline

Definition at line 496 of file Importer.h.

References replicate_count_.

496 { return replicate_count_; }
std::vector<ArrayDatum>* Importer_NS::TypedImportBuffer::getArrayBuffer ( ) const
inline

Definition at line 369 of file Importer.h.

References array_buffer_.

369 { return array_buffer_; }
std::vector< ArrayDatum > * array_buffer_
Definition: Importer.h:523
int8_t* Importer_NS::TypedImportBuffer::getAsBytes ( ) const
inline

Definition at line 311 of file Importer.h.

References column_desc_, ColumnDescriptor::columnType, SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), kBIGINT, kBOOLEAN, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kNUMERIC, kSMALLINT, kTIME, kTIMESTAMP, and kTINYINT.

Referenced by Importer_NS::anonymous_namespace{Importer.cpp}::double_value_at(), Importer_NS::anonymous_namespace{Importer.cpp}::float_value_at(), and Importer_NS::anonymous_namespace{Importer.cpp}::int_value_at().

311  {
312  switch (column_desc_->columnType.get_type()) {
313  case kBOOLEAN:
314  return reinterpret_cast<int8_t*>(&((*bool_buffer_)[0]));
315  case kTINYINT:
316  return reinterpret_cast<int8_t*>(&((*tinyint_buffer_)[0]));
317  case kSMALLINT:
318  return reinterpret_cast<int8_t*>(&((*smallint_buffer_)[0]));
319  case kINT:
320  return reinterpret_cast<int8_t*>(&((*int_buffer_)[0]));
321  case kBIGINT:
322  case kNUMERIC:
323  case kDECIMAL:
324  return reinterpret_cast<int8_t*>(&((*bigint_buffer_)[0]));
325  case kFLOAT:
326  return reinterpret_cast<int8_t*>(&((*float_buffer_)[0]));
327  case kDOUBLE:
328  return reinterpret_cast<int8_t*>(&((*double_buffer_)[0]));
329  case kDATE:
330  case kTIME:
331  case kTIMESTAMP:
332  return reinterpret_cast<int8_t*>(&((*bigint_buffer_)[0]));
333  default:
334  abort();
335  }
336  }
Definition: sqltypes.h:52
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:326
Definition: sqltypes.h:56
const ColumnDescriptor * column_desc_
Definition: Importer.h:532
Definition: sqltypes.h:48
SQLTypeInfo columnType

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const ColumnDescriptor* Importer_NS::TypedImportBuffer::getColumnDesc ( ) const
inline

Definition at line 307 of file Importer.h.

References column_desc_.

307 { return column_desc_; }
const ColumnDescriptor * column_desc_
Definition: Importer.h:532
size_t Importer_NS::TypedImportBuffer::getElementSize ( ) const
inline

Definition at line 338 of file Importer.h.

References column_desc_, ColumnDescriptor::columnType, SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), kBIGINT, kBOOLEAN, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kNUMERIC, kSMALLINT, kTIME, kTIMESTAMP, and kTINYINT.

338  {
339  switch (column_desc_->columnType.get_type()) {
340  case kBOOLEAN:
341  return sizeof((*bool_buffer_)[0]);
342  case kTINYINT:
343  return sizeof((*tinyint_buffer_)[0]);
344  case kSMALLINT:
345  return sizeof((*smallint_buffer_)[0]);
346  case kINT:
347  return sizeof((*int_buffer_)[0]);
348  case kBIGINT:
349  case kNUMERIC:
350  case kDECIMAL:
351  return sizeof((*bigint_buffer_)[0]);
352  case kFLOAT:
353  return sizeof((*float_buffer_)[0]);
354  case kDOUBLE:
355  return sizeof((*double_buffer_)[0]);
356  case kDATE:
357  case kTIME:
358  case kTIMESTAMP:
359  return sizeof((*bigint_buffer_)[0]);
360  default:
361  abort();
362  }
363  }
Definition: sqltypes.h:52
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:326
Definition: sqltypes.h:56
const ColumnDescriptor * column_desc_
Definition: Importer.h:532
Definition: sqltypes.h:48
SQLTypeInfo columnType

+ Here is the call graph for this function:

std::vector<std::string>* Importer_NS::TypedImportBuffer::getGeoStringBuffer ( ) const
inline

Definition at line 367 of file Importer.h.

References geo_string_buffer_.

367 { return geo_string_buffer_; }
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:522
std::vector<std::vector<std::string> >* Importer_NS::TypedImportBuffer::getStringArrayBuffer ( ) const
inline

Definition at line 371 of file Importer.h.

References string_array_buffer_.

371  {
372  return string_array_buffer_;
373  }
std::vector< std::vector< std::string > > * string_array_buffer_
Definition: Importer.h:524
std::vector<ArrayDatum>* Importer_NS::TypedImportBuffer::getStringArrayDictBuffer ( ) const
inline

Definition at line 375 of file Importer.h.

References string_array_dict_buffer_.

375  {
377  }
std::vector< ArrayDatum > * string_array_dict_buffer_
Definition: Importer.h:530
std::vector<std::string>* Importer_NS::TypedImportBuffer::getStringBuffer ( ) const
inline

Definition at line 365 of file Importer.h.

References string_buffer_.

365 { return string_buffer_; }
std::vector< std::string > * string_buffer_
Definition: Importer.h:521
int8_t* Importer_NS::TypedImportBuffer::getStringDictBuffer ( ) const
inline

Definition at line 379 of file Importer.h.

References column_desc_, ColumnDescriptor::columnType, and SQLTypeInfoCore< TYPE_FACET_PACK >::get_size().

Referenced by Importer_NS::anonymous_namespace{Importer.cpp}::int_value_at().

379  {
380  switch (column_desc_->columnType.get_size()) {
381  case 1:
382  return reinterpret_cast<int8_t*>(&((*string_dict_i8_buffer_)[0]));
383  case 2:
384  return reinterpret_cast<int8_t*>(&((*string_dict_i16_buffer_)[0]));
385  case 4:
386  return reinterpret_cast<int8_t*>(&((*string_dict_i32_buffer_)[0]));
387  default:
388  abort();
389  }
390  }
HOST DEVICE int get_size() const
Definition: sqltypes.h:336
const ColumnDescriptor * column_desc_
Definition: Importer.h:532
SQLTypeInfo columnType

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

StringDictionary* Importer_NS::TypedImportBuffer::getStringDictionary ( ) const
inline

Definition at line 309 of file Importer.h.

References string_dict_.

309 { return string_dict_; }
StringDictionary * string_dict_
Definition: Importer.h:533
const SQLTypeInfo& Importer_NS::TypedImportBuffer::getTypeInfo ( ) const
inline

Definition at line 305 of file Importer.h.

References column_desc_, and ColumnDescriptor::columnType.

Referenced by Importer_NS::anonymous_namespace{Importer.cpp}::double_value_at(), Importer_NS::anonymous_namespace{Importer.cpp}::float_value_at(), and Importer_NS::anonymous_namespace{Importer.cpp}::int_value_at().

305 { return column_desc_->columnType; }
const ColumnDescriptor * column_desc_
Definition: Importer.h:532
SQLTypeInfo columnType

+ Here is the caller graph for this function:

void Importer_NS::TypedImportBuffer::pop_value ( )

Definition at line 702 of file Importer.cpp.

References array_buffer_, bigint_buffer_, bool_buffer_, CHECK(), column_desc_, ColumnDescriptor::columnType, decimal_to_int_type(), double_buffer_, float_buffer_, geo_string_buffer_, SQLTypeInfoCore< TYPE_FACET_PACK >::get_subtype(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), int_buffer_, SQLTypeInfoCore< TYPE_FACET_PACK >::is_decimal(), IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, smallint_buffer_, string_array_buffer_, string_buffer_, tinyint_buffer_, and run_benchmark_import::type.

702  {
703  const auto type = column_desc_->columnType.is_decimal()
705  : column_desc_->columnType.get_type();
706  switch (type) {
707  case kBOOLEAN:
708  bool_buffer_->pop_back();
709  break;
710  case kTINYINT:
711  tinyint_buffer_->pop_back();
712  break;
713  case kSMALLINT:
714  smallint_buffer_->pop_back();
715  break;
716  case kINT:
717  int_buffer_->pop_back();
718  break;
719  case kBIGINT:
720  bigint_buffer_->pop_back();
721  break;
722  case kFLOAT:
723  float_buffer_->pop_back();
724  break;
725  case kDOUBLE:
726  double_buffer_->pop_back();
727  break;
728  case kTEXT:
729  case kVARCHAR:
730  case kCHAR:
731  string_buffer_->pop_back();
732  break;
733  case kDATE:
734  case kTIME:
735  case kTIMESTAMP:
736  bigint_buffer_->pop_back();
737  break;
738  case kARRAY:
740  string_array_buffer_->pop_back();
741  } else {
742  array_buffer_->pop_back();
743  }
744  break;
745  case kPOINT:
746  case kLINESTRING:
747  case kPOLYGON:
748  case kMULTIPOLYGON:
749  geo_string_buffer_->pop_back();
750  break;
751  default:
752  CHECK(false) << "TypedImportBuffer::pop_value() does not support type " << type;
753  }
754 }
std::vector< int16_t > * smallint_buffer_
Definition: Importer.h:516
std::vector< int64_t > * bigint_buffer_
Definition: Importer.h:518
Definition: sqltypes.h:52
std::vector< float > * float_buffer_
Definition: Importer.h:519
std::vector< int8_t > * tinyint_buffer_
Definition: Importer.h:515
CHECK(cgen_state)
std::vector< std::string > * string_buffer_
Definition: Importer.h:521
std::vector< std::vector< std::string > > * string_array_buffer_
Definition: Importer.h:524
std::vector< int32_t > * int_buffer_
Definition: Importer.h:517
std::vector< int8_t > * bool_buffer_
Definition: Importer.h:514
SQLTypes decimal_to_int_type(const SQLTypeInfo &ti)
Definition: Datum.cpp:268
Definition: sqltypes.h:55
Definition: sqltypes.h:56
std::vector< std::string > * geo_string_buffer_
Definition: Importer.h:522
std::vector< double > * double_buffer_
Definition: Importer.h:520
Definition: sqltypes.h:44
const ColumnDescriptor * column_desc_
Definition: Importer.h:532
#define IS_STRING(T)
Definition: sqltypes.h:166
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:327
Definition: sqltypes.h:48
SQLTypeInfo columnType
bool is_decimal() const
Definition: sqltypes.h:480
std::vector< ArrayDatum > * array_buffer_
Definition: Importer.h:523

+ Here is the call graph for this function:

void Importer_NS::TypedImportBuffer::set_replicate_count ( const int64_t  replicate_count)
inline

Definition at line 497 of file Importer.h.

References replicate_count_.

Referenced by add_value().

497  {
498  replicate_count_ = replicate_count;
499  }

+ Here is the caller graph for this function:

bool Importer_NS::TypedImportBuffer::stringDictCheckpoint ( )
inline

Definition at line 392 of file Importer.h.

References StringDictionary::checkpoint(), and string_dict_.

392  {
393  if (string_dict_ == nullptr) {
394  return true;
395  }
396  return string_dict_->checkpoint();
397  }
StringDictionary * string_dict_
Definition: Importer.h:533
bool checkpoint() noexcept

+ Here is the call graph for this function:

Member Data Documentation

union { ... }
union { ... }
std::vector<ArrayDatum>* Importer_NS::TypedImportBuffer::array_buffer_
std::vector<int64_t>* Importer_NS::TypedImportBuffer::bigint_buffer_
std::vector<int8_t>* Importer_NS::TypedImportBuffer::bool_buffer_
size_t Importer_NS::TypedImportBuffer::col_idx

Definition at line 510 of file Importer.h.

Referenced by convert_arrow_val_to_import_buffer().

const ColumnDescriptor* Importer_NS::TypedImportBuffer::column_desc_
private
std::vector<double>* Importer_NS::TypedImportBuffer::double_buffer_
std::vector<float>* Importer_NS::TypedImportBuffer::float_buffer_
std::vector<std::string>* Importer_NS::TypedImportBuffer::geo_string_buffer_
std::vector<std::unique_ptr<TypedImportBuffer> >* Importer_NS::TypedImportBuffer::import_buffers

Definition at line 509 of file Importer.h.

Referenced by convert_arrow_val_to_import_buffer().

std::vector<int32_t>* Importer_NS::TypedImportBuffer::int_buffer_
size_t Importer_NS::TypedImportBuffer::replicate_count_ = 0
private

Definition at line 534 of file Importer.h.

Referenced by get_replicate_count(), and set_replicate_count().

std::vector<int16_t>* Importer_NS::TypedImportBuffer::smallint_buffer_
std::vector<std::vector<std::string> >* Importer_NS::TypedImportBuffer::string_array_buffer_
std::vector<ArrayDatum>* Importer_NS::TypedImportBuffer::string_array_dict_buffer_
std::vector<std::string>* Importer_NS::TypedImportBuffer::string_buffer_
StringDictionary* Importer_NS::TypedImportBuffer::string_dict_
private
std::vector<uint16_t>* Importer_NS::TypedImportBuffer::string_dict_i16_buffer_

Definition at line 528 of file Importer.h.

Referenced by addDictEncodedString(), clear(), TypedImportBuffer(), and ~TypedImportBuffer().

std::vector<int32_t>* Importer_NS::TypedImportBuffer::string_dict_i32_buffer_

Definition at line 529 of file Importer.h.

Referenced by addDictEncodedString(), clear(), TypedImportBuffer(), and ~TypedImportBuffer().

std::vector<uint8_t>* Importer_NS::TypedImportBuffer::string_dict_i8_buffer_

Definition at line 527 of file Importer.h.

Referenced by addDictEncodedString(), clear(), TypedImportBuffer(), and ~TypedImportBuffer().

std::vector<int8_t>* Importer_NS::TypedImportBuffer::tinyint_buffer_

The documentation for this class was generated from the following files: