import_export::TypedImportBuffer Class Reference

#include <Importer.h>

Inheritance diagram for import_export::TypedImportBuffer:

Collaboration diagram for import_export::TypedImportBuffer:

Public Member Functions
	TypedImportBuffer (const ColumnDescriptor *col_desc, StringDictionary *string_dict)
	~TypedImportBuffer ()
void	addBoolean (const int8_t v)
void	addTinyint (const int8_t v)
void	addSmallint (const int16_t v)
void	addInt (const int32_t v)
void	addBigint (const int64_t v)
void	addFloat (const float v)
void	addDouble (const double v)
void	addString (const std::string_view v)
void	addGeoString (const std::string_view v)
void	addArray (const ArrayDatum &v)
std::vector< std::string > &	addStringArray ()
void	addStringArray (const std::vector< std::string > &arr)
void	addDictEncodedString (const std::vector< std::string > &string_vec)
void	addDictEncodedStringArray (const std::vector< std::vector< std::string >> &string_array_vec)
const SQLTypeInfo &	getTypeInfo () const
const ColumnDescriptor *	getColumnDesc () const
StringDictionary *	getStringDictionary () const
int8_t *	getAsBytes () const
size_t	getElementSize () const
std::vector< std::string > *	getStringBuffer () const
std::vector< std::string > *	getGeoStringBuffer () const
std::vector< ArrayDatum > *	getArrayBuffer () const
std::vector< std::vector < std::string > > *	getStringArrayBuffer () const
std::vector< ArrayDatum > *	getStringArrayDictBuffer () const
int8_t *	getStringDictBuffer () const
bool	stringDictCheckpoint ()
void	clear ()
size_t	add_values (const ColumnDescriptor *cd, const TColumn &data)
size_t	add_arrow_values (const ColumnDescriptor *cd, const arrow::Array &data, const bool exact_type_match, const ArraySliceRange &slice_range, BadRowsTracker *bad_rows_tracker)
void	add_value (const ColumnDescriptor *cd, const std::string_view val, const bool is_null, const CopyParams &copy_params, const int64_t replicate_count=0)
void	add_value (const ColumnDescriptor *cd, const TDatum &val, const bool is_null, const int64_t replicate_count=0)
void	pop_value ()
int64_t	get_replicate_count () const
void	set_replicate_count (const int64_t replicate_count)
template<typename DATA_TYPE >
size_t	convert_arrow_val_to_import_buffer (const ColumnDescriptor *cd, const arrow::Array &array, std::vector< DATA_TYPE > &buffer, const ArraySliceRange &slice_range, BadRowsTracker *const bad_rows_tracker)
template<typename DATA_TYPE >
auto	del_values (std::vector< DATA_TYPE > &buffer, BadRowsTracker *const bad_rows_tracker)
auto	del_values (const SQLTypes type, BadRowsTracker *const bad_rows_tracker)
template<typename DATA_TYPE >
size_t	convert_arrow_val_to_import_buffer (const ColumnDescriptor *cd, const Array &array, std::vector< DATA_TYPE > &buffer, const ArraySliceRange &slice_range, import_export::BadRowsTracker *const bad_rows_tracker)

Public Attributes
std::vector< std::unique_ptr < TypedImportBuffer > > *	import_buffers
size_t	col_idx
std::vector< int8_t > *	bool_buffer_
std::vector< int8_t > *	tinyint_buffer_
std::vector< int16_t > *	smallint_buffer_
std::vector< int32_t > *	int_buffer_
std::vector< int64_t > *	bigint_buffer_
std::vector< float > *	float_buffer_
std::vector< double > *	double_buffer_
std::vector< std::string > *	string_buffer_
std::vector< std::string > *	geo_string_buffer_
std::vector< ArrayDatum > *	array_buffer_
std::vector< std::vector < std::string > > *	string_array_buffer_
std::vector< uint8_t > *	string_dict_i8_buffer_
std::vector< uint16_t > *	string_dict_i16_buffer_
std::vector< int32_t > *	string_dict_i32_buffer_
std::vector< ArrayDatum > *	string_array_dict_buffer_

Private Attributes
union {
std::vector< int8_t > *   bool_buffer_
std::vector< int8_t > *   tinyint_buffer_
std::vector< int16_t > *   smallint_buffer_
std::vector< int32_t > *   int_buffer_
std::vector< int64_t > *   bigint_buffer_
std::vector< float > *   float_buffer_
std::vector< double > *   double_buffer_
std::vector< std::string > *   string_buffer_
std::vector< std::string > *   geo_string_buffer_
std::vector< ArrayDatum > *   array_buffer_
std::vector< std::vector < std::string > > *   string_array_buffer_
};
union {
std::vector< uint8_t > *   string_dict_i8_buffer_
std::vector< uint16_t > *   string_dict_i16_buffer_
std::vector< int32_t > *   string_dict_i32_buffer_
std::vector< ArrayDatum > *   string_array_dict_buffer_
};
const ColumnDescriptor *	column_desc_
StringDictionary *	string_dict_
size_t	replicate_count_ = 0

Detailed Description

Definition at line 83 of file Importer.h.

Constructor & Destructor Documentation

import_export::TypedImportBuffer::TypedImportBuffer ( const ColumnDescriptor *  col_desc, StringDictionary *  string_dict  )

inline

Definition at line 85 of file Importer.h.

References array_buffer_, bigint_buffer_, bool_buffer_, CHECK, ColumnDescriptor::columnType, double_buffer_, float_buffer_, geo_string_buffer_, SQLTypeInfo::get_compression(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), int_buffer_, IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kENCODING_DICT, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, smallint_buffer_, string_array_buffer_, string_array_dict_buffer_, string_buffer_, string_dict_i16_buffer_, string_dict_i32_buffer_, string_dict_i8_buffer_, and tinyint_buffer_.

      : column_desc_(col_desc), string_dict_(string_dict) {
    switch (col_desc->columnType.get_type()) {
      case kBOOLEAN:
        bool_buffer_ = new std::vector<int8_t>();
        break;
      case kTINYINT:
        tinyint_buffer_ = new std::vector<int8_t>();
        break;
      case kSMALLINT:
        smallint_buffer_ = new std::vector<int16_t>();
        break;
      case kINT:
        int_buffer_ = new std::vector<int32_t>();
        break;
      case kBIGINT:
      case kNUMERIC:
      case kDECIMAL:
        bigint_buffer_ = new std::vector<int64_t>();
        break;
      case kFLOAT:
        float_buffer_ = new std::vector<float>();
        break;
      case kDOUBLE:
        double_buffer_ = new std::vector<double>();
        break;
      case kTEXT:
      case kVARCHAR:
      case kCHAR:
        string_buffer_ = new std::vector<std::string>();
        if (col_desc->columnType.get_compression() == kENCODING_DICT) {
          switch (col_desc->columnType.get_size()) {
            case 1:
              string_dict_i8_buffer_ = new std::vector<uint8_t>();
              break;
            case 2:
              string_dict_i16_buffer_ = new std::vector<uint16_t>();
              break;
            case 4:
              string_dict_i32_buffer_ = new std::vector<int32_t>();
              break;
            default:
              CHECK(false);
          }
        }
        break;
      case kDATE:
      case kTIME:
      case kTIMESTAMP:
        bigint_buffer_ = new std::vector<int64_t>();
        break;
      case kARRAY:
        if (IS_STRING(col_desc->columnType.get_subtype())) {
          CHECK(col_desc->columnType.get_compression() == kENCODING_DICT);
          string_array_buffer_ = new std::vector<std::vector<std::string>>();
          string_array_dict_buffer_ = new std::vector<ArrayDatum>();
        } else {
          array_buffer_ = new std::vector<ArrayDatum>();
        }
        break;
      case kPOINT:
      case kLINESTRING:
      case kPOLYGON:
      case kMULTIPOLYGON:
        geo_string_buffer_ = new std::vector<std::string>();
        break;
      default:
        CHECK(false);
    }
  }

Here is the call graph for this function:

import_export::TypedImportBuffer::~TypedImportBuffer ( )

inline

Definition at line 156 of file Importer.h.

References array_buffer_, bigint_buffer_, bool_buffer_, CHECK, column_desc_, ColumnDescriptor::columnType, double_buffer_, float_buffer_, geo_string_buffer_, SQLTypeInfo::get_compression(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), int_buffer_, IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kENCODING_DICT, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, smallint_buffer_, string_array_buffer_, string_array_dict_buffer_, string_buffer_, string_dict_i16_buffer_, string_dict_i32_buffer_, string_dict_i8_buffer_, and tinyint_buffer_.

                       {
    switch (column_desc_->columnType.get_type()) {
      case kBOOLEAN:
        delete bool_buffer_;
        break;
      case kTINYINT:
        delete tinyint_buffer_;
        break;
      case kSMALLINT:
        delete smallint_buffer_;
        break;
      case kINT:
        delete int_buffer_;
        break;
      case kBIGINT:
      case kNUMERIC:
      case kDECIMAL:
        delete bigint_buffer_;
        break;
      case kFLOAT:
        delete float_buffer_;
        break;
      case kDOUBLE:
        delete double_buffer_;
        break;
      case kTEXT:
      case kVARCHAR:
      case kCHAR:
        delete string_buffer_;
        if (column_desc_->columnType.get_compression() == kENCODING_DICT) {
          switch (column_desc_->columnType.get_size()) {
            case 1:
              delete string_dict_i8_buffer_;
              break;
            case 2:
              delete string_dict_i16_buffer_;
              break;
            case 4:
              delete string_dict_i32_buffer_;
              break;
          }
        }
        break;
      case kDATE:
      case kTIME:
      case kTIMESTAMP:
        delete bigint_buffer_;
        break;
      case kARRAY:
        if (IS_STRING(column_desc_->columnType.get_subtype())) {
          delete string_array_buffer_;
          delete string_array_dict_buffer_;
        } else {
          delete array_buffer_;
        }
        break;
      case kPOINT:
      case kLINESTRING:
      case kPOLYGON:
      case kMULTIPOLYGON:
        delete geo_string_buffer_;
        break;
      default:
        CHECK(false);
    }
  }

Here is the call graph for this function:

Member Function Documentation

size_t import_export::TypedImportBuffer::add_arrow_values	(	const ColumnDescriptor *	cd,
		const arrow::Array &	data,
		const bool	exact_type_match,
		const ArraySliceRange &	slice_range,
		BadRowsTracker *	bad_rows_tracker
	)

Definition at line 853 of file Importer.cpp.

References arrow_throw_if(), bigint_buffer_, bool_buffer_, ColumnDescriptor::columnName, ColumnDescriptor::columnType, convert_arrow_val_to_import_buffer(), double_buffer_, float_buffer_, geo_string_buffer_, SQLTypeInfo::get_notnull(), SQLTypeInfo::get_type(), int_buffer_, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, smallint_buffer_, omnisci.dtypes::STRING, string_buffer_, tinyint_buffer_, and run_benchmark_import::type.

                                                                                   {
  const auto type = cd->columnType.get_type();
  if (cd->columnType.get_notnull()) {
    // We can't have any null values for this column; to have them is an error
    arrow_throw_if(col.null_count() > 0, "NULL not allowed for column " + cd->columnName);
  }

  switch (type) {
    case kBOOLEAN:
      if (exact_type_match) {
        arrow_throw_if(col.type_id() != Type::BOOL, "Expected boolean type");
      }
      return convert_arrow_val_to_import_buffer(
          cd, col, *bool_buffer_, slice_range, bad_rows_tracker);
    case kTINYINT:
      if (exact_type_match) {
        arrow_throw_if(col.type_id() != Type::INT8, "Expected int8 type");
      }
      return convert_arrow_val_to_import_buffer(
          cd, col, *tinyint_buffer_, slice_range, bad_rows_tracker);
    case kSMALLINT:
      if (exact_type_match) {
        arrow_throw_if(col.type_id() != Type::INT16, "Expected int16 type");
      }
      return convert_arrow_val_to_import_buffer(
          cd, col, *smallint_buffer_, slice_range, bad_rows_tracker);
    case kINT:
      if (exact_type_match) {
        arrow_throw_if(col.type_id() != Type::INT32, "Expected int32 type");
      }
      return convert_arrow_val_to_import_buffer(
          cd, col, *int_buffer_, slice_range, bad_rows_tracker);
    case kBIGINT:
    case kNUMERIC:
    case kDECIMAL:
      if (exact_type_match) {
        arrow_throw_if(col.type_id() != Type::INT64, "Expected int64 type");
      }
      return convert_arrow_val_to_import_buffer(
          cd, col, *bigint_buffer_, slice_range, bad_rows_tracker);
    case kFLOAT:
      if (exact_type_match) {
        arrow_throw_if(col.type_id() != Type::FLOAT, "Expected float type");
      }
      return convert_arrow_val_to_import_buffer(
          cd, col, *float_buffer_, slice_range, bad_rows_tracker);
    case kDOUBLE:
      if (exact_type_match) {
        arrow_throw_if(col.type_id() != Type::DOUBLE, "Expected double type");
      }
      return convert_arrow_val_to_import_buffer(
          cd, col, *double_buffer_, slice_range, bad_rows_tracker);
    case kTEXT:
    case kVARCHAR:
    case kCHAR:
      if (exact_type_match) {
        arrow_throw_if(col.type_id() != Type::BINARY && col.type_id() != Type::STRING,
                       "Expected string type");
      }
      return convert_arrow_val_to_import_buffer(
          cd, col, *string_buffer_, slice_range, bad_rows_tracker);
    case kTIME:
      if (exact_type_match) {
        arrow_throw_if(col.type_id() != Type::TIME32 && col.type_id() != Type::TIME64,
                       "Expected time32 or time64 type");
      }
      return convert_arrow_val_to_import_buffer(
          cd, col, *bigint_buffer_, slice_range, bad_rows_tracker);
    case kTIMESTAMP:
      if (exact_type_match) {
        arrow_throw_if(col.type_id() != Type::TIMESTAMP, "Expected timestamp type");
      }
      return convert_arrow_val_to_import_buffer(
          cd, col, *bigint_buffer_, slice_range, bad_rows_tracker);
    case kDATE:
      if (exact_type_match) {
        arrow_throw_if(col.type_id() != Type::DATE32 && col.type_id() != Type::DATE64,
                       "Expected date32 or date64 type");
      }
      return convert_arrow_val_to_import_buffer(
          cd, col, *bigint_buffer_, slice_range, bad_rows_tracker);
    case kPOINT:
    case kLINESTRING:
    case kPOLYGON:
    case kMULTIPOLYGON:
      arrow_throw_if(col.type_id() != Type::BINARY && col.type_id() != Type::STRING,
                     "Expected string type");
      return convert_arrow_val_to_import_buffer(
          cd, col, *geo_string_buffer_, slice_range, bad_rows_tracker);
    case kARRAY:
      throw std::runtime_error("Arrow array appends not yet supported");
    default:
      throw std::runtime_error("Invalid Type");
  }
}

Here is the call graph for this function:

void import_export::TypedImportBuffer::add_value	(	const ColumnDescriptor *	cd,
		const std::string_view	val,
		const bool	is_null,
		const CopyParams &	copy_params,
		const int64_t	replicate_count = 0
	)

Definition at line 503 of file Importer.cpp.

References addArray(), addBigint(), addBoolean(), addDouble(), addFloat(), addGeoString(), addInt(), addSmallint(), addString(), addStringArray(), addTinyint(), Datum::bigintval, Datum::boolval, CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, convert_decimal_value_to_scale(), SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), inline_fixed_encoding_null_val(), Datum::intval, IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, StringDictionary::MAX_STRLEN, NULL_DOUBLE, NULL_FLOAT, import_export::NullArray(), import_export::delimited_parser::parse_string_array(), set_replicate_count(), Datum::smallintval, import_export::StringToArray(), StringToDatum(), Datum::tinyintval, to_string(), and run_benchmark_import::type.

                                                                 {
  set_replicate_count(replicate_count);
  const auto type = cd->columnType.get_type();
  switch (type) {
    case kBOOLEAN: {
      if (is_null) {
        if (cd->columnType.get_notnull()) {
          throw std::runtime_error("NULL for column " + cd->columnName);
        }
        addBoolean(inline_fixed_encoding_null_val(cd->columnType));
      } else {
        auto ti = cd->columnType;
        Datum d = StringToDatum(val, ti);
        addBoolean(static_cast<int8_t>(d.boolval));
      }
      break;
    }
    case kTINYINT: {
      if (!is_null && (isdigit(val[0]) || val[0] == '-')) {
        auto ti = cd->columnType;
        Datum d = StringToDatum(val, ti);
        addTinyint(d.tinyintval);
      } else {
        if (cd->columnType.get_notnull()) {
          throw std::runtime_error("NULL for column " + cd->columnName);
        }
        addTinyint(inline_fixed_encoding_null_val(cd->columnType));
      }
      break;
    }
    case kSMALLINT: {
      if (!is_null && (isdigit(val[0]) || val[0] == '-')) {
        auto ti = cd->columnType;
        Datum d = StringToDatum(val, ti);
        addSmallint(d.smallintval);
      } else {
        if (cd->columnType.get_notnull()) {
          throw std::runtime_error("NULL for column " + cd->columnName);
        }
        addSmallint(inline_fixed_encoding_null_val(cd->columnType));
      }
      break;
    }
    case kINT: {
      if (!is_null && (isdigit(val[0]) || val[0] == '-')) {
        auto ti = cd->columnType;
        Datum d = StringToDatum(val, ti);
        addInt(d.intval);
      } else {
        if (cd->columnType.get_notnull()) {
          throw std::runtime_error("NULL for column " + cd->columnName);
        }
        addInt(inline_fixed_encoding_null_val(cd->columnType));
      }
      break;
    }
    case kBIGINT: {
      if (!is_null && (isdigit(val[0]) || val[0] == '-')) {
        auto ti = cd->columnType;
        Datum d = StringToDatum(val, ti);
        addBigint(d.bigintval);
      } else {
        if (cd->columnType.get_notnull()) {
          throw std::runtime_error("NULL for column " + cd->columnName);
        }
        addBigint(inline_fixed_encoding_null_val(cd->columnType));
      }
      break;
    }
    case kDECIMAL:
    case kNUMERIC: {
      if (!is_null) {
        SQLTypeInfo ti(kNUMERIC, 0, 0, false);
        Datum d = StringToDatum(val, ti);
        const auto converted_decimal_value =
            convert_decimal_value_to_scale(d.bigintval, ti, cd->columnType);
        addBigint(converted_decimal_value);
      } else {
        if (cd->columnType.get_notnull()) {
          throw std::runtime_error("NULL for column " + cd->columnName);
        }
        addBigint(inline_fixed_encoding_null_val(cd->columnType));
      }
      break;
    }
    case kFLOAT:
      if (!is_null && (val[0] == '.' || isdigit(val[0]) || val[0] == '-')) {
        addFloat(static_cast<float>(std::atof(std::string(val).c_str())));
      } else {
        if (cd->columnType.get_notnull()) {
          throw std::runtime_error("NULL for column " + cd->columnName);
        }
        addFloat(NULL_FLOAT);
      }
      break;
    case kDOUBLE:
      if (!is_null && (val[0] == '.' || isdigit(val[0]) || val[0] == '-')) {
        addDouble(std::atof(std::string(val).c_str()));
      } else {
        if (cd->columnType.get_notnull()) {
          throw std::runtime_error("NULL for column " + cd->columnName);
        }
        addDouble(NULL_DOUBLE);
      }
      break;
    case kTEXT:
    case kVARCHAR:
    case kCHAR: {
      // @TODO(wei) for now, use empty string for nulls
      if (is_null) {
        if (cd->columnType.get_notnull()) {
          throw std::runtime_error("NULL for column " + cd->columnName);
        }
        addString(std::string());
      } else {
        if (val.length() > StringDictionary::MAX_STRLEN) {
          throw std::runtime_error("String too long for column " + cd->columnName +
                                   " was " + std::to_string(val.length()) + " max is " +
                                   std::to_string(StringDictionary::MAX_STRLEN));
        }
        addString(val);
      }
      break;
    }
    case kTIME:
    case kTIMESTAMP:
    case kDATE:
      if (!is_null && (isdigit(val[0]) || val[0] == '-')) {
        SQLTypeInfo ti = cd->columnType;
        Datum d = StringToDatum(val, ti);
        addBigint(d.bigintval);
      } else {
        if (cd->columnType.get_notnull()) {
          throw std::runtime_error("NULL for column " + cd->columnName);
        }
        addBigint(inline_fixed_encoding_null_val(cd->columnType));
      }
      break;
    case kARRAY: {
      if (is_null && cd->columnType.get_notnull()) {
        throw std::runtime_error("NULL for column " + cd->columnName);
      }
      SQLTypeInfo ti = cd->columnType;
      if (IS_STRING(ti.get_subtype())) {
        std::vector<std::string> string_vec;
        // Just parse string array, don't push it to buffer yet as we might throw
        import_export::delimited_parser::parse_string_array(
            std::string(val), copy_params, string_vec);
        if (!is_null) {
          // TODO: add support for NULL string arrays
          if (ti.get_size() > 0) {
            auto sti = ti.get_elem_type();
            size_t expected_size = ti.get_size() / sti.get_size();
            size_t actual_size = string_vec.size();
            if (actual_size != expected_size) {
              throw std::runtime_error("Fixed length array column " + cd->columnName +
                                       " expects " + std::to_string(expected_size) +
                                       " values, received " +
                                       std::to_string(actual_size));
            }
          }
          addStringArray(string_vec);
        } else {
          if (ti.get_size() > 0) {
            // TODO: remove once NULL fixlen arrays are allowed
            throw std::runtime_error("Fixed length array column " + cd->columnName +
                                     " currently cannot accept NULL arrays");
          }
          // TODO: add support for NULL string arrays, replace with addStringArray(),
          //       for now add whatever parseStringArray() outputs for NULLs ("NULL")
          addStringArray(string_vec);
        }
      } else {
        if (!is_null) {
          ArrayDatum d = StringToArray(std::string(val), ti, copy_params);
          if (d.is_null) {  // val could be "NULL"
            addArray(NullArray(ti));
          } else {
            if (ti.get_size() > 0 && static_cast<size_t>(ti.get_size()) != d.length) {
              throw std::runtime_error("Fixed length array for column " + cd->columnName +
                                       " has incorrect length: " + std::string(val));
            }
            addArray(d);
          }
        } else {
          addArray(NullArray(ti));
        }
      }
      break;
    }
    case kPOINT:
    case kLINESTRING:
    case kPOLYGON:
    case kMULTIPOLYGON:
      addGeoString(val);
      break;
    default:
      CHECK(false) << "TypedImportBuffer::add_value() does not support type " << type;
  }
}

Here is the call graph for this function:

void import_export::TypedImportBuffer::add_value	(	const ColumnDescriptor *	cd,
		const TDatum &	val,
		const bool	is_null,
		const int64_t	replicate_count = 0
	)

Definition at line 1275 of file Importer.cpp.

References addArray(), addBigint(), import_export::addBinaryStringArray(), addBoolean(), addDouble(), addFloat(), addGeoString(), addInt(), addSmallint(), addString(), addStringArray(), addTinyint(), CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, decimal_to_int_type(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), inline_fixed_encoding_null_val(), SQLTypeInfo::is_decimal(), IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, NULL_DOUBLE, NULL_FLOAT, import_export::NullArray(), set_replicate_count(), import_export::TDatumToArrayDatum(), and run_benchmark_import::type.

                                                                 {
  set_replicate_count(replicate_count);
  const auto type = cd->columnType.is_decimal() ? decimal_to_int_type(cd->columnType)
                                                : cd->columnType.get_type();
  switch (type) {
    case kBOOLEAN: {
      if (is_null) {
        if (cd->columnType.get_notnull()) {
          throw std::runtime_error("NULL for column " + cd->columnName);
        }
        addBoolean(inline_fixed_encoding_null_val(cd->columnType));
      } else {
        addBoolean((int8_t)datum.val.int_val);
      }
      break;
    }
    case kTINYINT:
      if (!is_null) {
        addTinyint((int8_t)datum.val.int_val);
      } else {
        if (cd->columnType.get_notnull()) {
          throw std::runtime_error("NULL for column " + cd->columnName);
        }
        addTinyint(inline_fixed_encoding_null_val(cd->columnType));
      }
      break;
    case kSMALLINT:
      if (!is_null) {
        addSmallint((int16_t)datum.val.int_val);
      } else {
        if (cd->columnType.get_notnull()) {
          throw std::runtime_error("NULL for column " + cd->columnName);
        }
        addSmallint(inline_fixed_encoding_null_val(cd->columnType));
      }
      break;
    case kINT:
      if (!is_null) {
        addInt((int32_t)datum.val.int_val);
      } else {
        if (cd->columnType.get_notnull()) {
          throw std::runtime_error("NULL for column " + cd->columnName);
        }
        addInt(inline_fixed_encoding_null_val(cd->columnType));
      }
      break;
    case kBIGINT:
      if (!is_null) {
        addBigint(datum.val.int_val);
      } else {
        if (cd->columnType.get_notnull()) {
          throw std::runtime_error("NULL for column " + cd->columnName);
        }
        addBigint(inline_fixed_encoding_null_val(cd->columnType));
      }
      break;
    case kFLOAT:
      if (!is_null) {
        addFloat((float)datum.val.real_val);
      } else {
        if (cd->columnType.get_notnull()) {
          throw std::runtime_error("NULL for column " + cd->columnName);
        }
        addFloat(NULL_FLOAT);
      }
      break;
    case kDOUBLE:
      if (!is_null) {
        addDouble(datum.val.real_val);
      } else {
        if (cd->columnType.get_notnull()) {
          throw std::runtime_error("NULL for column " + cd->columnName);
        }
        addDouble(NULL_DOUBLE);
      }
      break;
    case kTEXT:
    case kVARCHAR:
    case kCHAR: {
      // @TODO(wei) for now, use empty string for nulls
      if (is_null) {
        if (cd->columnType.get_notnull()) {
          throw std::runtime_error("NULL for column " + cd->columnName);
        }
        addString(std::string());
      } else {
        addString(datum.val.str_val);
      }
      break;
    }
    case kTIME:
    case kTIMESTAMP:
    case kDATE: {
      if (!is_null) {
        addBigint(datum.val.int_val);
      } else {
        if (cd->columnType.get_notnull()) {
          throw std::runtime_error("NULL for column " + cd->columnName);
        }
        addBigint(inline_fixed_encoding_null_val(cd->columnType));
      }
      break;
    }
    case kARRAY:
      if (is_null && cd->columnType.get_notnull()) {
        throw std::runtime_error("NULL for column " + cd->columnName);
      }
      if (IS_STRING(cd->columnType.get_subtype())) {
        std::vector<std::string>& string_vec = addStringArray();
        addBinaryStringArray(datum, string_vec);
      } else {
        if (!is_null) {
          addArray(TDatumToArrayDatum(datum, cd->columnType));
        } else {
          addArray(NullArray(cd->columnType));
        }
      }
      break;
    case kPOINT:
    case kLINESTRING:
    case kPOLYGON:
    case kMULTIPOLYGON:
      if (is_null) {
        if (cd->columnType.get_notnull()) {
          throw std::runtime_error("NULL for column " + cd->columnName);
        }
        addGeoString(std::string());
      } else {
        addGeoString(datum.val.str_val);
      }
      break;
    default:
      CHECK(false) << "TypedImportBuffer::add_value() does not support type " << type;
  }
}

Here is the call graph for this function:

size_t import_export::TypedImportBuffer::add_values	(	const ColumnDescriptor *	cd,
		const TColumn &	data
	)

Definition at line 954 of file Importer.cpp.

References addArray(), addStringArray(), bigint_buffer_, bool_buffer_, checked_malloc(), ColumnDescriptor::columnName, ColumnDescriptor::columnType, double_buffer_, float_buffer_, geo_string_buffer_, SQLTypeInfo::get_notnull(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), generate_TableFunctionsFactory_init::i, inline_fixed_encoding_null_val(), int_buffer_, IS_STRING, generate_TableFunctionsFactory_init::j, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, NULL_DOUBLE, NULL_FLOAT, import_export::NullArray(), smallint_buffer_, string_buffer_, and tinyint_buffer_.

                                                                                   {
  size_t dataSize = 0;
  if (cd->columnType.get_notnull()) {
    // We can't have any null values for this column; to have them is an error
    if (std::any_of(col.nulls.begin(), col.nulls.end(), [](int i) { return i != 0; })) {
      throw std::runtime_error("NULL for column " + cd->columnName);
    }
  }

  switch (cd->columnType.get_type()) {
    case kBOOLEAN: {
      dataSize = col.data.int_col.size();
      bool_buffer_->reserve(dataSize);
      for (size_t i = 0; i < dataSize; i++) {
        if (col.nulls[i]) {
          bool_buffer_->push_back(inline_fixed_encoding_null_val(cd->columnType));
        } else {
          bool_buffer_->push_back((int8_t)col.data.int_col[i]);
        }
      }
      break;
    }
    case kTINYINT: {
      dataSize = col.data.int_col.size();
      tinyint_buffer_->reserve(dataSize);
      for (size_t i = 0; i < dataSize; i++) {
        if (col.nulls[i]) {
          tinyint_buffer_->push_back(inline_fixed_encoding_null_val(cd->columnType));
        } else {
          tinyint_buffer_->push_back((int8_t)col.data.int_col[i]);
        }
      }
      break;
    }
    case kSMALLINT: {
      dataSize = col.data.int_col.size();
      smallint_buffer_->reserve(dataSize);
      for (size_t i = 0; i < dataSize; i++) {
        if (col.nulls[i]) {
          smallint_buffer_->push_back(inline_fixed_encoding_null_val(cd->columnType));
        } else {
          smallint_buffer_->push_back((int16_t)col.data.int_col[i]);
        }
      }
      break;
    }
    case kINT: {
      dataSize = col.data.int_col.size();
      int_buffer_->reserve(dataSize);
      for (size_t i = 0; i < dataSize; i++) {
        if (col.nulls[i]) {
          int_buffer_->push_back(inline_fixed_encoding_null_val(cd->columnType));
        } else {
          int_buffer_->push_back((int32_t)col.data.int_col[i]);
        }
      }
      break;
    }
    case kBIGINT:
    case kNUMERIC:
    case kDECIMAL: {
      dataSize = col.data.int_col.size();
      bigint_buffer_->reserve(dataSize);
      for (size_t i = 0; i < dataSize; i++) {
        if (col.nulls[i]) {
          bigint_buffer_->push_back(inline_fixed_encoding_null_val(cd->columnType));
        } else {
          bigint_buffer_->push_back((int64_t)col.data.int_col[i]);
        }
      }
      break;
    }
    case kFLOAT: {
      dataSize = col.data.real_col.size();
      float_buffer_->reserve(dataSize);
      for (size_t i = 0; i < dataSize; i++) {
        if (col.nulls[i]) {
          float_buffer_->push_back(NULL_FLOAT);
        } else {
          float_buffer_->push_back((float)col.data.real_col[i]);
        }
      }
      break;
    }
    case kDOUBLE: {
      dataSize = col.data.real_col.size();
      double_buffer_->reserve(dataSize);
      for (size_t i = 0; i < dataSize; i++) {
        if (col.nulls[i]) {
          double_buffer_->push_back(NULL_DOUBLE);
        } else {
          double_buffer_->push_back((double)col.data.real_col[i]);
        }
      }
      break;
    }
    case kTEXT:
    case kVARCHAR:
    case kCHAR: {
      // TODO: for now, use empty string for nulls
      dataSize = col.data.str_col.size();
      string_buffer_->reserve(dataSize);
      for (size_t i = 0; i < dataSize; i++) {
        if (col.nulls[i]) {
          string_buffer_->push_back(std::string());
        } else {
          string_buffer_->push_back(col.data.str_col[i]);
        }
      }
      break;
    }
    case kTIME:
    case kTIMESTAMP:
    case kDATE: {
      dataSize = col.data.int_col.size();
      bigint_buffer_->reserve(dataSize);
      for (size_t i = 0; i < dataSize; i++) {
        if (col.nulls[i]) {
          bigint_buffer_->push_back(inline_fixed_encoding_null_val(cd->columnType));
        } else {
          bigint_buffer_->push_back(static_cast<int64_t>(col.data.int_col[i]));
        }
      }
      break;
    }
    case kPOINT:
    case kLINESTRING:
    case kPOLYGON:
    case kMULTIPOLYGON: {
      dataSize = col.data.str_col.size();
      geo_string_buffer_->reserve(dataSize);
      for (size_t i = 0; i < dataSize; i++) {
        if (col.nulls[i]) {
          // TODO: add support for NULL geo
          geo_string_buffer_->push_back(std::string());
        } else {
          geo_string_buffer_->push_back(col.data.str_col[i]);
        }
      }
      break;
    }
    case kARRAY: {
      dataSize = col.data.arr_col.size();
      if (IS_STRING(cd->columnType.get_subtype())) {
        for (size_t i = 0; i < dataSize; i++) {
          std::vector<std::string>& string_vec = addStringArray();
          if (!col.nulls[i]) {
            size_t stringArrSize = col.data.arr_col[i].data.str_col.size();
            for (size_t str_idx = 0; str_idx != stringArrSize; ++str_idx) {
              string_vec.push_back(col.data.arr_col[i].data.str_col[str_idx]);
            }
          }
        }
      } else {
        auto elem_ti = cd->columnType.get_subtype();
        switch (elem_ti) {
          case kBOOLEAN: {
            for (size_t i = 0; i < dataSize; i++) {
              if (col.nulls[i]) {
                addArray(NullArray(cd->columnType));
              } else {
                size_t len = col.data.arr_col[i].data.int_col.size();
                size_t byteSize = len * sizeof(int8_t);
                int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
                int8_t* p = buf;
                for (size_t j = 0; j < len; ++j) {
                  *(bool*)p = static_cast<bool>(col.data.arr_col[i].data.int_col[j]);
                  p += sizeof(bool);
                }
                addArray(ArrayDatum(byteSize, buf, false));
              }
            }
            break;
          }
          case kTINYINT: {
            for (size_t i = 0; i < dataSize; i++) {
              if (col.nulls[i]) {
                addArray(NullArray(cd->columnType));
              } else {
                size_t len = col.data.arr_col[i].data.int_col.size();
                size_t byteSize = len * sizeof(int8_t);
                int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
                int8_t* p = buf;
                for (size_t j = 0; j < len; ++j) {
                  *(int8_t*)p = static_cast<int8_t>(col.data.arr_col[i].data.int_col[j]);
                  p += sizeof(int8_t);
                }
                addArray(ArrayDatum(byteSize, buf, false));
              }
            }
            break;
          }
          case kSMALLINT: {
            for (size_t i = 0; i < dataSize; i++) {
              if (col.nulls[i]) {
                addArray(NullArray(cd->columnType));
              } else {
                size_t len = col.data.arr_col[i].data.int_col.size();
                size_t byteSize = len * sizeof(int16_t);
                int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
                int8_t* p = buf;
                for (size_t j = 0; j < len; ++j) {
                  *(int16_t*)p =
                      static_cast<int16_t>(col.data.arr_col[i].data.int_col[j]);
                  p += sizeof(int16_t);
                }
                addArray(ArrayDatum(byteSize, buf, false));
              }
            }
            break;
          }
          case kINT: {
            for (size_t i = 0; i < dataSize; i++) {
              if (col.nulls[i]) {
                addArray(NullArray(cd->columnType));
              } else {
                size_t len = col.data.arr_col[i].data.int_col.size();
                size_t byteSize = len * sizeof(int32_t);
                int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
                int8_t* p = buf;
                for (size_t j = 0; j < len; ++j) {
                  *(int32_t*)p =
                      static_cast<int32_t>(col.data.arr_col[i].data.int_col[j]);
                  p += sizeof(int32_t);
                }
                addArray(ArrayDatum(byteSize, buf, false));
              }
            }
            break;
          }
          case kBIGINT:
          case kNUMERIC:
          case kDECIMAL: {
            for (size_t i = 0; i < dataSize; i++) {
              if (col.nulls[i]) {
                addArray(NullArray(cd->columnType));
              } else {
                size_t len = col.data.arr_col[i].data.int_col.size();
                size_t byteSize = len * sizeof(int64_t);
                int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
                int8_t* p = buf;
                for (size_t j = 0; j < len; ++j) {
                  *(int64_t*)p =
                      static_cast<int64_t>(col.data.arr_col[i].data.int_col[j]);
                  p += sizeof(int64_t);
                }
                addArray(ArrayDatum(byteSize, buf, false));
              }
            }
            break;
          }
          case kFLOAT: {
            for (size_t i = 0; i < dataSize; i++) {
              if (col.nulls[i]) {
                addArray(NullArray(cd->columnType));
              } else {
                size_t len = col.data.arr_col[i].data.real_col.size();
                size_t byteSize = len * sizeof(float);
                int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
                int8_t* p = buf;
                for (size_t j = 0; j < len; ++j) {
                  *(float*)p = static_cast<float>(col.data.arr_col[i].data.real_col[j]);
                  p += sizeof(float);
                }
                addArray(ArrayDatum(byteSize, buf, false));
              }
            }
            break;
          }
          case kDOUBLE: {
            for (size_t i = 0; i < dataSize; i++) {
              if (col.nulls[i]) {
                addArray(NullArray(cd->columnType));
              } else {
                size_t len = col.data.arr_col[i].data.real_col.size();
                size_t byteSize = len * sizeof(double);
                int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
                int8_t* p = buf;
                for (size_t j = 0; j < len; ++j) {
                  *(double*)p = static_cast<double>(col.data.arr_col[i].data.real_col[j]);
                  p += sizeof(double);
                }
                addArray(ArrayDatum(byteSize, buf, false));
              }
            }
            break;
          }
          case kTIME:
          case kTIMESTAMP:
          case kDATE: {
            for (size_t i = 0; i < dataSize; i++) {
              if (col.nulls[i]) {
                addArray(NullArray(cd->columnType));
              } else {
                size_t len = col.data.arr_col[i].data.int_col.size();
                size_t byteWidth = sizeof(int64_t);
                size_t byteSize = len * byteWidth;
                int8_t* buf = (int8_t*)checked_malloc(len * byteSize);
                int8_t* p = buf;
                for (size_t j = 0; j < len; ++j) {
                  *reinterpret_cast<int64_t*>(p) =
                      static_cast<int64_t>(col.data.arr_col[i].data.int_col[j]);
                  p += sizeof(int64_t);
                }
                addArray(ArrayDatum(byteSize, buf, false));
              }
            }
            break;
          }
          default:
            throw std::runtime_error("Invalid Array Type");
        }
      }
      break;
    }
    default:
      throw std::runtime_error("Invalid Type");
  }
  return dataSize;
}

Here is the call graph for this function:

void import_export::TypedImportBuffer::addArray ( const ArrayDatum &  v )

inline

Definition at line 241 of file Importer.h.

References array_buffer_.

Referenced by add_value(), and add_values().

{ array_buffer_->push_back(v); }

Here is the caller graph for this function:

void import_export::TypedImportBuffer::addBigint ( const int64_t  v )

inline

Definition at line 231 of file Importer.h.

References bigint_buffer_.

Referenced by add_value().

{ bigint_buffer_->push_back(v); }

Here is the caller graph for this function:

void import_export::TypedImportBuffer::addBoolean ( const int8_t  v )

inline

Definition at line 223 of file Importer.h.

References bool_buffer_.

Referenced by add_value().

{ bool_buffer_->push_back(v); }

Here is the caller graph for this function:

void import_export::TypedImportBuffer::addDictEncodedString

(

const std::vector< std::string > &

string_vec

)

Definition at line 475 of file Importer.cpp.

References CHECK, column_desc_, ColumnDescriptor::columnType, SQLTypeInfo::get_size(), StringDictionary::getOrAddBulk(), StringDictionary::MAX_STRLEN, string_dict_, string_dict_i16_buffer_, string_dict_i32_buffer_, and string_dict_i8_buffer_.

                                                                                   {
  CHECK(string_dict_);
  std::vector<std::string_view> string_view_vec;
  string_view_vec.reserve(string_vec.size());
  for (const auto& str : string_vec) {
    if (str.size() > StringDictionary::MAX_STRLEN) {
      throw std::runtime_error("String too long for dictionary encoding.");
    }
    string_view_vec.push_back(str);
  }
  switch (column_desc_->columnType.get_size()) {
    case 1:
      string_dict_i8_buffer_->resize(string_view_vec.size());
      string_dict_->getOrAddBulk(string_view_vec, string_dict_i8_buffer_->data());
      break;
    case 2:
      string_dict_i16_buffer_->resize(string_view_vec.size());
      string_dict_->getOrAddBulk(string_view_vec, string_dict_i16_buffer_->data());
      break;
    case 4:
      string_dict_i32_buffer_->resize(string_view_vec.size());
      string_dict_->getOrAddBulk(string_view_vec, string_dict_i32_buffer_->data());
      break;
    default:
      CHECK(false);
  }
}

Here is the call graph for this function:

void import_export::TypedImportBuffer::addDictEncodedStringArray ( const std::vector< std::vector< std::string >> &  string_array_vec )

inline

Definition at line 254 of file Importer.h.

References CHECK, checked_malloc(), StringDictionary::getOrAddBulkArray(), StringDictionary::MAX_STRLEN, string_array_dict_buffer_, and string_dict_.

                                                               {
    CHECK(string_dict_);

    // first check data is ok
    for (auto& p : string_array_vec) {
      for (const auto& str : p) {
        if (str.size() > StringDictionary::MAX_STRLEN) {
          throw std::runtime_error("String too long for dictionary encoding.");
        }
      }
    }

    std::vector<std::vector<int32_t>> ids_array(0);
    string_dict_->getOrAddBulkArray(string_array_vec, ids_array);

    for (auto& p : ids_array) {
      size_t len = p.size() * sizeof(int32_t);
      auto a = static_cast<int32_t*>(checked_malloc(len));
      memcpy(a, &p[0], len);
      // TODO: distinguish between empty and NULL
      string_array_dict_buffer_->push_back(
          ArrayDatum(len, reinterpret_cast<int8_t*>(a), len == 0));
    }
  }

Here is the call graph for this function:

void import_export::TypedImportBuffer::addDouble ( const double  v )

inline

Definition at line 235 of file Importer.h.

References double_buffer_.

Referenced by add_value().

{ double_buffer_->push_back(v); }

Here is the caller graph for this function:

void import_export::TypedImportBuffer::addFloat ( const float  v )

inline

Definition at line 233 of file Importer.h.

References float_buffer_.

Referenced by add_value().

{ float_buffer_->push_back(v); }

Here is the caller graph for this function:

void import_export::TypedImportBuffer::addGeoString ( const std::string_view  v )

inline

Definition at line 239 of file Importer.h.

References geo_string_buffer_.

Referenced by add_value().

{ geo_string_buffer_->emplace_back(v); }

Here is the caller graph for this function:

void import_export::TypedImportBuffer::addInt ( const int32_t  v )

inline

Definition at line 229 of file Importer.h.

References int_buffer_.

Referenced by add_value().

{ int_buffer_->push_back(v); }

Here is the caller graph for this function:

void import_export::TypedImportBuffer::addSmallint ( const int16_t  v )

inline

Definition at line 227 of file Importer.h.

References smallint_buffer_.

Referenced by add_value().

{ smallint_buffer_->push_back(v); }

Here is the caller graph for this function:

void import_export::TypedImportBuffer::addString ( const std::string_view  v )

inline

Definition at line 237 of file Importer.h.

References string_buffer_.

Referenced by add_value().

{ string_buffer_->emplace_back(v); }

Here is the caller graph for this function:

std::vector<std::string>& import_export::TypedImportBuffer::addStringArray ( )

inline

Definition at line 243 of file Importer.h.

References string_array_buffer_.

Referenced by add_value(), and add_values().

                                         {
    string_array_buffer_->emplace_back();
    return string_array_buffer_->back();
  }

Here is the caller graph for this function:

void import_export::TypedImportBuffer::addStringArray ( const std::vector< std::string > &  arr )

inline

Definition at line 248 of file Importer.h.

References string_array_buffer_.

                                                       {
    string_array_buffer_->push_back(arr);
  }

void import_export::TypedImportBuffer::addTinyint ( const int8_t  v )

inline

Definition at line 225 of file Importer.h.

References tinyint_buffer_.

Referenced by add_value().

{ tinyint_buffer_->push_back(v); }

Here is the caller graph for this function:

void import_export::TypedImportBuffer::clear ( )

inline

Definition at line 374 of file Importer.h.

References array_buffer_, bigint_buffer_, bool_buffer_, CHECK, column_desc_, ColumnDescriptor::columnType, double_buffer_, float_buffer_, geo_string_buffer_, SQLTypeInfo::get_compression(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), int_buffer_, IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDECIMAL, kDOUBLE, kENCODING_DICT, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kNUMERIC, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, smallint_buffer_, string_array_buffer_, string_array_dict_buffer_, string_buffer_, string_dict_i16_buffer_, string_dict_i32_buffer_, string_dict_i8_buffer_, and tinyint_buffer_.

               {
    switch (column_desc_->columnType.get_type()) {
      case kBOOLEAN: {
        bool_buffer_->clear();
        break;
      }
      case kTINYINT: {
        tinyint_buffer_->clear();
        break;
      }
      case kSMALLINT: {
        smallint_buffer_->clear();
        break;
      }
      case kINT: {
        int_buffer_->clear();
        break;
      }
      case kBIGINT:
      case kNUMERIC:
      case kDECIMAL: {
        bigint_buffer_->clear();
        break;
      }
      case kFLOAT: {
        float_buffer_->clear();
        break;
      }
      case kDOUBLE: {
        double_buffer_->clear();
        break;
      }
      case kTEXT:
      case kVARCHAR:
      case kCHAR: {
        string_buffer_->clear();
        if (column_desc_->columnType.get_compression() == kENCODING_DICT) {
          switch (column_desc_->columnType.get_size()) {
            case 1:
              string_dict_i8_buffer_->clear();
              break;
            case 2:
              string_dict_i16_buffer_->clear();
              break;
            case 4:
              string_dict_i32_buffer_->clear();
              break;
            default:
              CHECK(false);
          }
        }
        break;
      }
      case kDATE:
      case kTIME:
      case kTIMESTAMP:
        bigint_buffer_->clear();
        break;
      case kARRAY: {
        if (IS_STRING(column_desc_->columnType.get_subtype())) {
          string_array_buffer_->clear();
          string_array_dict_buffer_->clear();
        } else {
          array_buffer_->clear();
        }
        break;
      }
      case kPOINT:
      case kLINESTRING:
      case kPOLYGON:
      case kMULTIPOLYGON:
        geo_string_buffer_->clear();
        break;
      default:
        CHECK(false);
    }
  }

Here is the call graph for this function:

template<typename DATA_TYPE >

size_t import_export::TypedImportBuffer::convert_arrow_val_to_import_buffer	(	const ColumnDescriptor *	cd,
		const arrow::Array &	array,
		std::vector< DATA_TYPE > &	buffer,
		const ArraySliceRange &	slice_range,
		BadRowsTracker *const	bad_rows_tracker
	)

Referenced by add_arrow_values().

Here is the caller graph for this function:

template<typename DATA_TYPE >

size_t import_export::TypedImportBuffer::convert_arrow_val_to_import_buffer	(	const ColumnDescriptor *	cd,
		const Array &	array,
		std::vector< DATA_TYPE > &	buffer,
		const ArraySliceRange &	slice_range,
		import_export::BadRowsTracker *const	bad_rows_tracker
	)

Definition at line 768 of file Importer.cpp.

References col_idx, anonymous_namespace{ArrowImporter.h}::error_context(), geo_string_buffer_, SQLTypeInfo::get_type(), import_export::Importer::getCatalog(), Geospatial::GeoTypesFactory::getGeoColumns(), Geospatial::GeoTypesFactory::getNullGeoColumns(), import_buffers, import_export::BadRowsTracker::importer, import_export::BadRowsTracker::mutex, import_export::BadRowsTracker::rows, import_export::Importer::set_geo_physical_import_buffer(), and anonymous_namespace{ArrowImporter.h}::value_getter().

                                                         {
  auto data =
      std::make_unique<DataBuffer<DATA_TYPE>>(cd, array, buffer, bad_rows_tracker);
  auto f_value_getter = value_getter(array, cd, bad_rows_tracker);
  std::function<void(const int64_t)> f_add_geo_phy_cols = [&](const int64_t row) {};
  if (bad_rows_tracker && cd->columnType.is_geometry()) {
    f_add_geo_phy_cols = [&](const int64_t row) {
      // Populate physical columns (ref. DBHandler::load_table)
      std::vector<double> coords, bounds;
      std::vector<int> ring_sizes, poly_rings;
      int render_group = 0;
      SQLTypeInfo ti;
      // replace any unexpected exception from getGeoColumns or other
      // on this path with a GeoImportException so that we wont over
      // push a null to the logical column...
      try {
        SQLTypeInfo import_ti{ti};
        if (array.IsNull(row)) {
          Geospatial::GeoTypesFactory::getNullGeoColumns(
              import_ti, coords, bounds, ring_sizes, poly_rings, false);
        } else {
          arrow_throw_if<GeoImportException>(
              !Geospatial::GeoTypesFactory::getGeoColumns(geo_string_buffer_->back(),
                                                          ti,
                                                          coords,
                                                          bounds,
                                                          ring_sizes,
                                                          poly_rings,
                                                          false),
              error_context(cd, bad_rows_tracker) + "Invalid geometry");
          arrow_throw_if<GeoImportException>(
              cd->columnType.get_type() != ti.get_type(),
              error_context(cd, bad_rows_tracker) + "Geometry type mismatch");
        }
        auto col_idx_workpad = col_idx;  // what a pitfall!!
        import_export::Importer::set_geo_physical_import_buffer(
            bad_rows_tracker->importer->getCatalog(),
            cd,
            *import_buffers,
            col_idx_workpad,
            coords,
            bounds,
            ring_sizes,
            poly_rings,
            render_group);
      } catch (GeoImportException&) {
        throw;
      } catch (std::runtime_error& e) {
        throw GeoImportException(e.what());
      } catch (const std::exception& e) {
        throw GeoImportException(e.what());
      } catch (...) {
        throw GeoImportException("unknown exception");
      }
    };
  }
  auto f_mark_a_bad_row = [&](const auto row) {
    std::unique_lock<std::mutex> lck(bad_rows_tracker->mutex);
    bad_rows_tracker->rows.insert(row - slice_range.first);
  };
  buffer.reserve(slice_range.second - slice_range.first);
  for (size_t row = slice_range.first; row < slice_range.second; ++row) {
    try {
      *data << (array.IsNull(row) ? nullptr : f_value_getter(array, row));
      f_add_geo_phy_cols(row);
    } catch (GeoImportException&) {
      f_mark_a_bad_row(row);
    } catch (ArrowImporterException&) {
      // trace bad rows of each column; otherwise rethrow.
      if (bad_rows_tracker) {
        *data << nullptr;
        f_mark_a_bad_row(row);
      } else {
        throw;
      }
    }
  }
  return buffer.size();
}

Here is the call graph for this function:

template<typename DATA_TYPE >

auto import_export::TypedImportBuffer::del_values	(	std::vector< DATA_TYPE > &	buffer,
		BadRowsTracker *const	bad_rows_tracker
	)

auto import_export::TypedImportBuffer::del_values	(	const SQLTypes	type,
		BadRowsTracker *const	bad_rows_tracker
	)

int64_t import_export::TypedImportBuffer::get_replicate_count ( ) const

inline

Definition at line 473 of file Importer.h.

References replicate_count_.

{ return replicate_count_; }

std::vector<ArrayDatum>* import_export::TypedImportBuffer::getArrayBuffer ( ) const

inline

Definition at line 344 of file Importer.h.

References array_buffer_.

{ return array_buffer_; }

int8_t* import_export::TypedImportBuffer::getAsBytes ( ) const

inline

Definition at line 286 of file Importer.h.

References column_desc_, ColumnDescriptor::columnType, SQLTypeInfo::get_type(), kBIGINT, kBOOLEAN, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kNUMERIC, kSMALLINT, kTIME, kTIMESTAMP, and kTINYINT.

Referenced by import_export::anonymous_namespace{Importer.cpp}::double_value_at(), import_export::anonymous_namespace{Importer.cpp}::float_value_at(), and import_export::anonymous_namespace{Importer.cpp}::int_value_at().

                             {
    switch (column_desc_->columnType.get_type()) {
      case kBOOLEAN:
        return reinterpret_cast<int8_t*>(&((*bool_buffer_)[0]));
      case kTINYINT:
        return reinterpret_cast<int8_t*>(&((*tinyint_buffer_)[0]));
      case kSMALLINT:
        return reinterpret_cast<int8_t*>(&((*smallint_buffer_)[0]));
      case kINT:
        return reinterpret_cast<int8_t*>(&((*int_buffer_)[0]));
      case kBIGINT:
      case kNUMERIC:
      case kDECIMAL:
        return reinterpret_cast<int8_t*>(&((*bigint_buffer_)[0]));
      case kFLOAT:
        return reinterpret_cast<int8_t*>(&((*float_buffer_)[0]));
      case kDOUBLE:
        return reinterpret_cast<int8_t*>(&((*double_buffer_)[0]));
      case kDATE:
      case kTIME:
      case kTIMESTAMP:
        return reinterpret_cast<int8_t*>(&((*bigint_buffer_)[0]));
      default:
        abort();
    }
  }

Here is the call graph for this function:

Here is the caller graph for this function:

const ColumnDescriptor* import_export::TypedImportBuffer::getColumnDesc ( ) const

inline

Definition at line 282 of file Importer.h.

References column_desc_.

{ return column_desc_; }

size_t import_export::TypedImportBuffer::getElementSize ( ) const

inline

Definition at line 313 of file Importer.h.

References column_desc_, ColumnDescriptor::columnType, SQLTypeInfo::get_type(), kBIGINT, kBOOLEAN, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kNUMERIC, kSMALLINT, kTIME, kTIMESTAMP, and kTINYINT.

                                {
    switch (column_desc_->columnType.get_type()) {
      case kBOOLEAN:
        return sizeof((*bool_buffer_)[0]);
      case kTINYINT:
        return sizeof((*tinyint_buffer_)[0]);
      case kSMALLINT:
        return sizeof((*smallint_buffer_)[0]);
      case kINT:
        return sizeof((*int_buffer_)[0]);
      case kBIGINT:
      case kNUMERIC:
      case kDECIMAL:
        return sizeof((*bigint_buffer_)[0]);
      case kFLOAT:
        return sizeof((*float_buffer_)[0]);
      case kDOUBLE:
        return sizeof((*double_buffer_)[0]);
      case kDATE:
      case kTIME:
      case kTIMESTAMP:
        return sizeof((*bigint_buffer_)[0]);
      default:
        abort();
    }
  }

Here is the call graph for this function:

std::vector<std::string>* import_export::TypedImportBuffer::getGeoStringBuffer ( ) const

inline

Definition at line 342 of file Importer.h.

References geo_string_buffer_.

{ return geo_string_buffer_; }

std::vector<std::vector<std::string> >* import_export::TypedImportBuffer::getStringArrayBuffer ( ) const

inline

Definition at line 346 of file Importer.h.

References string_array_buffer_.

                                                                {
    return string_array_buffer_;
  }

std::vector<ArrayDatum>* import_export::TypedImportBuffer::getStringArrayDictBuffer ( ) const

inline

Definition at line 350 of file Importer.h.

References string_array_dict_buffer_.

                                                          {
    return string_array_dict_buffer_;
  }

std::vector<std::string>* import_export::TypedImportBuffer::getStringBuffer ( ) const

inline

Definition at line 340 of file Importer.h.

References string_buffer_.

{ return string_buffer_; }

int8_t* import_export::TypedImportBuffer::getStringDictBuffer ( ) const

inline

Definition at line 354 of file Importer.h.

References column_desc_, ColumnDescriptor::columnType, and SQLTypeInfo::get_size().

Referenced by import_export::anonymous_namespace{Importer.cpp}::int_value_at().

                                      {
    switch (column_desc_->columnType.get_size()) {
      case 1:
        return reinterpret_cast<int8_t*>(&((*string_dict_i8_buffer_)[0]));
      case 2:
        return reinterpret_cast<int8_t*>(&((*string_dict_i16_buffer_)[0]));
      case 4:
        return reinterpret_cast<int8_t*>(&((*string_dict_i32_buffer_)[0]));
      default:
        abort();
    }
  }

Here is the call graph for this function:

Here is the caller graph for this function:

StringDictionary* import_export::TypedImportBuffer::getStringDictionary ( ) const

inline

Definition at line 284 of file Importer.h.

References string_dict_.

{ return string_dict_; }

const SQLTypeInfo& import_export::TypedImportBuffer::getTypeInfo ( ) const

inline

Definition at line 280 of file Importer.h.

References column_desc_, and ColumnDescriptor::columnType.

Referenced by import_export::anonymous_namespace{Importer.cpp}::double_value_at(), import_export::anonymous_namespace{Importer.cpp}::float_value_at(), and import_export::anonymous_namespace{Importer.cpp}::int_value_at().

{ return column_desc_->columnType; }

Here is the caller graph for this function:

void import_export::TypedImportBuffer::pop_value

(

)

Definition at line 708 of file Importer.cpp.

References array_buffer_, bigint_buffer_, bool_buffer_, CHECK, column_desc_, ColumnDescriptor::columnType, decimal_to_int_type(), double_buffer_, float_buffer_, geo_string_buffer_, SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), int_buffer_, SQLTypeInfo::is_decimal(), IS_STRING, kARRAY, kBIGINT, kBOOLEAN, kCHAR, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, kVARCHAR, smallint_buffer_, string_array_buffer_, string_buffer_, tinyint_buffer_, and run_benchmark_import::type.

                                  {
  const auto type = column_desc_->columnType.is_decimal()
                        ? decimal_to_int_type(column_desc_->columnType)
                        : column_desc_->columnType.get_type();
  switch (type) {
    case kBOOLEAN:
      bool_buffer_->pop_back();
      break;
    case kTINYINT:
      tinyint_buffer_->pop_back();
      break;
    case kSMALLINT:
      smallint_buffer_->pop_back();
      break;
    case kINT:
      int_buffer_->pop_back();
      break;
    case kBIGINT:
      bigint_buffer_->pop_back();
      break;
    case kFLOAT:
      float_buffer_->pop_back();
      break;
    case kDOUBLE:
      double_buffer_->pop_back();
      break;
    case kTEXT:
    case kVARCHAR:
    case kCHAR:
      string_buffer_->pop_back();
      break;
    case kDATE:
    case kTIME:
    case kTIMESTAMP:
      bigint_buffer_->pop_back();
      break;
    case kARRAY:
      if (IS_STRING(column_desc_->columnType.get_subtype())) {
        string_array_buffer_->pop_back();
      } else {
        array_buffer_->pop_back();
      }
      break;
    case kPOINT:
    case kLINESTRING:
    case kPOLYGON:
    case kMULTIPOLYGON:
      geo_string_buffer_->pop_back();
      break;
    default:
      CHECK(false) << "TypedImportBuffer::pop_value() does not support type " << type;
  }
}

Here is the call graph for this function:

void import_export::TypedImportBuffer::set_replicate_count ( const int64_t  replicate_count )

inline

Definition at line 474 of file Importer.h.

References replicate_count_.

Referenced by add_value().

                                                          {
    replicate_count_ = replicate_count;
  }

Here is the caller graph for this function:

bool import_export::TypedImportBuffer::stringDictCheckpoint ( )

inline

Definition at line 367 of file Importer.h.

References StringDictionary::checkpoint(), and string_dict_.

                              {
    if (string_dict_ == nullptr) {
      return true;
    }
    return string_dict_->checkpoint();
  }

Here is the call graph for this function:

Member Data Documentation

union { ... }

union { ... }

std::vector<ArrayDatum>* import_export::TypedImportBuffer::array_buffer_

Definition at line 500 of file Importer.h.

Referenced by addArray(), clear(), getArrayBuffer(), pop_value(), TypedImportBuffer(), and ~TypedImportBuffer().

std::vector<int64_t>* import_export::TypedImportBuffer::bigint_buffer_

Definition at line 495 of file Importer.h.

Referenced by add_arrow_values(), add_values(), addBigint(), clear(), pop_value(), TypedImportBuffer(), and ~TypedImportBuffer().

std::vector<int8_t>* import_export::TypedImportBuffer::bool_buffer_

Definition at line 491 of file Importer.h.

Referenced by add_arrow_values(), add_values(), addBoolean(), clear(), pop_value(), TypedImportBuffer(), and ~TypedImportBuffer().

size_t import_export::TypedImportBuffer::col_idx

Definition at line 487 of file Importer.h.

Referenced by convert_arrow_val_to_import_buffer().

const ColumnDescriptor* import_export::TypedImportBuffer::column_desc_

private

Definition at line 509 of file Importer.h.

Referenced by addDictEncodedString(), clear(), getAsBytes(), getColumnDesc(), getElementSize(), getStringDictBuffer(), getTypeInfo(), pop_value(), and ~TypedImportBuffer().

std::vector<double>* import_export::TypedImportBuffer::double_buffer_

Definition at line 497 of file Importer.h.

Referenced by add_arrow_values(), add_values(), addDouble(), clear(), pop_value(), TypedImportBuffer(), and ~TypedImportBuffer().

std::vector<float>* import_export::TypedImportBuffer::float_buffer_

Definition at line 496 of file Importer.h.

Referenced by add_arrow_values(), add_values(), addFloat(), clear(), pop_value(), TypedImportBuffer(), and ~TypedImportBuffer().

std::vector<std::string>* import_export::TypedImportBuffer::geo_string_buffer_

Definition at line 499 of file Importer.h.

Referenced by add_arrow_values(), add_values(), addGeoString(), clear(), convert_arrow_val_to_import_buffer(), getGeoStringBuffer(), pop_value(), TypedImportBuffer(), and ~TypedImportBuffer().

std::vector<std::unique_ptr<TypedImportBuffer> >* import_export::TypedImportBuffer::import_buffers

Definition at line 486 of file Importer.h.

Referenced by convert_arrow_val_to_import_buffer().

std::vector<int32_t>* import_export::TypedImportBuffer::int_buffer_

Definition at line 494 of file Importer.h.

Referenced by add_arrow_values(), add_values(), addInt(), clear(), pop_value(), TypedImportBuffer(), and ~TypedImportBuffer().

size_t import_export::TypedImportBuffer::replicate_count_ = 0

private

Definition at line 511 of file Importer.h.

Referenced by get_replicate_count(), and set_replicate_count().

std::vector<int16_t>* import_export::TypedImportBuffer::smallint_buffer_

Definition at line 493 of file Importer.h.

Referenced by add_arrow_values(), add_values(), addSmallint(), clear(), pop_value(), TypedImportBuffer(), and ~TypedImportBuffer().

std::vector<std::vector<std::string> >* import_export::TypedImportBuffer::string_array_buffer_

Definition at line 501 of file Importer.h.

Referenced by addStringArray(), clear(), getStringArrayBuffer(), pop_value(), TypedImportBuffer(), and ~TypedImportBuffer().

std::vector<ArrayDatum>* import_export::TypedImportBuffer::string_array_dict_buffer_

Definition at line 507 of file Importer.h.

Referenced by addDictEncodedStringArray(), clear(), getStringArrayDictBuffer(), TypedImportBuffer(), and ~TypedImportBuffer().

std::vector<std::string>* import_export::TypedImportBuffer::string_buffer_

Definition at line 498 of file Importer.h.

Referenced by add_arrow_values(), add_values(), addString(), clear(), getStringBuffer(), pop_value(), TypedImportBuffer(), and ~TypedImportBuffer().

StringDictionary* import_export::TypedImportBuffer::string_dict_

private

Definition at line 510 of file Importer.h.

Referenced by addDictEncodedString(), addDictEncodedStringArray(), getStringDictionary(), and stringDictCheckpoint().

std::vector<uint16_t>* import_export::TypedImportBuffer::string_dict_i16_buffer_

Definition at line 505 of file Importer.h.

Referenced by addDictEncodedString(), clear(), TypedImportBuffer(), and ~TypedImportBuffer().

std::vector<int32_t>* import_export::TypedImportBuffer::string_dict_i32_buffer_

Definition at line 506 of file Importer.h.

Referenced by addDictEncodedString(), clear(), TypedImportBuffer(), and ~TypedImportBuffer().

std::vector<uint8_t>* import_export::TypedImportBuffer::string_dict_i8_buffer_

Definition at line 504 of file Importer.h.

Referenced by addDictEncodedString(), clear(), TypedImportBuffer(), and ~TypedImportBuffer().

std::vector<int8_t>* import_export::TypedImportBuffer::tinyint_buffer_

Definition at line 492 of file Importer.h.

Referenced by add_arrow_values(), add_values(), addTinyint(), clear(), pop_value(), TypedImportBuffer(), and ~TypedImportBuffer().

---

The documentation for this class was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/ImportExport/Importer.h
• /home/jenkins-slave/workspace/core-os-doxygen/ImportExport/Importer.cpp