Namespaces
	anonymous_namespace{Importer.cpp}

	anonymous_namespace{QueryExporterGDAL.cpp}

	delimited_parser

Classes
struct	CopyParams

struct	GeoImportException

struct	BadRowsTracker

class	TypedImportBuffer

class	Loader

struct	ImportStatus

class	DataStreamSink

class	Detector

class	ImporterUtils

class	RenderGroupAnalyzer

class	Importer

class	QueryExporter

class	QueryExporterCSV

class	QueryExporterGDAL

Typedefs
using	FieldNameToIndexMapType = std::map< std::string, size_t >

using	ColumnNameToSourceNameMapType = std::map< std::string, std::string >

using	ColumnIdToRenderGroupAnalyzerMapType = std::map< int, std::shared_ptr< RenderGroupAnalyzer >>

using	FeaturePtrVector = std::vector< OGRFeatureUqPtr >

using	ArraySliceRange = std::pair< size_t, size_t >

Enumerations
enum	FileType { FileType::DELIMITED, FileType::POLYGON }

enum	ImportHeaderRow { ImportHeaderRow::AUTODETECT, ImportHeaderRow::NO_HEADER, ImportHeaderRow::HAS_HEADER }

Functions
static const std::string	trim_space (const char *field, const size_t len)

Datum	NullDatum (SQLTypeInfo &ti)

Datum	NullArrayDatum (SQLTypeInfo &ti)

ArrayDatum	StringToArray (const std::string &s, const SQLTypeInfo &ti, const CopyParams &copy_params)

ArrayDatum	NullArray (const SQLTypeInfo &ti)

void	addBinaryStringArray (const TDatum &datum, std::vector< std::string > &string_vec)

Datum	TDatumToDatum (const TDatum &datum, SQLTypeInfo &ti)

ArrayDatum	TDatumToArrayDatum (const TDatum &datum, const SQLTypeInfo &ti)

bool	importGeoFromLonLat (double lon, double lat, std::vector< double > &coords, SQLTypeInfo &ti)

static ImportStatus	import_thread_delimited (int thread_id, Importer importer, std::unique_ptr< char[]> scratch_buffer, size_t begin_pos, size_t end_pos, size_t total_size, const ColumnIdToRenderGroupAnalyzerMapType &columnIdToRenderGroupAnalyzerMap, size_t first_row_index_this_buffer, const Catalog_Namespace::SessionInfo session_info, Executor *executor)

static ImportStatus	import_thread_shapefile (int thread_id, Importer importer, OGRSpatialReference poGeographicSR, const FeaturePtrVector &features, size_t firstFeature, size_t numFeatures, const FieldNameToIndexMapType &fieldNameToIndexMap, const ColumnNameToSourceNameMapType &columnNameToSourceNameMap, const ColumnIdToRenderGroupAnalyzerMapType &columnIdToRenderGroupAnalyzerMap, const Catalog_Namespace::SessionInfo session_info, Executor executor)

template<class T >
bool	try_cast (const std::string &str)

std::pair< SQLTypes, bool >	ogr_to_type (const OGRFieldType &ogr_type)

SQLTypes	ogr_to_type (const OGRwkbGeometryType &ogr_type)

void	gdalGatherFilesInArchiveRecursive (const std::string &archive_path, std::vector< std::string > &files)

std::vector< std::unique_ptr < TypedImportBuffer > >	setup_column_loaders (const TableDescriptor td, Loader loader)

std::vector< std::unique_ptr < TypedImportBuffer > >	fill_missing_columns (const Catalog_Namespace::Catalog *cat, Fragmenter_Namespace::InsertData &insert_data)

Variables
static constexpr size_t	kImportFileBufferSize = (1 << 23)

static constexpr bool	PROMOTE_POLYGON_TO_MULTIPOLYGON = true

static mapd_shared_mutex	status_mutex

static std::map< std::string, ImportStatus >	import_status_map

Typedef Documentation

using import_export::ArraySliceRange = typedef std::pair<size_t, size_t>

Definition at line 72 of file Importer.h.

using import_export::ColumnIdToRenderGroupAnalyzerMapType = typedef std::map<int, std::shared_ptr<RenderGroupAnalyzer>>

Definition at line 157 of file Importer.cpp.

using import_export::ColumnNameToSourceNameMapType = typedef std::map<std::string, std::string>

Definition at line 155 of file Importer.cpp.

using import_export::FeaturePtrVector = typedef std::vector<OGRFeatureUqPtr>

Definition at line 158 of file Importer.cpp.

using import_export::FieldNameToIndexMapType = typedef std::map<std::string, size_t>

Definition at line 154 of file Importer.cpp.

Enumeration Type Documentation

enum import_export::FileType

strong

Enumerator
DELIMITED
POLYGON

Definition at line 34 of file CopyParams.h.

                     {
   DELIMITED,
   POLYGON
 #ifdef ENABLE_IMPORT_PARQUET
   ,
   PARQUET
 #endif
 };

enum import_export::ImportHeaderRow

strong

Enumerator
AUTODETECT
NO_HEADER
HAS_HEADER

Definition at line 43 of file CopyParams.h.

43 { AUTODETECT, NO_HEADER, HAS_HEADER };

import_export::ImportHeaderRow::HAS_HEADER

import_export::ImportHeaderRow::AUTODETECT

import_export::ImportHeaderRow::NO_HEADER

Function Documentation

void import_export::addBinaryStringArray	(	const TDatum &	datum,
		std::vector< std::string > &	string_vec
	)

Definition at line 421 of file Importer.cpp.

Referenced by import_export::TypedImportBuffer::add_value().

                                                                                  {
   const auto& arr = datum.val.arr_val;
   for (const auto& elem_datum : arr) {
     string_vec.push_back(elem_datum.val.str_val);
   }
 }

Here is the caller graph for this function:

std::vector< std::unique_ptr< TypedImportBuffer > > import_export::fill_missing_columns	(	const Catalog_Namespace::Catalog *	cat,
		Fragmenter_Namespace::InsertData &	insert_data
	)

Definition at line 5365 of file Importer.cpp.

References CHECK, ColumnDescriptor::columnId, Fragmenter_Namespace::InsertData::columnIds, ColumnDescriptor::columnName, ColumnDescriptor::columnType, Fragmenter_Namespace::InsertData::data, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), import_export::TypedImportBuffer::get_data_block_pointers(), SQLTypeInfo::get_physical_cols(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), Catalog_Namespace::Catalog::getAllColumnMetadataForTable(), Geospatial::GeoTypesFactory::getGeoColumns(), Catalog_Namespace::Catalog::getMetadataForDict(), i, Fragmenter_Namespace::InsertData::is_default, SQLTypeInfo::is_geometry(), IS_STRING, SQLTypeInfo::is_string(), kARRAY, kENCODING_DICT, import_export::Importer::set_geo_physical_import_buffer(), gpu_enabled::sort(), and Fragmenter_Namespace::InsertData::tableId.

Referenced by RelAlgExecutor::executeSimpleInsert(), DBHandler::insert_data(), and Parser::InsertIntoTableAsSelectStmt::populateData().

                                                  {
   std::vector<std::unique_ptr<import_export::TypedImportBuffer>> defaults_buffers;
   if (insert_data.is_default.size() == 0) {
     insert_data.is_default.resize(insert_data.columnIds.size(), false);
   }
   CHECK(insert_data.is_default.size() == insert_data.is_default.size());
   auto cds = cat->getAllColumnMetadataForTable(insert_data.tableId, false, false, true);
   if (cds.size() == insert_data.columnIds.size()) {
     // all columns specified
     return defaults_buffers;
   }
   for (auto cd : cds) {
     if (std::find(insert_data.columnIds.begin(),
                   insert_data.columnIds.end(),
                   cd->columnId) == insert_data.columnIds.end()) {
       StringDictionary* dict = nullptr;
       if (cd->columnType.get_type() == kARRAY &&
           IS_STRING(cd->columnType.get_subtype())) {
         throw std::runtime_error("Cannot omit column \"" + cd->columnName +
                                  "\": omitting TEXT arrays is not supported yet");
       }
       if (cd->columnType.is_string() &&
           cd->columnType.get_compression() == kENCODING_DICT) {
         dict = cat->getMetadataForDict(cd->columnType.get_comp_param())->stringDict.get();
       }
       defaults_buffers.emplace_back(std::make_unique<TypedImportBuffer>(cd, dict));
     }
   }
   // put buffers in order to fill geo sub-columns properly
   std::sort(defaults_buffers.begin(),
             defaults_buffers.end(),
             [](decltype(defaults_buffers[0])& a, decltype(defaults_buffers[0])& b) {
               return a->getColumnDesc()->columnId < b->getColumnDesc()->columnId;
             });
   for (size_t i = 0; i < defaults_buffers.size(); ++i) {
     auto cd = defaults_buffers[i]->getColumnDesc();
     defaults_buffers[i]->add_value(cd, "NULL", true, import_export::CopyParams());
     if (cd->columnType.is_geometry()) {
       std::vector<double> coords, bounds;
       std::vector<int> ring_sizes, poly_rings;
       int render_group = 0;
       SQLTypeInfo tinfo{cd->columnType};
       CHECK(Geospatial::GeoTypesFactory::getGeoColumns(
           "NULL", tinfo, coords, bounds, ring_sizes, poly_rings, false));
       // set physical columns starting with the following ID
       auto next_col = i + 1;
       import_export::Importer::set_geo_physical_import_buffer(*cat,
                                                               cd,
                                                               defaults_buffers,
                                                               next_col,
                                                               coords,
                                                               bounds,
                                                               ring_sizes,
                                                               poly_rings,
                                                               render_group);
       // skip physical columns filled with the call above
       i += cd->columnType.get_physical_cols();
     }
   }
   auto data = import_export::TypedImportBuffer::get_data_block_pointers(defaults_buffers);
   CHECK(data.size() == defaults_buffers.size());
   for (size_t i = 0; i < defaults_buffers.size(); ++i) {
     insert_data.data.push_back(data[i]);
     insert_data.columnIds.push_back(defaults_buffers[i]->getColumnDesc()->columnId);
     insert_data.is_default.push_back(true);
   }
   return defaults_buffers;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void import_export::gdalGatherFilesInArchiveRecursive	(	const std::string &	archive_path,
		std::vector< std::string > &	files
	)

Definition at line 4745 of file Importer.cpp.

References LOG, run_benchmark_import::result, and logger::WARNING.

Referenced by import_export::Importer::gdalGetAllFilesInArchive().

                                                                     {
   // prepare to gather subdirectories
   std::vector<std::string> subdirectories;
 
   // get entries
   char** entries = VSIReadDir(archive_path.c_str());
   if (!entries) {
     LOG(WARNING) << "Failed to get file listing at archive: " << archive_path;
     return;
   }
 
   // force scope
   {
     // request clean-up
     ScopeGuard entries_guard = [&] { CSLDestroy(entries); };
 
     // check all the entries
     int index = 0;
     while (true) {
       // get next entry, or drop out if there isn't one
       char* entry_c = entries[index++];
       if (!entry_c) {
         break;
       }
       std::string entry(entry_c);
 
       // ignore '.' and '..'
       if (entry == "." || entry == "..") {
         continue;
       }
 
       // build the full path
       std::string entry_path = archive_path + std::string("/") + entry;
 
       // is it a file or a sub-folder
       VSIStatBufL sb;
       int result = VSIStatExL(entry_path.c_str(), &sb, VSI_STAT_NATURE_FLAG);
       if (result < 0) {
         break;
       }
 
       if (VSI_ISDIR(sb.st_mode)) {
         // a directory that ends with .gdb could be a Geodatabase bundle
         // arguably dangerous to decide this purely by name, but any further
         // validation would be very complex especially at this scope
         if (boost::iends_with(entry_path, ".gdb")) {
           // add the directory as if it was a file and don't recurse into it
           files.push_back(entry_path);
         } else {
           // add subdirectory to be recursed into
           subdirectories.push_back(entry_path);
         }
       } else {
         // add this file
         files.push_back(entry_path);
       }
     }
   }
 
   // recurse into each subdirectories we found
   for (const auto& subdirectory : subdirectories) {
     gdalGatherFilesInArchiveRecursive(subdirectory, files);
   }
 }

Here is the caller graph for this function:

static ImportStatus import_export::import_thread_delimited	(	int	thread_id,
		Importer *	importer,
		std::unique_ptr< char[]>	scratch_buffer,
		size_t	begin_pos,
		size_t	end_pos,
		size_t	total_size,
		const ColumnIdToRenderGroupAnalyzerMapType &	columnIdToRenderGroupAnalyzerMap,
		size_t	first_row_index_this_buffer,
		const Catalog_Namespace::SessionInfo *	session_info,
		Executor *	executor
	)

static

Definition at line 1826 of file Importer.cpp.

Referenced by import_export::Importer::importDelimited().

                         {
   ImportStatus thread_import_status;
   int64_t total_get_row_time_us = 0;
   int64_t total_str_to_val_time_us = 0;
   auto query_session = session_info ? session_info->get_session_id() : "";
   CHECK(scratch_buffer);
   auto buffer = scratch_buffer.get();
   auto load_ms = measure<>::execution([]() {});
 
   thread_import_status.thread_id = thread_id;
 
   auto ms = measure<>::execution([&]() {
     const CopyParams& copy_params = importer->get_copy_params();
     const std::list<const ColumnDescriptor*>& col_descs = importer->get_column_descs();
     size_t begin =
         delimited_parser::find_beginning(buffer, begin_pos, end_pos, copy_params);
     const char* thread_buf = buffer + begin_pos + begin;
     const char* thread_buf_end = buffer + end_pos;
     const char* buf_end = buffer + total_size;
     bool try_single_thread = false;
     std::vector<std::unique_ptr<TypedImportBuffer>>& import_buffers =
         importer->get_import_buffers(thread_id);
     auto us = measure<std::chrono::microseconds>::execution([&]() {});
     int phys_cols = 0;
     int point_cols = 0;
     for (const auto cd : col_descs) {
       const auto& col_ti = cd->columnType;
       phys_cols += col_ti.get_physical_cols();
       if (cd->columnType.get_type() == kPOINT) {
         point_cols++;
       }
     }
     auto num_cols = col_descs.size() - phys_cols;
     for (const auto& p : import_buffers) {
       p->clear();
     }
     std::vector<std::string_view> row;
     size_t row_index_plus_one = 0;
     for (const char* p = thread_buf; p < thread_buf_end; p++) {
       row.clear();
       std::vector<std::unique_ptr<char[]>>
           tmp_buffers;  // holds string w/ removed escape chars, etc
       if (DEBUG_TIMING) {
         us = measure<std::chrono::microseconds>::execution([&]() {
           p = import_export::delimited_parser::get_row(p,
                                                        thread_buf_end,
                                                        buf_end,
                                                        copy_params,
                                                        importer->get_is_array(),
                                                        row,
                                                        tmp_buffers,
                                                        try_single_thread,
                                                        true);
         });
         total_get_row_time_us += us;
       } else {
         p = import_export::delimited_parser::get_row(p,
                                                      thread_buf_end,
                                                      buf_end,
                                                      copy_params,
                                                      importer->get_is_array(),
                                                      row,
                                                      tmp_buffers,
                                                      try_single_thread,
                                                      true);
       }
       row_index_plus_one++;
       // Each POINT could consume two separate coords instead of a single WKT
       if (row.size() < num_cols || (num_cols + point_cols) < row.size()) {
         thread_import_status.rows_rejected++;
         LOG(ERROR) << "Incorrect Row (expected " << num_cols << " columns, has "
                    << row.size() << "): " << shared::printContainer(row);
         if (thread_import_status.rows_rejected > copy_params.max_reject) {
           break;
         }
         continue;
       }
 
       //
       // lambda for importing a row (perhaps multiple times if exploding a collection)
       //
 
       auto execute_import_row = [&](OGRGeometry* import_geometry) {
         size_t import_idx = 0;
         size_t col_idx = 0;
         try {
           for (auto cd_it = col_descs.begin(); cd_it != col_descs.end(); cd_it++) {
             auto cd = *cd_it;
             const auto& col_ti = cd->columnType;
 
             bool is_null =
                 (row[import_idx] == copy_params.null_str || row[import_idx] == "NULL");
             // Note: default copy_params.null_str is "\N", but everyone uses "NULL".
             // So initially nullness may be missed and not passed to add_value,
             // which then might also check and still decide it's actually a NULL, e.g.
             // if kINT doesn't start with a digit or a '-' then it's considered NULL.
             // So "NULL" is not recognized as NULL but then it's not recognized as
             // a valid kINT, so it's a NULL after all.
             // Checking for "NULL" here too, as a widely accepted notation for NULL.
 
             // Treating empty as NULL
             if (!cd->columnType.is_string() && row[import_idx].empty()) {
               is_null = true;
             }
 
             if (col_ti.get_physical_cols() == 0) {
               // not geo
 
               import_buffers[col_idx]->add_value(
                   cd, row[import_idx], is_null, copy_params);
 
               // next
               ++import_idx;
               ++col_idx;
             } else {
               // geo
 
               // store null string in the base column
               import_buffers[col_idx]->add_value(
                   cd, copy_params.null_str, true, copy_params);
 
               // WKT from string we're not storing
               auto const& geo_string = row[import_idx];
 
               // next
               ++import_idx;
               ++col_idx;
 
               SQLTypes col_type = col_ti.get_type();
               CHECK(IS_GEO(col_type));
 
               std::vector<double> coords;
               std::vector<double> bounds;
               std::vector<int> ring_sizes;
               std::vector<int> poly_rings;
               int render_group = 0;
 
               // if this is a POINT column, and the field is not null, and
               // looks like a scalar numeric value (and not a hex blob)
               // attempt to import two columns as lon/lat (or lat/lon)
               if (col_type == kPOINT && !is_null && geo_string.size() > 0 &&
                   (geo_string[0] == '.' || isdigit(geo_string[0]) ||
                    geo_string[0] == '-') &&
                   geo_string.find_first_of("ABCDEFabcdef") == std::string::npos) {
                 double lon = std::atof(std::string(geo_string).c_str());
                 double lat = NAN;
                 auto lat_str = row[import_idx];
                 ++import_idx;
                 if (lat_str.size() > 0 &&
                     (lat_str[0] == '.' || isdigit(lat_str[0]) || lat_str[0] == '-')) {
                   lat = std::atof(std::string(lat_str).c_str());
                 }
                 // Swap coordinates if this table uses a reverse order: lat/lon
                 if (!copy_params.lonlat) {
                   std::swap(lat, lon);
                 }
                 // TODO: should check if POINT column should have been declared with
                 // SRID WGS 84, EPSG 4326 ? if (col_ti.get_dimension() != 4326) {
                 //  throw std::runtime_error("POINT column " + cd->columnName + " is
                 //  not WGS84, cannot insert lon/lat");
                 // }
                 SQLTypeInfo import_ti{col_ti};
                 if (copy_params.file_type == FileType::DELIMITED &&
                     import_ti.get_output_srid() == 4326) {
                   auto srid0 = copy_params.source_srid;
                   if (srid0 > 0) {
                     // srid0 -> 4326 transform is requested on import
                     import_ti.set_input_srid(srid0);
                   }
                 }
                 if (!importGeoFromLonLat(lon, lat, coords, import_ti)) {
                   throw std::runtime_error(
                       "Cannot read lon/lat to insert into POINT column " +
                       cd->columnName);
                 }
               } else {
                 // import it
                 SQLTypeInfo import_ti{col_ti};
                 if (copy_params.file_type == FileType::DELIMITED &&
                     import_ti.get_output_srid() == 4326) {
                   auto srid0 = copy_params.source_srid;
                   if (srid0 > 0) {
                     // srid0 -> 4326 transform is requested on import
                     import_ti.set_input_srid(srid0);
                   }
                 }
                 if (is_null) {
                   if (col_ti.get_notnull()) {
                     throw std::runtime_error("NULL geo for column " + cd->columnName);
                   }
                   Geospatial::GeoTypesFactory::getNullGeoColumns(
                       import_ti,
                       coords,
                       bounds,
                       ring_sizes,
                       poly_rings,
                       PROMOTE_POLYGON_TO_MULTIPOLYGON);
                 } else {
                   if (import_geometry) {
                     // geometry already exploded
                     if (!Geospatial::GeoTypesFactory::getGeoColumns(
                             import_geometry,
                             import_ti,
                             coords,
                             bounds,
                             ring_sizes,
                             poly_rings,
                             PROMOTE_POLYGON_TO_MULTIPOLYGON)) {
                       std::string msg =
                           "Failed to extract valid geometry from exploded row " +
                           std::to_string(first_row_index_this_buffer +
                                          row_index_plus_one) +
                           " for column " + cd->columnName;
                       throw std::runtime_error(msg);
                     }
                   } else {
                     // extract geometry directly from WKT
                     if (!Geospatial::GeoTypesFactory::getGeoColumns(
                             std::string(geo_string),
                             import_ti,
                             coords,
                             bounds,
                             ring_sizes,
                             poly_rings,
                             PROMOTE_POLYGON_TO_MULTIPOLYGON)) {
                       std::string msg = "Failed to extract valid geometry from row " +
                                         std::to_string(first_row_index_this_buffer +
                                                        row_index_plus_one) +
                                         " for column " + cd->columnName;
                       throw std::runtime_error(msg);
                     }
                   }
 
                   // validate types
                   if (col_type != import_ti.get_type()) {
                     if (!PROMOTE_POLYGON_TO_MULTIPOLYGON ||
                         !(import_ti.get_type() == SQLTypes::kPOLYGON &&
                           col_type == SQLTypes::kMULTIPOLYGON)) {
                       throw std::runtime_error(
                           "Imported geometry doesn't match the type of column " +
                           cd->columnName);
                     }
                   }
                 }
 
                 // assign render group?
                 if (columnIdToRenderGroupAnalyzerMap.size()) {
                   if (col_type == kPOLYGON || col_type == kMULTIPOLYGON) {
                     if (ring_sizes.size()) {
                       // get a suitable render group for these poly coords
                       auto rga_it = columnIdToRenderGroupAnalyzerMap.find(cd->columnId);
                       CHECK(rga_it != columnIdToRenderGroupAnalyzerMap.end());
                       render_group =
                           (*rga_it).second->insertBoundsAndReturnRenderGroup(bounds);
                     } else {
                       // empty poly
                       render_group = -1;
                     }
                   }
                 }
               }
 
               // import extracted geo
               Importer::set_geo_physical_import_buffer(importer->getCatalog(),
                                                        cd,
                                                        import_buffers,
                                                        col_idx,
                                                        coords,
                                                        bounds,
                                                        ring_sizes,
                                                        poly_rings,
                                                        render_group);
 
               // skip remaining physical columns
               for (int i = 0; i < cd->columnType.get_physical_cols(); ++i) {
                 ++cd_it;
               }
             }
           }
           if (UNLIKELY((thread_import_status.rows_completed & 0xFFFF) == 0 &&
                        checkInterrupt(query_session, executor))) {
             thread_import_status.load_failed = true;
             thread_import_status.load_msg =
                 "Table load was cancelled via Query Interrupt";
             return;
           }
           thread_import_status.rows_completed++;
         } catch (const std::exception& e) {
           for (size_t col_idx_to_pop = 0; col_idx_to_pop < col_idx; ++col_idx_to_pop) {
             import_buffers[col_idx_to_pop]->pop_value();
           }
           thread_import_status.rows_rejected++;
           LOG(ERROR) << "Input exception thrown: " << e.what()
                      << ". Row discarded. Data: " << shared::printContainer(row);
           if (thread_import_status.rows_rejected > copy_params.max_reject) {
             LOG(ERROR) << "Load was cancelled due to max reject rows being reached";
             thread_import_status.load_failed = true;
             thread_import_status.load_msg =
                 "Load was cancelled due to max reject rows being reached";
           }
         }
       };  // End of lambda
 
       if (copy_params.geo_explode_collections) {
         // explode and import
         auto const [collection_col_idx, collection_child_type, collection_col_name] =
             explode_collections_step1(col_descs);
         // pull out the collection WKT or WKB hex
         CHECK_LT(collection_col_idx, (int)row.size()) << "column index out of range";
         auto const& collection_geo_string = row[collection_col_idx];
         // convert to OGR
         OGRGeometry* ogr_geometry = nullptr;
         ScopeGuard destroy_ogr_geometry = [&] {
           if (ogr_geometry) {
             OGRGeometryFactory::destroyGeometry(ogr_geometry);
           }
         };
         ogr_geometry = Geospatial::GeoTypesFactory::createOGRGeometry(
             std::string(collection_geo_string));
         // do the explode and import
         us = explode_collections_step2(ogr_geometry,
                                        collection_child_type,
                                        collection_col_name,
                                        first_row_index_this_buffer + row_index_plus_one,
                                        execute_import_row);
       } else {
         // import non-collection row just once
         us = measure<std::chrono::microseconds>::execution(
             [&] { execute_import_row(nullptr); });
       }
 
       if (thread_import_status.load_failed) {
         break;
       }
     }  // end thread
     total_str_to_val_time_us += us;
     if (!thread_import_status.load_failed && thread_import_status.rows_completed > 0) {
       load_ms = measure<>::execution([&]() {
         importer->load(import_buffers, thread_import_status.rows_completed, session_info);
       });
     }
   });  // end execution
 
   if (DEBUG_TIMING && !thread_import_status.load_failed &&
       thread_import_status.rows_completed > 0) {
     LOG(INFO) << "Thread" << std::this_thread::get_id() << ":"
               << thread_import_status.rows_completed << " rows inserted in "
               << (double)ms / 1000.0 << "sec, Insert Time: " << (double)load_ms / 1000.0
               << "sec, get_row: " << (double)total_get_row_time_us / 1000000.0
               << "sec, str_to_val: " << (double)total_str_to_val_time_us / 1000000.0
               << "sec" << std::endl;
   }
 
   return thread_import_status;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

static ImportStatus import_export::import_thread_shapefile	(	int	thread_id,
		Importer *	importer,
		OGRSpatialReference *	poGeographicSR,
		const FeaturePtrVector &	features,
		size_t	firstFeature,
		size_t	numFeatures,
		const FieldNameToIndexMapType &	fieldNameToIndexMap,
		const ColumnNameToSourceNameMapType &	columnNameToSourceNameMap,
		const ColumnIdToRenderGroupAnalyzerMapType &	columnIdToRenderGroupAnalyzerMap,
		const Catalog_Namespace::SessionInfo *	session_info,
		Executor *	executor
	)

static

Definition at line 2192 of file Importer.cpp.

Referenced by import_export::Importer::importGDAL().

                         {
   ImportStatus thread_import_status;
   const CopyParams& copy_params = importer->get_copy_params();
   const std::list<const ColumnDescriptor*>& col_descs = importer->get_column_descs();
   std::vector<std::unique_ptr<TypedImportBuffer>>& import_buffers =
       importer->get_import_buffers(thread_id);
   auto query_session = session_info ? session_info->get_session_id() : "";
   for (const auto& p : import_buffers) {
     p->clear();
   }
 
   auto convert_timer = timer_start();
 
   // we create this on the fly based on the first feature's SR
   std::unique_ptr<OGRCoordinateTransformation> coordinate_transformation;
   for (size_t iFeature = 0; iFeature < numFeatures; iFeature++) {
     if (!features[iFeature]) {
       continue;
     }
 
     // get this feature's geometry
     OGRGeometry* pGeometry = features[iFeature]->GetGeometryRef();
     if (pGeometry) {
       // for geodatabase, we need to consider features with no geometry
       // as we still want to create a table, even if it has no geo column
 
       // transform it
       // avoid GDAL error if not transformable
       auto geometry_sr = pGeometry->getSpatialReference();
       if (geometry_sr) {
         // create an OGRCoordinateTransformation (CT) on the fly
         // we must assume that all geo in this file will have
         // the same source SR, so the CT will be valid for all
         // transforming to a reusable CT is faster than to an SR
         if (coordinate_transformation == nullptr) {
           coordinate_transformation.reset(
               OGRCreateCoordinateTransformation(geometry_sr, poGeographicSR));
           if (coordinate_transformation == nullptr) {
             throw std::runtime_error(
                 "Failed to create a GDAL CoordinateTransformation for incoming geo");
           }
         }
         pGeometry->transform(coordinate_transformation.get());
       }
     }
 
     //
     // lambda for importing a feature (perhaps multiple times if exploding a collection)
     //
 
     auto execute_import_feature = [&](OGRGeometry* import_geometry) {
       size_t col_idx = 0;
       try {
         if (UNLIKELY((thread_import_status.rows_completed & 0xFFFF) == 0 &&
                      checkInterrupt(query_session, executor))) {
           thread_import_status.load_failed = true;
           thread_import_status.load_msg = "Table load was cancelled via Query Interrupt";
           throw QueryExecutionError(Executor::ERR_INTERRUPTED);
         }
         for (auto cd_it = col_descs.begin(); cd_it != col_descs.end(); cd_it++) {
           auto cd = *cd_it;
 
           // is this a geo column?
           const auto& col_ti = cd->columnType;
           if (col_ti.is_geometry()) {
             // Note that this assumes there is one and only one geo column in the
             // table. Currently, the importer only supports reading a single
             // geospatial feature from an input shapefile / geojson file, but this
             // code will need to be modified if that changes
             SQLTypes col_type = col_ti.get_type();
 
             // store null string in the base column
             import_buffers[col_idx]->add_value(
                 cd, copy_params.null_str, true, copy_params);
             ++col_idx;
 
             // the data we now need to extract for the other columns
             std::vector<double> coords;
             std::vector<double> bounds;
             std::vector<int> ring_sizes;
             std::vector<int> poly_rings;
             int render_group = 0;
 
             // extract it
             SQLTypeInfo import_ti{col_ti};
             bool is_null_geo = !import_geometry;
             if (is_null_geo) {
               if (col_ti.get_notnull()) {
                 throw std::runtime_error("NULL geo for column " + cd->columnName);
               }
               Geospatial::GeoTypesFactory::getNullGeoColumns(
                   import_ti,
                   coords,
                   bounds,
                   ring_sizes,
                   poly_rings,
                   PROMOTE_POLYGON_TO_MULTIPOLYGON);
             } else {
               if (!Geospatial::GeoTypesFactory::getGeoColumns(
                       import_geometry,
                       import_ti,
                       coords,
                       bounds,
                       ring_sizes,
                       poly_rings,
                       PROMOTE_POLYGON_TO_MULTIPOLYGON)) {
                 std::string msg = "Failed to extract valid geometry from feature " +
                                   std::to_string(firstFeature + iFeature + 1) +
                                   " for column " + cd->columnName;
                 throw std::runtime_error(msg);
               }
 
               // validate types
               if (col_type != import_ti.get_type()) {
                 if (!PROMOTE_POLYGON_TO_MULTIPOLYGON ||
                     !(import_ti.get_type() == SQLTypes::kPOLYGON &&
                       col_type == SQLTypes::kMULTIPOLYGON)) {
                   throw std::runtime_error(
                       "Imported geometry doesn't match the type of column " +
                       cd->columnName);
                 }
               }
             }
 
             if (col_type == kPOLYGON || col_type == kMULTIPOLYGON) {
               if (ring_sizes.size()) {
                 // get a suitable render group for these poly coords
                 auto rga_it = columnIdToRenderGroupAnalyzerMap.find(cd->columnId);
                 CHECK(rga_it != columnIdToRenderGroupAnalyzerMap.end());
                 render_group = (*rga_it).second->insertBoundsAndReturnRenderGroup(bounds);
               } else {
                 // empty poly
                 render_group = -1;
               }
             }
 
             // create coords array value and add it to the physical column
             ++cd_it;
             auto cd_coords = *cd_it;
             std::vector<TDatum> td_coord_data;
             if (!is_null_geo) {
               std::vector<uint8_t> compressed_coords =
                   Geospatial::compress_coords(coords, col_ti);
               for (auto cc : compressed_coords) {
                 TDatum td_byte;
                 td_byte.val.int_val = cc;
                 td_coord_data.push_back(td_byte);
               }
             }
             TDatum tdd_coords;
             tdd_coords.val.arr_val = td_coord_data;
             tdd_coords.is_null = is_null_geo;
             import_buffers[col_idx]->add_value(cd_coords, tdd_coords, false);
             ++col_idx;
 
             if (col_type == kPOLYGON || col_type == kMULTIPOLYGON) {
               // Create ring_sizes array value and add it to the physical column
               ++cd_it;
               auto cd_ring_sizes = *cd_it;
               std::vector<TDatum> td_ring_sizes;
               if (!is_null_geo) {
                 for (auto ring_size : ring_sizes) {
                   TDatum td_ring_size;
                   td_ring_size.val.int_val = ring_size;
                   td_ring_sizes.push_back(td_ring_size);
                 }
               }
               TDatum tdd_ring_sizes;
               tdd_ring_sizes.val.arr_val = td_ring_sizes;
               tdd_ring_sizes.is_null = is_null_geo;
               import_buffers[col_idx]->add_value(cd_ring_sizes, tdd_ring_sizes, false);
               ++col_idx;
             }
 
             if (col_type == kMULTIPOLYGON) {
               // Create poly_rings array value and add it to the physical column
               ++cd_it;
               auto cd_poly_rings = *cd_it;
               std::vector<TDatum> td_poly_rings;
               if (!is_null_geo) {
                 for (auto num_rings : poly_rings) {
                   TDatum td_num_rings;
                   td_num_rings.val.int_val = num_rings;
                   td_poly_rings.push_back(td_num_rings);
                 }
               }
               TDatum tdd_poly_rings;
               tdd_poly_rings.val.arr_val = td_poly_rings;
               tdd_poly_rings.is_null = is_null_geo;
               import_buffers[col_idx]->add_value(cd_poly_rings, tdd_poly_rings, false);
               ++col_idx;
             }
 
             if (col_type == kLINESTRING || col_type == kPOLYGON ||
                 col_type == kMULTIPOLYGON) {
               // Create bounds array value and add it to the physical column
               ++cd_it;
               auto cd_bounds = *cd_it;
               std::vector<TDatum> td_bounds_data;
               if (!is_null_geo) {
                 for (auto b : bounds) {
                   TDatum td_double;
                   td_double.val.real_val = b;
                   td_bounds_data.push_back(td_double);
                 }
               }
               TDatum tdd_bounds;
               tdd_bounds.val.arr_val = td_bounds_data;
               tdd_bounds.is_null = is_null_geo;
               import_buffers[col_idx]->add_value(cd_bounds, tdd_bounds, false);
               ++col_idx;
             }
 
             if (col_type == kPOLYGON || col_type == kMULTIPOLYGON) {
               // Create render_group value and add it to the physical column
               ++cd_it;
               auto cd_render_group = *cd_it;
               TDatum td_render_group;
               td_render_group.val.int_val = render_group;
               td_render_group.is_null = is_null_geo;
               import_buffers[col_idx]->add_value(cd_render_group, td_render_group, false);
               ++col_idx;
             }
           } else {
             // regular column
             // pull from GDAL metadata
             auto const cit = columnNameToSourceNameMap.find(cd->columnName);
             CHECK(cit != columnNameToSourceNameMap.end());
             auto const& field_name = cit->second;
 
             auto const fit = fieldNameToIndexMap.find(field_name);
             CHECK(fit != fieldNameToIndexMap.end());
             auto const& field_index = fit->second;
             CHECK(field_index < fieldNameToIndexMap.size());
 
             auto const& feature = features[iFeature];
 
             auto field_defn = feature->GetFieldDefnRef(field_index);
             CHECK(field_defn);
 
             // OGRFeature::GetFieldAsString() can only return 80 characters
             // so for array columns, we are obliged to fetch the actual values
             // and construct the concatenated string ourselves
 
             std::string value_string;
             int array_index = 0, array_size = 0;
 
             auto stringify_numeric_list = [&](auto* values) {
               value_string = "{";
               while (array_index < array_size) {
                 auto separator = (array_index > 0) ? "," : "";
                 value_string += separator + std::to_string(values[array_index]);
                 array_index++;
               }
               value_string += "}";
             };
 
             auto field_type = field_defn->GetType();
             switch (field_type) {
               case OFTInteger:
               case OFTInteger64:
               case OFTReal:
               case OFTString:
               case OFTBinary:
               case OFTDate:
               case OFTTime:
               case OFTDateTime: {
                 value_string = feature->GetFieldAsString(field_index);
               } break;
               case OFTIntegerList: {
                 auto* values = feature->GetFieldAsIntegerList(field_index, &array_size);
                 stringify_numeric_list(values);
               } break;
               case OFTInteger64List: {
                 auto* values = feature->GetFieldAsInteger64List(field_index, &array_size);
                 stringify_numeric_list(values);
               } break;
               case OFTRealList: {
                 auto* values = feature->GetFieldAsDoubleList(field_index, &array_size);
                 stringify_numeric_list(values);
               } break;
               case OFTStringList: {
                 auto** array_of_strings = feature->GetFieldAsStringList(field_index);
                 value_string = "{";
                 if (array_of_strings) {
                   while (auto* this_string = array_of_strings[array_index]) {
                     auto separator = (array_index > 0) ? "," : "";
                     value_string += separator + std::string(this_string);
                     array_index++;
                   }
                 }
                 value_string += "}";
               } break;
               default:
                 throw std::runtime_error("Unsupported geo file field type (" +
                                          std::to_string(static_cast<int>(field_type)) +
                                          ")");
             }
 
             static CopyParams default_copy_params;
             import_buffers[col_idx]->add_value(
                 cd, value_string, false, default_copy_params);
             ++col_idx;
           }
         }
         thread_import_status.rows_completed++;
       } catch (QueryExecutionError& e) {
         if (e.getErrorCode() == Executor::ERR_INTERRUPTED) {
           throw e;
         }
       } catch (const std::exception& e) {
         for (size_t col_idx_to_pop = 0; col_idx_to_pop < col_idx; ++col_idx_to_pop) {
           import_buffers[col_idx_to_pop]->pop_value();
         }
         thread_import_status.rows_rejected++;
         LOG(ERROR) << "Input exception thrown: " << e.what() << ". Row discarded.";
       }
     };
 
     if (pGeometry && copy_params.geo_explode_collections) {
       // explode and import
       auto const [collection_idx_type_name, collection_child_type, collection_col_name] =
           explode_collections_step1(col_descs);
       explode_collections_step2(pGeometry,
                                 collection_child_type,
                                 collection_col_name,
                                 firstFeature + iFeature + 1,
                                 execute_import_feature);
     } else {
       // import non-collection or null feature just once
       execute_import_feature(pGeometry);
     }
   }  // end features
 
   float convert_ms =
       float(timer_stop<std::chrono::steady_clock::time_point, std::chrono::microseconds>(
           convert_timer)) /
       1000.0f;
 
   float load_ms = 0.0f;
   if (thread_import_status.rows_completed > 0) {
     auto load_timer = timer_start();
     importer->load(import_buffers, thread_import_status.rows_completed, session_info);
     load_ms =
         float(
             timer_stop<std::chrono::steady_clock::time_point, std::chrono::microseconds>(
                 load_timer)) /
         1000.0f;
   }
 
   if (DEBUG_TIMING && thread_import_status.rows_completed > 0) {
     LOG(INFO) << "DEBUG:      Process " << convert_ms << "ms";
     LOG(INFO) << "DEBUG:      Load " << load_ms << "ms";
   }
 
   thread_import_status.thread_id = thread_id;
 
   if (DEBUG_TIMING) {
     LOG(INFO) << "DEBUG:      Total "
               << float(timer_stop<std::chrono::steady_clock::time_point,
                                   std::chrono::microseconds>(convert_timer)) /
                      1000.0f
               << "ms";
   }
 
   return thread_import_status;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool import_export::importGeoFromLonLat	(	double	lon,
		double	lat,
		std::vector< double > &	coords,
		SQLTypeInfo &	ti
	)

Definition at line 1439 of file Importer.cpp.

References Geospatial::GeoPoint::getColumns(), and SQLTypeInfo::transforms().

Referenced by import_thread_delimited().

                                           {
   if (std::isinf(lat) || std::isnan(lat) || std::isinf(lon) || std::isnan(lon)) {
     return false;
   }
   if (ti.transforms()) {
     Geospatial::GeoPoint pt{std::vector<double>{lon, lat}};
     if (!pt.transform(ti)) {
       return false;
     }
     pt.getColumns(coords);
     return true;
   }
   coords.push_back(lon);
   coords.push_back(lat);
   return true;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

ArrayDatum import_export::NullArray ( const SQLTypeInfo & ti )

Definition at line 389 of file Importer.cpp.

References appendDatum(), CHECK, checked_malloc(), test_fsi::d, SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_size(), SQLTypeInfo::is_string(), NullArrayDatum(), and NullDatum().

Referenced by import_export::TypedImportBuffer::add_value(), import_export::TypedImportBuffer::add_values(), import_export::ImporterUtils::composeNullArray(), and TDatumToArrayDatum().

                                             {
   SQLTypeInfo elem_ti = ti.get_elem_type();
   auto len = ti.get_size();
 
   if (elem_ti.is_string()) {
     // must not be called for array of strings
     CHECK(false);
     return ArrayDatum(0, NULL, true);
   }
 
   if (len > 0) {
     // Compose a NULL fixlen array
     int8_t* buf = (int8_t*)checked_malloc(len);
     // First scalar is a NULL_ARRAY sentinel
     Datum d = NullArrayDatum(elem_ti);
     int8_t* p = appendDatum(buf, d, elem_ti);
     // Rest is filled with normal NULL sentinels
     Datum d0 = NullDatum(elem_ti);
     while ((p - buf) < len) {
       p = appendDatum(p, d0, elem_ti);
     }
     CHECK((p - buf) == len);
     return ArrayDatum(len, buf, true);
   }
   // NULL varlen array
   return ArrayDatum(0, NULL, true);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

Datum import_export::NullArrayDatum ( SQLTypeInfo & ti )

Definition at line 295 of file Importer.cpp.

References Datum::bigintval, Datum::boolval, test_fsi::d, decimal_to_int_type(), Datum::doubleval, Datum::floatval, SQLTypeInfo::get_type(), inline_fixed_encoding_null_array_val(), Datum::intval, SQLTypeInfo::is_decimal(), kBIGINT, kBOOLEAN, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTIME, kTIMESTAMP, kTINYINT, NULL_ARRAY_DOUBLE, NULL_ARRAY_FLOAT, Datum::smallintval, Datum::tinyintval, and run_benchmark_import::type.

Referenced by NullArray().

                                       {
   Datum d;
   const auto type = ti.is_decimal() ? decimal_to_int_type(ti) : ti.get_type();
   switch (type) {
     case kBOOLEAN:
       d.boolval = inline_fixed_encoding_null_array_val(ti);
       break;
     case kBIGINT:
       d.bigintval = inline_fixed_encoding_null_array_val(ti);
       break;
     case kINT:
       d.intval = inline_fixed_encoding_null_array_val(ti);
       break;
     case kSMALLINT:
       d.smallintval = inline_fixed_encoding_null_array_val(ti);
       break;
     case kTINYINT:
       d.tinyintval = inline_fixed_encoding_null_array_val(ti);
       break;
     case kFLOAT:
       d.floatval = NULL_ARRAY_FLOAT;
       break;
     case kDOUBLE:
       d.doubleval = NULL_ARRAY_DOUBLE;
       break;
     case kTIME:
     case kTIMESTAMP:
     case kDATE:
       d.bigintval = inline_fixed_encoding_null_array_val(ti);
       break;
     case kPOINT:
     case kLINESTRING:
     case kPOLYGON:
     case kMULTIPOLYGON:
       throw std::runtime_error("Internal error: geometry type in NullArrayDatum.");
     default:
       throw std::runtime_error("Internal error: invalid type in NullArrayDatum.");
   }
   return d;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

Datum import_export::NullDatum ( SQLTypeInfo & ti )

Definition at line 254 of file Importer.cpp.

References Datum::bigintval, Datum::boolval, test_fsi::d, decimal_to_int_type(), Datum::doubleval, Datum::floatval, SQLTypeInfo::get_type(), inline_fixed_encoding_null_val(), Datum::intval, SQLTypeInfo::is_decimal(), kBIGINT, kBOOLEAN, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTIME, kTIMESTAMP, kTINYINT, NULL_DOUBLE, NULL_FLOAT, Datum::smallintval, Datum::tinyintval, and run_benchmark_import::type.

Referenced by NullArray(), and StringToArray().

                                  {
   Datum d;
   const auto type = ti.is_decimal() ? decimal_to_int_type(ti) : ti.get_type();
   switch (type) {
     case kBOOLEAN:
       d.boolval = inline_fixed_encoding_null_val(ti);
       break;
     case kBIGINT:
       d.bigintval = inline_fixed_encoding_null_val(ti);
       break;
     case kINT:
       d.intval = inline_fixed_encoding_null_val(ti);
       break;
     case kSMALLINT:
       d.smallintval = inline_fixed_encoding_null_val(ti);
       break;
     case kTINYINT:
       d.tinyintval = inline_fixed_encoding_null_val(ti);
       break;
     case kFLOAT:
       d.floatval = NULL_FLOAT;
       break;
     case kDOUBLE:
       d.doubleval = NULL_DOUBLE;
       break;
     case kTIME:
     case kTIMESTAMP:
     case kDATE:
       d.bigintval = inline_fixed_encoding_null_val(ti);
       break;
     case kPOINT:
     case kLINESTRING:
     case kPOLYGON:
     case kMULTIPOLYGON:
       throw std::runtime_error("Internal error: geometry type in NullDatum.");
     default:
       throw std::runtime_error("Internal error: invalid type in NullDatum.");
   }
   return d;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::pair<SQLTypes, bool> import_export::ogr_to_type ( const OGRFieldType & ogr_type )

Definition at line 4557 of file Importer.cpp.

References kBIGINT, kDATE, kDOUBLE, kINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, and to_string().

Referenced by import_export::Importer::gdalToColumnDescriptors().

                                                                   {
   switch (ogr_type) {
     case OFTInteger:
       return std::make_pair(kINT, false);
     case OFTIntegerList:
       return std::make_pair(kINT, true);
 #if GDAL_VERSION_MAJOR > 1
     case OFTInteger64:
       return std::make_pair(kBIGINT, false);
     case OFTInteger64List:
       return std::make_pair(kBIGINT, true);
 #endif
     case OFTReal:
       return std::make_pair(kDOUBLE, false);
     case OFTRealList:
       return std::make_pair(kDOUBLE, true);
     case OFTString:
       return std::make_pair(kTEXT, false);
     case OFTStringList:
       return std::make_pair(kTEXT, true);
     case OFTDate:
       return std::make_pair(kDATE, false);
     case OFTTime:
       return std::make_pair(kTIME, false);
     case OFTDateTime:
       return std::make_pair(kTIMESTAMP, false);
     case OFTBinary:
       // Interpret binary blobs as byte arrays here
       // but actual import will store NULL as GDAL will not
       // extract the blob (OGRFeature::GetFieldAsString will
       // result in the import buffers having an empty string)
       return std::make_pair(kTINYINT, true);
     default:
       break;
   }
   throw std::runtime_error("Unknown OGR field type: " + std::to_string(ogr_type));
 }

Here is the call graph for this function:

Here is the caller graph for this function:

SQLTypes import_export::ogr_to_type ( const OGRwkbGeometryType & ogr_type )

Definition at line 4595 of file Importer.cpp.

References kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, and to_string().

                                                          {
   switch (ogr_type) {
     case wkbPoint:
       return kPOINT;
     case wkbLineString:
       return kLINESTRING;
     case wkbPolygon:
       return kPOLYGON;
     case wkbMultiPolygon:
       return kMULTIPOLYGON;
     default:
       break;
   }
   throw std::runtime_error("Unknown OGR geom type: " + std::to_string(ogr_type));
 }

Here is the call graph for this function:

std::vector< std::unique_ptr< TypedImportBuffer > > import_export::setup_column_loaders	(	const TableDescriptor *	td,
		Loader *	loader
	)

Definition at line 5350 of file Importer.cpp.

References CHECK, import_export::Loader::get_column_descs(), and import_export::Loader::getStringDict().

Referenced by DBHandler::prepare_loader_generic().

                     {
   CHECK(td);
   auto col_descs = loader->get_column_descs();
 
   std::vector<std::unique_ptr<TypedImportBuffer>> import_buffers;
   for (auto cd : col_descs) {
     import_buffers.emplace_back(
         std::make_unique<TypedImportBuffer>(cd, loader->getStringDict(cd)));
   }
 
   return import_buffers;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

ArrayDatum import_export::StringToArray	(	const std::string &	s,
		const SQLTypeInfo &	ti,
		const CopyParams &	copy_params
	)

Definition at line 336 of file Importer.cpp.

References appendDatum(), import_export::CopyParams::array_begin, import_export::CopyParams::array_delim, import_export::CopyParams::array_end, CHECK, checked_malloc(), test_fsi::d, SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_size(), i, is_null(), SQLTypeInfo::is_number(), SQLTypeInfo::is_string(), SQLTypeInfo::is_time(), LOG, import_export::CopyParams::null_str, NullDatum(), StringToDatum(), trim_space(), and logger::WARNING.

Referenced by import_export::TypedImportBuffer::add_value().

                                                         {
   SQLTypeInfo elem_ti = ti.get_elem_type();
   if (s == copy_params.null_str || s == "NULL" || s.empty()) {
     return ArrayDatum(0, NULL, true);
   }
   if (s[0] != copy_params.array_begin || s[s.size() - 1] != copy_params.array_end) {
     LOG(WARNING) << "Malformed array: " << s;
     return ArrayDatum(0, NULL, true);
   }
   std::vector<std::string> elem_strs;
   size_t last = 1;
   for (size_t i = s.find(copy_params.array_delim, 1); i != std::string::npos;
        i = s.find(copy_params.array_delim, last)) {
     elem_strs.push_back(s.substr(last, i - last));
     last = i + 1;
   }
   if (last + 1 <= s.size()) {
     elem_strs.push_back(s.substr(last, s.size() - 1 - last));
   }
   if (elem_strs.size() == 1) {
     auto str = elem_strs.front();
     auto str_trimmed = trim_space(str.c_str(), str.length());
     if (str_trimmed == "") {
       elem_strs.clear();  // Empty array
     }
   }
   if (!elem_ti.is_string()) {
     size_t len = elem_strs.size() * elem_ti.get_size();
     int8_t* buf = (int8_t*)checked_malloc(len);
     int8_t* p = buf;
     for (auto& es : elem_strs) {
       auto e = trim_space(es.c_str(), es.length());
       bool is_null = (e == copy_params.null_str) || e == "NULL";
       if (!elem_ti.is_string() && e == "") {
         is_null = true;
       }
       if (elem_ti.is_number() || elem_ti.is_time()) {
         if (!isdigit(e[0]) && e[0] != '-') {
           is_null = true;
         }
       }
       Datum d = is_null ? NullDatum(elem_ti) : StringToDatum(e, elem_ti);
       p = appendDatum(p, d, elem_ti);
     }
     return ArrayDatum(len, buf, false);
   }
   // must not be called for array of strings
   CHECK(false);
   return ArrayDatum(0, NULL, true);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

ArrayDatum import_export::TDatumToArrayDatum	(	const TDatum &	datum,
		const SQLTypeInfo &	ti
	)

Definition at line 473 of file Importer.cpp.

References appendDatum(), CHECK, checked_malloc(), SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_size(), SQLTypeInfo::is_string(), NullArray(), and TDatumToDatum().

Referenced by import_export::TypedImportBuffer::add_value().

                                                                           {
   SQLTypeInfo elem_ti = ti.get_elem_type();
 
   CHECK(!elem_ti.is_string());
 
   if (datum.is_null) {
     return NullArray(ti);
   }
 
   size_t len = datum.val.arr_val.size() * elem_ti.get_size();
   int8_t* buf = (int8_t*)checked_malloc(len);
   int8_t* p = buf;
   for (auto& e : datum.val.arr_val) {
     p = appendDatum(p, TDatumToDatum(e, elem_ti), elem_ti);
   }
 
   return ArrayDatum(len, buf, false);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

Datum import_export::TDatumToDatum	(	const TDatum &	datum,
		SQLTypeInfo &	ti
	)

Definition at line 428 of file Importer.cpp.

References Datum::bigintval, Datum::boolval, test_fsi::d, decimal_to_int_type(), Datum::doubleval, Datum::floatval, SQLTypeInfo::get_type(), inline_fixed_encoding_null_val(), Datum::intval, SQLTypeInfo::is_decimal(), kBIGINT, kBOOLEAN, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTIME, kTIMESTAMP, kTINYINT, NULL_DOUBLE, NULL_FLOAT, Datum::smallintval, Datum::tinyintval, and run_benchmark_import::type.

Referenced by TDatumToArrayDatum().

                                                           {
   Datum d;
   const auto type = ti.is_decimal() ? decimal_to_int_type(ti) : ti.get_type();
   switch (type) {
     case kBOOLEAN:
       d.boolval = datum.is_null ? inline_fixed_encoding_null_val(ti) : datum.val.int_val;
       break;
     case kBIGINT:
       d.bigintval =
           datum.is_null ? inline_fixed_encoding_null_val(ti) : datum.val.int_val;
       break;
     case kINT:
       d.intval = datum.is_null ? inline_fixed_encoding_null_val(ti) : datum.val.int_val;
       break;
     case kSMALLINT:
       d.smallintval =
           datum.is_null ? inline_fixed_encoding_null_val(ti) : datum.val.int_val;
       break;
     case kTINYINT:
       d.tinyintval =
           datum.is_null ? inline_fixed_encoding_null_val(ti) : datum.val.int_val;
       break;
     case kFLOAT:
       d.floatval = datum.is_null ? NULL_FLOAT : datum.val.real_val;
       break;
     case kDOUBLE:
       d.doubleval = datum.is_null ? NULL_DOUBLE : datum.val.real_val;
       break;
     case kTIME:
     case kTIMESTAMP:
     case kDATE:
       d.bigintval =
           datum.is_null ? inline_fixed_encoding_null_val(ti) : datum.val.int_val;
       break;
     case kPOINT:
     case kLINESTRING:
     case kPOLYGON:
     case kMULTIPOLYGON:
       throw std::runtime_error("Internal error: geometry type in TDatumToDatum.");
     default:
       throw std::runtime_error("Internal error: invalid type in TDatumToDatum.");
   }
   return d;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

static const std::string import_export::trim_space	(	const char *	field,
		const size_t	len
	)

static

Definition at line 242 of file Importer.cpp.

References i, and generate_TableFunctionsFactory_init::j.

Referenced by import_export::delimited_parser::get_row(), and StringToArray().

                                                                        {
   size_t i = 0;
   size_t j = len;
   while (i < j && (field[i] == ' ' || field[i] == '\r')) {
     i++;
   }
   while (i < j && (field[j - 1] == ' ' || field[j - 1] == '\r')) {
     j--;
   }
   return std::string(field + i, j - i);
 }

Here is the caller graph for this function:

template<class T >

bool import_export::try_cast ( const std::string & str )

Definition at line 3101 of file Importer.cpp.

References omnisci.dtypes::T.

                                     {
   try {
     boost::lexical_cast<T>(str);
   } catch (const boost::bad_lexical_cast& e) {
     return false;
   }
   return true;
 }

Variable Documentation

std::map<std::string, ImportStatus> import_export::import_status_map

static

Definition at line 169 of file Importer.cpp.

Referenced by import_export::Importer::get_import_status(), and import_export::Importer::set_import_status().

constexpr size_t import_export::kImportFileBufferSize = (1 << 23)

static

Definition at line 32 of file CopyParams.h.

constexpr bool import_export::PROMOTE_POLYGON_TO_MULTIPOLYGON = true

static

Definition at line 166 of file Importer.cpp.

Referenced by import_export::Detector::detect_sqltype(), import_export::Importer::gdalToColumnDescriptors(), import_thread_delimited(), and import_thread_shapefile().

mapd_shared_mutex import_export::status_mutex

static

Definition at line 168 of file Importer.cpp.

Referenced by import_export::Importer::get_import_status(), and import_export::Importer::set_import_status().

Namespaces

Classes

Typedefs

Enumerations

Functions

Variables

Typedef Documentation

Enumeration Type Documentation

Function Documentation

Variable Documentation