OmniSciDB  471d68cefb
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
import_export Namespace Reference

Namespaces

 anonymous_namespace{Importer.cpp}
 
 anonymous_namespace{QueryExporterCSV.cpp}
 
 anonymous_namespace{QueryExporterGDAL.cpp}
 
 delimited_parser
 

Classes

class  AbstractImporter
 
struct  CopyParams
 
class  ForeignDataImporter
 
class  ImportBatchResult
 
struct  GeoImportException
 
class  ColumnNotGeoError
 
struct  BadRowsTracker
 
class  ImporterUtils
 
class  TypedImportBuffer
 
class  Loader
 
struct  ImportStatus
 
class  DataStreamSink
 
class  Detector
 
class  Importer
 
class  QueryExporter
 
class  QueryExporterCSV
 
class  QueryExporterGDAL
 
class  RenderGroupAnalyzer
 

Typedefs

using FieldNameToIndexMapType = std::map< std::string, size_t >
 
using ColumnNameToSourceNameMapType = std::map< std::string, std::string >
 
using ColumnIdToRenderGroupAnalyzerMapType = std::map< int, std::shared_ptr< RenderGroupAnalyzer >>
 
using FeaturePtrVector = std::vector< OGRFeatureUqPtr >
 
using ArraySliceRange = std::pair< size_t, size_t >
 

Enumerations

enum  FileType { FileType::DELIMITED, FileType::POLYGON }
 
enum  ImportHeaderRow { ImportHeaderRow::AUTODETECT, ImportHeaderRow::NO_HEADER, ImportHeaderRow::HAS_HEADER }
 

Functions

static const std::string trim_space (const char *field, const size_t len)
 
Datum NullDatum (SQLTypeInfo &ti)
 
Datum NullArrayDatum (SQLTypeInfo &ti)
 
ArrayDatum StringToArray (const std::string &s, const SQLTypeInfo &ti, const CopyParams &copy_params)
 
ArrayDatum NullArray (const SQLTypeInfo &ti)
 
void addBinaryStringArray (const TDatum &datum, std::vector< std::string > &string_vec)
 
Datum TDatumToDatum (const TDatum &datum, SQLTypeInfo &ti)
 
ArrayDatum TDatumToArrayDatum (const TDatum &datum, const SQLTypeInfo &ti)
 
bool importGeoFromLonLat (double lon, double lat, std::vector< double > &coords, SQLTypeInfo &ti)
 
static ImportStatus import_thread_delimited (int thread_id, Importer *importer, std::unique_ptr< char[]> scratch_buffer, size_t begin_pos, size_t end_pos, size_t total_size, const ColumnIdToRenderGroupAnalyzerMapType &columnIdToRenderGroupAnalyzerMap, size_t first_row_index_this_buffer, const Catalog_Namespace::SessionInfo *session_info, Executor *executor)
 
static ImportStatus import_thread_shapefile (int thread_id, Importer *importer, OGRSpatialReference *poGeographicSR, const FeaturePtrVector &features, size_t firstFeature, size_t numFeatures, const FieldNameToIndexMapType &fieldNameToIndexMap, const ColumnNameToSourceNameMapType &columnNameToSourceNameMap, const ColumnIdToRenderGroupAnalyzerMapType &columnIdToRenderGroupAnalyzerMap, const Catalog_Namespace::SessionInfo *session_info, Executor *executor)
 
template<class T >
bool try_cast (const std::string &str)
 
std::pair< SQLTypes, bool > ogr_to_type (const OGRFieldType &ogr_type)
 
SQLTypes ogr_to_type (const OGRwkbGeometryType &ogr_type)
 
void gdalGatherFilesInArchiveRecursive (const std::string &archive_path, std::vector< std::string > &files)
 
std::vector< std::unique_ptr
< TypedImportBuffer > > 
setup_column_loaders (const TableDescriptor *td, Loader *loader)
 
std::vector< std::unique_ptr
< TypedImportBuffer > > 
fill_missing_columns (const Catalog_Namespace::Catalog *cat, Fragmenter_Namespace::InsertData &insert_data)
 

Variables

static constexpr size_t kImportFileBufferSize = (1 << 23)
 
static constexpr size_t max_import_buffer_resize_byte_size = 1024 * 1024 * 1024
 
static constexpr bool PROMOTE_POLYGON_TO_MULTIPOLYGON = true
 
static mapd_shared_mutex status_mutex
 
static std::map< std::string,
ImportStatus
import_status_map
 

Typedef Documentation

using import_export::ArraySliceRange = typedef std::pair<size_t, size_t>

Definition at line 73 of file Importer.h.

using import_export::ColumnIdToRenderGroupAnalyzerMapType = typedef std::map<int, std::shared_ptr<RenderGroupAnalyzer>>

Definition at line 156 of file Importer.cpp.

using import_export::ColumnNameToSourceNameMapType = typedef std::map<std::string, std::string>

Definition at line 154 of file Importer.cpp.

using import_export::FeaturePtrVector = typedef std::vector<OGRFeatureUqPtr>

Definition at line 157 of file Importer.cpp.

using import_export::FieldNameToIndexMapType = typedef std::map<std::string, size_t>

Definition at line 153 of file Importer.cpp.

Enumeration Type Documentation

Enumerator
DELIMITED 
POLYGON 

Definition at line 37 of file CopyParams.h.

37  {
38  DELIMITED,
39  POLYGON
40 #ifdef ENABLE_IMPORT_PARQUET
41  ,
42  PARQUET
43 #endif
44 };
#define POLYGON

Function Documentation

void import_export::addBinaryStringArray ( const TDatum &  datum,
std::vector< std::string > &  string_vec 
)

Definition at line 430 of file Importer.cpp.

Referenced by import_export::TypedImportBuffer::add_value().

430  {
431  const auto& arr = datum.val.arr_val;
432  for (const auto& elem_datum : arr) {
433  string_vec.push_back(elem_datum.val.str_val);
434  }
435 }

+ Here is the caller graph for this function:

std::vector< std::unique_ptr< TypedImportBuffer > > import_export::fill_missing_columns ( const Catalog_Namespace::Catalog cat,
Fragmenter_Namespace::InsertData insert_data 
)

Definition at line 5409 of file Importer.cpp.

References anonymous_namespace{Utm.h}::a, CHECK, ColumnDescriptor::columnId, Fragmenter_Namespace::InsertData::columnIds, ColumnDescriptor::columnName, ColumnDescriptor::columnType, Fragmenter_Namespace::InsertData::data, ColumnDescriptor::default_value, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), import_export::TypedImportBuffer::get_data_block_pointers(), SQLTypeInfo::get_physical_cols(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), Catalog_Namespace::Catalog::getAllColumnMetadataForTable(), Geospatial::GeoTypesFactory::getGeoColumns(), Catalog_Namespace::Catalog::getMetadataForDict(), i, Fragmenter_Namespace::InsertData::is_default, SQLTypeInfo::is_geometry(), IS_STRING, kARRAY, kENCODING_DICT, import_export::Importer::set_geo_physical_import_buffer(), gpu_enabled::sort(), and Fragmenter_Namespace::InsertData::tableId.

Referenced by RelAlgExecutor::executeSimpleInsert(), DBHandler::insert_data(), and Parser::InsertIntoTableAsSelectStmt::populateData().

5411  {
5412  std::vector<std::unique_ptr<import_export::TypedImportBuffer>> defaults_buffers;
5413  if (insert_data.is_default.size() == 0) {
5414  insert_data.is_default.resize(insert_data.columnIds.size(), false);
5415  }
5416  CHECK(insert_data.is_default.size() == insert_data.is_default.size());
5417  auto cds = cat->getAllColumnMetadataForTable(insert_data.tableId, false, false, true);
5418  if (cds.size() == insert_data.columnIds.size()) {
5419  // all columns specified
5420  return defaults_buffers;
5421  }
5422  for (auto cd : cds) {
5423  if (std::find(insert_data.columnIds.begin(),
5424  insert_data.columnIds.end(),
5425  cd->columnId) == insert_data.columnIds.end()) {
5426  StringDictionary* dict = nullptr;
5427  if (cd->columnType.get_type() == kARRAY &&
5428  IS_STRING(cd->columnType.get_subtype()) && !cd->default_value.has_value()) {
5429  throw std::runtime_error("Cannot omit column \"" + cd->columnName +
5430  "\": omitting TEXT arrays is not supported yet");
5431  }
5432  if (cd->columnType.get_compression() == kENCODING_DICT) {
5433  dict = cat->getMetadataForDict(cd->columnType.get_comp_param())->stringDict.get();
5434  }
5435  defaults_buffers.emplace_back(std::make_unique<TypedImportBuffer>(cd, dict));
5436  }
5437  }
5438  // put buffers in order to fill geo sub-columns properly
5439  std::sort(defaults_buffers.begin(),
5440  defaults_buffers.end(),
5441  [](decltype(defaults_buffers[0])& a, decltype(defaults_buffers[0])& b) {
5442  return a->getColumnDesc()->columnId < b->getColumnDesc()->columnId;
5443  });
5444  for (size_t i = 0; i < defaults_buffers.size(); ++i) {
5445  auto cd = defaults_buffers[i]->getColumnDesc();
5446  std::string default_value = cd->default_value.value_or("NULL");
5447  defaults_buffers[i]->add_value(
5448  cd, default_value, !cd->default_value.has_value(), import_export::CopyParams());
5449  if (cd->columnType.is_geometry()) {
5450  std::vector<double> coords, bounds;
5451  std::vector<int> ring_sizes, poly_rings;
5452  int render_group = 0;
5453  SQLTypeInfo tinfo{cd->columnType};
5455  default_value, tinfo, coords, bounds, ring_sizes, poly_rings, false));
5456  // set physical columns starting with the following ID
5457  auto next_col = i + 1;
5459  cd,
5460  defaults_buffers,
5461  next_col,
5462  coords,
5463  bounds,
5464  ring_sizes,
5465  poly_rings,
5466  render_group);
5467  // skip physical columns filled with the call above
5468  i += cd->columnType.get_physical_cols();
5469  }
5470  }
5471  auto data = import_export::TypedImportBuffer::get_data_block_pointers(defaults_buffers);
5472  CHECK(data.size() == defaults_buffers.size());
5473  for (size_t i = 0; i < defaults_buffers.size(); ++i) {
5474  insert_data.data.push_back(data[i]);
5475  insert_data.columnIds.push_back(defaults_buffers[i]->getColumnDesc()->columnId);
5476  insert_data.is_default.push_back(true);
5477  }
5478  return defaults_buffers;
5479 }
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
std::vector< bool > is_default
Definition: Fragmenter.h:66
constexpr double a
Definition: Utm.h:38
int tableId
identifies the database into which the data is being inserted
Definition: Fragmenter.h:61
static std::vector< DataBlockPtr > get_data_block_pointers(const std::vector< std::unique_ptr< TypedImportBuffer >> &import_buffers)
Definition: Importer.cpp:3033
static void set_geo_physical_import_buffer(const Catalog_Namespace::Catalog &catalog, const ColumnDescriptor *cd, std::vector< std::unique_ptr< TypedImportBuffer >> &import_buffers, size_t &col_idx, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, int render_group)
Definition: Importer.cpp:1630
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1537
static bool getGeoColumns(const std::string &wkt_or_wkb_hex, SQLTypeInfo &ti, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, const bool promote_poly_to_mpoly=false)
Definition: Types.cpp:937
std::vector< DataBlockPtr > data
the number of rows being inserted
Definition: Fragmenter.h:64
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1794
#define IS_STRING(T)
Definition: sqltypes.h:250
#define CHECK(condition)
Definition: Logger.h:209
std::vector< int > columnIds
identifies the table into which the data is being inserted
Definition: Fragmenter.h:62

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void import_export::gdalGatherFilesInArchiveRecursive ( const std::string &  archive_path,
std::vector< std::string > &  files 
)

Definition at line 4968 of file Importer.cpp.

References LOG, run_benchmark_import::result, and logger::WARNING.

Referenced by import_export::Importer::gdalGetAllFilesInArchive().

4969  {
4970  // prepare to gather subdirectories
4971  std::vector<std::string> subdirectories;
4972 
4973  // get entries
4974  char** entries = VSIReadDir(archive_path.c_str());
4975  if (!entries) {
4976  LOG(WARNING) << "Failed to get file listing at archive: " << archive_path;
4977  return;
4978  }
4979 
4980  // force scope
4981  {
4982  // request clean-up
4983  ScopeGuard entries_guard = [&] { CSLDestroy(entries); };
4984 
4985  // check all the entries
4986  int index = 0;
4987  while (true) {
4988  // get next entry, or drop out if there isn't one
4989  char* entry_c = entries[index++];
4990  if (!entry_c) {
4991  break;
4992  }
4993  std::string entry(entry_c);
4994 
4995  // ignore '.' and '..'
4996  if (entry == "." || entry == "..") {
4997  continue;
4998  }
4999 
5000  // build the full path
5001  std::string entry_path = archive_path + std::string("/") + entry;
5002 
5003  // is it a file or a sub-folder
5004  VSIStatBufL sb;
5005  int result = VSIStatExL(entry_path.c_str(), &sb, VSI_STAT_NATURE_FLAG);
5006  if (result < 0) {
5007  break;
5008  }
5009 
5010  if (VSI_ISDIR(sb.st_mode)) {
5011  // a directory that ends with .gdb could be a Geodatabase bundle
5012  // arguably dangerous to decide this purely by name, but any further
5013  // validation would be very complex especially at this scope
5014  if (boost::iends_with(entry_path, ".gdb")) {
5015  // add the directory as if it was a file and don't recurse into it
5016  files.push_back(entry_path);
5017  } else {
5018  // add subdirectory to be recursed into
5019  subdirectories.push_back(entry_path);
5020  }
5021  } else {
5022  // add this file
5023  files.push_back(entry_path);
5024  }
5025  }
5026  }
5027 
5028  // recurse into each subdirectories we found
5029  for (const auto& subdirectory : subdirectories) {
5030  gdalGatherFilesInArchiveRecursive(subdirectory, files);
5031  }
5032 }
#define LOG(tag)
Definition: Logger.h:203
void gdalGatherFilesInArchiveRecursive(const std::string &archive_path, std::vector< std::string > &files)
Definition: Importer.cpp:4968

+ Here is the caller graph for this function:

static ImportStatus import_export::import_thread_delimited ( int  thread_id,
Importer *  importer,
std::unique_ptr< char[]>  scratch_buffer,
size_t  begin_pos,
size_t  end_pos,
size_t  total_size,
const ColumnIdToRenderGroupAnalyzerMapType &  columnIdToRenderGroupAnalyzerMap,
size_t  first_row_index_this_buffer,
const Catalog_Namespace::SessionInfo session_info,
Executor executor 
)
static

Definition at line 1997 of file Importer.cpp.

References CHECK, CHECK_LT, anonymous_namespace{Importer.cpp}::check_session_interrupted(), Geospatial::GeoTypesFactory::createOGRGeometry(), DEBUG_TIMING, DELIMITED, logger::ERROR, measure< TimeT >::execution(), import_export::anonymous_namespace{Importer.cpp}::explode_collections_step1(), import_export::anonymous_namespace{Importer.cpp}::explode_collections_step2(), import_export::CopyParams::file_type, import_export::delimited_parser::find_beginning(), import_export::CopyParams::geo_explode_collections, import_export::Importer::get_column_descs(), import_export::Importer::get_copy_params(), import_export::Importer::get_import_buffers(), import_export::Importer::get_is_array(), import_export::delimited_parser::get_row(), Catalog_Namespace::SessionInfo::get_session_id(), import_export::Importer::getCatalog(), Geospatial::GeoTypesFactory::getGeoColumns(), Geospatial::GeoTypesFactory::getNullGeoColumns(), i, importGeoFromLonLat(), logger::INFO, IS_GEO, is_null(), kMULTIPOLYGON, kPOINT, kPOLYGON, import_export::Importer::load(), import_export::ImportStatus::load_failed, import_export::ImportStatus::load_msg, LOG, import_export::CopyParams::lonlat, import_export::CopyParams::max_reject, import_export::CopyParams::null_str, shared::printContainer(), PROMOTE_POLYGON_TO_MULTIPOLYGON, import_export::ImportStatus::rows_completed, import_export::ImportStatus::rows_rejected, import_export::Importer::set_geo_physical_import_buffer(), import_export::CopyParams::source_srid, gpu_enabled::swap(), import_export::ImportStatus::thread_id, logger::thread_id(), to_string(), and UNLIKELY.

Referenced by import_export::Importer::importDelimited().

2007  {
2008  ImportStatus thread_import_status;
2009  int64_t total_get_row_time_us = 0;
2010  int64_t total_str_to_val_time_us = 0;
2011  auto query_session = session_info ? session_info->get_session_id() : "";
2012  CHECK(scratch_buffer);
2013  auto buffer = scratch_buffer.get();
2014  auto load_ms = measure<>::execution([]() {});
2015 
2016  thread_import_status.thread_id = thread_id;
2017 
2018  auto ms = measure<>::execution([&]() {
2019  const CopyParams& copy_params = importer->get_copy_params();
2020  const std::list<const ColumnDescriptor*>& col_descs = importer->get_column_descs();
2021  size_t begin =
2022  delimited_parser::find_beginning(buffer, begin_pos, end_pos, copy_params);
2023  const char* thread_buf = buffer + begin_pos + begin;
2024  const char* thread_buf_end = buffer + end_pos;
2025  const char* buf_end = buffer + total_size;
2026  bool try_single_thread = false;
2027  std::vector<std::unique_ptr<TypedImportBuffer>>& import_buffers =
2028  importer->get_import_buffers(thread_id);
2030  int phys_cols = 0;
2031  int point_cols = 0;
2032  for (const auto cd : col_descs) {
2033  const auto& col_ti = cd->columnType;
2034  phys_cols += col_ti.get_physical_cols();
2035  if (cd->columnType.get_type() == kPOINT) {
2036  point_cols++;
2037  }
2038  }
2039  auto num_cols = col_descs.size() - phys_cols;
2040  for (const auto& p : import_buffers) {
2041  p->clear();
2042  }
2043  std::vector<std::string_view> row;
2044  size_t row_index_plus_one = 0;
2045  for (const char* p = thread_buf; p < thread_buf_end; p++) {
2046  row.clear();
2047  std::vector<std::unique_ptr<char[]>>
2048  tmp_buffers; // holds string w/ removed escape chars, etc
2049  if (DEBUG_TIMING) {
2052  thread_buf_end,
2053  buf_end,
2054  copy_params,
2055  importer->get_is_array(),
2056  row,
2057  tmp_buffers,
2058  try_single_thread,
2059  true);
2060  });
2061  total_get_row_time_us += us;
2062  } else {
2064  thread_buf_end,
2065  buf_end,
2066  copy_params,
2067  importer->get_is_array(),
2068  row,
2069  tmp_buffers,
2070  try_single_thread,
2071  true);
2072  }
2073  row_index_plus_one++;
2074  // Each POINT could consume two separate coords instead of a single WKT
2075  if (row.size() < num_cols || (num_cols + point_cols) < row.size()) {
2076  thread_import_status.rows_rejected++;
2077  LOG(ERROR) << "Incorrect Row (expected " << num_cols << " columns, has "
2078  << row.size() << "): " << shared::printContainer(row);
2079  if (thread_import_status.rows_rejected > copy_params.max_reject) {
2080  break;
2081  }
2082  continue;
2083  }
2084 
2085  //
2086  // lambda for importing a row (perhaps multiple times if exploding a collection)
2087  //
2088 
2089  auto execute_import_row = [&](OGRGeometry* import_geometry) {
2090  size_t import_idx = 0;
2091  size_t col_idx = 0;
2092  try {
2093  for (auto cd_it = col_descs.begin(); cd_it != col_descs.end(); cd_it++) {
2094  auto cd = *cd_it;
2095  const auto& col_ti = cd->columnType;
2096 
2097  bool is_null =
2098  (row[import_idx] == copy_params.null_str || row[import_idx] == "NULL");
2099  // Note: default copy_params.null_str is "\N", but everyone uses "NULL".
2100  // So initially nullness may be missed and not passed to add_value,
2101  // which then might also check and still decide it's actually a NULL, e.g.
2102  // if kINT doesn't start with a digit or a '-' then it's considered NULL.
2103  // So "NULL" is not recognized as NULL but then it's not recognized as
2104  // a valid kINT, so it's a NULL after all.
2105  // Checking for "NULL" here too, as a widely accepted notation for NULL.
2106 
2107  // Treating empty as NULL
2108  if (!cd->columnType.is_string() && row[import_idx].empty()) {
2109  is_null = true;
2110  }
2111 
2112  if (col_ti.get_physical_cols() == 0) {
2113  // not geo
2114 
2115  import_buffers[col_idx]->add_value(
2116  cd, row[import_idx], is_null, copy_params);
2117 
2118  // next
2119  ++import_idx;
2120  ++col_idx;
2121  } else {
2122  // geo
2123 
2124  // store null string in the base column
2125  import_buffers[col_idx]->add_value(
2126  cd, copy_params.null_str, true, copy_params);
2127 
2128  // WKT from string we're not storing
2129  auto const& geo_string = row[import_idx];
2130 
2131  // next
2132  ++import_idx;
2133  ++col_idx;
2134 
2135  SQLTypes col_type = col_ti.get_type();
2136  CHECK(IS_GEO(col_type));
2137 
2138  std::vector<double> coords;
2139  std::vector<double> bounds;
2140  std::vector<int> ring_sizes;
2141  std::vector<int> poly_rings;
2142  int render_group = 0;
2143 
2144  // if this is a POINT column, and the field is not null, and
2145  // looks like a scalar numeric value (and not a hex blob)
2146  // attempt to import two columns as lon/lat (or lat/lon)
2147  if (col_type == kPOINT && !is_null && geo_string.size() > 0 &&
2148  (geo_string[0] == '.' || isdigit(geo_string[0]) ||
2149  geo_string[0] == '-') &&
2150  geo_string.find_first_of("ABCDEFabcdef") == std::string::npos) {
2151  double lon = std::atof(std::string(geo_string).c_str());
2152  double lat = NAN;
2153  auto lat_str = row[import_idx];
2154  ++import_idx;
2155  if (lat_str.size() > 0 &&
2156  (lat_str[0] == '.' || isdigit(lat_str[0]) || lat_str[0] == '-')) {
2157  lat = std::atof(std::string(lat_str).c_str());
2158  }
2159  // Swap coordinates if this table uses a reverse order: lat/lon
2160  if (!copy_params.lonlat) {
2161  std::swap(lat, lon);
2162  }
2163  // TODO: should check if POINT column should have been declared with
2164  // SRID WGS 84, EPSG 4326 ? if (col_ti.get_dimension() != 4326) {
2165  // throw std::runtime_error("POINT column " + cd->columnName + " is
2166  // not WGS84, cannot insert lon/lat");
2167  // }
2168  SQLTypeInfo import_ti{col_ti};
2169  if (copy_params.file_type == FileType::DELIMITED &&
2170  import_ti.get_output_srid() == 4326) {
2171  auto srid0 = copy_params.source_srid;
2172  if (srid0 > 0) {
2173  // srid0 -> 4326 transform is requested on import
2174  import_ti.set_input_srid(srid0);
2175  }
2176  }
2177  if (!importGeoFromLonLat(lon, lat, coords, import_ti)) {
2178  throw std::runtime_error(
2179  "Cannot read lon/lat to insert into POINT column " +
2180  cd->columnName);
2181  }
2182  } else {
2183  // import it
2184  SQLTypeInfo import_ti{col_ti};
2185  if (copy_params.file_type == FileType::DELIMITED &&
2186  import_ti.get_output_srid() == 4326) {
2187  auto srid0 = copy_params.source_srid;
2188  if (srid0 > 0) {
2189  // srid0 -> 4326 transform is requested on import
2190  import_ti.set_input_srid(srid0);
2191  }
2192  }
2193  if (is_null) {
2194  if (col_ti.get_notnull()) {
2195  throw std::runtime_error("NULL geo for column " + cd->columnName);
2196  }
2198  import_ti,
2199  coords,
2200  bounds,
2201  ring_sizes,
2202  poly_rings,
2204  } else {
2205  if (import_geometry) {
2206  // geometry already exploded
2208  import_geometry,
2209  import_ti,
2210  coords,
2211  bounds,
2212  ring_sizes,
2213  poly_rings,
2215  std::string msg =
2216  "Failed to extract valid geometry from exploded row " +
2217  std::to_string(first_row_index_this_buffer +
2218  row_index_plus_one) +
2219  " for column " + cd->columnName;
2220  throw std::runtime_error(msg);
2221  }
2222  } else {
2223  // extract geometry directly from WKT
2225  std::string(geo_string),
2226  import_ti,
2227  coords,
2228  bounds,
2229  ring_sizes,
2230  poly_rings,
2232  std::string msg = "Failed to extract valid geometry from row " +
2233  std::to_string(first_row_index_this_buffer +
2234  row_index_plus_one) +
2235  " for column " + cd->columnName;
2236  throw std::runtime_error(msg);
2237  }
2238  }
2239 
2240  // validate types
2241  if (col_type != import_ti.get_type()) {
2243  !(import_ti.get_type() == SQLTypes::kPOLYGON &&
2244  col_type == SQLTypes::kMULTIPOLYGON)) {
2245  throw std::runtime_error(
2246  "Imported geometry doesn't match the type of column " +
2247  cd->columnName);
2248  }
2249  }
2250  }
2251 
2252  // assign render group?
2253  if (columnIdToRenderGroupAnalyzerMap.size()) {
2254  if (col_type == kPOLYGON || col_type == kMULTIPOLYGON) {
2255  if (ring_sizes.size()) {
2256  // get a suitable render group for these poly coords
2257  auto rga_it = columnIdToRenderGroupAnalyzerMap.find(cd->columnId);
2258  CHECK(rga_it != columnIdToRenderGroupAnalyzerMap.end());
2259  render_group =
2260  (*rga_it).second->insertBoundsAndReturnRenderGroup(bounds);
2261  } else {
2262  // empty poly
2263  render_group = -1;
2264  }
2265  }
2266  }
2267  }
2268 
2269  // import extracted geo
2270  Importer::set_geo_physical_import_buffer(importer->getCatalog(),
2271  cd,
2272  import_buffers,
2273  col_idx,
2274  coords,
2275  bounds,
2276  ring_sizes,
2277  poly_rings,
2278  render_group);
2279 
2280  // skip remaining physical columns
2281  for (int i = 0; i < cd->columnType.get_physical_cols(); ++i) {
2282  ++cd_it;
2283  }
2284  }
2285  }
2286  if (UNLIKELY((thread_import_status.rows_completed & 0xFFFF) == 0 &&
2287  check_session_interrupted(query_session, executor))) {
2288  thread_import_status.load_failed = true;
2289  thread_import_status.load_msg =
2290  "Table load was cancelled via Query Interrupt";
2291  return;
2292  }
2293  thread_import_status.rows_completed++;
2294  } catch (const std::exception& e) {
2295  for (size_t col_idx_to_pop = 0; col_idx_to_pop < col_idx; ++col_idx_to_pop) {
2296  import_buffers[col_idx_to_pop]->pop_value();
2297  }
2298  thread_import_status.rows_rejected++;
2299  LOG(ERROR) << "Input exception thrown: " << e.what()
2300  << ". Row discarded. Data: " << shared::printContainer(row);
2301  if (thread_import_status.rows_rejected > copy_params.max_reject) {
2302  LOG(ERROR) << "Load was cancelled due to max reject rows being reached";
2303  thread_import_status.load_failed = true;
2304  thread_import_status.load_msg =
2305  "Load was cancelled due to max reject rows being reached";
2306  }
2307  }
2308  }; // End of lambda
2309 
2310  if (copy_params.geo_explode_collections) {
2311  // explode and import
2312  auto const [collection_col_idx, collection_child_type, collection_col_name] =
2313  explode_collections_step1(col_descs);
2314  // pull out the collection WKT or WKB hex
2315  CHECK_LT(collection_col_idx, (int)row.size()) << "column index out of range";
2316  auto const& collection_geo_string = row[collection_col_idx];
2317  // convert to OGR
2318  OGRGeometry* ogr_geometry = nullptr;
2319  ScopeGuard destroy_ogr_geometry = [&] {
2320  if (ogr_geometry) {
2321  OGRGeometryFactory::destroyGeometry(ogr_geometry);
2322  }
2323  };
2325  std::string(collection_geo_string));
2326  // do the explode and import
2327  us = explode_collections_step2(ogr_geometry,
2328  collection_child_type,
2329  collection_col_name,
2330  first_row_index_this_buffer + row_index_plus_one,
2331  execute_import_row);
2332  } else {
2333  // import non-collection row just once
2335  [&] { execute_import_row(nullptr); });
2336  }
2337 
2338  if (thread_import_status.load_failed) {
2339  break;
2340  }
2341  } // end thread
2342  total_str_to_val_time_us += us;
2343  if (!thread_import_status.load_failed && thread_import_status.rows_completed > 0) {
2344  load_ms = measure<>::execution([&]() {
2345  importer->load(import_buffers, thread_import_status.rows_completed, session_info);
2346  });
2347  }
2348  }); // end execution
2349 
2350  if (DEBUG_TIMING && !thread_import_status.load_failed &&
2351  thread_import_status.rows_completed > 0) {
2352  LOG(INFO) << "Thread" << std::this_thread::get_id() << ":"
2353  << thread_import_status.rows_completed << " rows inserted in "
2354  << (double)ms / 1000.0 << "sec, Insert Time: " << (double)load_ms / 1000.0
2355  << "sec, get_row: " << (double)total_get_row_time_us / 1000000.0
2356  << "sec, str_to_val: " << (double)total_str_to_val_time_us / 1000000.0
2357  << "sec" << std::endl;
2358  }
2359 
2360  return thread_import_status;
2361 }
bool check_session_interrupted(const QuerySessionId &query_session, Executor *executor)
Definition: Importer.cpp:99
SQLTypes
Definition: sqltypes.h:38
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
int64_t explode_collections_step2(OGRGeometry *ogr_geometry, SQLTypes collection_child_type, const std::string &collection_col_name, size_t row_or_feature_idx, std::function< void(OGRGeometry *)> execute_import_lambda)
Definition: Importer.cpp:1907
#define LOG(tag)
Definition: Logger.h:203
size_t find_beginning(const char *buffer, size_t begin, size_t end, const import_export::CopyParams &copy_params)
Finds the closest possible row beginning in the given buffer.
static void getNullGeoColumns(SQLTypeInfo &ti, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, const bool promote_poly_to_mpoly=false)
Definition: Types.cpp:1144
std::string to_string(char const *&&v)
#define DEBUG_TIMING
Definition: Importer.cpp:159
void set_input_srid(int d)
Definition: sqltypes.h:423
CONSTEXPR DEVICE bool is_null(const T &value)
const char * get_row(const char *buf, const char *buf_end, const char *entire_buf_end, const import_export::CopyParams &copy_params, const bool *is_array, std::vector< T > &row, std::vector< std::unique_ptr< char[]>> &tmp_buffers, bool &try_single_thread, bool filter_empty_lines)
Parses the first row in the given buffer and inserts fields into given vector.
#define UNLIKELY(x)
Definition: likely.h:25
std::string get_session_id() const
Definition: SessionInfo.h:78
static bool getGeoColumns(const std::string &wkt_or_wkb_hex, SQLTypeInfo &ti, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, const bool promote_poly_to_mpoly=false)
Definition: Types.cpp:937
#define CHECK_LT(x, y)
Definition: Logger.h:219
static OGRGeometry * createOGRGeometry(const std::string &wkt_or_wkb_hex)
Definition: Types.cpp:903
std::tuple< int, SQLTypes, std::string > explode_collections_step1(const std::list< const ColumnDescriptor * > &col_descs)
Definition: Importer.cpp:1873
ThreadId thread_id()
Definition: Logger.cpp:791
#define CHECK(condition)
Definition: Logger.h:209
bool importGeoFromLonLat(double lon, double lat, std::vector< double > &coords, SQLTypeInfo &ti)
Definition: Importer.cpp:1610
PrintContainer< CONTAINER > printContainer(CONTAINER &container)
Definition: misc.h:105
DEVICE void swap(ARGS &&...args)
Definition: gpu_enabled.h:114
#define IS_GEO(T)
Definition: sqltypes.h:251

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

static ImportStatus import_export::import_thread_shapefile ( int  thread_id,
Importer *  importer,
OGRSpatialReference *  poGeographicSR,
const FeaturePtrVector &  features,
size_t  firstFeature,
size_t  numFeatures,
const FieldNameToIndexMapType &  fieldNameToIndexMap,
const ColumnNameToSourceNameMapType &  columnNameToSourceNameMap,
const ColumnIdToRenderGroupAnalyzerMapType &  columnIdToRenderGroupAnalyzerMap,
const Catalog_Namespace::SessionInfo session_info,
Executor executor 
)
static

Definition at line 2369 of file Importer.cpp.

References CHECK, anonymous_namespace{Importer.cpp}::check_session_interrupted(), Geospatial::compress_coords(), DEBUG_TIMING, Executor::ERR_INTERRUPTED, logger::ERROR, import_export::anonymous_namespace{Importer.cpp}::explode_collections_step1(), import_export::anonymous_namespace{Importer.cpp}::explode_collections_step2(), import_export::CopyParams::geo_explode_collections, import_export::Importer::get_column_descs(), import_export::Importer::get_copy_params(), import_export::Importer::get_import_buffers(), Catalog_Namespace::SessionInfo::get_session_id(), QueryExecutionError::getErrorCode(), Geospatial::GeoTypesFactory::getGeoColumns(), Geospatial::GeoTypesFactory::getNullGeoColumns(), logger::INFO, kLINESTRING, kMULTIPOLYGON, kPOLYGON, import_export::Importer::load(), import_export::ImportStatus::load_failed, import_export::ImportStatus::load_msg, LOG, import_export::CopyParams::null_str, PROMOTE_POLYGON_TO_MULTIPOLYGON, import_export::ImportStatus::rows_completed, import_export::ImportStatus::rows_rejected, import_export::ImportStatus::thread_id, logger::thread_id(), timer_start(), timer_stop(), to_string(), and UNLIKELY.

Referenced by import_export::Importer::importGDAL().

2380  {
2381  ImportStatus thread_import_status;
2382  const CopyParams& copy_params = importer->get_copy_params();
2383  const std::list<const ColumnDescriptor*>& col_descs = importer->get_column_descs();
2384  std::vector<std::unique_ptr<TypedImportBuffer>>& import_buffers =
2385  importer->get_import_buffers(thread_id);
2386  auto query_session = session_info ? session_info->get_session_id() : "";
2387  for (const auto& p : import_buffers) {
2388  p->clear();
2389  }
2390 
2391  auto convert_timer = timer_start();
2392 
2393  // we create this on the fly based on the first feature's SR
2394  std::unique_ptr<OGRCoordinateTransformation> coordinate_transformation;
2395  bool checked_transformation{false};
2396 
2397  // for all the features in this chunk...
2398  for (size_t iFeature = 0; iFeature < numFeatures; iFeature++) {
2399  // ignore null features
2400  if (!features[iFeature]) {
2401  continue;
2402  }
2403 
2404  // get this feature's geometry
2405  // for geodatabase, we need to consider features with no geometry
2406  // as we still want to create a table, even if it has no geo column
2407  OGRGeometry* pGeometry = features[iFeature]->GetGeometryRef();
2408  if (pGeometry) {
2409  // check if we need to make a CoordinateTransformation
2410  // we assume that all features in this chunk will have
2411  // the same source SR, so the CT will be valid for all
2412  // transforming to a reusable CT is faster than to an SR
2413  if (!checked_transformation) {
2414  // get the SR of the incoming geo
2415  auto geometry_sr = pGeometry->getSpatialReference();
2416  // if the SR is non-null and non-empty and different from what we want
2417  // we need to make a reusable CoordinateTransformation
2418  if (geometry_sr &&
2419 #if GDAL_VERSION_MAJOR >= 3
2420  !geometry_sr->IsEmpty() &&
2421 #endif
2422  !geometry_sr->IsSame(poGeographicSR)) {
2423  // validate the SR before trying to use it
2424  if (geometry_sr->Validate() != OGRERR_NONE) {
2425  throw std::runtime_error("Incoming geo has invalid Spatial Reference");
2426  }
2427  // create the OGRCoordinateTransformation that will be used for
2428  // all the features in this chunk
2429  if (coordinate_transformation == nullptr) {
2430  coordinate_transformation.reset(
2431  OGRCreateCoordinateTransformation(geometry_sr, poGeographicSR));
2432  if (coordinate_transformation == nullptr) {
2433  throw std::runtime_error(
2434  "Failed to create a GDAL CoordinateTransformation for incoming geo");
2435  }
2436  }
2437  }
2438  checked_transformation = true;
2439  }
2440 
2441  // if we have a transformation, use it
2442  if (coordinate_transformation) {
2443  pGeometry->transform(coordinate_transformation.get());
2444  }
2445  }
2446 
2447  //
2448  // lambda for importing a feature (perhaps multiple times if exploding a collection)
2449  //
2450 
2451  auto execute_import_feature = [&](OGRGeometry* import_geometry) {
2452  size_t col_idx = 0;
2453  try {
2454  if (UNLIKELY((thread_import_status.rows_completed & 0xFFFF) == 0 &&
2455  check_session_interrupted(query_session, executor))) {
2456  thread_import_status.load_failed = true;
2457  thread_import_status.load_msg = "Table load was cancelled via Query Interrupt";
2459  }
2460  for (auto cd_it = col_descs.begin(); cd_it != col_descs.end(); cd_it++) {
2461  auto cd = *cd_it;
2462 
2463  // is this a geo column?
2464  const auto& col_ti = cd->columnType;
2465  if (col_ti.is_geometry()) {
2466  // Note that this assumes there is one and only one geo column in the
2467  // table. Currently, the importer only supports reading a single
2468  // geospatial feature from an input shapefile / geojson file, but this
2469  // code will need to be modified if that changes
2470  SQLTypes col_type = col_ti.get_type();
2471 
2472  // store null string in the base column
2473  import_buffers[col_idx]->add_value(
2474  cd, copy_params.null_str, true, copy_params);
2475  ++col_idx;
2476 
2477  // the data we now need to extract for the other columns
2478  std::vector<double> coords;
2479  std::vector<double> bounds;
2480  std::vector<int> ring_sizes;
2481  std::vector<int> poly_rings;
2482  int render_group = 0;
2483 
2484  // extract it
2485  SQLTypeInfo import_ti{col_ti};
2486  bool is_null_geo = !import_geometry;
2487  if (is_null_geo) {
2488  if (col_ti.get_notnull()) {
2489  throw std::runtime_error("NULL geo for column " + cd->columnName);
2490  }
2492  import_ti,
2493  coords,
2494  bounds,
2495  ring_sizes,
2496  poly_rings,
2498  } else {
2500  import_geometry,
2501  import_ti,
2502  coords,
2503  bounds,
2504  ring_sizes,
2505  poly_rings,
2507  std::string msg = "Failed to extract valid geometry from feature " +
2508  std::to_string(firstFeature + iFeature + 1) +
2509  " for column " + cd->columnName;
2510  throw std::runtime_error(msg);
2511  }
2512 
2513  // validate types
2514  if (col_type != import_ti.get_type()) {
2516  !(import_ti.get_type() == SQLTypes::kPOLYGON &&
2517  col_type == SQLTypes::kMULTIPOLYGON)) {
2518  throw std::runtime_error(
2519  "Imported geometry doesn't match the type of column " +
2520  cd->columnName);
2521  }
2522  }
2523  }
2524 
2525  if (col_type == kPOLYGON || col_type == kMULTIPOLYGON) {
2526  if (ring_sizes.size()) {
2527  // get a suitable render group for these poly coords
2528  auto rga_it = columnIdToRenderGroupAnalyzerMap.find(cd->columnId);
2529  CHECK(rga_it != columnIdToRenderGroupAnalyzerMap.end());
2530  render_group = (*rga_it).second->insertBoundsAndReturnRenderGroup(bounds);
2531  } else {
2532  // empty poly
2533  render_group = -1;
2534  }
2535  }
2536 
2537  // create coords array value and add it to the physical column
2538  ++cd_it;
2539  auto cd_coords = *cd_it;
2540  std::vector<TDatum> td_coord_data;
2541  if (!is_null_geo) {
2542  std::vector<uint8_t> compressed_coords =
2543  Geospatial::compress_coords(coords, col_ti);
2544  for (auto cc : compressed_coords) {
2545  TDatum td_byte;
2546  td_byte.val.int_val = cc;
2547  td_coord_data.push_back(td_byte);
2548  }
2549  }
2550  TDatum tdd_coords;
2551  tdd_coords.val.arr_val = td_coord_data;
2552  tdd_coords.is_null = is_null_geo;
2553  import_buffers[col_idx]->add_value(cd_coords, tdd_coords, false);
2554  ++col_idx;
2555 
2556  if (col_type == kPOLYGON || col_type == kMULTIPOLYGON) {
2557  // Create ring_sizes array value and add it to the physical column
2558  ++cd_it;
2559  auto cd_ring_sizes = *cd_it;
2560  std::vector<TDatum> td_ring_sizes;
2561  if (!is_null_geo) {
2562  for (auto ring_size : ring_sizes) {
2563  TDatum td_ring_size;
2564  td_ring_size.val.int_val = ring_size;
2565  td_ring_sizes.push_back(td_ring_size);
2566  }
2567  }
2568  TDatum tdd_ring_sizes;
2569  tdd_ring_sizes.val.arr_val = td_ring_sizes;
2570  tdd_ring_sizes.is_null = is_null_geo;
2571  import_buffers[col_idx]->add_value(cd_ring_sizes, tdd_ring_sizes, false);
2572  ++col_idx;
2573  }
2574 
2575  if (col_type == kMULTIPOLYGON) {
2576  // Create poly_rings array value and add it to the physical column
2577  ++cd_it;
2578  auto cd_poly_rings = *cd_it;
2579  std::vector<TDatum> td_poly_rings;
2580  if (!is_null_geo) {
2581  for (auto num_rings : poly_rings) {
2582  TDatum td_num_rings;
2583  td_num_rings.val.int_val = num_rings;
2584  td_poly_rings.push_back(td_num_rings);
2585  }
2586  }
2587  TDatum tdd_poly_rings;
2588  tdd_poly_rings.val.arr_val = td_poly_rings;
2589  tdd_poly_rings.is_null = is_null_geo;
2590  import_buffers[col_idx]->add_value(cd_poly_rings, tdd_poly_rings, false);
2591  ++col_idx;
2592  }
2593 
2594  if (col_type == kLINESTRING || col_type == kPOLYGON ||
2595  col_type == kMULTIPOLYGON) {
2596  // Create bounds array value and add it to the physical column
2597  ++cd_it;
2598  auto cd_bounds = *cd_it;
2599  std::vector<TDatum> td_bounds_data;
2600  if (!is_null_geo) {
2601  for (auto b : bounds) {
2602  TDatum td_double;
2603  td_double.val.real_val = b;
2604  td_bounds_data.push_back(td_double);
2605  }
2606  }
2607  TDatum tdd_bounds;
2608  tdd_bounds.val.arr_val = td_bounds_data;
2609  tdd_bounds.is_null = is_null_geo;
2610  import_buffers[col_idx]->add_value(cd_bounds, tdd_bounds, false);
2611  ++col_idx;
2612  }
2613 
2614  if (col_type == kPOLYGON || col_type == kMULTIPOLYGON) {
2615  // Create render_group value and add it to the physical column
2616  ++cd_it;
2617  auto cd_render_group = *cd_it;
2618  TDatum td_render_group;
2619  td_render_group.val.int_val = render_group;
2620  td_render_group.is_null = is_null_geo;
2621  import_buffers[col_idx]->add_value(cd_render_group, td_render_group, false);
2622  ++col_idx;
2623  }
2624  } else {
2625  // regular column
2626  // pull from GDAL metadata
2627  auto const cit = columnNameToSourceNameMap.find(cd->columnName);
2628  CHECK(cit != columnNameToSourceNameMap.end());
2629  auto const& field_name = cit->second;
2630 
2631  auto const fit = fieldNameToIndexMap.find(field_name);
2632  if (fit == fieldNameToIndexMap.end()) {
2633  throw ColumnNotGeoError(cd->columnName);
2634  }
2635 
2636  auto const& field_index = fit->second;
2637  CHECK(field_index < fieldNameToIndexMap.size());
2638 
2639  auto const& feature = features[iFeature];
2640 
2641  auto field_defn = feature->GetFieldDefnRef(field_index);
2642  CHECK(field_defn);
2643 
2644  // OGRFeature::GetFieldAsString() can only return 80 characters
2645  // so for array columns, we are obliged to fetch the actual values
2646  // and construct the concatenated string ourselves
2647 
2648  std::string value_string;
2649  int array_index = 0, array_size = 0;
2650 
2651  auto stringify_numeric_list = [&](auto* values) {
2652  value_string = "{";
2653  while (array_index < array_size) {
2654  auto separator = (array_index > 0) ? "," : "";
2655  value_string += separator + std::to_string(values[array_index]);
2656  array_index++;
2657  }
2658  value_string += "}";
2659  };
2660 
2661  auto field_type = field_defn->GetType();
2662  switch (field_type) {
2663  case OFTInteger:
2664  case OFTInteger64:
2665  case OFTReal:
2666  case OFTString:
2667  case OFTBinary:
2668  case OFTDate:
2669  case OFTTime:
2670  case OFTDateTime: {
2671  value_string = feature->GetFieldAsString(field_index);
2672  } break;
2673  case OFTIntegerList: {
2674  auto* values = feature->GetFieldAsIntegerList(field_index, &array_size);
2675  stringify_numeric_list(values);
2676  } break;
2677  case OFTInteger64List: {
2678  auto* values = feature->GetFieldAsInteger64List(field_index, &array_size);
2679  stringify_numeric_list(values);
2680  } break;
2681  case OFTRealList: {
2682  auto* values = feature->GetFieldAsDoubleList(field_index, &array_size);
2683  stringify_numeric_list(values);
2684  } break;
2685  case OFTStringList: {
2686  auto** array_of_strings = feature->GetFieldAsStringList(field_index);
2687  value_string = "{";
2688  if (array_of_strings) {
2689  while (auto* this_string = array_of_strings[array_index]) {
2690  auto separator = (array_index > 0) ? "," : "";
2691  value_string += separator + std::string(this_string);
2692  array_index++;
2693  }
2694  }
2695  value_string += "}";
2696  } break;
2697  default:
2698  throw std::runtime_error("Unsupported geo file field type (" +
2699  std::to_string(static_cast<int>(field_type)) +
2700  ")");
2701  }
2702 
2703  static CopyParams default_copy_params;
2704  import_buffers[col_idx]->add_value(
2705  cd, value_string, false, default_copy_params);
2706  ++col_idx;
2707  }
2708  }
2709  thread_import_status.rows_completed++;
2710  } catch (QueryExecutionError& e) {
2712  throw e;
2713  }
2714  } catch (ColumnNotGeoError& e) {
2715  LOG(ERROR) << "Input exception thrown: " << e.what() << ". Aborting import.";
2716  throw std::runtime_error(e.what());
2717  } catch (const std::exception& e) {
2718  for (size_t col_idx_to_pop = 0; col_idx_to_pop < col_idx; ++col_idx_to_pop) {
2719  import_buffers[col_idx_to_pop]->pop_value();
2720  }
2721  thread_import_status.rows_rejected++;
2722  LOG(ERROR) << "Input exception thrown: " << e.what() << ". Row discarded.";
2723  }
2724  };
2725 
2726  if (pGeometry && copy_params.geo_explode_collections) {
2727  // explode and import
2728  auto const [collection_idx_type_name, collection_child_type, collection_col_name] =
2729  explode_collections_step1(col_descs);
2730  explode_collections_step2(pGeometry,
2731  collection_child_type,
2732  collection_col_name,
2733  firstFeature + iFeature + 1,
2734  execute_import_feature);
2735  } else {
2736  // import non-collection or null feature just once
2737  execute_import_feature(pGeometry);
2738  }
2739  } // end features
2740 
2741  float convert_ms =
2742  float(timer_stop<std::chrono::steady_clock::time_point, std::chrono::microseconds>(
2743  convert_timer)) /
2744  1000.0f;
2745 
2746  float load_ms = 0.0f;
2747  if (thread_import_status.rows_completed > 0) {
2748  auto load_timer = timer_start();
2749  importer->load(import_buffers, thread_import_status.rows_completed, session_info);
2750  load_ms =
2751  float(
2752  timer_stop<std::chrono::steady_clock::time_point, std::chrono::microseconds>(
2753  load_timer)) /
2754  1000.0f;
2755  }
2756 
2757  if (DEBUG_TIMING && thread_import_status.rows_completed > 0) {
2758  LOG(INFO) << "DEBUG: Process " << convert_ms << "ms";
2759  LOG(INFO) << "DEBUG: Load " << load_ms << "ms";
2760  }
2761 
2762  thread_import_status.thread_id = thread_id;
2763 
2764  if (DEBUG_TIMING) {
2765  LOG(INFO) << "DEBUG: Total "
2766  << float(timer_stop<std::chrono::steady_clock::time_point,
2767  std::chrono::microseconds>(convert_timer)) /
2768  1000.0f
2769  << "ms";
2770  }
2771 
2772  return thread_import_status;
2773 }
bool check_session_interrupted(const QuerySessionId &query_session, Executor *executor)
Definition: Importer.cpp:99
int32_t getErrorCode() const
Definition: ErrorHandling.h:55
static const int32_t ERR_INTERRUPTED
Definition: Execute.h:1163
SQLTypes
Definition: sqltypes.h:38
int64_t explode_collections_step2(OGRGeometry *ogr_geometry, SQLTypes collection_child_type, const std::string &collection_col_name, size_t row_or_feature_idx, std::function< void(OGRGeometry *)> execute_import_lambda)
Definition: Importer.cpp:1907
#define LOG(tag)
Definition: Logger.h:203
TypeR::rep timer_stop(Type clock_begin)
Definition: measure.h:48
static void getNullGeoColumns(SQLTypeInfo &ti, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, const bool promote_poly_to_mpoly=false)
Definition: Types.cpp:1144
std::string to_string(char const *&&v)
#define DEBUG_TIMING
Definition: Importer.cpp:159
std::vector< uint8_t > compress_coords(const std::vector< double > &coords, const SQLTypeInfo &ti)
Definition: Compression.cpp:52
#define UNLIKELY(x)
Definition: likely.h:25
std::string get_session_id() const
Definition: SessionInfo.h:78
static bool getGeoColumns(const std::string &wkt_or_wkb_hex, SQLTypeInfo &ti, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, const bool promote_poly_to_mpoly=false)
Definition: Types.cpp:937
std::tuple< int, SQLTypes, std::string > explode_collections_step1(const std::list< const ColumnDescriptor * > &col_descs)
Definition: Importer.cpp:1873
ThreadId thread_id()
Definition: Logger.cpp:791
#define CHECK(condition)
Definition: Logger.h:209
Type timer_start()
Definition: measure.h:42

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool import_export::importGeoFromLonLat ( double  lon,
double  lat,
std::vector< double > &  coords,
SQLTypeInfo ti 
)

Definition at line 1610 of file Importer.cpp.

References Geospatial::GeoPoint::getColumns(), and SQLTypeInfo::transforms().

Referenced by import_thread_delimited().

1613  {
1614  if (std::isinf(lat) || std::isnan(lat) || std::isinf(lon) || std::isnan(lon)) {
1615  return false;
1616  }
1617  if (ti.transforms()) {
1618  Geospatial::GeoPoint pt{std::vector<double>{lon, lat}};
1619  if (!pt.transform(ti)) {
1620  return false;
1621  }
1622  pt.getColumns(coords);
1623  return true;
1624  }
1625  coords.push_back(lon);
1626  coords.push_back(lat);
1627  return true;
1628 }
void getColumns(std::vector< double > &coords) const
Definition: Types.cpp:567
bool transforms() const
Definition: sqltypes.h:530

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

ArrayDatum import_export::NullArray ( const SQLTypeInfo ti)

Definition at line 388 of file Importer.cpp.

References appendDatum(), CHECK, checked_malloc(), SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_size(), NullArrayDatum(), and NullDatum().

Referenced by import_export::TypedImportBuffer::add_value(), import_export::TypedImportBuffer::add_values(), import_export::TypedImportBuffer::addDefaultValues(), import_export::ImporterUtils::composeNullArray(), and TDatumToArrayDatum().

388  {
389  SQLTypeInfo elem_ti = ti.get_elem_type();
390  auto len = ti.get_size();
391 
392  if (len > 0) {
393  // Compose a NULL fixlen array
394  int8_t* buf = (int8_t*)checked_malloc(len);
395  // First scalar is a NULL_ARRAY sentinel
396  Datum d = NullArrayDatum(elem_ti);
397  int8_t* p = appendDatum(buf, d, elem_ti);
398  // Rest is filled with normal NULL sentinels
399  Datum d0 = NullDatum(elem_ti);
400  while ((p - buf) < len) {
401  p = appendDatum(p, d0, elem_ti);
402  }
403  CHECK((p - buf) == len);
404  return ArrayDatum(len, buf, true);
405  }
406  // NULL varlen array
407  return ArrayDatum(0, NULL, true);
408 }
HOST DEVICE int get_size() const
Definition: sqltypes.h:339
std::conditional_t< is_cuda_compiler(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:208
Datum NullDatum(SQLTypeInfo &ti)
Definition: Importer.cpp:253
void * checked_malloc(const size_t size)
Definition: checked_alloc.h:45
int8_t * appendDatum(int8_t *buf, Datum d, const SQLTypeInfo &ti)
Definition: sqltypes.h:1078
#define CHECK(condition)
Definition: Logger.h:209
Datum NullArrayDatum(SQLTypeInfo &ti)
Definition: Importer.cpp:294
SQLTypeInfo get_elem_type() const
Definition: sqltypes.h:850

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Datum import_export::NullArrayDatum ( SQLTypeInfo ti)

Definition at line 294 of file Importer.cpp.

References Datum::bigintval, Datum::boolval, decimal_to_int_type(), Datum::doubleval, Datum::floatval, SQLTypeInfo::get_type(), inline_fixed_encoding_null_array_val(), Datum::intval, SQLTypeInfo::is_decimal(), kBIGINT, kBOOLEAN, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTIME, kTIMESTAMP, kTINYINT, NULL_ARRAY_DOUBLE, NULL_ARRAY_FLOAT, Datum::smallintval, Datum::tinyintval, and run_benchmark_import::type.

Referenced by NullArray().

294  {
295  Datum d;
296  const auto type = ti.is_decimal() ? decimal_to_int_type(ti) : ti.get_type();
297  switch (type) {
298  case kBOOLEAN:
300  break;
301  case kBIGINT:
303  break;
304  case kINT:
306  break;
307  case kSMALLINT:
309  break;
310  case kTINYINT:
312  break;
313  case kFLOAT:
315  break;
316  case kDOUBLE:
318  break;
319  case kTIME:
320  case kTIMESTAMP:
321  case kDATE:
323  break;
324  case kPOINT:
325  case kLINESTRING:
326  case kPOLYGON:
327  case kMULTIPOLYGON:
328  throw std::runtime_error("Internal error: geometry type in NullArrayDatum.");
329  default:
330  throw std::runtime_error("Internal error: invalid type in NullArrayDatum.");
331  }
332  return d;
333 }
int8_t tinyintval
Definition: sqltypes.h:212
Definition: sqltypes.h:49
int8_t boolval
Definition: sqltypes.h:211
int32_t intval
Definition: sqltypes.h:214
float floatval
Definition: sqltypes.h:216
int64_t bigintval
Definition: sqltypes.h:215
#define NULL_ARRAY_FLOAT
int16_t smallintval
Definition: sqltypes.h:213
SQLTypes decimal_to_int_type(const SQLTypeInfo &ti)
Definition: Datum.cpp:455
Definition: sqltypes.h:53
int64_t inline_fixed_encoding_null_array_val(const SQL_TYPE_INFO &ti)
#define NULL_ARRAY_DOUBLE
Definition: sqltypes.h:45
bool is_decimal() const
Definition: sqltypes.h:512
double doubleval
Definition: sqltypes.h:217

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Datum import_export::NullDatum ( SQLTypeInfo ti)

Definition at line 253 of file Importer.cpp.

References Datum::bigintval, Datum::boolval, decimal_to_int_type(), Datum::doubleval, Datum::floatval, SQLTypeInfo::get_type(), inline_fixed_encoding_null_val(), Datum::intval, SQLTypeInfo::is_decimal(), kBIGINT, kBOOLEAN, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTIME, kTIMESTAMP, kTINYINT, NULL_DOUBLE, NULL_FLOAT, Datum::smallintval, Datum::tinyintval, and run_benchmark_import::type.

Referenced by NullArray(), and StringToArray().

253  {
254  Datum d;
255  const auto type = ti.is_decimal() ? decimal_to_int_type(ti) : ti.get_type();
256  switch (type) {
257  case kBOOLEAN:
259  break;
260  case kBIGINT:
262  break;
263  case kINT:
265  break;
266  case kSMALLINT:
268  break;
269  case kTINYINT:
271  break;
272  case kFLOAT:
273  d.floatval = NULL_FLOAT;
274  break;
275  case kDOUBLE:
277  break;
278  case kTIME:
279  case kTIMESTAMP:
280  case kDATE:
282  break;
283  case kPOINT:
284  case kLINESTRING:
285  case kPOLYGON:
286  case kMULTIPOLYGON:
287  throw std::runtime_error("Internal error: geometry type in NullDatum.");
288  default:
289  throw std::runtime_error("Internal error: invalid type in NullDatum.");
290  }
291  return d;
292 }
int8_t tinyintval
Definition: sqltypes.h:212
#define NULL_DOUBLE
Definition: sqltypes.h:49
#define NULL_FLOAT
int8_t boolval
Definition: sqltypes.h:211
int32_t intval
Definition: sqltypes.h:214
float floatval
Definition: sqltypes.h:216
int64_t bigintval
Definition: sqltypes.h:215
int16_t smallintval
Definition: sqltypes.h:213
SQLTypes decimal_to_int_type(const SQLTypeInfo &ti)
Definition: Datum.cpp:455
Definition: sqltypes.h:53
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
Definition: sqltypes.h:45
bool is_decimal() const
Definition: sqltypes.h:512
double doubleval
Definition: sqltypes.h:217

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::pair<SQLTypes, bool> import_export::ogr_to_type ( const OGRFieldType &  ogr_type)

Definition at line 4780 of file Importer.cpp.

References kBIGINT, kDATE, kDOUBLE, kINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, and to_string().

Referenced by import_export::Importer::gdalToColumnDescriptors().

4780  {
4781  switch (ogr_type) {
4782  case OFTInteger:
4783  return std::make_pair(kINT, false);
4784  case OFTIntegerList:
4785  return std::make_pair(kINT, true);
4786 #if GDAL_VERSION_MAJOR > 1
4787  case OFTInteger64:
4788  return std::make_pair(kBIGINT, false);
4789  case OFTInteger64List:
4790  return std::make_pair(kBIGINT, true);
4791 #endif
4792  case OFTReal:
4793  return std::make_pair(kDOUBLE, false);
4794  case OFTRealList:
4795  return std::make_pair(kDOUBLE, true);
4796  case OFTString:
4797  return std::make_pair(kTEXT, false);
4798  case OFTStringList:
4799  return std::make_pair(kTEXT, true);
4800  case OFTDate:
4801  return std::make_pair(kDATE, false);
4802  case OFTTime:
4803  return std::make_pair(kTIME, false);
4804  case OFTDateTime:
4805  return std::make_pair(kTIMESTAMP, false);
4806  case OFTBinary:
4807  // Interpret binary blobs as byte arrays here
4808  // but actual import will store NULL as GDAL will not
4809  // extract the blob (OGRFeature::GetFieldAsString will
4810  // result in the import buffers having an empty string)
4811  return std::make_pair(kTINYINT, true);
4812  default:
4813  break;
4814  }
4815  throw std::runtime_error("Unknown OGR field type: " + std::to_string(ogr_type));
4816 }
Definition: sqltypes.h:49
std::string to_string(char const *&&v)
Definition: sqltypes.h:52
Definition: sqltypes.h:53
Definition: sqltypes.h:45

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

SQLTypes import_export::ogr_to_type ( const OGRwkbGeometryType &  ogr_type)

Definition at line 4818 of file Importer.cpp.

References kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, and to_string().

4818  {
4819  switch (ogr_type) {
4820  case wkbPoint:
4821  return kPOINT;
4822  case wkbLineString:
4823  return kLINESTRING;
4824  case wkbPolygon:
4825  return kPOLYGON;
4826  case wkbMultiPolygon:
4827  return kMULTIPOLYGON;
4828  default:
4829  break;
4830  }
4831  throw std::runtime_error("Unknown OGR geom type: " + std::to_string(ogr_type));
4832 }
std::string to_string(char const *&&v)

+ Here is the call graph for this function:

std::vector< std::unique_ptr< TypedImportBuffer > > import_export::setup_column_loaders ( const TableDescriptor td,
Loader *  loader 
)

Definition at line 5394 of file Importer.cpp.

References CHECK, import_export::Loader::get_column_descs(), and import_export::Loader::getStringDict().

Referenced by DBHandler::prepare_loader_generic().

5396  {
5397  CHECK(td);
5398  auto col_descs = loader->get_column_descs();
5399 
5400  std::vector<std::unique_ptr<TypedImportBuffer>> import_buffers;
5401  for (auto cd : col_descs) {
5402  import_buffers.emplace_back(
5403  std::make_unique<TypedImportBuffer>(cd, loader->getStringDict(cd)));
5404  }
5405 
5406  return import_buffers;
5407 }
#define CHECK(condition)
Definition: Logger.h:209

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

ArrayDatum import_export::StringToArray ( const std::string &  s,
const SQLTypeInfo ti,
const CopyParams &  copy_params 
)

Definition at line 335 of file Importer.cpp.

References appendDatum(), import_export::CopyParams::array_begin, import_export::CopyParams::array_delim, import_export::CopyParams::array_end, CHECK, checked_malloc(), SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_size(), i, is_null(), SQLTypeInfo::is_number(), SQLTypeInfo::is_string(), SQLTypeInfo::is_time(), LOG, import_export::CopyParams::null_str, NullDatum(), StringToDatum(), trim_space(), and logger::WARNING.

Referenced by import_export::TypedImportBuffer::add_value(), and import_export::TypedImportBuffer::addDefaultValues().

337  {
338  SQLTypeInfo elem_ti = ti.get_elem_type();
339  if (s == copy_params.null_str || s == "NULL" || s.empty()) {
340  return ArrayDatum(0, NULL, true);
341  }
342  if (s[0] != copy_params.array_begin || s[s.size() - 1] != copy_params.array_end) {
343  LOG(WARNING) << "Malformed array: " << s;
344  return ArrayDatum(0, NULL, true);
345  }
346  std::vector<std::string> elem_strs;
347  size_t last = 1;
348  for (size_t i = s.find(copy_params.array_delim, 1); i != std::string::npos;
349  i = s.find(copy_params.array_delim, last)) {
350  elem_strs.push_back(s.substr(last, i - last));
351  last = i + 1;
352  }
353  if (last + 1 <= s.size()) {
354  elem_strs.push_back(s.substr(last, s.size() - 1 - last));
355  }
356  if (elem_strs.size() == 1) {
357  auto str = elem_strs.front();
358  auto str_trimmed = trim_space(str.c_str(), str.length());
359  if (str_trimmed == "") {
360  elem_strs.clear(); // Empty array
361  }
362  }
363  if (!elem_ti.is_string()) {
364  size_t len = elem_strs.size() * elem_ti.get_size();
365  int8_t* buf = (int8_t*)checked_malloc(len);
366  int8_t* p = buf;
367  for (auto& es : elem_strs) {
368  auto e = trim_space(es.c_str(), es.length());
369  bool is_null = (e == copy_params.null_str) || e == "NULL";
370  if (!elem_ti.is_string() && e == "") {
371  is_null = true;
372  }
373  if (elem_ti.is_number() || elem_ti.is_time()) {
374  if (!isdigit(e[0]) && e[0] != '-') {
375  is_null = true;
376  }
377  }
378  Datum d = is_null ? NullDatum(elem_ti) : StringToDatum(e, elem_ti);
379  p = appendDatum(p, d, elem_ti);
380  }
381  return ArrayDatum(len, buf, false);
382  }
383  // must not be called for array of strings
384  CHECK(false);
385  return ArrayDatum(0, NULL, true);
386 }
HOST DEVICE int get_size() const
Definition: sqltypes.h:339
#define LOG(tag)
Definition: Logger.h:203
bool is_number() const
Definition: sqltypes.h:514
bool is_time() const
Definition: sqltypes.h:515
std::conditional_t< is_cuda_compiler(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:208
Datum NullDatum(SQLTypeInfo &ti)
Definition: Importer.cpp:253
CONSTEXPR DEVICE bool is_null(const T &value)
void * checked_malloc(const size_t size)
Definition: checked_alloc.h:45
Datum StringToDatum(std::string_view s, SQLTypeInfo &ti)
Definition: Datum.cpp:275
int8_t * appendDatum(int8_t *buf, Datum d, const SQLTypeInfo &ti)
Definition: sqltypes.h:1078
void trim_space(const char *&field_begin, const char *&field_end)
#define CHECK(condition)
Definition: Logger.h:209
bool is_string() const
Definition: sqltypes.h:509
SQLTypeInfo get_elem_type() const
Definition: sqltypes.h:850

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

ArrayDatum import_export::TDatumToArrayDatum ( const TDatum &  datum,
const SQLTypeInfo ti 
)

Definition at line 482 of file Importer.cpp.

References appendDatum(), CHECK, checked_malloc(), SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_size(), SQLTypeInfo::is_string(), NullArray(), and TDatumToDatum().

Referenced by import_export::TypedImportBuffer::add_value().

482  {
483  SQLTypeInfo elem_ti = ti.get_elem_type();
484 
485  CHECK(!elem_ti.is_string());
486 
487  if (datum.is_null) {
488  return NullArray(ti);
489  }
490 
491  size_t len = datum.val.arr_val.size() * elem_ti.get_size();
492  int8_t* buf = (int8_t*)checked_malloc(len);
493  int8_t* p = buf;
494  for (auto& e : datum.val.arr_val) {
495  p = appendDatum(p, TDatumToDatum(e, elem_ti), elem_ti);
496  }
497 
498  return ArrayDatum(len, buf, false);
499 }
HOST DEVICE int get_size() const
Definition: sqltypes.h:339
ArrayDatum NullArray(const SQLTypeInfo &ti)
Definition: Importer.cpp:388
Datum TDatumToDatum(const TDatum &datum, SQLTypeInfo &ti)
Definition: Importer.cpp:437
std::conditional_t< is_cuda_compiler(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:208
void * checked_malloc(const size_t size)
Definition: checked_alloc.h:45
int8_t * appendDatum(int8_t *buf, Datum d, const SQLTypeInfo &ti)
Definition: sqltypes.h:1078
#define CHECK(condition)
Definition: Logger.h:209
bool is_string() const
Definition: sqltypes.h:509
SQLTypeInfo get_elem_type() const
Definition: sqltypes.h:850

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Datum import_export::TDatumToDatum ( const TDatum &  datum,
SQLTypeInfo ti 
)

Definition at line 437 of file Importer.cpp.

References Datum::bigintval, Datum::boolval, decimal_to_int_type(), Datum::doubleval, Datum::floatval, SQLTypeInfo::get_type(), inline_fixed_encoding_null_val(), Datum::intval, SQLTypeInfo::is_decimal(), kBIGINT, kBOOLEAN, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTIME, kTIMESTAMP, kTINYINT, NULL_DOUBLE, NULL_FLOAT, Datum::smallintval, Datum::tinyintval, and run_benchmark_import::type.

Referenced by TDatumToArrayDatum().

437  {
438  Datum d;
439  const auto type = ti.is_decimal() ? decimal_to_int_type(ti) : ti.get_type();
440  switch (type) {
441  case kBOOLEAN:
442  d.boolval = datum.is_null ? inline_fixed_encoding_null_val(ti) : datum.val.int_val;
443  break;
444  case kBIGINT:
445  d.bigintval =
446  datum.is_null ? inline_fixed_encoding_null_val(ti) : datum.val.int_val;
447  break;
448  case kINT:
449  d.intval = datum.is_null ? inline_fixed_encoding_null_val(ti) : datum.val.int_val;
450  break;
451  case kSMALLINT:
452  d.smallintval =
453  datum.is_null ? inline_fixed_encoding_null_val(ti) : datum.val.int_val;
454  break;
455  case kTINYINT:
456  d.tinyintval =
457  datum.is_null ? inline_fixed_encoding_null_val(ti) : datum.val.int_val;
458  break;
459  case kFLOAT:
460  d.floatval = datum.is_null ? NULL_FLOAT : datum.val.real_val;
461  break;
462  case kDOUBLE:
463  d.doubleval = datum.is_null ? NULL_DOUBLE : datum.val.real_val;
464  break;
465  case kTIME:
466  case kTIMESTAMP:
467  case kDATE:
468  d.bigintval =
469  datum.is_null ? inline_fixed_encoding_null_val(ti) : datum.val.int_val;
470  break;
471  case kPOINT:
472  case kLINESTRING:
473  case kPOLYGON:
474  case kMULTIPOLYGON:
475  throw std::runtime_error("Internal error: geometry type in TDatumToDatum.");
476  default:
477  throw std::runtime_error("Internal error: invalid type in TDatumToDatum.");
478  }
479  return d;
480 }
int8_t tinyintval
Definition: sqltypes.h:212
#define NULL_DOUBLE
Definition: sqltypes.h:49
#define NULL_FLOAT
int8_t boolval
Definition: sqltypes.h:211
int32_t intval
Definition: sqltypes.h:214
float floatval
Definition: sqltypes.h:216
int64_t bigintval
Definition: sqltypes.h:215
int16_t smallintval
Definition: sqltypes.h:213
SQLTypes decimal_to_int_type(const SQLTypeInfo &ti)
Definition: Datum.cpp:455
Definition: sqltypes.h:53
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
Definition: sqltypes.h:45
bool is_decimal() const
Definition: sqltypes.h:512
double doubleval
Definition: sqltypes.h:217

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

static const std::string import_export::trim_space ( const char *  field,
const size_t  len 
)
static

Definition at line 241 of file Importer.cpp.

References i.

Referenced by import_export::delimited_parser::get_row(), and StringToArray().

241  {
242  size_t i = 0;
243  size_t j = len;
244  while (i < j && (field[i] == ' ' || field[i] == '\r')) {
245  i++;
246  }
247  while (i < j && (field[j - 1] == ' ' || field[j - 1] == '\r')) {
248  j--;
249  }
250  return std::string(field + i, j - i);
251 }
const rapidjson::Value & field(const rapidjson::Value &obj, const char field[]) noexcept
Definition: JsonAccessors.h:31

+ Here is the caller graph for this function:

template<class T >
bool import_export::try_cast ( const std::string &  str)

Definition at line 3302 of file Importer.cpp.

References omnisci.dtypes::T.

3302  {
3303  try {
3304  boost::lexical_cast<T>(str);
3305  } catch (const boost::bad_lexical_cast& e) {
3306  return false;
3307  }
3308  return true;
3309 }

Variable Documentation

std::map<std::string, ImportStatus> import_export::import_status_map
static
constexpr size_t import_export::kImportFileBufferSize = (1 << 23)
static

Definition at line 32 of file CopyParams.h.

constexpr size_t import_export::max_import_buffer_resize_byte_size = 1024 * 1024 * 1024
static

Definition at line 35 of file CopyParams.h.

constexpr bool import_export::PROMOTE_POLYGON_TO_MULTIPOLYGON = true
static
mapd_shared_mutex import_export::status_mutex
static