OmniSciDB  a667adc9c8
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
import_export Namespace Reference

Namespaces

 anonymous_namespace{Importer.cpp}
 
 anonymous_namespace{QueryExporterGDAL.cpp}
 
 delimited_parser
 

Classes

struct  CopyParams
 
struct  GeoImportException
 
struct  BadRowsTracker
 
class  TypedImportBuffer
 
class  Loader
 
struct  ImportStatus
 
class  DataStreamSink
 
class  Detector
 
class  ImporterUtils
 
class  RenderGroupAnalyzer
 
class  Importer
 
class  QueryExporter
 
class  QueryExporterCSV
 
class  QueryExporterGDAL
 

Typedefs

using FieldNameToIndexMapType = std::map< std::string, size_t >
 
using ColumnNameToSourceNameMapType = std::map< std::string, std::string >
 
using ColumnIdToRenderGroupAnalyzerMapType = std::map< int, std::shared_ptr< RenderGroupAnalyzer >>
 
using FeaturePtrVector = std::vector< OGRFeatureUqPtr >
 
using ArraySliceRange = std::pair< size_t, size_t >
 

Enumerations

enum  FileType { FileType::DELIMITED, FileType::POLYGON }
 
enum  ImportHeaderRow { ImportHeaderRow::AUTODETECT, ImportHeaderRow::NO_HEADER, ImportHeaderRow::HAS_HEADER }
 

Functions

static const std::string trim_space (const char *field, const size_t len)
 
Datum NullDatum (SQLTypeInfo &ti)
 
Datum NullArrayDatum (SQLTypeInfo &ti)
 
ArrayDatum StringToArray (const std::string &s, const SQLTypeInfo &ti, const CopyParams &copy_params)
 
ArrayDatum NullArray (const SQLTypeInfo &ti)
 
void addBinaryStringArray (const TDatum &datum, std::vector< std::string > &string_vec)
 
Datum TDatumToDatum (const TDatum &datum, SQLTypeInfo &ti)
 
ArrayDatum TDatumToArrayDatum (const TDatum &datum, const SQLTypeInfo &ti)
 
bool importGeoFromLonLat (double lon, double lat, std::vector< double > &coords, SQLTypeInfo &ti)
 
static ImportStatus import_thread_delimited (int thread_id, Importer *importer, std::unique_ptr< char[]> scratch_buffer, size_t begin_pos, size_t end_pos, size_t total_size, const ColumnIdToRenderGroupAnalyzerMapType &columnIdToRenderGroupAnalyzerMap, size_t first_row_index_this_buffer, const Catalog_Namespace::SessionInfo *session_info, Executor *executor)
 
static ImportStatus import_thread_shapefile (int thread_id, Importer *importer, OGRSpatialReference *poGeographicSR, const FeaturePtrVector &features, size_t firstFeature, size_t numFeatures, const FieldNameToIndexMapType &fieldNameToIndexMap, const ColumnNameToSourceNameMapType &columnNameToSourceNameMap, const ColumnIdToRenderGroupAnalyzerMapType &columnIdToRenderGroupAnalyzerMap, const Catalog_Namespace::SessionInfo *session_info, Executor *executor)
 
template<class T >
bool try_cast (const std::string &str)
 
std::pair< SQLTypes, bool > ogr_to_type (const OGRFieldType &ogr_type)
 
SQLTypes ogr_to_type (const OGRwkbGeometryType &ogr_type)
 
void gdalGatherFilesInArchiveRecursive (const std::string &archive_path, std::vector< std::string > &files)
 
std::vector< std::unique_ptr
< TypedImportBuffer > > 
setup_column_loaders (const TableDescriptor *td, Loader *loader)
 
std::vector< std::unique_ptr
< TypedImportBuffer > > 
fill_missing_columns (const Catalog_Namespace::Catalog *cat, Fragmenter_Namespace::InsertData &insert_data)
 

Variables

static constexpr size_t kImportFileBufferSize = (1 << 23)
 
static constexpr bool PROMOTE_POLYGON_TO_MULTIPOLYGON = true
 
static mapd_shared_mutex status_mutex
 
static std::map< std::string,
ImportStatus
import_status_map
 

Typedef Documentation

using import_export::ArraySliceRange = typedef std::pair<size_t, size_t>

Definition at line 72 of file Importer.h.

using import_export::ColumnIdToRenderGroupAnalyzerMapType = typedef std::map<int, std::shared_ptr<RenderGroupAnalyzer>>

Definition at line 157 of file Importer.cpp.

using import_export::ColumnNameToSourceNameMapType = typedef std::map<std::string, std::string>

Definition at line 155 of file Importer.cpp.

using import_export::FeaturePtrVector = typedef std::vector<OGRFeatureUqPtr>

Definition at line 158 of file Importer.cpp.

using import_export::FieldNameToIndexMapType = typedef std::map<std::string, size_t>

Definition at line 154 of file Importer.cpp.

Enumeration Type Documentation

Enumerator
DELIMITED 
POLYGON 

Definition at line 34 of file CopyParams.h.

34  {
35  DELIMITED,
36  POLYGON
37 #ifdef ENABLE_IMPORT_PARQUET
38  ,
39  PARQUET
40 #endif
41 };
#define POLYGON

Function Documentation

void import_export::addBinaryStringArray ( const TDatum &  datum,
std::vector< std::string > &  string_vec 
)

Definition at line 421 of file Importer.cpp.

Referenced by import_export::TypedImportBuffer::add_value().

421  {
422  const auto& arr = datum.val.arr_val;
423  for (const auto& elem_datum : arr) {
424  string_vec.push_back(elem_datum.val.str_val);
425  }
426 }

+ Here is the caller graph for this function:

std::vector< std::unique_ptr< TypedImportBuffer > > import_export::fill_missing_columns ( const Catalog_Namespace::Catalog cat,
Fragmenter_Namespace::InsertData insert_data 
)

Definition at line 5365 of file Importer.cpp.

References CHECK, ColumnDescriptor::columnId, Fragmenter_Namespace::InsertData::columnIds, ColumnDescriptor::columnName, ColumnDescriptor::columnType, Fragmenter_Namespace::InsertData::data, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), import_export::TypedImportBuffer::get_data_block_pointers(), SQLTypeInfo::get_physical_cols(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), Catalog_Namespace::Catalog::getAllColumnMetadataForTable(), Geospatial::GeoTypesFactory::getGeoColumns(), Catalog_Namespace::Catalog::getMetadataForDict(), i, Fragmenter_Namespace::InsertData::is_default, SQLTypeInfo::is_geometry(), IS_STRING, SQLTypeInfo::is_string(), kARRAY, kENCODING_DICT, import_export::Importer::set_geo_physical_import_buffer(), gpu_enabled::sort(), and Fragmenter_Namespace::InsertData::tableId.

Referenced by RelAlgExecutor::executeSimpleInsert(), DBHandler::insert_data(), and Parser::InsertIntoTableAsSelectStmt::populateData().

5367  {
5368  std::vector<std::unique_ptr<import_export::TypedImportBuffer>> defaults_buffers;
5369  if (insert_data.is_default.size() == 0) {
5370  insert_data.is_default.resize(insert_data.columnIds.size(), false);
5371  }
5372  CHECK(insert_data.is_default.size() == insert_data.is_default.size());
5373  auto cds = cat->getAllColumnMetadataForTable(insert_data.tableId, false, false, true);
5374  if (cds.size() == insert_data.columnIds.size()) {
5375  // all columns specified
5376  return defaults_buffers;
5377  }
5378  for (auto cd : cds) {
5379  if (std::find(insert_data.columnIds.begin(),
5380  insert_data.columnIds.end(),
5381  cd->columnId) == insert_data.columnIds.end()) {
5382  StringDictionary* dict = nullptr;
5383  if (cd->columnType.get_type() == kARRAY &&
5384  IS_STRING(cd->columnType.get_subtype())) {
5385  throw std::runtime_error("Cannot omit column \"" + cd->columnName +
5386  "\": omitting TEXT arrays is not supported yet");
5387  }
5388  if (cd->columnType.is_string() &&
5389  cd->columnType.get_compression() == kENCODING_DICT) {
5390  dict = cat->getMetadataForDict(cd->columnType.get_comp_param())->stringDict.get();
5391  }
5392  defaults_buffers.emplace_back(std::make_unique<TypedImportBuffer>(cd, dict));
5393  }
5394  }
5395  // put buffers in order to fill geo sub-columns properly
5396  std::sort(defaults_buffers.begin(),
5397  defaults_buffers.end(),
5398  [](decltype(defaults_buffers[0])& a, decltype(defaults_buffers[0])& b) {
5399  return a->getColumnDesc()->columnId < b->getColumnDesc()->columnId;
5400  });
5401  for (size_t i = 0; i < defaults_buffers.size(); ++i) {
5402  auto cd = defaults_buffers[i]->getColumnDesc();
5403  defaults_buffers[i]->add_value(cd, "NULL", true, import_export::CopyParams());
5404  if (cd->columnType.is_geometry()) {
5405  std::vector<double> coords, bounds;
5406  std::vector<int> ring_sizes, poly_rings;
5407  int render_group = 0;
5408  SQLTypeInfo tinfo{cd->columnType};
5410  "NULL", tinfo, coords, bounds, ring_sizes, poly_rings, false));
5411  // set physical columns starting with the following ID
5412  auto next_col = i + 1;
5414  cd,
5415  defaults_buffers,
5416  next_col,
5417  coords,
5418  bounds,
5419  ring_sizes,
5420  poly_rings,
5421  render_group);
5422  // skip physical columns filled with the call above
5423  i += cd->columnType.get_physical_cols();
5424  }
5425  }
5426  auto data = import_export::TypedImportBuffer::get_data_block_pointers(defaults_buffers);
5427  CHECK(data.size() == defaults_buffers.size());
5428  for (size_t i = 0; i < defaults_buffers.size(); ++i) {
5429  insert_data.data.push_back(data[i]);
5430  insert_data.columnIds.push_back(defaults_buffers[i]->getColumnDesc()->columnId);
5431  insert_data.is_default.push_back(true);
5432  }
5433  return defaults_buffers;
5434 }
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
std::vector< bool > is_default
Definition: Fragmenter.h:66
int tableId
identifies the database into which the data is being inserted
Definition: Fragmenter.h:61
static std::vector< DataBlockPtr > get_data_block_pointers(const std::vector< std::unique_ptr< TypedImportBuffer >> &import_buffers)
Definition: Importer.cpp:2829
static void set_geo_physical_import_buffer(const Catalog_Namespace::Catalog &catalog, const ColumnDescriptor *cd, std::vector< std::unique_ptr< TypedImportBuffer >> &import_buffers, size_t &col_idx, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, int render_group)
Definition: Importer.cpp:1459
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1444
static bool getGeoColumns(const std::string &wkt_or_wkb_hex, SQLTypeInfo &ti, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, const bool promote_poly_to_mpoly=false)
Definition: Types.cpp:907
std::vector< DataBlockPtr > data
the number of rows being inserted
Definition: Fragmenter.h:64
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1721
#define IS_STRING(T)
Definition: sqltypes.h:244
#define CHECK(condition)
Definition: Logger.h:197
std::vector< int > columnIds
identifies the table into which the data is being inserted
Definition: Fragmenter.h:62

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void import_export::gdalGatherFilesInArchiveRecursive ( const std::string &  archive_path,
std::vector< std::string > &  files 
)

Definition at line 4745 of file Importer.cpp.

References LOG, run_benchmark_import::result, and logger::WARNING.

Referenced by import_export::Importer::gdalGetAllFilesInArchive().

4746  {
4747  // prepare to gather subdirectories
4748  std::vector<std::string> subdirectories;
4749 
4750  // get entries
4751  char** entries = VSIReadDir(archive_path.c_str());
4752  if (!entries) {
4753  LOG(WARNING) << "Failed to get file listing at archive: " << archive_path;
4754  return;
4755  }
4756 
4757  // force scope
4758  {
4759  // request clean-up
4760  ScopeGuard entries_guard = [&] { CSLDestroy(entries); };
4761 
4762  // check all the entries
4763  int index = 0;
4764  while (true) {
4765  // get next entry, or drop out if there isn't one
4766  char* entry_c = entries[index++];
4767  if (!entry_c) {
4768  break;
4769  }
4770  std::string entry(entry_c);
4771 
4772  // ignore '.' and '..'
4773  if (entry == "." || entry == "..") {
4774  continue;
4775  }
4776 
4777  // build the full path
4778  std::string entry_path = archive_path + std::string("/") + entry;
4779 
4780  // is it a file or a sub-folder
4781  VSIStatBufL sb;
4782  int result = VSIStatExL(entry_path.c_str(), &sb, VSI_STAT_NATURE_FLAG);
4783  if (result < 0) {
4784  break;
4785  }
4786 
4787  if (VSI_ISDIR(sb.st_mode)) {
4788  // a directory that ends with .gdb could be a Geodatabase bundle
4789  // arguably dangerous to decide this purely by name, but any further
4790  // validation would be very complex especially at this scope
4791  if (boost::iends_with(entry_path, ".gdb")) {
4792  // add the directory as if it was a file and don't recurse into it
4793  files.push_back(entry_path);
4794  } else {
4795  // add subdirectory to be recursed into
4796  subdirectories.push_back(entry_path);
4797  }
4798  } else {
4799  // add this file
4800  files.push_back(entry_path);
4801  }
4802  }
4803  }
4804 
4805  // recurse into each subdirectories we found
4806  for (const auto& subdirectory : subdirectories) {
4807  gdalGatherFilesInArchiveRecursive(subdirectory, files);
4808  }
4809 }
#define LOG(tag)
Definition: Logger.h:188
void gdalGatherFilesInArchiveRecursive(const std::string &archive_path, std::vector< std::string > &files)
Definition: Importer.cpp:4745

+ Here is the caller graph for this function:

static ImportStatus import_export::import_thread_delimited ( int  thread_id,
Importer *  importer,
std::unique_ptr< char[]>  scratch_buffer,
size_t  begin_pos,
size_t  end_pos,
size_t  total_size,
const ColumnIdToRenderGroupAnalyzerMapType &  columnIdToRenderGroupAnalyzerMap,
size_t  first_row_index_this_buffer,
const Catalog_Namespace::SessionInfo session_info,
Executor executor 
)
static

Definition at line 1826 of file Importer.cpp.

References CHECK, CHECK_LT, anonymous_namespace{Importer.cpp}::checkInterrupt(), Geospatial::GeoTypesFactory::createOGRGeometry(), DEBUG_TIMING, DELIMITED, logger::ERROR, measure< TimeT >::execution(), import_export::anonymous_namespace{Importer.cpp}::explode_collections_step1(), import_export::anonymous_namespace{Importer.cpp}::explode_collections_step2(), import_export::CopyParams::file_type, import_export::delimited_parser::find_beginning(), import_export::CopyParams::geo_explode_collections, import_export::Importer::get_column_descs(), import_export::Importer::get_copy_params(), import_export::Importer::get_import_buffers(), import_export::Importer::get_is_array(), import_export::delimited_parser::get_row(), Catalog_Namespace::SessionInfo::get_session_id(), import_export::Importer::getCatalog(), Geospatial::GeoTypesFactory::getGeoColumns(), Geospatial::GeoTypesFactory::getNullGeoColumns(), i, importGeoFromLonLat(), logger::INFO, IS_GEO, is_null(), kMULTIPOLYGON, kPOINT, kPOLYGON, import_export::Importer::load(), import_export::ImportStatus::load_failed, import_export::ImportStatus::load_msg, LOG, import_export::CopyParams::lonlat, import_export::CopyParams::max_reject, import_export::CopyParams::null_str, shared::printContainer(), PROMOTE_POLYGON_TO_MULTIPOLYGON, import_export::ImportStatus::rows_completed, import_export::ImportStatus::rows_rejected, import_export::Importer::set_geo_physical_import_buffer(), import_export::CopyParams::source_srid, gpu_enabled::swap(), import_export::ImportStatus::thread_id, logger::thread_id(), to_string(), and UNLIKELY.

Referenced by import_export::Importer::importDelimited().

1836  {
1837  ImportStatus thread_import_status;
1838  int64_t total_get_row_time_us = 0;
1839  int64_t total_str_to_val_time_us = 0;
1840  auto query_session = session_info ? session_info->get_session_id() : "";
1841  CHECK(scratch_buffer);
1842  auto buffer = scratch_buffer.get();
1843  auto load_ms = measure<>::execution([]() {});
1844 
1845  thread_import_status.thread_id = thread_id;
1846 
1847  auto ms = measure<>::execution([&]() {
1848  const CopyParams& copy_params = importer->get_copy_params();
1849  const std::list<const ColumnDescriptor*>& col_descs = importer->get_column_descs();
1850  size_t begin =
1851  delimited_parser::find_beginning(buffer, begin_pos, end_pos, copy_params);
1852  const char* thread_buf = buffer + begin_pos + begin;
1853  const char* thread_buf_end = buffer + end_pos;
1854  const char* buf_end = buffer + total_size;
1855  bool try_single_thread = false;
1856  std::vector<std::unique_ptr<TypedImportBuffer>>& import_buffers =
1857  importer->get_import_buffers(thread_id);
1859  int phys_cols = 0;
1860  int point_cols = 0;
1861  for (const auto cd : col_descs) {
1862  const auto& col_ti = cd->columnType;
1863  phys_cols += col_ti.get_physical_cols();
1864  if (cd->columnType.get_type() == kPOINT) {
1865  point_cols++;
1866  }
1867  }
1868  auto num_cols = col_descs.size() - phys_cols;
1869  for (const auto& p : import_buffers) {
1870  p->clear();
1871  }
1872  std::vector<std::string_view> row;
1873  size_t row_index_plus_one = 0;
1874  for (const char* p = thread_buf; p < thread_buf_end; p++) {
1875  row.clear();
1876  std::vector<std::unique_ptr<char[]>>
1877  tmp_buffers; // holds string w/ removed escape chars, etc
1878  if (DEBUG_TIMING) {
1881  thread_buf_end,
1882  buf_end,
1883  copy_params,
1884  importer->get_is_array(),
1885  row,
1886  tmp_buffers,
1887  try_single_thread,
1888  true);
1889  });
1890  total_get_row_time_us += us;
1891  } else {
1893  thread_buf_end,
1894  buf_end,
1895  copy_params,
1896  importer->get_is_array(),
1897  row,
1898  tmp_buffers,
1899  try_single_thread,
1900  true);
1901  }
1902  row_index_plus_one++;
1903  // Each POINT could consume two separate coords instead of a single WKT
1904  if (row.size() < num_cols || (num_cols + point_cols) < row.size()) {
1905  thread_import_status.rows_rejected++;
1906  LOG(ERROR) << "Incorrect Row (expected " << num_cols << " columns, has "
1907  << row.size() << "): " << shared::printContainer(row);
1908  if (thread_import_status.rows_rejected > copy_params.max_reject) {
1909  break;
1910  }
1911  continue;
1912  }
1913 
1914  //
1915  // lambda for importing a row (perhaps multiple times if exploding a collection)
1916  //
1917 
1918  auto execute_import_row = [&](OGRGeometry* import_geometry) {
1919  size_t import_idx = 0;
1920  size_t col_idx = 0;
1921  try {
1922  for (auto cd_it = col_descs.begin(); cd_it != col_descs.end(); cd_it++) {
1923  auto cd = *cd_it;
1924  const auto& col_ti = cd->columnType;
1925 
1926  bool is_null =
1927  (row[import_idx] == copy_params.null_str || row[import_idx] == "NULL");
1928  // Note: default copy_params.null_str is "\N", but everyone uses "NULL".
1929  // So initially nullness may be missed and not passed to add_value,
1930  // which then might also check and still decide it's actually a NULL, e.g.
1931  // if kINT doesn't start with a digit or a '-' then it's considered NULL.
1932  // So "NULL" is not recognized as NULL but then it's not recognized as
1933  // a valid kINT, so it's a NULL after all.
1934  // Checking for "NULL" here too, as a widely accepted notation for NULL.
1935 
1936  // Treating empty as NULL
1937  if (!cd->columnType.is_string() && row[import_idx].empty()) {
1938  is_null = true;
1939  }
1940 
1941  if (col_ti.get_physical_cols() == 0) {
1942  // not geo
1943 
1944  import_buffers[col_idx]->add_value(
1945  cd, row[import_idx], is_null, copy_params);
1946 
1947  // next
1948  ++import_idx;
1949  ++col_idx;
1950  } else {
1951  // geo
1952 
1953  // store null string in the base column
1954  import_buffers[col_idx]->add_value(
1955  cd, copy_params.null_str, true, copy_params);
1956 
1957  // WKT from string we're not storing
1958  auto const& geo_string = row[import_idx];
1959 
1960  // next
1961  ++import_idx;
1962  ++col_idx;
1963 
1964  SQLTypes col_type = col_ti.get_type();
1965  CHECK(IS_GEO(col_type));
1966 
1967  std::vector<double> coords;
1968  std::vector<double> bounds;
1969  std::vector<int> ring_sizes;
1970  std::vector<int> poly_rings;
1971  int render_group = 0;
1972 
1973  // if this is a POINT column, and the field is not null, and
1974  // looks like a scalar numeric value (and not a hex blob)
1975  // attempt to import two columns as lon/lat (or lat/lon)
1976  if (col_type == kPOINT && !is_null && geo_string.size() > 0 &&
1977  (geo_string[0] == '.' || isdigit(geo_string[0]) ||
1978  geo_string[0] == '-') &&
1979  geo_string.find_first_of("ABCDEFabcdef") == std::string::npos) {
1980  double lon = std::atof(std::string(geo_string).c_str());
1981  double lat = NAN;
1982  auto lat_str = row[import_idx];
1983  ++import_idx;
1984  if (lat_str.size() > 0 &&
1985  (lat_str[0] == '.' || isdigit(lat_str[0]) || lat_str[0] == '-')) {
1986  lat = std::atof(std::string(lat_str).c_str());
1987  }
1988  // Swap coordinates if this table uses a reverse order: lat/lon
1989  if (!copy_params.lonlat) {
1990  std::swap(lat, lon);
1991  }
1992  // TODO: should check if POINT column should have been declared with
1993  // SRID WGS 84, EPSG 4326 ? if (col_ti.get_dimension() != 4326) {
1994  // throw std::runtime_error("POINT column " + cd->columnName + " is
1995  // not WGS84, cannot insert lon/lat");
1996  // }
1997  SQLTypeInfo import_ti{col_ti};
1998  if (copy_params.file_type == FileType::DELIMITED &&
1999  import_ti.get_output_srid() == 4326) {
2000  auto srid0 = copy_params.source_srid;
2001  if (srid0 > 0) {
2002  // srid0 -> 4326 transform is requested on import
2003  import_ti.set_input_srid(srid0);
2004  }
2005  }
2006  if (!importGeoFromLonLat(lon, lat, coords, import_ti)) {
2007  throw std::runtime_error(
2008  "Cannot read lon/lat to insert into POINT column " +
2009  cd->columnName);
2010  }
2011  } else {
2012  // import it
2013  SQLTypeInfo import_ti{col_ti};
2014  if (copy_params.file_type == FileType::DELIMITED &&
2015  import_ti.get_output_srid() == 4326) {
2016  auto srid0 = copy_params.source_srid;
2017  if (srid0 > 0) {
2018  // srid0 -> 4326 transform is requested on import
2019  import_ti.set_input_srid(srid0);
2020  }
2021  }
2022  if (is_null) {
2023  if (col_ti.get_notnull()) {
2024  throw std::runtime_error("NULL geo for column " + cd->columnName);
2025  }
2027  import_ti,
2028  coords,
2029  bounds,
2030  ring_sizes,
2031  poly_rings,
2033  } else {
2034  if (import_geometry) {
2035  // geometry already exploded
2037  import_geometry,
2038  import_ti,
2039  coords,
2040  bounds,
2041  ring_sizes,
2042  poly_rings,
2044  std::string msg =
2045  "Failed to extract valid geometry from exploded row " +
2046  std::to_string(first_row_index_this_buffer +
2047  row_index_plus_one) +
2048  " for column " + cd->columnName;
2049  throw std::runtime_error(msg);
2050  }
2051  } else {
2052  // extract geometry directly from WKT
2054  std::string(geo_string),
2055  import_ti,
2056  coords,
2057  bounds,
2058  ring_sizes,
2059  poly_rings,
2061  std::string msg = "Failed to extract valid geometry from row " +
2062  std::to_string(first_row_index_this_buffer +
2063  row_index_plus_one) +
2064  " for column " + cd->columnName;
2065  throw std::runtime_error(msg);
2066  }
2067  }
2068 
2069  // validate types
2070  if (col_type != import_ti.get_type()) {
2072  !(import_ti.get_type() == SQLTypes::kPOLYGON &&
2073  col_type == SQLTypes::kMULTIPOLYGON)) {
2074  throw std::runtime_error(
2075  "Imported geometry doesn't match the type of column " +
2076  cd->columnName);
2077  }
2078  }
2079  }
2080 
2081  // assign render group?
2082  if (columnIdToRenderGroupAnalyzerMap.size()) {
2083  if (col_type == kPOLYGON || col_type == kMULTIPOLYGON) {
2084  if (ring_sizes.size()) {
2085  // get a suitable render group for these poly coords
2086  auto rga_it = columnIdToRenderGroupAnalyzerMap.find(cd->columnId);
2087  CHECK(rga_it != columnIdToRenderGroupAnalyzerMap.end());
2088  render_group =
2089  (*rga_it).second->insertBoundsAndReturnRenderGroup(bounds);
2090  } else {
2091  // empty poly
2092  render_group = -1;
2093  }
2094  }
2095  }
2096  }
2097 
2098  // import extracted geo
2099  Importer::set_geo_physical_import_buffer(importer->getCatalog(),
2100  cd,
2101  import_buffers,
2102  col_idx,
2103  coords,
2104  bounds,
2105  ring_sizes,
2106  poly_rings,
2107  render_group);
2108 
2109  // skip remaining physical columns
2110  for (int i = 0; i < cd->columnType.get_physical_cols(); ++i) {
2111  ++cd_it;
2112  }
2113  }
2114  }
2115  if (UNLIKELY((thread_import_status.rows_completed & 0xFFFF) == 0 &&
2116  checkInterrupt(query_session, executor))) {
2117  thread_import_status.load_failed = true;
2118  thread_import_status.load_msg =
2119  "Table load was cancelled via Query Interrupt";
2120  return;
2121  }
2122  thread_import_status.rows_completed++;
2123  } catch (const std::exception& e) {
2124  for (size_t col_idx_to_pop = 0; col_idx_to_pop < col_idx; ++col_idx_to_pop) {
2125  import_buffers[col_idx_to_pop]->pop_value();
2126  }
2127  thread_import_status.rows_rejected++;
2128  LOG(ERROR) << "Input exception thrown: " << e.what()
2129  << ". Row discarded. Data: " << shared::printContainer(row);
2130  if (thread_import_status.rows_rejected > copy_params.max_reject) {
2131  LOG(ERROR) << "Load was cancelled due to max reject rows being reached";
2132  thread_import_status.load_failed = true;
2133  thread_import_status.load_msg =
2134  "Load was cancelled due to max reject rows being reached";
2135  }
2136  }
2137  }; // End of lambda
2138 
2139  if (copy_params.geo_explode_collections) {
2140  // explode and import
2141  auto const [collection_col_idx, collection_child_type, collection_col_name] =
2142  explode_collections_step1(col_descs);
2143  // pull out the collection WKT or WKB hex
2144  CHECK_LT(collection_col_idx, (int)row.size()) << "column index out of range";
2145  auto const& collection_geo_string = row[collection_col_idx];
2146  // convert to OGR
2147  OGRGeometry* ogr_geometry = nullptr;
2148  ScopeGuard destroy_ogr_geometry = [&] {
2149  if (ogr_geometry) {
2150  OGRGeometryFactory::destroyGeometry(ogr_geometry);
2151  }
2152  };
2154  std::string(collection_geo_string));
2155  // do the explode and import
2156  us = explode_collections_step2(ogr_geometry,
2157  collection_child_type,
2158  collection_col_name,
2159  first_row_index_this_buffer + row_index_plus_one,
2160  execute_import_row);
2161  } else {
2162  // import non-collection row just once
2164  [&] { execute_import_row(nullptr); });
2165  }
2166 
2167  if (thread_import_status.load_failed) {
2168  break;
2169  }
2170  } // end thread
2171  total_str_to_val_time_us += us;
2172  if (!thread_import_status.load_failed && thread_import_status.rows_completed > 0) {
2173  load_ms = measure<>::execution([&]() {
2174  importer->load(import_buffers, thread_import_status.rows_completed, session_info);
2175  });
2176  }
2177  }); // end execution
2178 
2179  if (DEBUG_TIMING && !thread_import_status.load_failed &&
2180  thread_import_status.rows_completed > 0) {
2181  LOG(INFO) << "Thread" << std::this_thread::get_id() << ":"
2182  << thread_import_status.rows_completed << " rows inserted in "
2183  << (double)ms / 1000.0 << "sec, Insert Time: " << (double)load_ms / 1000.0
2184  << "sec, get_row: " << (double)total_get_row_time_us / 1000000.0
2185  << "sec, str_to_val: " << (double)total_str_to_val_time_us / 1000000.0
2186  << "sec" << std::endl;
2187  }
2188 
2189  return thread_import_status;
2190 }
SQLTypes
Definition: sqltypes.h:37
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
int64_t explode_collections_step2(OGRGeometry *ogr_geometry, SQLTypes collection_child_type, const std::string &collection_col_name, size_t row_or_feature_idx, std::function< void(OGRGeometry *)> execute_import_lambda)
Definition: Importer.cpp:1736
#define LOG(tag)
Definition: Logger.h:188
size_t find_beginning(const char *buffer, size_t begin, size_t end, const import_export::CopyParams &copy_params)
Finds the closest possible row beginning in the given buffer.
static void getNullGeoColumns(SQLTypeInfo &ti, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, const bool promote_poly_to_mpoly=false)
Definition: Types.cpp:1114
std::string to_string(char const *&&v)
#define DEBUG_TIMING
Definition: Importer.cpp:160
void set_input_srid(int d)
Definition: sqltypes.h:408
CONSTEXPR DEVICE bool is_null(const T &value)
const char * get_row(const char *buf, const char *buf_end, const char *entire_buf_end, const import_export::CopyParams &copy_params, const bool *is_array, std::vector< T > &row, std::vector< std::unique_ptr< char[]>> &tmp_buffers, bool &try_single_thread, bool filter_empty_lines)
Parses the first row in the given buffer and inserts fields into given vector.
#define UNLIKELY(x)
Definition: likely.h:25
std::string get_session_id() const
Definition: SessionInfo.h:77
static bool getGeoColumns(const std::string &wkt_or_wkb_hex, SQLTypeInfo &ti, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, const bool promote_poly_to_mpoly=false)
Definition: Types.cpp:907
#define CHECK_LT(x, y)
Definition: Logger.h:207
static OGRGeometry * createOGRGeometry(const std::string &wkt_or_wkb_hex)
Definition: Types.cpp:873
static constexpr bool PROMOTE_POLYGON_TO_MULTIPOLYGON
std::tuple< int, SQLTypes, std::string > explode_collections_step1(const std::list< const ColumnDescriptor * > &col_descs)
Definition: Importer.cpp:1702
ThreadId thread_id()
Definition: Logger.cpp:732
#define CHECK(condition)
Definition: Logger.h:197
bool importGeoFromLonLat(double lon, double lat, std::vector< double > &coords, SQLTypeInfo &ti)
Definition: Importer.cpp:1439
PrintContainer< CONTAINER > printContainer(CONTAINER &container)
Definition: misc.h:64
bool checkInterrupt(const QuerySessionId &query_session, Executor *executor)
Definition: Importer.cpp:98
DEVICE void swap(ARGS &&...args)
Definition: gpu_enabled.h:114
#define IS_GEO(T)
Definition: sqltypes.h:245

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

static ImportStatus import_export::import_thread_shapefile ( int  thread_id,
Importer *  importer,
OGRSpatialReference *  poGeographicSR,
const FeaturePtrVector &  features,
size_t  firstFeature,
size_t  numFeatures,
const FieldNameToIndexMapType &  fieldNameToIndexMap,
const ColumnNameToSourceNameMapType &  columnNameToSourceNameMap,
const ColumnIdToRenderGroupAnalyzerMapType &  columnIdToRenderGroupAnalyzerMap,
const Catalog_Namespace::SessionInfo session_info,
Executor executor 
)
static

Definition at line 2192 of file Importer.cpp.

References CHECK, anonymous_namespace{Importer.cpp}::checkInterrupt(), Geospatial::compress_coords(), DEBUG_TIMING, Executor::ERR_INTERRUPTED, logger::ERROR, import_export::anonymous_namespace{Importer.cpp}::explode_collections_step1(), import_export::anonymous_namespace{Importer.cpp}::explode_collections_step2(), import_export::CopyParams::geo_explode_collections, import_export::Importer::get_column_descs(), import_export::Importer::get_copy_params(), import_export::Importer::get_import_buffers(), Catalog_Namespace::SessionInfo::get_session_id(), QueryExecutionError::getErrorCode(), Geospatial::GeoTypesFactory::getGeoColumns(), Geospatial::GeoTypesFactory::getNullGeoColumns(), logger::INFO, kLINESTRING, kMULTIPOLYGON, kPOLYGON, import_export::Importer::load(), import_export::ImportStatus::load_failed, import_export::ImportStatus::load_msg, LOG, import_export::CopyParams::null_str, PROMOTE_POLYGON_TO_MULTIPOLYGON, import_export::ImportStatus::rows_completed, import_export::ImportStatus::rows_rejected, import_export::ImportStatus::thread_id, logger::thread_id(), timer_start(), timer_stop(), to_string(), and UNLIKELY.

Referenced by import_export::Importer::importGDAL().

2203  {
2204  ImportStatus thread_import_status;
2205  const CopyParams& copy_params = importer->get_copy_params();
2206  const std::list<const ColumnDescriptor*>& col_descs = importer->get_column_descs();
2207  std::vector<std::unique_ptr<TypedImportBuffer>>& import_buffers =
2208  importer->get_import_buffers(thread_id);
2209  auto query_session = session_info ? session_info->get_session_id() : "";
2210  for (const auto& p : import_buffers) {
2211  p->clear();
2212  }
2213 
2214  auto convert_timer = timer_start();
2215 
2216  // we create this on the fly based on the first feature's SR
2217  std::unique_ptr<OGRCoordinateTransformation> coordinate_transformation;
2218  for (size_t iFeature = 0; iFeature < numFeatures; iFeature++) {
2219  if (!features[iFeature]) {
2220  continue;
2221  }
2222 
2223  // get this feature's geometry
2224  OGRGeometry* pGeometry = features[iFeature]->GetGeometryRef();
2225  if (pGeometry) {
2226  // for geodatabase, we need to consider features with no geometry
2227  // as we still want to create a table, even if it has no geo column
2228 
2229  // transform it
2230  // avoid GDAL error if not transformable
2231  auto geometry_sr = pGeometry->getSpatialReference();
2232  if (geometry_sr) {
2233  // create an OGRCoordinateTransformation (CT) on the fly
2234  // we must assume that all geo in this file will have
2235  // the same source SR, so the CT will be valid for all
2236  // transforming to a reusable CT is faster than to an SR
2237  if (coordinate_transformation == nullptr) {
2238  coordinate_transformation.reset(
2239  OGRCreateCoordinateTransformation(geometry_sr, poGeographicSR));
2240  if (coordinate_transformation == nullptr) {
2241  throw std::runtime_error(
2242  "Failed to create a GDAL CoordinateTransformation for incoming geo");
2243  }
2244  }
2245  pGeometry->transform(coordinate_transformation.get());
2246  }
2247  }
2248 
2249  //
2250  // lambda for importing a feature (perhaps multiple times if exploding a collection)
2251  //
2252 
2253  auto execute_import_feature = [&](OGRGeometry* import_geometry) {
2254  size_t col_idx = 0;
2255  try {
2256  if (UNLIKELY((thread_import_status.rows_completed & 0xFFFF) == 0 &&
2257  checkInterrupt(query_session, executor))) {
2258  thread_import_status.load_failed = true;
2259  thread_import_status.load_msg = "Table load was cancelled via Query Interrupt";
2261  }
2262  for (auto cd_it = col_descs.begin(); cd_it != col_descs.end(); cd_it++) {
2263  auto cd = *cd_it;
2264 
2265  // is this a geo column?
2266  const auto& col_ti = cd->columnType;
2267  if (col_ti.is_geometry()) {
2268  // Note that this assumes there is one and only one geo column in the
2269  // table. Currently, the importer only supports reading a single
2270  // geospatial feature from an input shapefile / geojson file, but this
2271  // code will need to be modified if that changes
2272  SQLTypes col_type = col_ti.get_type();
2273 
2274  // store null string in the base column
2275  import_buffers[col_idx]->add_value(
2276  cd, copy_params.null_str, true, copy_params);
2277  ++col_idx;
2278 
2279  // the data we now need to extract for the other columns
2280  std::vector<double> coords;
2281  std::vector<double> bounds;
2282  std::vector<int> ring_sizes;
2283  std::vector<int> poly_rings;
2284  int render_group = 0;
2285 
2286  // extract it
2287  SQLTypeInfo import_ti{col_ti};
2288  bool is_null_geo = !import_geometry;
2289  if (is_null_geo) {
2290  if (col_ti.get_notnull()) {
2291  throw std::runtime_error("NULL geo for column " + cd->columnName);
2292  }
2294  import_ti,
2295  coords,
2296  bounds,
2297  ring_sizes,
2298  poly_rings,
2300  } else {
2302  import_geometry,
2303  import_ti,
2304  coords,
2305  bounds,
2306  ring_sizes,
2307  poly_rings,
2309  std::string msg = "Failed to extract valid geometry from feature " +
2310  std::to_string(firstFeature + iFeature + 1) +
2311  " for column " + cd->columnName;
2312  throw std::runtime_error(msg);
2313  }
2314 
2315  // validate types
2316  if (col_type != import_ti.get_type()) {
2318  !(import_ti.get_type() == SQLTypes::kPOLYGON &&
2319  col_type == SQLTypes::kMULTIPOLYGON)) {
2320  throw std::runtime_error(
2321  "Imported geometry doesn't match the type of column " +
2322  cd->columnName);
2323  }
2324  }
2325  }
2326 
2327  if (col_type == kPOLYGON || col_type == kMULTIPOLYGON) {
2328  if (ring_sizes.size()) {
2329  // get a suitable render group for these poly coords
2330  auto rga_it = columnIdToRenderGroupAnalyzerMap.find(cd->columnId);
2331  CHECK(rga_it != columnIdToRenderGroupAnalyzerMap.end());
2332  render_group = (*rga_it).second->insertBoundsAndReturnRenderGroup(bounds);
2333  } else {
2334  // empty poly
2335  render_group = -1;
2336  }
2337  }
2338 
2339  // create coords array value and add it to the physical column
2340  ++cd_it;
2341  auto cd_coords = *cd_it;
2342  std::vector<TDatum> td_coord_data;
2343  if (!is_null_geo) {
2344  std::vector<uint8_t> compressed_coords =
2345  Geospatial::compress_coords(coords, col_ti);
2346  for (auto cc : compressed_coords) {
2347  TDatum td_byte;
2348  td_byte.val.int_val = cc;
2349  td_coord_data.push_back(td_byte);
2350  }
2351  }
2352  TDatum tdd_coords;
2353  tdd_coords.val.arr_val = td_coord_data;
2354  tdd_coords.is_null = is_null_geo;
2355  import_buffers[col_idx]->add_value(cd_coords, tdd_coords, false);
2356  ++col_idx;
2357 
2358  if (col_type == kPOLYGON || col_type == kMULTIPOLYGON) {
2359  // Create ring_sizes array value and add it to the physical column
2360  ++cd_it;
2361  auto cd_ring_sizes = *cd_it;
2362  std::vector<TDatum> td_ring_sizes;
2363  if (!is_null_geo) {
2364  for (auto ring_size : ring_sizes) {
2365  TDatum td_ring_size;
2366  td_ring_size.val.int_val = ring_size;
2367  td_ring_sizes.push_back(td_ring_size);
2368  }
2369  }
2370  TDatum tdd_ring_sizes;
2371  tdd_ring_sizes.val.arr_val = td_ring_sizes;
2372  tdd_ring_sizes.is_null = is_null_geo;
2373  import_buffers[col_idx]->add_value(cd_ring_sizes, tdd_ring_sizes, false);
2374  ++col_idx;
2375  }
2376 
2377  if (col_type == kMULTIPOLYGON) {
2378  // Create poly_rings array value and add it to the physical column
2379  ++cd_it;
2380  auto cd_poly_rings = *cd_it;
2381  std::vector<TDatum> td_poly_rings;
2382  if (!is_null_geo) {
2383  for (auto num_rings : poly_rings) {
2384  TDatum td_num_rings;
2385  td_num_rings.val.int_val = num_rings;
2386  td_poly_rings.push_back(td_num_rings);
2387  }
2388  }
2389  TDatum tdd_poly_rings;
2390  tdd_poly_rings.val.arr_val = td_poly_rings;
2391  tdd_poly_rings.is_null = is_null_geo;
2392  import_buffers[col_idx]->add_value(cd_poly_rings, tdd_poly_rings, false);
2393  ++col_idx;
2394  }
2395 
2396  if (col_type == kLINESTRING || col_type == kPOLYGON ||
2397  col_type == kMULTIPOLYGON) {
2398  // Create bounds array value and add it to the physical column
2399  ++cd_it;
2400  auto cd_bounds = *cd_it;
2401  std::vector<TDatum> td_bounds_data;
2402  if (!is_null_geo) {
2403  for (auto b : bounds) {
2404  TDatum td_double;
2405  td_double.val.real_val = b;
2406  td_bounds_data.push_back(td_double);
2407  }
2408  }
2409  TDatum tdd_bounds;
2410  tdd_bounds.val.arr_val = td_bounds_data;
2411  tdd_bounds.is_null = is_null_geo;
2412  import_buffers[col_idx]->add_value(cd_bounds, tdd_bounds, false);
2413  ++col_idx;
2414  }
2415 
2416  if (col_type == kPOLYGON || col_type == kMULTIPOLYGON) {
2417  // Create render_group value and add it to the physical column
2418  ++cd_it;
2419  auto cd_render_group = *cd_it;
2420  TDatum td_render_group;
2421  td_render_group.val.int_val = render_group;
2422  td_render_group.is_null = is_null_geo;
2423  import_buffers[col_idx]->add_value(cd_render_group, td_render_group, false);
2424  ++col_idx;
2425  }
2426  } else {
2427  // regular column
2428  // pull from GDAL metadata
2429  auto const cit = columnNameToSourceNameMap.find(cd->columnName);
2430  CHECK(cit != columnNameToSourceNameMap.end());
2431  auto const& field_name = cit->second;
2432 
2433  auto const fit = fieldNameToIndexMap.find(field_name);
2434  CHECK(fit != fieldNameToIndexMap.end());
2435  auto const& field_index = fit->second;
2436  CHECK(field_index < fieldNameToIndexMap.size());
2437 
2438  auto const& feature = features[iFeature];
2439 
2440  auto field_defn = feature->GetFieldDefnRef(field_index);
2441  CHECK(field_defn);
2442 
2443  // OGRFeature::GetFieldAsString() can only return 80 characters
2444  // so for array columns, we are obliged to fetch the actual values
2445  // and construct the concatenated string ourselves
2446 
2447  std::string value_string;
2448  int array_index = 0, array_size = 0;
2449 
2450  auto stringify_numeric_list = [&](auto* values) {
2451  value_string = "{";
2452  while (array_index < array_size) {
2453  auto separator = (array_index > 0) ? "," : "";
2454  value_string += separator + std::to_string(values[array_index]);
2455  array_index++;
2456  }
2457  value_string += "}";
2458  };
2459 
2460  auto field_type = field_defn->GetType();
2461  switch (field_type) {
2462  case OFTInteger:
2463  case OFTInteger64:
2464  case OFTReal:
2465  case OFTString:
2466  case OFTBinary:
2467  case OFTDate:
2468  case OFTTime:
2469  case OFTDateTime: {
2470  value_string = feature->GetFieldAsString(field_index);
2471  } break;
2472  case OFTIntegerList: {
2473  auto* values = feature->GetFieldAsIntegerList(field_index, &array_size);
2474  stringify_numeric_list(values);
2475  } break;
2476  case OFTInteger64List: {
2477  auto* values = feature->GetFieldAsInteger64List(field_index, &array_size);
2478  stringify_numeric_list(values);
2479  } break;
2480  case OFTRealList: {
2481  auto* values = feature->GetFieldAsDoubleList(field_index, &array_size);
2482  stringify_numeric_list(values);
2483  } break;
2484  case OFTStringList: {
2485  auto** array_of_strings = feature->GetFieldAsStringList(field_index);
2486  value_string = "{";
2487  if (array_of_strings) {
2488  while (auto* this_string = array_of_strings[array_index]) {
2489  auto separator = (array_index > 0) ? "," : "";
2490  value_string += separator + std::string(this_string);
2491  array_index++;
2492  }
2493  }
2494  value_string += "}";
2495  } break;
2496  default:
2497  throw std::runtime_error("Unsupported geo file field type (" +
2498  std::to_string(static_cast<int>(field_type)) +
2499  ")");
2500  }
2501 
2502  static CopyParams default_copy_params;
2503  import_buffers[col_idx]->add_value(
2504  cd, value_string, false, default_copy_params);
2505  ++col_idx;
2506  }
2507  }
2508  thread_import_status.rows_completed++;
2509  } catch (QueryExecutionError& e) {
2511  throw e;
2512  }
2513  } catch (const std::exception& e) {
2514  for (size_t col_idx_to_pop = 0; col_idx_to_pop < col_idx; ++col_idx_to_pop) {
2515  import_buffers[col_idx_to_pop]->pop_value();
2516  }
2517  thread_import_status.rows_rejected++;
2518  LOG(ERROR) << "Input exception thrown: " << e.what() << ". Row discarded.";
2519  }
2520  };
2521 
2522  if (pGeometry && copy_params.geo_explode_collections) {
2523  // explode and import
2524  auto const [collection_idx_type_name, collection_child_type, collection_col_name] =
2525  explode_collections_step1(col_descs);
2526  explode_collections_step2(pGeometry,
2527  collection_child_type,
2528  collection_col_name,
2529  firstFeature + iFeature + 1,
2530  execute_import_feature);
2531  } else {
2532  // import non-collection or null feature just once
2533  execute_import_feature(pGeometry);
2534  }
2535  } // end features
2536 
2537  float convert_ms =
2538  float(timer_stop<std::chrono::steady_clock::time_point, std::chrono::microseconds>(
2539  convert_timer)) /
2540  1000.0f;
2541 
2542  float load_ms = 0.0f;
2543  if (thread_import_status.rows_completed > 0) {
2544  auto load_timer = timer_start();
2545  importer->load(import_buffers, thread_import_status.rows_completed, session_info);
2546  load_ms =
2547  float(
2548  timer_stop<std::chrono::steady_clock::time_point, std::chrono::microseconds>(
2549  load_timer)) /
2550  1000.0f;
2551  }
2552 
2553  if (DEBUG_TIMING && thread_import_status.rows_completed > 0) {
2554  LOG(INFO) << "DEBUG: Process " << convert_ms << "ms";
2555  LOG(INFO) << "DEBUG: Load " << load_ms << "ms";
2556  }
2557 
2558  thread_import_status.thread_id = thread_id;
2559 
2560  if (DEBUG_TIMING) {
2561  LOG(INFO) << "DEBUG: Total "
2562  << float(timer_stop<std::chrono::steady_clock::time_point,
2563  std::chrono::microseconds>(convert_timer)) /
2564  1000.0f
2565  << "ms";
2566  }
2567 
2568  return thread_import_status;
2569 }
int32_t getErrorCode() const
Definition: ErrorHandling.h:55
static const int32_t ERR_INTERRUPTED
Definition: Execute.h:1117
SQLTypes
Definition: sqltypes.h:37
std::vector< uint8_t > compress_coords(std::vector< double > &coords, const SQLTypeInfo &ti)
Definition: Compression.cpp:52
int64_t explode_collections_step2(OGRGeometry *ogr_geometry, SQLTypes collection_child_type, const std::string &collection_col_name, size_t row_or_feature_idx, std::function< void(OGRGeometry *)> execute_import_lambda)
Definition: Importer.cpp:1736
#define LOG(tag)
Definition: Logger.h:188
TypeR::rep timer_stop(Type clock_begin)
Definition: measure.h:48
static void getNullGeoColumns(SQLTypeInfo &ti, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, const bool promote_poly_to_mpoly=false)
Definition: Types.cpp:1114
std::string to_string(char const *&&v)
#define DEBUG_TIMING
Definition: Importer.cpp:160
#define UNLIKELY(x)
Definition: likely.h:25
std::string get_session_id() const
Definition: SessionInfo.h:77
static bool getGeoColumns(const std::string &wkt_or_wkb_hex, SQLTypeInfo &ti, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, const bool promote_poly_to_mpoly=false)
Definition: Types.cpp:907
static constexpr bool PROMOTE_POLYGON_TO_MULTIPOLYGON
std::tuple< int, SQLTypes, std::string > explode_collections_step1(const std::list< const ColumnDescriptor * > &col_descs)
Definition: Importer.cpp:1702
ThreadId thread_id()
Definition: Logger.cpp:732
#define CHECK(condition)
Definition: Logger.h:197
bool checkInterrupt(const QuerySessionId &query_session, Executor *executor)
Definition: Importer.cpp:98
Type timer_start()
Definition: measure.h:42

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool import_export::importGeoFromLonLat ( double  lon,
double  lat,
std::vector< double > &  coords,
SQLTypeInfo ti 
)

Definition at line 1439 of file Importer.cpp.

References Geospatial::GeoPoint::getColumns(), and SQLTypeInfo::transforms().

Referenced by import_thread_delimited().

1442  {
1443  if (std::isinf(lat) || std::isnan(lat) || std::isinf(lon) || std::isnan(lon)) {
1444  return false;
1445  }
1446  if (ti.transforms()) {
1447  Geospatial::GeoPoint pt{std::vector<double>{lon, lat}};
1448  if (!pt.transform(ti)) {
1449  return false;
1450  }
1451  pt.getColumns(coords);
1452  return true;
1453  }
1454  coords.push_back(lon);
1455  coords.push_back(lat);
1456  return true;
1457 }
void getColumns(std::vector< double > &coords) const
Definition: Types.cpp:562
bool transforms() const
Definition: sqltypes.h:509

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

ArrayDatum import_export::NullArray ( const SQLTypeInfo ti)

Definition at line 389 of file Importer.cpp.

References appendDatum(), CHECK, checked_malloc(), test_fsi::d, SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_size(), SQLTypeInfo::is_string(), NullArrayDatum(), and NullDatum().

Referenced by import_export::TypedImportBuffer::add_value(), import_export::TypedImportBuffer::add_values(), import_export::ImporterUtils::composeNullArray(), and TDatumToArrayDatum().

389  {
390  SQLTypeInfo elem_ti = ti.get_elem_type();
391  auto len = ti.get_size();
392 
393  if (elem_ti.is_string()) {
394  // must not be called for array of strings
395  CHECK(false);
396  return ArrayDatum(0, NULL, true);
397  }
398 
399  if (len > 0) {
400  // Compose a NULL fixlen array
401  int8_t* buf = (int8_t*)checked_malloc(len);
402  // First scalar is a NULL_ARRAY sentinel
403  Datum d = NullArrayDatum(elem_ti);
404  int8_t* p = appendDatum(buf, d, elem_ti);
405  // Rest is filled with normal NULL sentinels
406  Datum d0 = NullDatum(elem_ti);
407  while ((p - buf) < len) {
408  p = appendDatum(p, d0, elem_ti);
409  }
410  CHECK((p - buf) == len);
411  return ArrayDatum(len, buf, true);
412  }
413  // NULL varlen array
414  return ArrayDatum(0, NULL, true);
415 }
HOST DEVICE int get_size() const
Definition: sqltypes.h:324
tuple d
Definition: test_fsi.py:9
std::conditional_t< is_cuda_compiler(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:202
Datum NullDatum(SQLTypeInfo &ti)
Definition: Importer.cpp:254
void * checked_malloc(const size_t size)
Definition: checked_alloc.h:45
int8_t * appendDatum(int8_t *buf, Datum d, const SQLTypeInfo &ti)
Definition: sqltypes.h:939
#define CHECK(condition)
Definition: Logger.h:197
bool is_string() const
Definition: sqltypes.h:488
Datum NullArrayDatum(SQLTypeInfo &ti)
Definition: Importer.cpp:295
SQLTypeInfo get_elem_type() const
Definition: sqltypes.h:712

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Datum import_export::NullArrayDatum ( SQLTypeInfo ti)

Definition at line 295 of file Importer.cpp.

References Datum::bigintval, Datum::boolval, test_fsi::d, decimal_to_int_type(), Datum::doubleval, Datum::floatval, SQLTypeInfo::get_type(), inline_fixed_encoding_null_array_val(), Datum::intval, SQLTypeInfo::is_decimal(), kBIGINT, kBOOLEAN, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTIME, kTIMESTAMP, kTINYINT, NULL_ARRAY_DOUBLE, NULL_ARRAY_FLOAT, Datum::smallintval, Datum::tinyintval, and run_benchmark_import::type.

Referenced by NullArray().

295  {
296  Datum d;
297  const auto type = ti.is_decimal() ? decimal_to_int_type(ti) : ti.get_type();
298  switch (type) {
299  case kBOOLEAN:
301  break;
302  case kBIGINT:
304  break;
305  case kINT:
307  break;
308  case kSMALLINT:
310  break;
311  case kTINYINT:
313  break;
314  case kFLOAT:
316  break;
317  case kDOUBLE:
319  break;
320  case kTIME:
321  case kTIMESTAMP:
322  case kDATE:
324  break;
325  case kPOINT:
326  case kLINESTRING:
327  case kPOLYGON:
328  case kMULTIPOLYGON:
329  throw std::runtime_error("Internal error: geometry type in NullArrayDatum.");
330  default:
331  throw std::runtime_error("Internal error: invalid type in NullArrayDatum.");
332  }
333  return d;
334 }
int8_t tinyintval
Definition: sqltypes.h:206
Definition: sqltypes.h:48
tuple d
Definition: test_fsi.py:9
bool boolval
Definition: sqltypes.h:205
int32_t intval
Definition: sqltypes.h:208
float floatval
Definition: sqltypes.h:210
int64_t bigintval
Definition: sqltypes.h:209
#define NULL_ARRAY_FLOAT
int16_t smallintval
Definition: sqltypes.h:207
SQLTypes decimal_to_int_type(const SQLTypeInfo &ti)
Definition: Datum.cpp:419
Definition: sqltypes.h:52
int64_t inline_fixed_encoding_null_array_val(const SQL_TYPE_INFO &ti)
#define NULL_ARRAY_DOUBLE
Definition: sqltypes.h:44
bool is_decimal() const
Definition: sqltypes.h:491
double doubleval
Definition: sqltypes.h:211

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Datum import_export::NullDatum ( SQLTypeInfo ti)

Definition at line 254 of file Importer.cpp.

References Datum::bigintval, Datum::boolval, test_fsi::d, decimal_to_int_type(), Datum::doubleval, Datum::floatval, SQLTypeInfo::get_type(), inline_fixed_encoding_null_val(), Datum::intval, SQLTypeInfo::is_decimal(), kBIGINT, kBOOLEAN, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTIME, kTIMESTAMP, kTINYINT, NULL_DOUBLE, NULL_FLOAT, Datum::smallintval, Datum::tinyintval, and run_benchmark_import::type.

Referenced by NullArray(), and StringToArray().

254  {
255  Datum d;
256  const auto type = ti.is_decimal() ? decimal_to_int_type(ti) : ti.get_type();
257  switch (type) {
258  case kBOOLEAN:
260  break;
261  case kBIGINT:
263  break;
264  case kINT:
266  break;
267  case kSMALLINT:
269  break;
270  case kTINYINT:
272  break;
273  case kFLOAT:
274  d.floatval = NULL_FLOAT;
275  break;
276  case kDOUBLE:
278  break;
279  case kTIME:
280  case kTIMESTAMP:
281  case kDATE:
283  break;
284  case kPOINT:
285  case kLINESTRING:
286  case kPOLYGON:
287  case kMULTIPOLYGON:
288  throw std::runtime_error("Internal error: geometry type in NullDatum.");
289  default:
290  throw std::runtime_error("Internal error: invalid type in NullDatum.");
291  }
292  return d;
293 }
int8_t tinyintval
Definition: sqltypes.h:206
#define NULL_DOUBLE
Definition: sqltypes.h:48
tuple d
Definition: test_fsi.py:9
#define NULL_FLOAT
bool boolval
Definition: sqltypes.h:205
int32_t intval
Definition: sqltypes.h:208
float floatval
Definition: sqltypes.h:210
int64_t bigintval
Definition: sqltypes.h:209
int16_t smallintval
Definition: sqltypes.h:207
SQLTypes decimal_to_int_type(const SQLTypeInfo &ti)
Definition: Datum.cpp:419
Definition: sqltypes.h:52
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
Definition: sqltypes.h:44
bool is_decimal() const
Definition: sqltypes.h:491
double doubleval
Definition: sqltypes.h:211

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::pair<SQLTypes, bool> import_export::ogr_to_type ( const OGRFieldType &  ogr_type)

Definition at line 4557 of file Importer.cpp.

References kBIGINT, kDATE, kDOUBLE, kINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, and to_string().

Referenced by import_export::Importer::gdalToColumnDescriptors().

4557  {
4558  switch (ogr_type) {
4559  case OFTInteger:
4560  return std::make_pair(kINT, false);
4561  case OFTIntegerList:
4562  return std::make_pair(kINT, true);
4563 #if GDAL_VERSION_MAJOR > 1
4564  case OFTInteger64:
4565  return std::make_pair(kBIGINT, false);
4566  case OFTInteger64List:
4567  return std::make_pair(kBIGINT, true);
4568 #endif
4569  case OFTReal:
4570  return std::make_pair(kDOUBLE, false);
4571  case OFTRealList:
4572  return std::make_pair(kDOUBLE, true);
4573  case OFTString:
4574  return std::make_pair(kTEXT, false);
4575  case OFTStringList:
4576  return std::make_pair(kTEXT, true);
4577  case OFTDate:
4578  return std::make_pair(kDATE, false);
4579  case OFTTime:
4580  return std::make_pair(kTIME, false);
4581  case OFTDateTime:
4582  return std::make_pair(kTIMESTAMP, false);
4583  case OFTBinary:
4584  // Interpret binary blobs as byte arrays here
4585  // but actual import will store NULL as GDAL will not
4586  // extract the blob (OGRFeature::GetFieldAsString will
4587  // result in the import buffers having an empty string)
4588  return std::make_pair(kTINYINT, true);
4589  default:
4590  break;
4591  }
4592  throw std::runtime_error("Unknown OGR field type: " + std::to_string(ogr_type));
4593 }
Definition: sqltypes.h:48
std::string to_string(char const *&&v)
Definition: sqltypes.h:51
Definition: sqltypes.h:52
Definition: sqltypes.h:44

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

SQLTypes import_export::ogr_to_type ( const OGRwkbGeometryType &  ogr_type)

Definition at line 4595 of file Importer.cpp.

References kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, and to_string().

4595  {
4596  switch (ogr_type) {
4597  case wkbPoint:
4598  return kPOINT;
4599  case wkbLineString:
4600  return kLINESTRING;
4601  case wkbPolygon:
4602  return kPOLYGON;
4603  case wkbMultiPolygon:
4604  return kMULTIPOLYGON;
4605  default:
4606  break;
4607  }
4608  throw std::runtime_error("Unknown OGR geom type: " + std::to_string(ogr_type));
4609 }
std::string to_string(char const *&&v)

+ Here is the call graph for this function:

std::vector< std::unique_ptr< TypedImportBuffer > > import_export::setup_column_loaders ( const TableDescriptor td,
Loader *  loader 
)

Definition at line 5350 of file Importer.cpp.

References CHECK, import_export::Loader::get_column_descs(), and import_export::Loader::getStringDict().

Referenced by DBHandler::prepare_loader_generic().

5352  {
5353  CHECK(td);
5354  auto col_descs = loader->get_column_descs();
5355 
5356  std::vector<std::unique_ptr<TypedImportBuffer>> import_buffers;
5357  for (auto cd : col_descs) {
5358  import_buffers.emplace_back(
5359  std::make_unique<TypedImportBuffer>(cd, loader->getStringDict(cd)));
5360  }
5361 
5362  return import_buffers;
5363 }
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

ArrayDatum import_export::StringToArray ( const std::string &  s,
const SQLTypeInfo ti,
const CopyParams &  copy_params 
)

Definition at line 336 of file Importer.cpp.

References appendDatum(), import_export::CopyParams::array_begin, import_export::CopyParams::array_delim, import_export::CopyParams::array_end, CHECK, checked_malloc(), test_fsi::d, SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_size(), i, is_null(), SQLTypeInfo::is_number(), SQLTypeInfo::is_string(), SQLTypeInfo::is_time(), LOG, import_export::CopyParams::null_str, NullDatum(), StringToDatum(), trim_space(), and logger::WARNING.

Referenced by import_export::TypedImportBuffer::add_value().

338  {
339  SQLTypeInfo elem_ti = ti.get_elem_type();
340  if (s == copy_params.null_str || s == "NULL" || s.empty()) {
341  return ArrayDatum(0, NULL, true);
342  }
343  if (s[0] != copy_params.array_begin || s[s.size() - 1] != copy_params.array_end) {
344  LOG(WARNING) << "Malformed array: " << s;
345  return ArrayDatum(0, NULL, true);
346  }
347  std::vector<std::string> elem_strs;
348  size_t last = 1;
349  for (size_t i = s.find(copy_params.array_delim, 1); i != std::string::npos;
350  i = s.find(copy_params.array_delim, last)) {
351  elem_strs.push_back(s.substr(last, i - last));
352  last = i + 1;
353  }
354  if (last + 1 <= s.size()) {
355  elem_strs.push_back(s.substr(last, s.size() - 1 - last));
356  }
357  if (elem_strs.size() == 1) {
358  auto str = elem_strs.front();
359  auto str_trimmed = trim_space(str.c_str(), str.length());
360  if (str_trimmed == "") {
361  elem_strs.clear(); // Empty array
362  }
363  }
364  if (!elem_ti.is_string()) {
365  size_t len = elem_strs.size() * elem_ti.get_size();
366  int8_t* buf = (int8_t*)checked_malloc(len);
367  int8_t* p = buf;
368  for (auto& es : elem_strs) {
369  auto e = trim_space(es.c_str(), es.length());
370  bool is_null = (e == copy_params.null_str) || e == "NULL";
371  if (!elem_ti.is_string() && e == "") {
372  is_null = true;
373  }
374  if (elem_ti.is_number() || elem_ti.is_time()) {
375  if (!isdigit(e[0]) && e[0] != '-') {
376  is_null = true;
377  }
378  }
379  Datum d = is_null ? NullDatum(elem_ti) : StringToDatum(e, elem_ti);
380  p = appendDatum(p, d, elem_ti);
381  }
382  return ArrayDatum(len, buf, false);
383  }
384  // must not be called for array of strings
385  CHECK(false);
386  return ArrayDatum(0, NULL, true);
387 }
HOST DEVICE int get_size() const
Definition: sqltypes.h:324
tuple d
Definition: test_fsi.py:9
#define LOG(tag)
Definition: Logger.h:188
bool is_number() const
Definition: sqltypes.h:493
bool is_time() const
Definition: sqltypes.h:494
std::conditional_t< is_cuda_compiler(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:202
Datum NullDatum(SQLTypeInfo &ti)
Definition: Importer.cpp:254
CONSTEXPR DEVICE bool is_null(const T &value)
void * checked_malloc(const size_t size)
Definition: checked_alloc.h:45
Datum StringToDatum(std::string_view s, SQLTypeInfo &ti)
Definition: Datum.cpp:239
int8_t * appendDatum(int8_t *buf, Datum d, const SQLTypeInfo &ti)
Definition: sqltypes.h:939
void trim_space(const char *&field_begin, const char *&field_end)
#define CHECK(condition)
Definition: Logger.h:197
bool is_string() const
Definition: sqltypes.h:488
SQLTypeInfo get_elem_type() const
Definition: sqltypes.h:712

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

ArrayDatum import_export::TDatumToArrayDatum ( const TDatum &  datum,
const SQLTypeInfo ti 
)

Definition at line 473 of file Importer.cpp.

References appendDatum(), CHECK, checked_malloc(), SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_size(), SQLTypeInfo::is_string(), NullArray(), and TDatumToDatum().

Referenced by import_export::TypedImportBuffer::add_value().

473  {
474  SQLTypeInfo elem_ti = ti.get_elem_type();
475 
476  CHECK(!elem_ti.is_string());
477 
478  if (datum.is_null) {
479  return NullArray(ti);
480  }
481 
482  size_t len = datum.val.arr_val.size() * elem_ti.get_size();
483  int8_t* buf = (int8_t*)checked_malloc(len);
484  int8_t* p = buf;
485  for (auto& e : datum.val.arr_val) {
486  p = appendDatum(p, TDatumToDatum(e, elem_ti), elem_ti);
487  }
488 
489  return ArrayDatum(len, buf, false);
490 }
HOST DEVICE int get_size() const
Definition: sqltypes.h:324
ArrayDatum NullArray(const SQLTypeInfo &ti)
Definition: Importer.cpp:389
Datum TDatumToDatum(const TDatum &datum, SQLTypeInfo &ti)
Definition: Importer.cpp:428
std::conditional_t< is_cuda_compiler(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:202
void * checked_malloc(const size_t size)
Definition: checked_alloc.h:45
int8_t * appendDatum(int8_t *buf, Datum d, const SQLTypeInfo &ti)
Definition: sqltypes.h:939
#define CHECK(condition)
Definition: Logger.h:197
bool is_string() const
Definition: sqltypes.h:488
SQLTypeInfo get_elem_type() const
Definition: sqltypes.h:712

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Datum import_export::TDatumToDatum ( const TDatum &  datum,
SQLTypeInfo ti 
)

Definition at line 428 of file Importer.cpp.

References Datum::bigintval, Datum::boolval, test_fsi::d, decimal_to_int_type(), Datum::doubleval, Datum::floatval, SQLTypeInfo::get_type(), inline_fixed_encoding_null_val(), Datum::intval, SQLTypeInfo::is_decimal(), kBIGINT, kBOOLEAN, kDATE, kDOUBLE, kFLOAT, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kSMALLINT, kTIME, kTIMESTAMP, kTINYINT, NULL_DOUBLE, NULL_FLOAT, Datum::smallintval, Datum::tinyintval, and run_benchmark_import::type.

Referenced by TDatumToArrayDatum().

428  {
429  Datum d;
430  const auto type = ti.is_decimal() ? decimal_to_int_type(ti) : ti.get_type();
431  switch (type) {
432  case kBOOLEAN:
433  d.boolval = datum.is_null ? inline_fixed_encoding_null_val(ti) : datum.val.int_val;
434  break;
435  case kBIGINT:
436  d.bigintval =
437  datum.is_null ? inline_fixed_encoding_null_val(ti) : datum.val.int_val;
438  break;
439  case kINT:
440  d.intval = datum.is_null ? inline_fixed_encoding_null_val(ti) : datum.val.int_val;
441  break;
442  case kSMALLINT:
443  d.smallintval =
444  datum.is_null ? inline_fixed_encoding_null_val(ti) : datum.val.int_val;
445  break;
446  case kTINYINT:
447  d.tinyintval =
448  datum.is_null ? inline_fixed_encoding_null_val(ti) : datum.val.int_val;
449  break;
450  case kFLOAT:
451  d.floatval = datum.is_null ? NULL_FLOAT : datum.val.real_val;
452  break;
453  case kDOUBLE:
454  d.doubleval = datum.is_null ? NULL_DOUBLE : datum.val.real_val;
455  break;
456  case kTIME:
457  case kTIMESTAMP:
458  case kDATE:
459  d.bigintval =
460  datum.is_null ? inline_fixed_encoding_null_val(ti) : datum.val.int_val;
461  break;
462  case kPOINT:
463  case kLINESTRING:
464  case kPOLYGON:
465  case kMULTIPOLYGON:
466  throw std::runtime_error("Internal error: geometry type in TDatumToDatum.");
467  default:
468  throw std::runtime_error("Internal error: invalid type in TDatumToDatum.");
469  }
470  return d;
471 }
int8_t tinyintval
Definition: sqltypes.h:206
#define NULL_DOUBLE
Definition: sqltypes.h:48
tuple d
Definition: test_fsi.py:9
#define NULL_FLOAT
bool boolval
Definition: sqltypes.h:205
int32_t intval
Definition: sqltypes.h:208
float floatval
Definition: sqltypes.h:210
int64_t bigintval
Definition: sqltypes.h:209
int16_t smallintval
Definition: sqltypes.h:207
SQLTypes decimal_to_int_type(const SQLTypeInfo &ti)
Definition: Datum.cpp:419
Definition: sqltypes.h:52
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
Definition: sqltypes.h:44
bool is_decimal() const
Definition: sqltypes.h:491
double doubleval
Definition: sqltypes.h:211

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

static const std::string import_export::trim_space ( const char *  field,
const size_t  len 
)
static

Definition at line 242 of file Importer.cpp.

References i, and generate_TableFunctionsFactory_init::j.

Referenced by import_export::delimited_parser::get_row(), and StringToArray().

242  {
243  size_t i = 0;
244  size_t j = len;
245  while (i < j && (field[i] == ' ' || field[i] == '\r')) {
246  i++;
247  }
248  while (i < j && (field[j - 1] == ' ' || field[j - 1] == '\r')) {
249  j--;
250  }
251  return std::string(field + i, j - i);
252 }
const rapidjson::Value & field(const rapidjson::Value &obj, const char field[]) noexcept
Definition: JsonAccessors.h:31

+ Here is the caller graph for this function:

template<class T >
bool import_export::try_cast ( const std::string &  str)

Definition at line 3101 of file Importer.cpp.

References omnisci.dtypes::T.

3101  {
3102  try {
3103  boost::lexical_cast<T>(str);
3104  } catch (const boost::bad_lexical_cast& e) {
3105  return false;
3106  }
3107  return true;
3108 }

Variable Documentation

std::map<std::string, ImportStatus> import_export::import_status_map
static
constexpr size_t import_export::kImportFileBufferSize = (1 << 23)
static

Definition at line 32 of file CopyParams.h.

constexpr bool import_export::PROMOTE_POLYGON_TO_MULTIPOLYGON = true
static
mapd_shared_mutex import_export::status_mutex
static