OmniSciDB  085a039ca4
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
import_export::anonymous_namespace{Importer.cpp} Namespace Reference

Functions

SQLTypes get_type_for_datum (const SQLTypeInfo &ti)
 
std::tuple< int, SQLTypes,
std::string > 
explode_collections_step1 (const std::list< const ColumnDescriptor * > &col_descs)
 
int64_t explode_collections_step2 (OGRGeometry *ogr_geometry, SQLTypes collection_child_type, const std::string &collection_col_name, size_t row_or_feature_idx, std::function< void(OGRGeometry *)> execute_import_lambda)
 
int64_t int_value_at (const TypedImportBuffer &import_buffer, const size_t index)
 
float float_value_at (const TypedImportBuffer &import_buffer, const size_t index)
 
double double_value_at (const TypedImportBuffer &import_buffer, const size_t index)
 
OGRLayer & getLayerWithSpecifiedName (const std::string &geo_layer_name, const Geospatial::GDAL::DataSourceUqPtr &poDS, const std::string &file_name)
 
std::pair< SQLTypes, bool > ogr_to_type (const OGRFieldType &ogr_type)
 
SQLTypes ogr_to_type (const OGRwkbGeometryType &ogr_type)
 
RasterImporter::PointType convert_raster_point_type (const import_export::RasterPointType raster_point_type)
 
RasterImporter::PointTransform convert_raster_point_transform (const import_export::RasterPointTransform raster_point_transform)
 

Function Documentation

RasterImporter::PointTransform import_export::anonymous_namespace{Importer.cpp}::convert_raster_point_transform ( const import_export::RasterPointTransform  raster_point_transform)

Definition at line 4892 of file Importer.cpp.

References import_export::kAuto, import_export::RasterImporter::kAuto, import_export::kFile, import_export::RasterImporter::kFile, import_export::kNone, import_export::RasterImporter::kNone, import_export::kWorld, import_export::RasterImporter::kWorld, and UNREACHABLE.

Referenced by import_export::Importer::gdalToColumnDescriptorsRaster(), and import_export::Importer::importGDALRaster().

4893  {
4894  switch (raster_point_transform) {
4896  return RasterImporter::PointTransform::kNone;
4898  return RasterImporter::PointTransform::kAuto;
4900  return RasterImporter::PointTransform::kFile;
4902  return RasterImporter::PointTransform::kWorld;
4903  }
4904  UNREACHABLE();
4905  return RasterImporter::PointTransform::kNone;
4906 }
#define UNREACHABLE()
Definition: Logger.h:267

+ Here is the caller graph for this function:

RasterImporter::PointType import_export::anonymous_namespace{Importer.cpp}::convert_raster_point_type ( const import_export::RasterPointType  raster_point_type)

Definition at line 4870 of file Importer.cpp.

References import_export::kAuto, import_export::RasterImporter::kAuto, import_export::kDouble, import_export::RasterImporter::kDouble, import_export::kFloat, import_export::RasterImporter::kFloat, import_export::kInt, import_export::RasterImporter::kInt, import_export::kNone, import_export::RasterImporter::kNone, import_export::kPoint, import_export::RasterImporter::kPoint, import_export::kSmallInt, import_export::RasterImporter::kSmallInt, and UNREACHABLE.

Referenced by import_export::Importer::gdalToColumnDescriptorsRaster(), and import_export::Importer::importGDALRaster().

4871  {
4872  switch (raster_point_type) {
4874  return RasterImporter::PointType::kNone;
4876  return RasterImporter::PointType::kAuto;
4878  return RasterImporter::PointType::kSmallInt;
4880  return RasterImporter::PointType::kInt;
4882  return RasterImporter::PointType::kFloat;
4884  return RasterImporter::PointType::kDouble;
4886  return RasterImporter::PointType::kPoint;
4887  }
4888  UNREACHABLE();
4889  return RasterImporter::PointType::kNone;
4890 }
#define UNREACHABLE()
Definition: Logger.h:267

+ Here is the caller graph for this function:

double import_export::anonymous_namespace{Importer.cpp}::double_value_at ( const TypedImportBuffer &  import_buffer,
const size_t  index 
)

Definition at line 2825 of file Importer.cpp.

References CHECK_EQ, import_export::TypedImportBuffer::getAsBytes(), import_export::TypedImportBuffer::getTypeInfo(), and kDOUBLE.

Referenced by import_export::Loader::fillShardRow().

2825  {
2826  const auto& ti = import_buffer.getTypeInfo();
2827  CHECK_EQ(kDOUBLE, ti.get_type());
2828  const auto values_buffer = import_buffer.getAsBytes();
2829  return reinterpret_cast<const double*>(may_alias_ptr(values_buffer))[index];
2830 }
#define CHECK_EQ(x, y)
Definition: Logger.h:231

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::tuple<int, SQLTypes, std::string> import_export::anonymous_namespace{Importer.cpp}::explode_collections_step1 ( const std::list< const ColumnDescriptor * > &  col_descs)

Definition at line 1900 of file Importer.cpp.

References kLINESTRING, kNULLT, kPOINT, and kPOLYGON.

Referenced by import_export::import_thread_delimited(), and import_export::import_thread_shapefile().

1901  {
1902  // validate the columns
1903  // for now we can only explode into a single destination column
1904  // which must be of the child type (POLYGON, LINESTRING, POINT)
1905  int collection_col_idx = -1;
1906  int col_idx = 0;
1907  std::string collection_col_name;
1908  SQLTypes collection_child_type = kNULLT;
1909  for (auto cd_it = col_descs.begin(); cd_it != col_descs.end(); cd_it++) {
1910  auto const& cd = *cd_it;
1911  auto const col_type = cd->columnType.get_type();
1912  if (col_type == kPOLYGON || col_type == kLINESTRING || col_type == kPOINT) {
1913  if (collection_col_idx >= 0) {
1914  throw std::runtime_error(
1915  "Explode Collections: Found more than one destination column");
1916  }
1917  collection_col_idx = col_idx;
1918  collection_child_type = col_type;
1919  collection_col_name = cd->columnName;
1920  }
1921  for (int i = 0; i < cd->columnType.get_physical_cols(); ++i) {
1922  ++cd_it;
1923  }
1924  col_idx++;
1925  }
1926  if (collection_col_idx < 0) {
1927  throw std::runtime_error(
1928  "Explode Collections: Failed to find a supported column type to explode "
1929  "into");
1930  }
1931  return std::make_tuple(collection_col_idx, collection_child_type, collection_col_name);
1932 }
SQLTypes
Definition: sqltypes.h:38

+ Here is the caller graph for this function:

int64_t import_export::anonymous_namespace{Importer.cpp}::explode_collections_step2 ( OGRGeometry *  ogr_geometry,
SQLTypes  collection_child_type,
const std::string &  collection_col_name,
size_t  row_or_feature_idx,
std::function< void(OGRGeometry *)>  execute_import_lambda 
)

Definition at line 1934 of file Importer.cpp.

References CHECK, measure< TimeT >::execution(), logger::INFO, kLINESTRING, kPOINT, kPOLYGON, and LOG.

Referenced by import_export::import_thread_delimited(), and import_export::import_thread_shapefile().

1939  {
1940  auto ogr_geometry_type = wkbFlatten(ogr_geometry->getGeometryType());
1941  bool is_collection = false;
1942  switch (collection_child_type) {
1943  case kPOINT:
1944  switch (ogr_geometry_type) {
1945  case wkbMultiPoint:
1946  is_collection = true;
1947  break;
1948  case wkbPoint:
1949  break;
1950  default:
1951  throw std::runtime_error(
1952  "Explode Collections: Source geo type must be MULTIPOINT or POINT");
1953  }
1954  break;
1955  case kLINESTRING:
1956  switch (ogr_geometry_type) {
1957  case wkbMultiLineString:
1958  is_collection = true;
1959  break;
1960  case wkbLineString:
1961  break;
1962  default:
1963  throw std::runtime_error(
1964  "Explode Collections: Source geo type must be MULTILINESTRING or "
1965  "LINESTRING");
1966  }
1967  break;
1968  case kPOLYGON:
1969  switch (ogr_geometry_type) {
1970  case wkbMultiPolygon:
1971  is_collection = true;
1972  break;
1973  case wkbPolygon:
1974  break;
1975  default:
1976  throw std::runtime_error(
1977  "Explode Collections: Source geo type must be MULTIPOLYGON or POLYGON");
1978  }
1979  break;
1980  default:
1981  CHECK(false) << "Unsupported geo child type " << collection_child_type;
1982  }
1983 
1984  int64_t us = 0LL;
1985 
1986  // explode or just import
1987  if (is_collection) {
1988  // cast to collection
1989  OGRGeometryCollection* collection_geometry = ogr_geometry->toGeometryCollection();
1990  CHECK(collection_geometry);
1991 
1992 #if LOG_EXPLODE_COLLECTIONS
1993  // log number of children
1994  LOG(INFO) << "Exploding row/feature " << row_or_feature_idx << " for column '"
1995  << explode_col_name << "' into " << collection_geometry->getNumGeometries()
1996  << " child rows";
1997 #endif
1998 
1999  // loop over children
2000  uint32_t child_geometry_count = 0;
2001  auto child_geometry_it = collection_geometry->begin();
2002  while (child_geometry_it != collection_geometry->end()) {
2003  // get and import this child
2004  OGRGeometry* import_geometry = *child_geometry_it;
2006  [&] { execute_import_lambda(import_geometry); });
2007 
2008  // next child
2009  child_geometry_it++;
2010  child_geometry_count++;
2011  }
2012  } else {
2013  // import non-collection row just once
2015  [&] { execute_import_lambda(ogr_geometry); });
2016  }
2017 
2018  // done
2019  return us;
2020 }
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:217
#define CHECK(condition)
Definition: Logger.h:223

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

float import_export::anonymous_namespace{Importer.cpp}::float_value_at ( const TypedImportBuffer &  import_buffer,
const size_t  index 
)

Definition at line 2818 of file Importer.cpp.

References CHECK_EQ, import_export::TypedImportBuffer::getAsBytes(), import_export::TypedImportBuffer::getTypeInfo(), and kFLOAT.

Referenced by import_export::Loader::fillShardRow().

2818  {
2819  const auto& ti = import_buffer.getTypeInfo();
2820  CHECK_EQ(kFLOAT, ti.get_type());
2821  const auto values_buffer = import_buffer.getAsBytes();
2822  return reinterpret_cast<const float*>(may_alias_ptr(values_buffer))[index];
2823 }
#define CHECK_EQ(x, y)
Definition: Logger.h:231

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

SQLTypes import_export::anonymous_namespace{Importer.cpp}::get_type_for_datum ( const SQLTypeInfo ti)
inline

Definition at line 260 of file Importer.cpp.

References decimal_to_int_type(), SQLTypeInfo::get_type(), SQLTypeInfo::is_decimal(), SQLTypeInfo::is_dict_encoded_string(), string_dict_to_int_type(), and run_benchmark_import::type.

Referenced by import_export::NullArrayDatum(), and import_export::NullDatum().

260  {
261  SQLTypes type;
262  if (ti.is_decimal()) {
263  type = decimal_to_int_type(ti);
264  } else if (ti.is_dict_encoded_string()) {
265  type = string_dict_to_int_type(ti);
266  } else {
267  type = ti.get_type();
268  }
269  return type;
270 }
SQLTypes
Definition: sqltypes.h:38
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:329
SQLTypes decimal_to_int_type(const SQLTypeInfo &ti)
Definition: Datum.cpp:493
bool is_dict_encoded_string() const
Definition: sqltypes.h:548
bool is_decimal() const
Definition: sqltypes.h:513
SQLTypes string_dict_to_int_type(const SQLTypeInfo &ti)
Definition: Datum.cpp:497

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

OGRLayer& import_export::anonymous_namespace{Importer.cpp}::getLayerWithSpecifiedName ( const std::string &  geo_layer_name,
const Geospatial::GDAL::DataSourceUqPtr poDS,
const std::string &  file_name 
)

Definition at line 4691 of file Importer.cpp.

Referenced by import_export::Importer::gdalToColumnDescriptorsGeo(), import_export::Importer::importGDALGeo(), and import_export::Importer::readMetadataSampleGDAL().

4693  {
4694  // get layer with specified name, or default to first layer
4695  OGRLayer* poLayer = nullptr;
4696  if (geo_layer_name.size()) {
4697  poLayer = poDS->GetLayerByName(geo_layer_name.c_str());
4698  if (poLayer == nullptr) {
4699  throw std::runtime_error("Layer '" + geo_layer_name + "' not found in " +
4700  file_name);
4701  }
4702  } else {
4703  poLayer = poDS->GetLayer(0);
4704  if (poLayer == nullptr) {
4705  throw std::runtime_error("No layers found in " + file_name);
4706  }
4707  }
4708  return *poLayer;
4709 }

+ Here is the caller graph for this function:

int64_t import_export::anonymous_namespace{Importer.cpp}::int_value_at ( const TypedImportBuffer &  import_buffer,
const size_t  index 
)

Definition at line 2787 of file Importer.cpp.

References CHECK, CHECK_EQ, logger::FATAL, import_export::TypedImportBuffer::getAsBytes(), import_export::TypedImportBuffer::getStringDictBuffer(), import_export::TypedImportBuffer::getTypeInfo(), kENCODING_DICT, LOG, and UNREACHABLE.

Referenced by import_export::Loader::distributeToShardsExistingColumns(), and import_export::Loader::fillShardRow().

2787  {
2788  const auto& ti = import_buffer.getTypeInfo();
2789  const int8_t* values_buffer{nullptr};
2790  if (ti.is_string()) {
2791  CHECK_EQ(kENCODING_DICT, ti.get_compression());
2792  values_buffer = import_buffer.getStringDictBuffer();
2793  } else {
2794  values_buffer = import_buffer.getAsBytes();
2795  }
2796  CHECK(values_buffer);
2797  const int logical_size = ti.is_string() ? ti.get_size() : ti.get_logical_size();
2798  switch (logical_size) {
2799  case 1: {
2800  return values_buffer[index];
2801  }
2802  case 2: {
2803  return reinterpret_cast<const int16_t*>(values_buffer)[index];
2804  }
2805  case 4: {
2806  return reinterpret_cast<const int32_t*>(values_buffer)[index];
2807  }
2808  case 8: {
2809  return reinterpret_cast<const int64_t*>(values_buffer)[index];
2810  }
2811  default:
2812  LOG(FATAL) << "Unexpected size for shard key: " << logical_size;
2813  }
2814  UNREACHABLE();
2815  return 0;
2816 }
#define CHECK_EQ(x, y)
Definition: Logger.h:231
#define LOG(tag)
Definition: Logger.h:217
#define UNREACHABLE()
Definition: Logger.h:267
#define CHECK(condition)
Definition: Logger.h:223

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::pair<SQLTypes, bool> import_export::anonymous_namespace{Importer.cpp}::ogr_to_type ( const OGRFieldType &  ogr_type)

Definition at line 4816 of file Importer.cpp.

References kBIGINT, kDATE, kDOUBLE, kINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, and to_string().

Referenced by import_export::Importer::gdalToColumnDescriptorsGeo().

4816  {
4817  switch (ogr_type) {
4818  case OFTInteger:
4819  return std::make_pair(kINT, false);
4820  case OFTIntegerList:
4821  return std::make_pair(kINT, true);
4822 #if GDAL_VERSION_MAJOR > 1
4823  case OFTInteger64:
4824  return std::make_pair(kBIGINT, false);
4825  case OFTInteger64List:
4826  return std::make_pair(kBIGINT, true);
4827 #endif
4828  case OFTReal:
4829  return std::make_pair(kDOUBLE, false);
4830  case OFTRealList:
4831  return std::make_pair(kDOUBLE, true);
4832  case OFTString:
4833  return std::make_pair(kTEXT, false);
4834  case OFTStringList:
4835  return std::make_pair(kTEXT, true);
4836  case OFTDate:
4837  return std::make_pair(kDATE, false);
4838  case OFTTime:
4839  return std::make_pair(kTIME, false);
4840  case OFTDateTime:
4841  return std::make_pair(kTIMESTAMP, false);
4842  case OFTBinary:
4843  // Interpret binary blobs as byte arrays here
4844  // but actual import will store NULL as GDAL will not
4845  // extract the blob (OGRFeature::GetFieldAsString will
4846  // result in the import buffers having an empty string)
4847  return std::make_pair(kTINYINT, true);
4848  default:
4849  break;
4850  }
4851  throw std::runtime_error("Unknown OGR field type: " + std::to_string(ogr_type));
4852 }
Definition: sqltypes.h:49
std::string to_string(char const *&&v)
Definition: sqltypes.h:52
Definition: sqltypes.h:53
Definition: sqltypes.h:45

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

SQLTypes import_export::anonymous_namespace{Importer.cpp}::ogr_to_type ( const OGRwkbGeometryType &  ogr_type)

Definition at line 4854 of file Importer.cpp.

References kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, and to_string().

4854  {
4855  switch (ogr_type) {
4856  case wkbPoint:
4857  return kPOINT;
4858  case wkbLineString:
4859  return kLINESTRING;
4860  case wkbPolygon:
4861  return kPOLYGON;
4862  case wkbMultiPolygon:
4863  return kMULTIPOLYGON;
4864  default:
4865  break;
4866  }
4867  throw std::runtime_error("Unknown OGR geom type: " + std::to_string(ogr_type));
4868 }
std::string to_string(char const *&&v)

+ Here is the call graph for this function: