OmniSciDB  ca0c39ec8f
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
import_export::anonymous_namespace{Importer.cpp} Namespace Reference

Functions

SQLTypes get_type_for_datum (const SQLTypeInfo &ti)
 
std::tuple< int, SQLTypes,
std::string > 
explode_collections_step1 (const std::list< const ColumnDescriptor * > &col_descs)
 
int64_t explode_collections_step2 (OGRGeometry *ogr_geometry, SQLTypes collection_child_type, const std::string &collection_col_name, size_t row_or_feature_idx, std::function< void(OGRGeometry *)> execute_import_lambda)
 
int64_t int_value_at (const TypedImportBuffer &import_buffer, const size_t index)
 
float float_value_at (const TypedImportBuffer &import_buffer, const size_t index)
 
double double_value_at (const TypedImportBuffer &import_buffer, const size_t index)
 
OGRLayer & getLayerWithSpecifiedName (const std::string &geo_layer_name, const Geospatial::GDAL::DataSourceUqPtr &poDS, const std::string &file_name)
 
std::pair< SQLTypes, bool > ogr_to_type (const OGRFieldType &ogr_type)
 
SQLTypes ogr_to_type (const OGRwkbGeometryType &ogr_type)
 
RasterImporter::PointType convert_raster_point_type (const import_export::RasterPointType raster_point_type)
 
RasterImporter::PointTransform convert_raster_point_transform (const import_export::RasterPointTransform raster_point_transform)
 

Function Documentation

RasterImporter::PointTransform import_export::anonymous_namespace{Importer.cpp}::convert_raster_point_transform ( const import_export::RasterPointTransform  raster_point_transform)

Definition at line 4899 of file Importer.cpp.

References import_export::kAuto, import_export::RasterImporter::kAuto, import_export::kFile, import_export::RasterImporter::kFile, import_export::kNone, import_export::RasterImporter::kNone, import_export::kWorld, import_export::RasterImporter::kWorld, and UNREACHABLE.

Referenced by import_export::Importer::gdalToColumnDescriptorsRaster(), and import_export::Importer::importGDALRaster().

4900  {
4901  switch (raster_point_transform) {
4903  return RasterImporter::PointTransform::kNone;
4905  return RasterImporter::PointTransform::kAuto;
4907  return RasterImporter::PointTransform::kFile;
4909  return RasterImporter::PointTransform::kWorld;
4910  }
4911  UNREACHABLE();
4912  return RasterImporter::PointTransform::kNone;
4913 }
#define UNREACHABLE()
Definition: Logger.h:266

+ Here is the caller graph for this function:

RasterImporter::PointType import_export::anonymous_namespace{Importer.cpp}::convert_raster_point_type ( const import_export::RasterPointType  raster_point_type)

Definition at line 4877 of file Importer.cpp.

References import_export::kAuto, import_export::RasterImporter::kAuto, import_export::kDouble, import_export::RasterImporter::kDouble, import_export::kFloat, import_export::RasterImporter::kFloat, import_export::kInt, import_export::RasterImporter::kInt, import_export::kNone, import_export::RasterImporter::kNone, import_export::kPoint, import_export::RasterImporter::kPoint, import_export::kSmallInt, import_export::RasterImporter::kSmallInt, and UNREACHABLE.

Referenced by import_export::Importer::gdalToColumnDescriptorsRaster(), and import_export::Importer::importGDALRaster().

4878  {
4879  switch (raster_point_type) {
4881  return RasterImporter::PointType::kNone;
4883  return RasterImporter::PointType::kAuto;
4885  return RasterImporter::PointType::kSmallInt;
4887  return RasterImporter::PointType::kInt;
4889  return RasterImporter::PointType::kFloat;
4891  return RasterImporter::PointType::kDouble;
4893  return RasterImporter::PointType::kPoint;
4894  }
4895  UNREACHABLE();
4896  return RasterImporter::PointType::kNone;
4897 }
#define UNREACHABLE()
Definition: Logger.h:266

+ Here is the caller graph for this function:

double import_export::anonymous_namespace{Importer.cpp}::double_value_at ( const TypedImportBuffer &  import_buffer,
const size_t  index 
)

Definition at line 2803 of file Importer.cpp.

References CHECK_EQ, import_export::TypedImportBuffer::getAsBytes(), import_export::TypedImportBuffer::getTypeInfo(), and kDOUBLE.

Referenced by import_export::Loader::fillShardRow().

2803  {
2804  const auto& ti = import_buffer.getTypeInfo();
2805  CHECK_EQ(kDOUBLE, ti.get_type());
2806  const auto values_buffer = import_buffer.getAsBytes();
2807  return reinterpret_cast<const double*>(may_alias_ptr(values_buffer))[index];
2808 }
#define CHECK_EQ(x, y)
Definition: Logger.h:230

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::tuple<int, SQLTypes, std::string> import_export::anonymous_namespace{Importer.cpp}::explode_collections_step1 ( const std::list< const ColumnDescriptor * > &  col_descs)

Definition at line 1872 of file Importer.cpp.

References kLINESTRING, kNULLT, kPOINT, and kPOLYGON.

Referenced by import_export::import_thread_delimited(), and import_export::import_thread_shapefile().

1873  {
1874  // validate the columns
1875  // for now we can only explode into a single destination column
1876  // which must be of the child type (POLYGON, LINESTRING, POINT)
1877  int collection_col_idx = -1;
1878  int col_idx = 0;
1879  std::string collection_col_name;
1880  SQLTypes collection_child_type = kNULLT;
1881  for (auto cd_it = col_descs.begin(); cd_it != col_descs.end(); cd_it++) {
1882  auto const& cd = *cd_it;
1883  auto const col_type = cd->columnType.get_type();
1884  if (col_type == kPOLYGON || col_type == kLINESTRING || col_type == kPOINT) {
1885  if (collection_col_idx >= 0) {
1886  throw std::runtime_error(
1887  "Explode Collections: Found more than one destination column");
1888  }
1889  collection_col_idx = col_idx;
1890  collection_child_type = col_type;
1891  collection_col_name = cd->columnName;
1892  }
1893  for (int i = 0; i < cd->columnType.get_physical_cols(); ++i) {
1894  ++cd_it;
1895  }
1896  col_idx++;
1897  }
1898  if (collection_col_idx < 0) {
1899  throw std::runtime_error(
1900  "Explode Collections: Failed to find a supported column type to explode "
1901  "into");
1902  }
1903  return std::make_tuple(collection_col_idx, collection_child_type, collection_col_name);
1904 }
SQLTypes
Definition: sqltypes.h:53

+ Here is the caller graph for this function:

int64_t import_export::anonymous_namespace{Importer.cpp}::explode_collections_step2 ( OGRGeometry *  ogr_geometry,
SQLTypes  collection_child_type,
const std::string &  collection_col_name,
size_t  row_or_feature_idx,
std::function< void(OGRGeometry *)>  execute_import_lambda 
)

Definition at line 1906 of file Importer.cpp.

References CHECK, measure< TimeT >::execution(), logger::INFO, kLINESTRING, kPOINT, kPOLYGON, and LOG.

Referenced by import_export::import_thread_delimited(), and import_export::import_thread_shapefile().

1911  {
1912  auto ogr_geometry_type = wkbFlatten(ogr_geometry->getGeometryType());
1913  bool is_collection = false;
1914  switch (collection_child_type) {
1915  case kPOINT:
1916  switch (ogr_geometry_type) {
1917  case wkbMultiPoint:
1918  is_collection = true;
1919  break;
1920  case wkbPoint:
1921  break;
1922  default:
1923  throw std::runtime_error(
1924  "Explode Collections: Source geo type must be MULTIPOINT or POINT");
1925  }
1926  break;
1927  case kLINESTRING:
1928  switch (ogr_geometry_type) {
1929  case wkbMultiLineString:
1930  is_collection = true;
1931  break;
1932  case wkbLineString:
1933  break;
1934  default:
1935  throw std::runtime_error(
1936  "Explode Collections: Source geo type must be MULTILINESTRING or "
1937  "LINESTRING");
1938  }
1939  break;
1940  case kPOLYGON:
1941  switch (ogr_geometry_type) {
1942  case wkbMultiPolygon:
1943  is_collection = true;
1944  break;
1945  case wkbPolygon:
1946  break;
1947  default:
1948  throw std::runtime_error(
1949  "Explode Collections: Source geo type must be MULTIPOLYGON or POLYGON");
1950  }
1951  break;
1952  default:
1953  CHECK(false) << "Unsupported geo child type " << collection_child_type;
1954  }
1955 
1956  int64_t us = 0LL;
1957 
1958  // explode or just import
1959  if (is_collection) {
1960  // cast to collection
1961  OGRGeometryCollection* collection_geometry = ogr_geometry->toGeometryCollection();
1962  CHECK(collection_geometry);
1963 
1964 #if LOG_EXPLODE_COLLECTIONS
1965  // log number of children
1966  LOG(INFO) << "Exploding row/feature " << row_or_feature_idx << " for column '"
1967  << explode_col_name << "' into " << collection_geometry->getNumGeometries()
1968  << " child rows";
1969 #endif
1970 
1971  // loop over children
1972  uint32_t child_geometry_count = 0;
1973  auto child_geometry_it = collection_geometry->begin();
1974  while (child_geometry_it != collection_geometry->end()) {
1975  // get and import this child
1976  OGRGeometry* import_geometry = *child_geometry_it;
1978  [&] { execute_import_lambda(import_geometry); });
1979 
1980  // next child
1981  child_geometry_it++;
1982  child_geometry_count++;
1983  }
1984  } else {
1985  // import non-collection row just once
1987  [&] { execute_import_lambda(ogr_geometry); });
1988  }
1989 
1990  // done
1991  return us;
1992 }
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:216
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

float import_export::anonymous_namespace{Importer.cpp}::float_value_at ( const TypedImportBuffer &  import_buffer,
const size_t  index 
)

Definition at line 2796 of file Importer.cpp.

References CHECK_EQ, import_export::TypedImportBuffer::getAsBytes(), import_export::TypedImportBuffer::getTypeInfo(), and kFLOAT.

Referenced by import_export::Loader::fillShardRow().

2796  {
2797  const auto& ti = import_buffer.getTypeInfo();
2798  CHECK_EQ(kFLOAT, ti.get_type());
2799  const auto values_buffer = import_buffer.getAsBytes();
2800  return reinterpret_cast<const float*>(may_alias_ptr(values_buffer))[index];
2801 }
#define CHECK_EQ(x, y)
Definition: Logger.h:230

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

SQLTypes import_export::anonymous_namespace{Importer.cpp}::get_type_for_datum ( const SQLTypeInfo ti)
inline

Definition at line 255 of file Importer.cpp.

References decimal_to_int_type(), SQLTypeInfo::get_type(), SQLTypeInfo::is_decimal(), SQLTypeInfo::is_dict_encoded_string(), string_dict_to_int_type(), and run_benchmark_import::type.

Referenced by import_export::NullArrayDatum(), and NullDatum().

255  {
256  SQLTypes type;
257  if (ti.is_decimal()) {
258  type = decimal_to_int_type(ti);
259  } else if (ti.is_dict_encoded_string()) {
260  type = string_dict_to_int_type(ti);
261  } else {
262  type = ti.get_type();
263  }
264  return type;
265 }
SQLTypes
Definition: sqltypes.h:53
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:379
SQLTypes decimal_to_int_type(const SQLTypeInfo &ti)
Definition: Datum.cpp:559
bool is_dict_encoded_string() const
Definition: sqltypes.h:627
bool is_decimal() const
Definition: sqltypes.h:578
SQLTypes string_dict_to_int_type(const SQLTypeInfo &ti)
Definition: Datum.cpp:563

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

OGRLayer& import_export::anonymous_namespace{Importer.cpp}::getLayerWithSpecifiedName ( const std::string &  geo_layer_name,
const Geospatial::GDAL::DataSourceUqPtr poDS,
const std::string &  file_name 
)

Definition at line 4700 of file Importer.cpp.

Referenced by import_export::Importer::gdalToColumnDescriptorsGeo(), import_export::Importer::importGDALGeo(), and import_export::Importer::readMetadataSampleGDAL().

4702  {
4703  // get layer with specified name, or default to first layer
4704  OGRLayer* poLayer = nullptr;
4705  if (geo_layer_name.size()) {
4706  poLayer = poDS->GetLayerByName(geo_layer_name.c_str());
4707  if (poLayer == nullptr) {
4708  throw std::runtime_error("Layer '" + geo_layer_name + "' not found in " +
4709  file_name);
4710  }
4711  } else {
4712  poLayer = poDS->GetLayer(0);
4713  if (poLayer == nullptr) {
4714  throw std::runtime_error("No layers found in " + file_name);
4715  }
4716  }
4717  return *poLayer;
4718 }

+ Here is the caller graph for this function:

int64_t import_export::anonymous_namespace{Importer.cpp}::int_value_at ( const TypedImportBuffer &  import_buffer,
const size_t  index 
)

Definition at line 2765 of file Importer.cpp.

References CHECK, CHECK_EQ, logger::FATAL, import_export::TypedImportBuffer::getAsBytes(), import_export::TypedImportBuffer::getStringDictBuffer(), import_export::TypedImportBuffer::getTypeInfo(), kENCODING_DICT, LOG, and UNREACHABLE.

Referenced by import_export::Loader::distributeToShardsExistingColumns(), and import_export::Loader::fillShardRow().

2765  {
2766  const auto& ti = import_buffer.getTypeInfo();
2767  const int8_t* values_buffer{nullptr};
2768  if (ti.is_string()) {
2769  CHECK_EQ(kENCODING_DICT, ti.get_compression());
2770  values_buffer = import_buffer.getStringDictBuffer();
2771  } else {
2772  values_buffer = import_buffer.getAsBytes();
2773  }
2774  CHECK(values_buffer);
2775  const int logical_size = ti.is_string() ? ti.get_size() : ti.get_logical_size();
2776  switch (logical_size) {
2777  case 1: {
2778  return values_buffer[index];
2779  }
2780  case 2: {
2781  return reinterpret_cast<const int16_t*>(values_buffer)[index];
2782  }
2783  case 4: {
2784  return reinterpret_cast<const int32_t*>(values_buffer)[index];
2785  }
2786  case 8: {
2787  return reinterpret_cast<const int64_t*>(values_buffer)[index];
2788  }
2789  default:
2790  LOG(FATAL) << "Unexpected size for shard key: " << logical_size;
2791  }
2792  UNREACHABLE();
2793  return 0;
2794 }
#define CHECK_EQ(x, y)
Definition: Logger.h:230
#define LOG(tag)
Definition: Logger.h:216
#define UNREACHABLE()
Definition: Logger.h:266
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::pair<SQLTypes, bool> import_export::anonymous_namespace{Importer.cpp}::ogr_to_type ( const OGRFieldType &  ogr_type)

Definition at line 4819 of file Importer.cpp.

References kBIGINT, kDATE, kDOUBLE, kINT, kTEXT, kTIME, kTIMESTAMP, kTINYINT, and to_string().

Referenced by import_export::Importer::gdalToColumnDescriptorsGeo().

4819  {
4820  switch (ogr_type) {
4821  case OFTInteger:
4822  return std::make_pair(kINT, false);
4823  case OFTIntegerList:
4824  return std::make_pair(kINT, true);
4825 #if GDAL_VERSION_MAJOR > 1
4826  case OFTInteger64:
4827  return std::make_pair(kBIGINT, false);
4828  case OFTInteger64List:
4829  return std::make_pair(kBIGINT, true);
4830 #endif
4831  case OFTReal:
4832  return std::make_pair(kDOUBLE, false);
4833  case OFTRealList:
4834  return std::make_pair(kDOUBLE, true);
4835  case OFTString:
4836  return std::make_pair(kTEXT, false);
4837  case OFTStringList:
4838  return std::make_pair(kTEXT, true);
4839  case OFTDate:
4840  return std::make_pair(kDATE, false);
4841  case OFTTime:
4842  return std::make_pair(kTIME, false);
4843  case OFTDateTime:
4844  return std::make_pair(kTIMESTAMP, false);
4845  case OFTBinary:
4846  // Interpret binary blobs as byte arrays here
4847  // but actual import will store NULL as GDAL will not
4848  // extract the blob (OGRFeature::GetFieldAsString will
4849  // result in the import buffers having an empty string)
4850  return std::make_pair(kTINYINT, true);
4851  default:
4852  break;
4853  }
4854  throw std::runtime_error("Unknown OGR field type: " + std::to_string(ogr_type));
4855 }
Definition: sqltypes.h:64
std::string to_string(char const *&&v)
Definition: sqltypes.h:67
Definition: sqltypes.h:68
Definition: sqltypes.h:60

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

SQLTypes import_export::anonymous_namespace{Importer.cpp}::ogr_to_type ( const OGRwkbGeometryType &  ogr_type)

Definition at line 4857 of file Importer.cpp.

References kLINESTRING, kMULTILINESTRING, kMULTIPOINT, kMULTIPOLYGON, kPOINT, kPOLYGON, and to_string().

4857  {
4858  switch (ogr_type) {
4859  case wkbPoint:
4860  return kPOINT;
4861  case wkbMultiPoint:
4862  return kMULTIPOINT;
4863  case wkbLineString:
4864  return kLINESTRING;
4865  case wkbMultiLineString:
4866  return kMULTILINESTRING;
4867  case wkbPolygon:
4868  return kPOLYGON;
4869  case wkbMultiPolygon:
4870  return kMULTIPOLYGON;
4871  default:
4872  break;
4873  }
4874  throw std::runtime_error("Unknown OGR geom type: " + std::to_string(ogr_type));
4875 }
std::string to_string(char const *&&v)

+ Here is the call graph for this function: