OmniSciDB  471d68cefb
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
import_export::anonymous_namespace{Importer.cpp} Namespace Reference

Functions

std::tuple< int, SQLTypes,
std::string > 
explode_collections_step1 (const std::list< const ColumnDescriptor * > &col_descs)
 
int64_t explode_collections_step2 (OGRGeometry *ogr_geometry, SQLTypes collection_child_type, const std::string &collection_col_name, size_t row_or_feature_idx, std::function< void(OGRGeometry *)> execute_import_lambda)
 
int64_t int_value_at (const TypedImportBuffer &import_buffer, const size_t index)
 
float float_value_at (const TypedImportBuffer &import_buffer, const size_t index)
 
double double_value_at (const TypedImportBuffer &import_buffer, const size_t index)
 
OGRLayer & getLayerWithSpecifiedName (const std::string &geo_layer_name, const OGRDataSourceUqPtr &poDS, const std::string &file_name)
 

Function Documentation

double import_export::anonymous_namespace{Importer.cpp}::double_value_at ( const TypedImportBuffer &  import_buffer,
const size_t  index 
)

Definition at line 2828 of file Importer.cpp.

References CHECK_EQ, import_export::TypedImportBuffer::getAsBytes(), import_export::TypedImportBuffer::getTypeInfo(), and kDOUBLE.

Referenced by import_export::Loader::fillShardRow().

2828  {
2829  const auto& ti = import_buffer.getTypeInfo();
2830  CHECK_EQ(kDOUBLE, ti.get_type());
2831  const auto values_buffer = import_buffer.getAsBytes();
2832  return reinterpret_cast<const double*>(may_alias_ptr(values_buffer))[index];
2833 }
#define CHECK_EQ(x, y)
Definition: Logger.h:217

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::tuple<int, SQLTypes, std::string> import_export::anonymous_namespace{Importer.cpp}::explode_collections_step1 ( const std::list< const ColumnDescriptor * > &  col_descs)

Definition at line 1873 of file Importer.cpp.

References i, kLINESTRING, kNULLT, kPOINT, and kPOLYGON.

Referenced by import_export::import_thread_delimited(), and import_export::import_thread_shapefile().

1874  {
1875  // validate the columns
1876  // for now we can only explode into a single destination column
1877  // which must be of the child type (POLYGON, LINESTRING, POINT)
1878  int collection_col_idx = -1;
1879  int col_idx = 0;
1880  std::string collection_col_name;
1881  SQLTypes collection_child_type = kNULLT;
1882  for (auto cd_it = col_descs.begin(); cd_it != col_descs.end(); cd_it++) {
1883  auto const& cd = *cd_it;
1884  auto const col_type = cd->columnType.get_type();
1885  if (col_type == kPOLYGON || col_type == kLINESTRING || col_type == kPOINT) {
1886  if (collection_col_idx >= 0) {
1887  throw std::runtime_error(
1888  "Explode Collections: Found more than one destination column");
1889  }
1890  collection_col_idx = col_idx;
1891  collection_child_type = col_type;
1892  collection_col_name = cd->columnName;
1893  }
1894  for (int i = 0; i < cd->columnType.get_physical_cols(); ++i) {
1895  ++cd_it;
1896  }
1897  col_idx++;
1898  }
1899  if (collection_col_idx < 0) {
1900  throw std::runtime_error(
1901  "Explode Collections: Failed to find a supported column type to explode "
1902  "into");
1903  }
1904  return std::make_tuple(collection_col_idx, collection_child_type, collection_col_name);
1905 }
SQLTypes
Definition: sqltypes.h:38

+ Here is the caller graph for this function:

int64_t import_export::anonymous_namespace{Importer.cpp}::explode_collections_step2 ( OGRGeometry *  ogr_geometry,
SQLTypes  collection_child_type,
const std::string &  collection_col_name,
size_t  row_or_feature_idx,
std::function< void(OGRGeometry *)>  execute_import_lambda 
)

Definition at line 1907 of file Importer.cpp.

References CHECK, measure< TimeT >::execution(), logger::INFO, kLINESTRING, kPOINT, kPOLYGON, and LOG.

Referenced by import_export::import_thread_delimited(), and import_export::import_thread_shapefile().

1912  {
1913  auto ogr_geometry_type = wkbFlatten(ogr_geometry->getGeometryType());
1914  bool is_collection = false;
1915  switch (collection_child_type) {
1916  case kPOINT:
1917  switch (ogr_geometry_type) {
1918  case wkbMultiPoint:
1919  is_collection = true;
1920  break;
1921  case wkbPoint:
1922  break;
1923  default:
1924  throw std::runtime_error(
1925  "Explode Collections: Source geo type must be MULTIPOINT or POINT");
1926  }
1927  break;
1928  case kLINESTRING:
1929  switch (ogr_geometry_type) {
1930  case wkbMultiLineString:
1931  is_collection = true;
1932  break;
1933  case wkbLineString:
1934  break;
1935  default:
1936  throw std::runtime_error(
1937  "Explode Collections: Source geo type must be MULTILINESTRING or "
1938  "LINESTRING");
1939  }
1940  break;
1941  case kPOLYGON:
1942  switch (ogr_geometry_type) {
1943  case wkbMultiPolygon:
1944  is_collection = true;
1945  break;
1946  case wkbPolygon:
1947  break;
1948  default:
1949  throw std::runtime_error(
1950  "Explode Collections: Source geo type must be MULTIPOLYGON or POLYGON");
1951  }
1952  break;
1953  default:
1954  CHECK(false) << "Unsupported geo child type " << collection_child_type;
1955  }
1956 
1957  int64_t us = 0LL;
1958 
1959  // explode or just import
1960  if (is_collection) {
1961  // cast to collection
1962  OGRGeometryCollection* collection_geometry = ogr_geometry->toGeometryCollection();
1963  CHECK(collection_geometry);
1964 
1965 #if LOG_EXPLODE_COLLECTIONS
1966  // log number of children
1967  LOG(INFO) << "Exploding row/feature " << row_or_feature_idx << " for column '"
1968  << explode_col_name << "' into " << collection_geometry->getNumGeometries()
1969  << " child rows";
1970 #endif
1971 
1972  // loop over children
1973  uint32_t child_geometry_count = 0;
1974  auto child_geometry_it = collection_geometry->begin();
1975  while (child_geometry_it != collection_geometry->end()) {
1976  // get and import this child
1977  OGRGeometry* import_geometry = *child_geometry_it;
1979  [&] { execute_import_lambda(import_geometry); });
1980 
1981  // next child
1982  child_geometry_it++;
1983  child_geometry_count++;
1984  }
1985  } else {
1986  // import non-collection row just once
1988  [&] { execute_import_lambda(ogr_geometry); });
1989  }
1990 
1991  // done
1992  return us;
1993 }
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:203
#define CHECK(condition)
Definition: Logger.h:209

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

float import_export::anonymous_namespace{Importer.cpp}::float_value_at ( const TypedImportBuffer &  import_buffer,
const size_t  index 
)

Definition at line 2821 of file Importer.cpp.

References CHECK_EQ, import_export::TypedImportBuffer::getAsBytes(), import_export::TypedImportBuffer::getTypeInfo(), and kFLOAT.

Referenced by import_export::Loader::fillShardRow().

2821  {
2822  const auto& ti = import_buffer.getTypeInfo();
2823  CHECK_EQ(kFLOAT, ti.get_type());
2824  const auto values_buffer = import_buffer.getAsBytes();
2825  return reinterpret_cast<const float*>(may_alias_ptr(values_buffer))[index];
2826 }
#define CHECK_EQ(x, y)
Definition: Logger.h:217

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

OGRLayer& import_export::anonymous_namespace{Importer.cpp}::getLayerWithSpecifiedName ( const std::string &  geo_layer_name,
const OGRDataSourceUqPtr &  poDS,
const std::string &  file_name 
)

Definition at line 4678 of file Importer.cpp.

Referenced by import_export::Importer::gdalToColumnDescriptors(), import_export::Importer::importGDAL(), and import_export::Importer::readMetadataSampleGDAL().

4680  {
4681  // get layer with specified name, or default to first layer
4682  OGRLayer* poLayer = nullptr;
4683  if (geo_layer_name.size()) {
4684  poLayer = poDS->GetLayerByName(geo_layer_name.c_str());
4685  if (poLayer == nullptr) {
4686  throw std::runtime_error("Layer '" + geo_layer_name + "' not found in " +
4687  file_name);
4688  }
4689  } else {
4690  poLayer = poDS->GetLayer(0);
4691  if (poLayer == nullptr) {
4692  throw std::runtime_error("No layers found in " + file_name);
4693  }
4694  }
4695  return *poLayer;
4696 }

+ Here is the caller graph for this function:

int64_t import_export::anonymous_namespace{Importer.cpp}::int_value_at ( const TypedImportBuffer &  import_buffer,
const size_t  index 
)

Definition at line 2790 of file Importer.cpp.

References CHECK, CHECK_EQ, logger::FATAL, import_export::TypedImportBuffer::getAsBytes(), import_export::TypedImportBuffer::getStringDictBuffer(), import_export::TypedImportBuffer::getTypeInfo(), kENCODING_DICT, LOG, and UNREACHABLE.

Referenced by import_export::Loader::distributeToShardsExistingColumns(), and import_export::Loader::fillShardRow().

2790  {
2791  const auto& ti = import_buffer.getTypeInfo();
2792  const int8_t* values_buffer{nullptr};
2793  if (ti.is_string()) {
2794  CHECK_EQ(kENCODING_DICT, ti.get_compression());
2795  values_buffer = import_buffer.getStringDictBuffer();
2796  } else {
2797  values_buffer = import_buffer.getAsBytes();
2798  }
2799  CHECK(values_buffer);
2800  const int logical_size = ti.is_string() ? ti.get_size() : ti.get_logical_size();
2801  switch (logical_size) {
2802  case 1: {
2803  return values_buffer[index];
2804  }
2805  case 2: {
2806  return reinterpret_cast<const int16_t*>(values_buffer)[index];
2807  }
2808  case 4: {
2809  return reinterpret_cast<const int32_t*>(values_buffer)[index];
2810  }
2811  case 8: {
2812  return reinterpret_cast<const int64_t*>(values_buffer)[index];
2813  }
2814  default:
2815  LOG(FATAL) << "Unexpected size for shard key: " << logical_size;
2816  }
2817  UNREACHABLE();
2818  return 0;
2819 }
#define CHECK_EQ(x, y)
Definition: Logger.h:217
#define LOG(tag)
Definition: Logger.h:203
#define UNREACHABLE()
Definition: Logger.h:253
#define CHECK(condition)
Definition: Logger.h:209

+ Here is the call graph for this function:

+ Here is the caller graph for this function: