OmniSciDB  1dac507f6e
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Importer_NS::anonymous_namespace{Importer.cpp} Namespace Reference

Functions

std::tuple< int, SQLTypes,
std::string > 
explode_collections_step1 (const std::list< const ColumnDescriptor * > &col_descs)
 
int64_t explode_collections_step2 (OGRGeometry *ogr_geometry, SQLTypes collection_child_type, const std::string &collection_col_name, size_t row_or_feature_idx, std::function< void(OGRGeometry *)> execute_import_lambda)
 
int64_t int_value_at (const TypedImportBuffer &import_buffer, const size_t index)
 
float float_value_at (const TypedImportBuffer &import_buffer, const size_t index)
 
double double_value_at (const TypedImportBuffer &import_buffer, const size_t index)
 
OGRLayer & getLayerWithSpecifiedName (const std::string &geo_layer_name, const OGRDataSourceUqPtr &poDS, const std::string &file_name)
 

Function Documentation

double Importer_NS::anonymous_namespace{Importer.cpp}::double_value_at ( const TypedImportBuffer &  import_buffer,
const size_t  index 
)

Definition at line 2424 of file Importer.cpp.

References CHECK_EQ, Importer_NS::TypedImportBuffer::getAsBytes(), Importer_NS::TypedImportBuffer::getTypeInfo(), and kDOUBLE.

Referenced by Importer_NS::Loader::distributeToShards().

2424  {
2425  const auto& ti = import_buffer.getTypeInfo();
2426  CHECK_EQ(kDOUBLE, ti.get_type());
2427  const auto values_buffer = import_buffer.getAsBytes();
2428  return reinterpret_cast<const double*>(may_alias_ptr(values_buffer))[index];
2429 }
#define CHECK_EQ(x, y)
Definition: Logger.h:198

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::tuple<int, SQLTypes, std::string> Importer_NS::anonymous_namespace{Importer.cpp}::explode_collections_step1 ( const std::list< const ColumnDescriptor * > &  col_descs)

Definition at line 1653 of file Importer.cpp.

References kLINESTRING, kNULLT, kPOINT, and kPOLYGON.

Referenced by Importer_NS::import_thread_delimited(), and Importer_NS::import_thread_shapefile().

1654  {
1655  // validate the columns
1656  // for now we can only explode into a single destination column
1657  // which must be of the child type (POLYGON, LINESTRING, POINT)
1658  int collection_col_idx = -1;
1659  int col_idx = 0;
1660  std::string collection_col_name;
1661  SQLTypes collection_child_type = kNULLT;
1662  for (auto cd_it = col_descs.begin(); cd_it != col_descs.end(); cd_it++) {
1663  auto const& cd = *cd_it;
1664  auto const col_type = cd->columnType.get_type();
1665  if (col_type == kPOLYGON || col_type == kLINESTRING || col_type == kPOINT) {
1666  if (collection_col_idx >= 0) {
1667  throw std::runtime_error(
1668  "Explode Collections: Found more than one destination column");
1669  }
1670  collection_col_idx = col_idx;
1671  collection_child_type = col_type;
1672  collection_col_name = cd->columnName;
1673  }
1674  for (int i = 0; i < cd->columnType.get_physical_cols(); ++i) {
1675  ++cd_it;
1676  }
1677  col_idx++;
1678  }
1679  if (collection_col_idx < 0) {
1680  throw std::runtime_error(
1681  "Explode Collections: Failed to find a supported column type to explode "
1682  "into");
1683  }
1684  return std::make_tuple(collection_col_idx, collection_child_type, collection_col_name);
1685 }
SQLTypes
Definition: sqltypes.h:41

+ Here is the caller graph for this function:

int64_t Importer_NS::anonymous_namespace{Importer.cpp}::explode_collections_step2 ( OGRGeometry *  ogr_geometry,
SQLTypes  collection_child_type,
const std::string &  collection_col_name,
size_t  row_or_feature_idx,
std::function< void(OGRGeometry *)>  execute_import_lambda 
)

Definition at line 1687 of file Importer.cpp.

References CHECK(), measure< TimeT >::execution(), logger::INFO, kLINESTRING, kPOINT, kPOLYGON, and LOG.

Referenced by Importer_NS::import_thread_delimited(), and Importer_NS::import_thread_shapefile().

1692  {
1693  auto ogr_geometry_type = wkbFlatten(ogr_geometry->getGeometryType());
1694  bool is_collection = false;
1695  switch (collection_child_type) {
1696  case kPOINT:
1697  switch (ogr_geometry_type) {
1698  case wkbMultiPoint:
1699  is_collection = true;
1700  break;
1701  case wkbPoint:
1702  break;
1703  default:
1704  throw std::runtime_error(
1705  "Explode Collections: Source geo type must be MULTIPOINT or POINT");
1706  }
1707  break;
1708  case kLINESTRING:
1709  switch (ogr_geometry_type) {
1710  case wkbMultiLineString:
1711  is_collection = true;
1712  break;
1713  case wkbLineString:
1714  break;
1715  default:
1716  throw std::runtime_error(
1717  "Explode Collections: Source geo type must be MULTILINESTRING or "
1718  "LINESTRING");
1719  }
1720  break;
1721  case kPOLYGON:
1722  switch (ogr_geometry_type) {
1723  case wkbMultiPolygon:
1724  is_collection = true;
1725  break;
1726  case wkbPolygon:
1727  break;
1728  default:
1729  throw std::runtime_error(
1730  "Explode Collections: Source geo type must be MULTIPOLYGON or POLYGON");
1731  }
1732  break;
1733  default:
1734  CHECK(false) << "Unsupported geo child type " << collection_child_type;
1735  }
1736 
1737  int64_t us = 0LL;
1738 
1739  // explode or just import
1740  if (is_collection) {
1741  // cast to collection
1742  OGRGeometryCollection* collection_geometry = ogr_geometry->toGeometryCollection();
1743  CHECK(collection_geometry);
1744 
1745 #if LOG_EXPLODE_COLLECTIONS
1746  // log number of children
1747  LOG(INFO) << "Exploding row/feature " << row_or_feature_idx << " for column '"
1748  << explode_col_name << "' into " << collection_geometry->getNumGeometries()
1749  << " child rows";
1750 #endif
1751 
1752  // loop over children
1753  uint32_t child_geometry_count = 0;
1754  auto child_geometry_it = collection_geometry->begin();
1755  while (child_geometry_it != collection_geometry->end()) {
1756  // get and import this child
1757  OGRGeometry* import_geometry = *child_geometry_it;
1759  [&] { execute_import_lambda(import_geometry); });
1760 
1761  // next child
1762  child_geometry_it++;
1763  child_geometry_count++;
1764  }
1765  } else {
1766  // import non-collection row just once
1768  [&] { execute_import_lambda(ogr_geometry); });
1769  }
1770 
1771  // done
1772  return us;
1773 }
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:185
CHECK(cgen_state)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

float Importer_NS::anonymous_namespace{Importer.cpp}::float_value_at ( const TypedImportBuffer &  import_buffer,
const size_t  index 
)

Definition at line 2417 of file Importer.cpp.

References CHECK_EQ, Importer_NS::TypedImportBuffer::getAsBytes(), Importer_NS::TypedImportBuffer::getTypeInfo(), and kFLOAT.

Referenced by Importer_NS::Loader::distributeToShards().

2417  {
2418  const auto& ti = import_buffer.getTypeInfo();
2419  CHECK_EQ(kFLOAT, ti.get_type());
2420  const auto values_buffer = import_buffer.getAsBytes();
2421  return reinterpret_cast<const float*>(may_alias_ptr(values_buffer))[index];
2422 }
#define CHECK_EQ(x, y)
Definition: Logger.h:198

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

OGRLayer& Importer_NS::anonymous_namespace{Importer.cpp}::getLayerWithSpecifiedName ( const std::string &  geo_layer_name,
const OGRDataSourceUqPtr &  poDS,
const std::string &  file_name 
)

Definition at line 4190 of file Importer.cpp.

Referenced by Importer_NS::Importer::gdalToColumnDescriptors(), Importer_NS::Importer::importGDAL(), and Importer_NS::Importer::readMetadataSampleGDAL().

4192  {
4193  // get layer with specified name, or default to first layer
4194  OGRLayer* poLayer = nullptr;
4195  if (geo_layer_name.size()) {
4196  poLayer = poDS->GetLayerByName(geo_layer_name.c_str());
4197  if (poLayer == nullptr) {
4198  throw std::runtime_error("Layer '" + geo_layer_name + "' not found in " +
4199  file_name);
4200  }
4201  } else {
4202  poLayer = poDS->GetLayer(0);
4203  if (poLayer == nullptr) {
4204  throw std::runtime_error("No layers found in " + file_name);
4205  }
4206  }
4207  return *poLayer;
4208 }

+ Here is the caller graph for this function:

int64_t Importer_NS::anonymous_namespace{Importer.cpp}::int_value_at ( const TypedImportBuffer &  import_buffer,
const size_t  index 
)

Definition at line 2386 of file Importer.cpp.

References CHECK(), CHECK_EQ, logger::FATAL, Importer_NS::TypedImportBuffer::getAsBytes(), Importer_NS::TypedImportBuffer::getStringDictBuffer(), Importer_NS::TypedImportBuffer::getTypeInfo(), kENCODING_DICT, LOG, and UNREACHABLE.

Referenced by Importer_NS::Loader::distributeToShards().

2386  {
2387  const auto& ti = import_buffer.getTypeInfo();
2388  const int8_t* values_buffer{nullptr};
2389  if (ti.is_string()) {
2390  CHECK_EQ(kENCODING_DICT, ti.get_compression());
2391  values_buffer = import_buffer.getStringDictBuffer();
2392  } else {
2393  values_buffer = import_buffer.getAsBytes();
2394  }
2395  CHECK(values_buffer);
2396  const int logical_size = ti.is_string() ? ti.get_size() : ti.get_logical_size();
2397  switch (logical_size) {
2398  case 1: {
2399  return values_buffer[index];
2400  }
2401  case 2: {
2402  return reinterpret_cast<const int16_t*>(values_buffer)[index];
2403  }
2404  case 4: {
2405  return reinterpret_cast<const int32_t*>(values_buffer)[index];
2406  }
2407  case 8: {
2408  return reinterpret_cast<const int64_t*>(values_buffer)[index];
2409  }
2410  default:
2411  LOG(FATAL) << "Unexpected size for shard key: " << logical_size;
2412  }
2413  UNREACHABLE();
2414  return 0;
2415 }
#define CHECK_EQ(x, y)
Definition: Logger.h:198
#define LOG(tag)
Definition: Logger.h:185
#define UNREACHABLE()
Definition: Logger.h:234
CHECK(cgen_state)

+ Here is the call graph for this function:

+ Here is the caller graph for this function: