OmniSciDB  2e3a973ef4
import_export::anonymous_namespace{Importer.cpp} Namespace Reference

Functions

std::tuple< int, SQLTypes, std::string > explode_collections_step1 (const std::list< const ColumnDescriptor *> &col_descs)
 
int64_t explode_collections_step2 (OGRGeometry *ogr_geometry, SQLTypes collection_child_type, const std::string &collection_col_name, size_t row_or_feature_idx, std::function< void(OGRGeometry *)> execute_import_lambda)
 
int64_t int_value_at (const TypedImportBuffer &import_buffer, const size_t index)
 
float float_value_at (const TypedImportBuffer &import_buffer, const size_t index)
 
double double_value_at (const TypedImportBuffer &import_buffer, const size_t index)
 
OGRLayer & getLayerWithSpecifiedName (const std::string &geo_layer_name, const OGRDataSourceUqPtr &poDS, const std::string &file_name)
 

Function Documentation

◆ double_value_at()

double import_export::anonymous_namespace{Importer.cpp}::double_value_at ( const TypedImportBuffer import_buffer,
const size_t  index 
)

Definition at line 2539 of file Importer.cpp.

References CHECK_EQ, import_export::TypedImportBuffer::getAsBytes(), import_export::TypedImportBuffer::getTypeInfo(), and kDOUBLE.

Referenced by import_export::Loader::distributeToShards().

2539  {
2540  const auto& ti = import_buffer.getTypeInfo();
2541  CHECK_EQ(kDOUBLE, ti.get_type());
2542  const auto values_buffer = import_buffer.getAsBytes();
2543  return reinterpret_cast<const double*>(may_alias_ptr(values_buffer))[index];
2544 }
#define CHECK_EQ(x, y)
Definition: Logger.h:205
int8_t * getAsBytes() const
Definition: Importer.h:286
const SQLTypeInfo & getTypeInfo() const
Definition: Importer.h:280
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ explode_collections_step1()

std::tuple<int, SQLTypes, std::string> import_export::anonymous_namespace{Importer.cpp}::explode_collections_step1 ( const std::list< const ColumnDescriptor *> &  col_descs)

Definition at line 1672 of file Importer.cpp.

References kLINESTRING, kNULLT, kPOINT, and kPOLYGON.

Referenced by import_export::import_thread_delimited(), and import_export::import_thread_shapefile().

1673  {
1674  // validate the columns
1675  // for now we can only explode into a single destination column
1676  // which must be of the child type (POLYGON, LINESTRING, POINT)
1677  int collection_col_idx = -1;
1678  int col_idx = 0;
1679  std::string collection_col_name;
1680  SQLTypes collection_child_type = kNULLT;
1681  for (auto cd_it = col_descs.begin(); cd_it != col_descs.end(); cd_it++) {
1682  auto const& cd = *cd_it;
1683  auto const col_type = cd->columnType.get_type();
1684  if (col_type == kPOLYGON || col_type == kLINESTRING || col_type == kPOINT) {
1685  if (collection_col_idx >= 0) {
1686  throw std::runtime_error(
1687  "Explode Collections: Found more than one destination column");
1688  }
1689  collection_col_idx = col_idx;
1690  collection_child_type = col_type;
1691  collection_col_name = cd->columnName;
1692  }
1693  for (int i = 0; i < cd->columnType.get_physical_cols(); ++i) {
1694  ++cd_it;
1695  }
1696  col_idx++;
1697  }
1698  if (collection_col_idx < 0) {
1699  throw std::runtime_error(
1700  "Explode Collections: Failed to find a supported column type to explode "
1701  "into");
1702  }
1703  return std::make_tuple(collection_col_idx, collection_child_type, collection_col_name);
1704 }
SQLTypes
Definition: sqltypes.h:40
+ Here is the caller graph for this function:

◆ explode_collections_step2()

int64_t import_export::anonymous_namespace{Importer.cpp}::explode_collections_step2 ( OGRGeometry *  ogr_geometry,
SQLTypes  collection_child_type,
const std::string &  collection_col_name,
size_t  row_or_feature_idx,
std::function< void(OGRGeometry *)>  execute_import_lambda 
)

Definition at line 1706 of file Importer.cpp.

References CHECK, measure< TimeT >::execution(), logger::INFO, kLINESTRING, kPOINT, kPOLYGON, and LOG.

Referenced by import_export::import_thread_delimited(), and import_export::import_thread_shapefile().

1711  {
1712  auto ogr_geometry_type = wkbFlatten(ogr_geometry->getGeometryType());
1713  bool is_collection = false;
1714  switch (collection_child_type) {
1715  case kPOINT:
1716  switch (ogr_geometry_type) {
1717  case wkbMultiPoint:
1718  is_collection = true;
1719  break;
1720  case wkbPoint:
1721  break;
1722  default:
1723  throw std::runtime_error(
1724  "Explode Collections: Source geo type must be MULTIPOINT or POINT");
1725  }
1726  break;
1727  case kLINESTRING:
1728  switch (ogr_geometry_type) {
1729  case wkbMultiLineString:
1730  is_collection = true;
1731  break;
1732  case wkbLineString:
1733  break;
1734  default:
1735  throw std::runtime_error(
1736  "Explode Collections: Source geo type must be MULTILINESTRING or "
1737  "LINESTRING");
1738  }
1739  break;
1740  case kPOLYGON:
1741  switch (ogr_geometry_type) {
1742  case wkbMultiPolygon:
1743  is_collection = true;
1744  break;
1745  case wkbPolygon:
1746  break;
1747  default:
1748  throw std::runtime_error(
1749  "Explode Collections: Source geo type must be MULTIPOLYGON or POLYGON");
1750  }
1751  break;
1752  default:
1753  CHECK(false) << "Unsupported geo child type " << collection_child_type;
1754  }
1755 
1756  int64_t us = 0LL;
1757 
1758  // explode or just import
1759  if (is_collection) {
1760  // cast to collection
1761  OGRGeometryCollection* collection_geometry = ogr_geometry->toGeometryCollection();
1762  CHECK(collection_geometry);
1763 
1764 #if LOG_EXPLODE_COLLECTIONS
1765  // log number of children
1766  LOG(INFO) << "Exploding row/feature " << row_or_feature_idx << " for column '"
1767  << explode_col_name << "' into " << collection_geometry->getNumGeometries()
1768  << " child rows";
1769 #endif
1770 
1771  // loop over children
1772  uint32_t child_geometry_count = 0;
1773  auto child_geometry_it = collection_geometry->begin();
1774  while (child_geometry_it != collection_geometry->end()) {
1775  // get and import this child
1776  OGRGeometry* import_geometry = *child_geometry_it;
1778  [&] { execute_import_lambda(import_geometry); });
1779 
1780  // next child
1781  child_geometry_it++;
1782  child_geometry_count++;
1783  }
1784  } else {
1785  // import non-collection row just once
1787  [&] { execute_import_lambda(ogr_geometry); });
1788  }
1789 
1790  // done
1791  return us;
1792 }
#define LOG(tag)
Definition: Logger.h:188
#define CHECK(condition)
Definition: Logger.h:197
static TimeT::rep execution(F func, Args &&... args)
Definition: sample.cpp:29
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ float_value_at()

float import_export::anonymous_namespace{Importer.cpp}::float_value_at ( const TypedImportBuffer import_buffer,
const size_t  index 
)

Definition at line 2532 of file Importer.cpp.

References CHECK_EQ, import_export::TypedImportBuffer::getAsBytes(), import_export::TypedImportBuffer::getTypeInfo(), and kFLOAT.

Referenced by import_export::Loader::distributeToShards().

2532  {
2533  const auto& ti = import_buffer.getTypeInfo();
2534  CHECK_EQ(kFLOAT, ti.get_type());
2535  const auto values_buffer = import_buffer.getAsBytes();
2536  return reinterpret_cast<const float*>(may_alias_ptr(values_buffer))[index];
2537 }
#define CHECK_EQ(x, y)
Definition: Logger.h:205
int8_t * getAsBytes() const
Definition: Importer.h:286
const SQLTypeInfo & getTypeInfo() const
Definition: Importer.h:280
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getLayerWithSpecifiedName()

OGRLayer& import_export::anonymous_namespace{Importer.cpp}::getLayerWithSpecifiedName ( const std::string &  geo_layer_name,
const OGRDataSourceUqPtr &  poDS,
const std::string &  file_name 
)

Definition at line 4281 of file Importer.cpp.

Referenced by import_export::Importer::gdalToColumnDescriptors(), import_export::Importer::importGDAL(), and import_export::Importer::readMetadataSampleGDAL().

4283  {
4284  // get layer with specified name, or default to first layer
4285  OGRLayer* poLayer = nullptr;
4286  if (geo_layer_name.size()) {
4287  poLayer = poDS->GetLayerByName(geo_layer_name.c_str());
4288  if (poLayer == nullptr) {
4289  throw std::runtime_error("Layer '" + geo_layer_name + "' not found in " +
4290  file_name);
4291  }
4292  } else {
4293  poLayer = poDS->GetLayer(0);
4294  if (poLayer == nullptr) {
4295  throw std::runtime_error("No layers found in " + file_name);
4296  }
4297  }
4298  return *poLayer;
4299 }
+ Here is the caller graph for this function:

◆ int_value_at()

int64_t import_export::anonymous_namespace{Importer.cpp}::int_value_at ( const TypedImportBuffer import_buffer,
const size_t  index 
)

Definition at line 2501 of file Importer.cpp.

References CHECK, CHECK_EQ, logger::FATAL, import_export::TypedImportBuffer::getAsBytes(), import_export::TypedImportBuffer::getStringDictBuffer(), import_export::TypedImportBuffer::getTypeInfo(), kENCODING_DICT, LOG, and UNREACHABLE.

Referenced by import_export::Loader::distributeToShards().

2501  {
2502  const auto& ti = import_buffer.getTypeInfo();
2503  const int8_t* values_buffer{nullptr};
2504  if (ti.is_string()) {
2505  CHECK_EQ(kENCODING_DICT, ti.get_compression());
2506  values_buffer = import_buffer.getStringDictBuffer();
2507  } else {
2508  values_buffer = import_buffer.getAsBytes();
2509  }
2510  CHECK(values_buffer);
2511  const int logical_size = ti.is_string() ? ti.get_size() : ti.get_logical_size();
2512  switch (logical_size) {
2513  case 1: {
2514  return values_buffer[index];
2515  }
2516  case 2: {
2517  return reinterpret_cast<const int16_t*>(values_buffer)[index];
2518  }
2519  case 4: {
2520  return reinterpret_cast<const int32_t*>(values_buffer)[index];
2521  }
2522  case 8: {
2523  return reinterpret_cast<const int64_t*>(values_buffer)[index];
2524  }
2525  default:
2526  LOG(FATAL) << "Unexpected size for shard key: " << logical_size;
2527  }
2528  UNREACHABLE();
2529  return 0;
2530 }
#define CHECK_EQ(x, y)
Definition: Logger.h:205
#define LOG(tag)
Definition: Logger.h:188
#define UNREACHABLE()
Definition: Logger.h:241
int8_t * getAsBytes() const
Definition: Importer.h:286
int8_t * getStringDictBuffer() const
Definition: Importer.h:354
#define CHECK(condition)
Definition: Logger.h:197
const SQLTypeInfo & getTypeInfo() const
Definition: Importer.h:280
+ Here is the call graph for this function:
+ Here is the caller graph for this function: