OmniSciDB  8fa3bf436f
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
import_export::RenderGroupAnalyzer Class Reference

#include <Importer.h>

Public Member Functions

 RenderGroupAnalyzer ()
 
void seedFromExistingTableContents (Catalog_Namespace::Catalog &cat, const std::string &tableName, const std::string &geoColumnBaseName)
 
int insertBoundsAndReturnRenderGroup (const std::vector< double > &bounds)
 

Private Types

using Point = boost::geometry::model::point< double, 2, boost::geometry::cs::cartesian >
 
using BoundingBox = boost::geometry::model::box< Point >
 
using Node = std::pair< BoundingBox, int >
 
using RTree = boost::geometry::index::rtree< Node, boost::geometry::index::quadratic< 16 >>
 

Private Attributes

std::unique_ptr< RTree_rtree
 
std::mutex _rtreeMutex
 
int _numRenderGroups
 

Detailed Description

Definition at line 742 of file Importer.h.

Member Typedef Documentation

using import_export::RenderGroupAnalyzer::BoundingBox = boost::geometry::model::box<Point>
private

Definition at line 752 of file Importer.h.

using import_export::RenderGroupAnalyzer::Node = std::pair<BoundingBox, int>
private

Definition at line 753 of file Importer.h.

using import_export::RenderGroupAnalyzer::Point = boost::geometry::model::point<double, 2, boost::geometry::cs::cartesian>
private

Definition at line 751 of file Importer.h.

using import_export::RenderGroupAnalyzer::RTree = boost::geometry::index::rtree<Node, boost::geometry::index::quadratic<16>>
private

Definition at line 755 of file Importer.h.

Constructor & Destructor Documentation

import_export::RenderGroupAnalyzer::RenderGroupAnalyzer ( )
inline

Definition at line 744 of file Importer.h.

744 : _rtree(std::make_unique<RTree>()), _numRenderGroups(0) {}
std::unique_ptr< RTree > _rtree
Definition: Importer.h:756

Member Function Documentation

int import_export::RenderGroupAnalyzer::insertBoundsAndReturnRenderGroup ( const std::vector< double > &  bounds)

Definition at line 5308 of file Importer.cpp.

References _numRenderGroups, _rtree, _rtreeMutex, and CHECK.

Referenced by GeoPolygonValueConverter::convertToColumnarFormat(), and GeoMultiPolygonValueConverter::convertToColumnarFormat().

5309  {
5310  // validate
5311  CHECK(bounds.size() == 4);
5312 
5313  // get bounds
5314  BoundingBox bounding_box;
5315  boost::geometry::assign_inverse(bounding_box);
5316  boost::geometry::expand(bounding_box, Point(bounds[0], bounds[1]));
5317  boost::geometry::expand(bounding_box, Point(bounds[2], bounds[3]));
5318 
5319  // remainder under mutex to allow this to be multi-threaded
5320  std::lock_guard<std::mutex> guard(_rtreeMutex);
5321 
5322  // get the intersecting nodes
5323  std::vector<Node> intersects;
5324  _rtree->query(boost::geometry::index::intersects(bounding_box),
5325  std::back_inserter(intersects));
5326 
5327  // build bitset of render groups of the intersecting rectangles
5328  // clear bit means available, allows use of find_first()
5329  boost::dynamic_bitset<> bits(_numRenderGroups);
5330  bits.set();
5331  for (const auto& intersection : intersects) {
5332  CHECK(intersection.second < _numRenderGroups);
5333  bits.reset(intersection.second);
5334  }
5335 
5336  // find first available group
5337  int firstAvailableRenderGroup = 0;
5338  size_t firstSetBit = bits.find_first();
5339  if (firstSetBit == boost::dynamic_bitset<>::npos) {
5340  // all known groups represented, add a new one
5341  firstAvailableRenderGroup = _numRenderGroups;
5342  _numRenderGroups++;
5343  } else {
5344  firstAvailableRenderGroup = (int)firstSetBit;
5345  }
5346 
5347  // insert new node
5348  _rtree->insert(std::make_pair(bounding_box, firstAvailableRenderGroup));
5349 
5350  // return it
5351  return firstAvailableRenderGroup;
5352 }
boost::geometry::model::box< Point > BoundingBox
Definition: Importer.h:752
std::unique_ptr< RTree > _rtree
Definition: Importer.h:756
#define CHECK(condition)
Definition: Logger.h:203
boost::geometry::model::point< double, 2, boost::geometry::cs::cartesian > Point
Definition: Importer.h:751

+ Here is the caller graph for this function:

void import_export::RenderGroupAnalyzer::seedFromExistingTableContents ( Catalog_Namespace::Catalog cat,
const std::string &  tableName,
const std::string &  geoColumnBaseName 
)

Definition at line 5171 of file Importer.cpp.

References _numRenderGroups, _rtree, CHECK, ChunkIter_get_nth(), DEBUG_RENDER_GROUP_ANALYZER, StorageType::FOREIGN_TABLE, getChunkAccessorTable(), getChunkItersAndRowOffset(), Catalog_Namespace::Catalog::getMetadataForColumn(), Catalog_Namespace::Catalog::getMetadataForTable(), logger::INFO, kARRAY, kDOUBLE, kINT, LOG, VarlenDatum::pointer, timer_start(), timer_stop(), and to_string().

5174  {
5175  // start timer
5176  auto seedTimer = timer_start();
5177 
5178  // start with a fresh tree
5179  _rtree = nullptr;
5180  _numRenderGroups = 0;
5181 
5182  // get the table descriptor
5183  auto const* td = cat.getMetadataForTable(tableName);
5184  CHECK(td);
5185 
5186  // foreign tables not supported
5187  if (td->storageType == StorageType::FOREIGN_TABLE) {
5189  LOG(INFO) << "DEBUG: Table is a foreign table";
5190  }
5191  _rtree = std::make_unique<RTree>();
5192  CHECK(_rtree);
5193  return;
5194  }
5195 
5196  // if the table is empty, just make an empty tree
5197  CHECK(td->fragmenter);
5198  if (td->fragmenter->getFragmentsForQuery().getPhysicalNumTuples() == 0) {
5200  LOG(INFO) << "DEBUG: Table is empty!";
5201  }
5202  _rtree = std::make_unique<RTree>();
5203  CHECK(_rtree);
5204  return;
5205  }
5206 
5207  // no seeding possible without these two columns
5208  const auto cd_bounds =
5209  cat.getMetadataForColumn(td->tableId, geoColumnBaseName + "_bounds");
5210  const auto cd_render_group =
5211  cat.getMetadataForColumn(td->tableId, geoColumnBaseName + "_render_group");
5212  if (!cd_bounds || !cd_render_group) {
5213  throw std::runtime_error("RenderGroupAnalyzer: Table " + tableName +
5214  " doesn't have bounds or render_group columns!");
5215  }
5216 
5217  // and validate their types
5218  if (cd_bounds->columnType.get_type() != kARRAY ||
5219  cd_bounds->columnType.get_subtype() != kDOUBLE) {
5220  throw std::runtime_error("RenderGroupAnalyzer: Table " + tableName +
5221  " bounds column is wrong type!");
5222  }
5223  if (cd_render_group->columnType.get_type() != kINT) {
5224  throw std::runtime_error("RenderGroupAnalyzer: Table " + tableName +
5225  " render_group column is wrong type!");
5226  }
5227 
5228  // get chunk accessor table
5229  auto chunkAccessorTable = getChunkAccessorTable(
5230  cat, td, {geoColumnBaseName + "_bounds", geoColumnBaseName + "_render_group"});
5231  const auto table_count = std::get<0>(chunkAccessorTable.back());
5232 
5234  LOG(INFO) << "DEBUG: Scanning existing table geo column set '" << geoColumnBaseName
5235  << "'";
5236  }
5237 
5238  std::vector<Node> nodes;
5239  try {
5240  nodes.resize(table_count);
5241  } catch (const std::exception& e) {
5242  throw std::runtime_error("RenderGroupAnalyzer failed to reserve memory for " +
5243  std::to_string(table_count) + " rows");
5244  }
5245 
5246  for (size_t row = 0; row < table_count; row++) {
5247  ArrayDatum ad;
5248  VarlenDatum vd;
5249  bool is_end;
5250 
5251  // get ChunkIters and fragment row offset
5252  size_t rowOffset = 0;
5253  auto& chunkIters = getChunkItersAndRowOffset(chunkAccessorTable, row, rowOffset);
5254  auto& boundsChunkIter = chunkIters[0];
5255  auto& renderGroupChunkIter = chunkIters[1];
5256 
5257  // get bounds values
5258  ChunkIter_get_nth(&boundsChunkIter, row - rowOffset, &ad, &is_end);
5259  CHECK(!is_end);
5260  CHECK(ad.pointer);
5261  int numBounds = (int)(ad.length / sizeof(double));
5262  CHECK(numBounds == 4);
5263 
5264  // convert to bounding box
5265  double* bounds = reinterpret_cast<double*>(ad.pointer);
5266  BoundingBox bounding_box;
5267  boost::geometry::assign_inverse(bounding_box);
5268  boost::geometry::expand(bounding_box, Point(bounds[0], bounds[1]));
5269  boost::geometry::expand(bounding_box, Point(bounds[2], bounds[3]));
5270 
5271  // get render group
5272  ChunkIter_get_nth(&renderGroupChunkIter, row - rowOffset, false, &vd, &is_end);
5273  CHECK(!is_end);
5274  CHECK(vd.pointer);
5275  int renderGroup = *reinterpret_cast<int32_t*>(vd.pointer);
5276 
5277  // skip rows with invalid render groups (e.g. EMPTY geometry)
5278  if (renderGroup < 0) {
5279  continue;
5280  }
5281 
5282  // store
5283  nodes[row] = std::make_pair(bounding_box, renderGroup);
5284 
5285  // how many render groups do we have now?
5286  if (renderGroup >= _numRenderGroups) {
5287  _numRenderGroups = renderGroup + 1;
5288  }
5289 
5291  LOG(INFO) << "DEBUG: Existing row " << row << " has Render Group " << renderGroup;
5292  }
5293  }
5294 
5295  // bulk-load the tree
5296  auto bulk_load_timer = timer_start();
5297  _rtree = std::make_unique<RTree>(nodes);
5298  CHECK(_rtree);
5299  LOG(INFO) << "Scanning render groups of poly column '" << geoColumnBaseName
5300  << "' of table '" << tableName << "' took " << timer_stop(seedTimer) << "ms ("
5301  << timer_stop(bulk_load_timer) << " ms for tree)";
5302 
5304  LOG(INFO) << "DEBUG: Done! Now have " << _numRenderGroups << " Render Groups";
5305  }
5306 }
ChunkAccessorTable getChunkAccessorTable(const Catalog_Namespace::Catalog &cat, const TableDescriptor *td, const std::vector< std::string > &columnNames)
#define LOG(tag)
Definition: Logger.h:194
TypeR::rep timer_stop(Type clock_begin)
Definition: measure.h:48
DEVICE void ChunkIter_get_nth(ChunkIter *it, int n, bool uncompress, VarlenDatum *result, bool *is_end)
Definition: ChunkIter.cpp:181
std::string to_string(char const *&&v)
int8_t * pointer
Definition: sqltypes.h:146
std::conditional_t< is_cuda_compiler(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:202
ChunkIterVector & getChunkItersAndRowOffset(ChunkAccessorTable &table, size_t rowid, size_t &rowOffset)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
#define DEBUG_RENDER_GROUP_ANALYZER
Definition: Importer.cpp:161
boost::geometry::model::box< Point > BoundingBox
Definition: Importer.h:752
std::unique_ptr< RTree > _rtree
Definition: Importer.h:756
#define CHECK(condition)
Definition: Logger.h:203
Definition: sqltypes.h:44
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
static constexpr char const * FOREIGN_TABLE
Type timer_start()
Definition: measure.h:42
boost::geometry::model::point< double, 2, boost::geometry::cs::cartesian > Point
Definition: Importer.h:751

+ Here is the call graph for this function:

Member Data Documentation

int import_export::RenderGroupAnalyzer::_numRenderGroups
private

Definition at line 758 of file Importer.h.

Referenced by insertBoundsAndReturnRenderGroup(), and seedFromExistingTableContents().

std::unique_ptr<RTree> import_export::RenderGroupAnalyzer::_rtree
private

Definition at line 756 of file Importer.h.

Referenced by insertBoundsAndReturnRenderGroup(), and seedFromExistingTableContents().

std::mutex import_export::RenderGroupAnalyzer::_rtreeMutex
private

Definition at line 757 of file Importer.h.

Referenced by insertBoundsAndReturnRenderGroup().


The documentation for this class was generated from the following files: