OmniSciDB  8a228a1076
import_export::RenderGroupAnalyzer Class Reference

#include <Importer.h>

Public Member Functions

 RenderGroupAnalyzer ()
 
void seedFromExistingTableContents (const std::unique_ptr< Loader > &loader, const std::string &geoColumnBaseName)
 
int insertBoundsAndReturnRenderGroup (const std::vector< double > &bounds)
 

Private Types

using Point = boost::geometry::model::point< double, 2, boost::geometry::cs::cartesian >
 
using BoundingBox = boost::geometry::model::box< Point >
 
using Node = std::pair< BoundingBox, int >
 
using RTree = boost::geometry::index::rtree< Node, boost::geometry::index::quadratic< 16 > >
 

Private Attributes

std::unique_ptr< RTree_rtree
 
std::mutex _rtreeMutex
 
int _numRenderGroups
 

Detailed Description

Definition at line 711 of file Importer.h.

Member Typedef Documentation

◆ BoundingBox

using import_export::RenderGroupAnalyzer::BoundingBox = boost::geometry::model::box<Point>
private

Definition at line 720 of file Importer.h.

◆ Node

using import_export::RenderGroupAnalyzer::Node = std::pair<BoundingBox, int>
private

Definition at line 721 of file Importer.h.

◆ Point

using import_export::RenderGroupAnalyzer::Point = boost::geometry::model::point<double, 2, boost::geometry::cs::cartesian>
private

Definition at line 719 of file Importer.h.

◆ RTree

using import_export::RenderGroupAnalyzer::RTree = boost::geometry::index::rtree<Node, boost::geometry::index::quadratic<16> >
private

Definition at line 723 of file Importer.h.

Constructor & Destructor Documentation

◆ RenderGroupAnalyzer()

import_export::RenderGroupAnalyzer::RenderGroupAnalyzer ( )
inline

Definition at line 713 of file Importer.h.

713 : _rtree(std::make_unique<RTree>()), _numRenderGroups(0) {}
std::unique_ptr< RTree > _rtree
Definition: Importer.h:724

Member Function Documentation

◆ insertBoundsAndReturnRenderGroup()

int import_export::RenderGroupAnalyzer::insertBoundsAndReturnRenderGroup ( const std::vector< double > &  bounds)

Definition at line 5102 of file Importer.cpp.

References CHECK.

Referenced by GeoPolygonValueConverter::convertToColumnarFormat(), and GeoMultiPolygonValueConverter::convertToColumnarFormat().

5103  {
5104  // validate
5105  CHECK(bounds.size() == 4);
5106 
5107  // get bounds
5108  BoundingBox bounding_box;
5109  boost::geometry::assign_inverse(bounding_box);
5110  boost::geometry::expand(bounding_box, Point(bounds[0], bounds[1]));
5111  boost::geometry::expand(bounding_box, Point(bounds[2], bounds[3]));
5112 
5113  // remainder under mutex to allow this to be multi-threaded
5114  std::lock_guard<std::mutex> guard(_rtreeMutex);
5115 
5116  // get the intersecting nodes
5117  std::vector<Node> intersects;
5118  _rtree->query(boost::geometry::index::intersects(bounding_box),
5119  std::back_inserter(intersects));
5120 
5121  // build bitset of render groups of the intersecting rectangles
5122  // clear bit means available, allows use of find_first()
5123  boost::dynamic_bitset<> bits(_numRenderGroups);
5124  bits.set();
5125  for (const auto& intersection : intersects) {
5126  CHECK(intersection.second < _numRenderGroups);
5127  bits.reset(intersection.second);
5128  }
5129 
5130  // find first available group
5131  int firstAvailableRenderGroup = 0;
5132  size_t firstSetBit = bits.find_first();
5133  if (firstSetBit == boost::dynamic_bitset<>::npos) {
5134  // all known groups represented, add a new one
5135  firstAvailableRenderGroup = _numRenderGroups;
5136  _numRenderGroups++;
5137  } else {
5138  firstAvailableRenderGroup = (int)firstSetBit;
5139  }
5140 
5141  // insert new node
5142  _rtree->insert(std::make_pair(bounding_box, firstAvailableRenderGroup));
5143 
5144  // return it
5145  return firstAvailableRenderGroup;
5146 }
boost::geometry::model::box< Point > BoundingBox
Definition: Importer.h:720
std::unique_ptr< RTree > _rtree
Definition: Importer.h:724
#define CHECK(condition)
Definition: Logger.h:197
boost::geometry::model::point< double, 2, boost::geometry::cs::cartesian > Point
Definition: Importer.h:719
+ Here is the caller graph for this function:

◆ seedFromExistingTableContents()

void import_export::RenderGroupAnalyzer::seedFromExistingTableContents ( const std::unique_ptr< Loader > &  loader,
const std::string &  geoColumnBaseName 
)

Definition at line 4965 of file Importer.cpp.

References cat(), CHECK, ChunkIter_get_nth(), DEBUG_RENDER_GROUP_ANALYZER, StorageType::FOREIGN_TABLE, getChunkAccessorTable(), getChunkItersAndRowOffset(), logger::INFO, kARRAY, kDOUBLE, kINT, LOG, VarlenDatum::pointer, timer_start(), timer_stop(), and to_string().

4967  {
4968  // start timer
4969  auto seedTimer = timer_start();
4970 
4971  // start with a fresh tree
4972  _rtree = nullptr;
4973  _numRenderGroups = 0;
4974 
4975  // get the table descriptor
4976  const auto& cat = loader->getCatalog();
4977  if (loader->getTableDesc()->storageType == StorageType::FOREIGN_TABLE) {
4979  LOG(INFO) << "DEBUG: Table is a foreign table";
4980  }
4981  _rtree = std::make_unique<RTree>();
4982  CHECK(_rtree);
4983  return;
4984  }
4985 
4986  const std::string& tableName = loader->getTableDesc()->tableName;
4987  const auto td = cat.getMetadataForTable(tableName);
4988  CHECK(td);
4989  CHECK(td->fragmenter);
4990 
4991  // if the table is empty, just make an empty tree
4992  if (td->fragmenter->getFragmentsForQuery().getPhysicalNumTuples() == 0) {
4994  LOG(INFO) << "DEBUG: Table is empty!";
4995  }
4996  _rtree = std::make_unique<RTree>();
4997  CHECK(_rtree);
4998  return;
4999  }
5000 
5001  // no seeding possible without these two columns
5002  const auto cd_bounds =
5003  cat.getMetadataForColumn(td->tableId, geoColumnBaseName + "_bounds");
5004  const auto cd_render_group =
5005  cat.getMetadataForColumn(td->tableId, geoColumnBaseName + "_render_group");
5006  if (!cd_bounds || !cd_render_group) {
5007  throw std::runtime_error("RenderGroupAnalyzer: Table " + tableName +
5008  " doesn't have bounds or render_group columns!");
5009  }
5010 
5011  // and validate their types
5012  if (cd_bounds->columnType.get_type() != kARRAY ||
5013  cd_bounds->columnType.get_subtype() != kDOUBLE) {
5014  throw std::runtime_error("RenderGroupAnalyzer: Table " + tableName +
5015  " bounds column is wrong type!");
5016  }
5017  if (cd_render_group->columnType.get_type() != kINT) {
5018  throw std::runtime_error("RenderGroupAnalyzer: Table " + tableName +
5019  " render_group column is wrong type!");
5020  }
5021 
5022  // get chunk accessor table
5023  auto chunkAccessorTable = getChunkAccessorTable(
5024  cat, td, {geoColumnBaseName + "_bounds", geoColumnBaseName + "_render_group"});
5025  const auto table_count = std::get<0>(chunkAccessorTable.back());
5026 
5028  LOG(INFO) << "DEBUG: Scanning existing table geo column set '" << geoColumnBaseName
5029  << "'";
5030  }
5031 
5032  std::vector<Node> nodes;
5033  try {
5034  nodes.resize(table_count);
5035  } catch (const std::exception& e) {
5036  throw std::runtime_error("RenderGroupAnalyzer failed to reserve memory for " +
5037  std::to_string(table_count) + " rows");
5038  }
5039 
5040  for (size_t row = 0; row < table_count; row++) {
5041  ArrayDatum ad;
5042  VarlenDatum vd;
5043  bool is_end;
5044 
5045  // get ChunkIters and fragment row offset
5046  size_t rowOffset = 0;
5047  auto& chunkIters = getChunkItersAndRowOffset(chunkAccessorTable, row, rowOffset);
5048  auto& boundsChunkIter = chunkIters[0];
5049  auto& renderGroupChunkIter = chunkIters[1];
5050 
5051  // get bounds values
5052  ChunkIter_get_nth(&boundsChunkIter, row - rowOffset, &ad, &is_end);
5053  CHECK(!is_end);
5054  CHECK(ad.pointer);
5055  int numBounds = (int)(ad.length / sizeof(double));
5056  CHECK(numBounds == 4);
5057 
5058  // convert to bounding box
5059  double* bounds = reinterpret_cast<double*>(ad.pointer);
5060  BoundingBox bounding_box;
5061  boost::geometry::assign_inverse(bounding_box);
5062  boost::geometry::expand(bounding_box, Point(bounds[0], bounds[1]));
5063  boost::geometry::expand(bounding_box, Point(bounds[2], bounds[3]));
5064 
5065  // get render group
5066  ChunkIter_get_nth(&renderGroupChunkIter, row - rowOffset, false, &vd, &is_end);
5067  CHECK(!is_end);
5068  CHECK(vd.pointer);
5069  int renderGroup = *reinterpret_cast<int32_t*>(vd.pointer);
5070 
5071  // skip rows with invalid render groups (e.g. EMPTY geometry)
5072  if (renderGroup < 0) {
5073  continue;
5074  }
5075 
5076  // store
5077  nodes[row] = std::make_pair(bounding_box, renderGroup);
5078 
5079  // how many render groups do we have now?
5080  if (renderGroup >= _numRenderGroups) {
5081  _numRenderGroups = renderGroup + 1;
5082  }
5083 
5085  LOG(INFO) << "DEBUG: Existing row " << row << " has Render Group " << renderGroup;
5086  }
5087  }
5088 
5089  // bulk-load the tree
5090  auto bulk_load_timer = timer_start();
5091  _rtree = std::make_unique<RTree>(nodes);
5092  CHECK(_rtree);
5093  LOG(INFO) << "Scanning render groups of poly column '" << geoColumnBaseName
5094  << "' of table '" << tableName << "' took " << timer_stop(seedTimer) << "ms ("
5095  << timer_stop(bulk_load_timer) << " ms for tree)";
5096 
5098  LOG(INFO) << "DEBUG: Done! Now have " << _numRenderGroups << " Render Groups";
5099  }
5100 }
ChunkAccessorTable getChunkAccessorTable(const Catalog_Namespace::Catalog &cat, const TableDescriptor *td, const std::vector< std::string > &columnNames)
#define LOG(tag)
Definition: Logger.h:188
TypeR::rep timer_stop(Type clock_begin)
Definition: measure.h:46
DEVICE void ChunkIter_get_nth(ChunkIter *it, int n, bool uncompress, VarlenDatum *result, bool *is_end)
Definition: ChunkIter.cpp:181
std::string to_string(char const *&&v)
int8_t * pointer
Definition: sqltypes.h:75
std::conditional_t< is_cuda_compiler(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:131
ChunkIterVector & getChunkItersAndRowOffset(ChunkAccessorTable &table, size_t rowid, size_t &rowOffset)
std::string cat(Ts &&... args)
#define DEBUG_RENDER_GROUP_ANALYZER
Definition: Importer.cpp:140
boost::geometry::model::box< Point > BoundingBox
Definition: Importer.h:720
std::unique_ptr< RTree > _rtree
Definition: Importer.h:724
#define CHECK(condition)
Definition: Logger.h:197
Definition: sqltypes.h:47
static constexpr char const * FOREIGN_TABLE
Type timer_start()
Definition: measure.h:40
boost::geometry::model::point< double, 2, boost::geometry::cs::cartesian > Point
Definition: Importer.h:719
+ Here is the call graph for this function:

Member Data Documentation

◆ _numRenderGroups

int import_export::RenderGroupAnalyzer::_numRenderGroups
private

Definition at line 726 of file Importer.h.

◆ _rtree

std::unique_ptr<RTree> import_export::RenderGroupAnalyzer::_rtree
private

Definition at line 724 of file Importer.h.

◆ _rtreeMutex

std::mutex import_export::RenderGroupAnalyzer::_rtreeMutex
private

Definition at line 725 of file Importer.h.


The documentation for this class was generated from the following files: