OmniSciDB  1dac507f6e
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Importer_NS::RenderGroupAnalyzer Class Reference

#include <Importer.h>

Public Member Functions

 RenderGroupAnalyzer ()
 
void seedFromExistingTableContents (const std::unique_ptr< Loader > &loader, const std::string &geoColumnBaseName)
 
int insertBoundsAndReturnRenderGroup (const std::vector< double > &bounds)
 

Private Types

using Point = boost::geometry::model::point< double, 2, boost::geometry::cs::cartesian >
 
using BoundingBox = boost::geometry::model::box< Point >
 
using Node = std::pair< BoundingBox, int >
 
using RTree = boost::geometry::index::rtree< Node, boost::geometry::index::quadratic< 16 >>
 

Private Attributes

std::unique_ptr< RTree_rtree
 
std::mutex _rtreeMutex
 
int _numRenderGroups
 

Detailed Description

Definition at line 718 of file Importer.h.

Member Typedef Documentation

using Importer_NS::RenderGroupAnalyzer::BoundingBox = boost::geometry::model::box<Point>
private

Definition at line 727 of file Importer.h.

using Importer_NS::RenderGroupAnalyzer::Node = std::pair<BoundingBox, int>
private

Definition at line 728 of file Importer.h.

using Importer_NS::RenderGroupAnalyzer::Point = boost::geometry::model::point<double, 2, boost::geometry::cs::cartesian>
private

Definition at line 726 of file Importer.h.

using Importer_NS::RenderGroupAnalyzer::RTree = boost::geometry::index::rtree<Node, boost::geometry::index::quadratic<16>>
private

Definition at line 730 of file Importer.h.

Constructor & Destructor Documentation

Importer_NS::RenderGroupAnalyzer::RenderGroupAnalyzer ( )
inline

Definition at line 720 of file Importer.h.

720 : _rtree(std::make_unique<RTree>()), _numRenderGroups(0) {}
std::unique_ptr< RTree > _rtree
Definition: Importer.h:731

Member Function Documentation

int Importer_NS::RenderGroupAnalyzer::insertBoundsAndReturnRenderGroup ( const std::vector< double > &  bounds)

Definition at line 4988 of file Importer.cpp.

References _numRenderGroups, _rtree, _rtreeMutex, and CHECK().

Referenced by GeoPolygonValueConverter::convertToColumnarFormat(), and GeoMultiPolygonValueConverter::convertToColumnarFormat().

4989  {
4990  // validate
4991  CHECK(bounds.size() == 4);
4992 
4993  // get bounds
4994  BoundingBox bounding_box;
4995  boost::geometry::assign_inverse(bounding_box);
4996  boost::geometry::expand(bounding_box, Point(bounds[0], bounds[1]));
4997  boost::geometry::expand(bounding_box, Point(bounds[2], bounds[3]));
4998 
4999  // remainder under mutex to allow this to be multi-threaded
5000  std::lock_guard<std::mutex> guard(_rtreeMutex);
5001 
5002  // get the intersecting nodes
5003  std::vector<Node> intersects;
5004  _rtree->query(boost::geometry::index::intersects(bounding_box),
5005  std::back_inserter(intersects));
5006 
5007  // build bitset of render groups of the intersecting rectangles
5008  // clear bit means available, allows use of find_first()
5009  boost::dynamic_bitset<> bits(_numRenderGroups);
5010  bits.set();
5011  for (const auto& intersection : intersects) {
5012  CHECK(intersection.second < _numRenderGroups);
5013  bits.reset(intersection.second);
5014  }
5015 
5016  // find first available group
5017  int firstAvailableRenderGroup = 0;
5018  size_t firstSetBit = bits.find_first();
5019  if (firstSetBit == boost::dynamic_bitset<>::npos) {
5020  // all known groups represented, add a new one
5021  firstAvailableRenderGroup = _numRenderGroups;
5022  _numRenderGroups++;
5023  } else {
5024  firstAvailableRenderGroup = (int)firstSetBit;
5025  }
5026 
5027  // insert new node
5028  _rtree->insert(std::make_pair(bounding_box, firstAvailableRenderGroup));
5029 
5030  // return it
5031  return firstAvailableRenderGroup;
5032 }
boost::geometry::model::point< double, 2, boost::geometry::cs::cartesian > Point
Definition: Importer.h:726
CHECK(cgen_state)
boost::geometry::model::box< Point > BoundingBox
Definition: Importer.h:727
std::unique_ptr< RTree > _rtree
Definition: Importer.h:731

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Importer_NS::RenderGroupAnalyzer::seedFromExistingTableContents ( const std::unique_ptr< Loader > &  loader,
const std::string &  geoColumnBaseName 
)

Definition at line 4864 of file Importer.cpp.

References _numRenderGroups, _rtree, CHECK(), CHECK_GE, ChunkIter_get_nth(), DEBUG_RENDER_GROUP_ANALYZER, getChunkAccessorTable(), getChunkItersAndRowOffset(), logger::INFO, kARRAY, kDOUBLE, kINT, LOG, VarlenDatum::pointer, timer_start(), timer_stop(), and to_string().

4866  {
4867  // start timer
4868  auto seedTimer = timer_start();
4869 
4870  // get the table descriptor
4871  const auto& cat = loader->getCatalog();
4872  const std::string& tableName = loader->getTableDesc()->tableName;
4873  const auto td = cat.getMetadataForTable(tableName);
4874  CHECK(td);
4875  CHECK(td->fragmenter);
4876 
4877  // start with a fresh tree
4878  _rtree = nullptr;
4879  _numRenderGroups = 0;
4880 
4881  // if the table is empty, just make an empty tree
4882  if (td->fragmenter->getFragmentsForQuery().getPhysicalNumTuples() == 0) {
4884  LOG(INFO) << "DEBUG: Table is empty!";
4885  }
4886  _rtree = std::make_unique<RTree>();
4887  CHECK(_rtree);
4888  return;
4889  }
4890 
4891  // no seeding possible without these two columns
4892  const auto cd_bounds =
4893  cat.getMetadataForColumn(td->tableId, geoColumnBaseName + "_bounds");
4894  const auto cd_render_group =
4895  cat.getMetadataForColumn(td->tableId, geoColumnBaseName + "_render_group");
4896  if (!cd_bounds || !cd_render_group) {
4897  throw std::runtime_error("RenderGroupAnalyzer: Table " + tableName +
4898  " doesn't have bounds or render_group columns!");
4899  }
4900 
4901  // and validate their types
4902  if (cd_bounds->columnType.get_type() != kARRAY ||
4903  cd_bounds->columnType.get_subtype() != kDOUBLE) {
4904  throw std::runtime_error("RenderGroupAnalyzer: Table " + tableName +
4905  " bounds column is wrong type!");
4906  }
4907  if (cd_render_group->columnType.get_type() != kINT) {
4908  throw std::runtime_error("RenderGroupAnalyzer: Table " + tableName +
4909  " render_group column is wrong type!");
4910  }
4911 
4912  // get chunk accessor table
4913  auto chunkAccessorTable = getChunkAccessorTable(
4914  cat, td, {geoColumnBaseName + "_bounds", geoColumnBaseName + "_render_group"});
4915  const auto table_count = std::get<0>(chunkAccessorTable.back());
4916 
4918  LOG(INFO) << "DEBUG: Scanning existing table geo column set '" << geoColumnBaseName
4919  << "'";
4920  }
4921 
4922  std::vector<Node> nodes;
4923  try {
4924  nodes.resize(table_count);
4925  } catch (const std::exception& e) {
4926  throw std::runtime_error("RenderGroupAnalyzer failed to reserve memory for " +
4927  std::to_string(table_count) + " rows");
4928  }
4929 
4930  for (size_t row = 0; row < table_count; row++) {
4931  ArrayDatum ad;
4932  VarlenDatum vd;
4933  bool is_end;
4934 
4935  // get ChunkIters and fragment row offset
4936  size_t rowOffset = 0;
4937  auto& chunkIters = getChunkItersAndRowOffset(chunkAccessorTable, row, rowOffset);
4938  auto& boundsChunkIter = chunkIters[0];
4939  auto& renderGroupChunkIter = chunkIters[1];
4940 
4941  // get bounds values
4942  ChunkIter_get_nth(&boundsChunkIter, row - rowOffset, &ad, &is_end);
4943  CHECK(!is_end);
4944  CHECK(ad.pointer);
4945  int numBounds = (int)(ad.length / sizeof(double));
4946  CHECK(numBounds == 4);
4947 
4948  // convert to bounding box
4949  double* bounds = reinterpret_cast<double*>(ad.pointer);
4950  BoundingBox bounding_box;
4951  boost::geometry::assign_inverse(bounding_box);
4952  boost::geometry::expand(bounding_box, Point(bounds[0], bounds[1]));
4953  boost::geometry::expand(bounding_box, Point(bounds[2], bounds[3]));
4954 
4955  // get render group
4956  ChunkIter_get_nth(&renderGroupChunkIter, row - rowOffset, false, &vd, &is_end);
4957  CHECK(!is_end);
4958  CHECK(vd.pointer);
4959  int renderGroup = *reinterpret_cast<int32_t*>(vd.pointer);
4960  CHECK_GE(renderGroup, 0);
4961 
4962  // store
4963  nodes[row] = std::make_pair(bounding_box, renderGroup);
4964 
4965  // how many render groups do we have now?
4966  if (renderGroup >= _numRenderGroups) {
4967  _numRenderGroups = renderGroup + 1;
4968  }
4969 
4971  LOG(INFO) << "DEBUG: Existing row " << row << " has Render Group " << renderGroup;
4972  }
4973  }
4974 
4975  // bulk-load the tree
4976  auto bulk_load_timer = timer_start();
4977  _rtree = std::make_unique<RTree>(nodes);
4978  CHECK(_rtree);
4979  LOG(INFO) << "Scanning render groups of poly column '" << geoColumnBaseName
4980  << "' of table '" << tableName << "' took " << timer_stop(seedTimer) << "ms ("
4981  << timer_stop(bulk_load_timer) << " ms for tree)";
4982 
4984  LOG(INFO) << "DEBUG: Done! Now have " << _numRenderGroups << " Render Groups";
4985  }
4986 }
ChunkAccessorTable getChunkAccessorTable(const Catalog_Namespace::Catalog &cat, const TableDescriptor *td, const std::vector< std::string > &columnNames)
#define LOG(tag)
Definition: Logger.h:185
#define CHECK_GE(x, y)
Definition: Logger.h:203
TypeR::rep timer_stop(Type clock_begin)
Definition: measure.h:46
boost::geometry::model::point< double, 2, boost::geometry::cs::cartesian > Point
Definition: Importer.h:726
DEVICE void ChunkIter_get_nth(ChunkIter *it, int n, bool uncompress, VarlenDatum *result, bool *is_end)
Definition: ChunkIter.cpp:181
std::string to_string(char const *&&v)
int8_t * pointer
Definition: sqltypes.h:75
ChunkIterVector & getChunkItersAndRowOffset(ChunkAccessorTable &table, size_t rowid, size_t &rowOffset)
CHECK(cgen_state)
boost::geometry::model::box< Point > BoundingBox
Definition: Importer.h:727
#define DEBUG_RENDER_GROUP_ANALYZER
Definition: Importer.cpp:139
Definition: sqltypes.h:48
std::unique_ptr< RTree > _rtree
Definition: Importer.h:731
Type timer_start()
Definition: measure.h:40
std::conditional_t< isCudaCC(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:122

+ Here is the call graph for this function:

Member Data Documentation

int Importer_NS::RenderGroupAnalyzer::_numRenderGroups
private

Definition at line 733 of file Importer.h.

Referenced by insertBoundsAndReturnRenderGroup(), and seedFromExistingTableContents().

std::unique_ptr<RTree> Importer_NS::RenderGroupAnalyzer::_rtree
private

Definition at line 731 of file Importer.h.

Referenced by insertBoundsAndReturnRenderGroup(), and seedFromExistingTableContents().

std::mutex Importer_NS::RenderGroupAnalyzer::_rtreeMutex
private

Definition at line 732 of file Importer.h.

Referenced by insertBoundsAndReturnRenderGroup().


The documentation for this class was generated from the following files: