import_export::RenderGroupAnalyzer Class Reference

#include <Importer.h>

Public Member Functions
	RenderGroupAnalyzer ()
void	seedFromExistingTableContents (const std::unique_ptr< Loader > &loader, const std::string &geoColumnBaseName)
int	insertBoundsAndReturnRenderGroup (const std::vector< double > &bounds)

Private Types
using	Point = boost::geometry::model::point< double, 2, boost::geometry::cs::cartesian >
using	BoundingBox = boost::geometry::model::box< Point >
using	Node = std::pair< BoundingBox, int >
using	RTree = boost::geometry::index::rtree< Node, boost::geometry::index::quadratic< 16 >>

Private Attributes
std::unique_ptr< RTree >	_rtree
std::mutex	_rtreeMutex
int	_numRenderGroups

Detailed Description

Definition at line 712 of file Importer.h.

Member Typedef Documentation

using import_export::RenderGroupAnalyzer::BoundingBox = boost::geometry::model::box<Point>

private

Definition at line 721 of file Importer.h.

using import_export::RenderGroupAnalyzer::Node = std::pair<BoundingBox, int>

private

Definition at line 722 of file Importer.h.

using import_export::RenderGroupAnalyzer::Point = boost::geometry::model::point<double, 2, boost::geometry::cs::cartesian>

private

Definition at line 720 of file Importer.h.

using import_export::RenderGroupAnalyzer::RTree = boost::geometry::index::rtree<Node, boost::geometry::index::quadratic<16>>

private

Definition at line 724 of file Importer.h.

Constructor & Destructor Documentation

import_export::RenderGroupAnalyzer::RenderGroupAnalyzer ( )

inline

Definition at line 714 of file Importer.h.

: _rtree(std::make_unique<RTree>()), _numRenderGroups(0) {}

Member Function Documentation

int import_export::RenderGroupAnalyzer::insertBoundsAndReturnRenderGroup

(

const std::vector< double > &

bounds

)

Definition at line 5120 of file Importer.cpp.

References _numRenderGroups, _rtree, _rtreeMutex, and CHECK.

Referenced by GeoPolygonValueConverter::convertToColumnarFormat(), and GeoMultiPolygonValueConverter::convertToColumnarFormat().

                                     {
  // validate
  CHECK(bounds.size() == 4);

  // get bounds
  BoundingBox bounding_box;
  boost::geometry::assign_inverse(bounding_box);
  boost::geometry::expand(bounding_box, Point(bounds[0], bounds[1]));
  boost::geometry::expand(bounding_box, Point(bounds[2], bounds[3]));

  // remainder under mutex to allow this to be multi-threaded
  std::lock_guard<std::mutex> guard(_rtreeMutex);

  // get the intersecting nodes
  std::vector<Node> intersects;
  _rtree->query(boost::geometry::index::intersects(bounding_box),
                std::back_inserter(intersects));

  // build bitset of render groups of the intersecting rectangles
  // clear bit means available, allows use of find_first()
  boost::dynamic_bitset<> bits(_numRenderGroups);
  bits.set();
  for (const auto& intersection : intersects) {
    CHECK(intersection.second < _numRenderGroups);
    bits.reset(intersection.second);
  }

  // find first available group
  int firstAvailableRenderGroup = 0;
  size_t firstSetBit = bits.find_first();
  if (firstSetBit == boost::dynamic_bitset<>::npos) {
    // all known groups represented, add a new one
    firstAvailableRenderGroup = _numRenderGroups;
    _numRenderGroups++;
  } else {
    firstAvailableRenderGroup = (int)firstSetBit;
  }

  // insert new node
  _rtree->insert(std::make_pair(bounding_box, firstAvailableRenderGroup));

  // return it
  return firstAvailableRenderGroup;
}

Here is the caller graph for this function:

void import_export::RenderGroupAnalyzer::seedFromExistingTableContents	(	const std::unique_ptr< Loader > &	loader,
		const std::string &	geoColumnBaseName
	)

Definition at line 4983 of file Importer.cpp.

References _numRenderGroups, _rtree, cat(), CHECK, ChunkIter_get_nth(), DEBUG_RENDER_GROUP_ANALYZER, StorageType::FOREIGN_TABLE, getChunkAccessorTable(), getChunkItersAndRowOffset(), logger::INFO, kARRAY, kDOUBLE, kINT, LOG, VarlenDatum::pointer, timer_start(), timer_stop(), and to_string().

                                        {
  // start timer
  auto seedTimer = timer_start();

  // start with a fresh tree
  _rtree = nullptr;
  _numRenderGroups = 0;

  // get the table descriptor
  const auto& cat = loader->getCatalog();
  if (loader->getTableDesc()->storageType == StorageType::FOREIGN_TABLE) {
    if (DEBUG_RENDER_GROUP_ANALYZER) {
      LOG(INFO) << "DEBUG: Table is a foreign table";
    }
    _rtree = std::make_unique<RTree>();
    CHECK(_rtree);
    return;
  }

  const std::string& tableName = loader->getTableDesc()->tableName;
  const auto td = cat.getMetadataForTable(tableName);
  CHECK(td);
  CHECK(td->fragmenter);

  // if the table is empty, just make an empty tree
  if (td->fragmenter->getFragmentsForQuery().getPhysicalNumTuples() == 0) {
    if (DEBUG_RENDER_GROUP_ANALYZER) {
      LOG(INFO) << "DEBUG: Table is empty!";
    }
    _rtree = std::make_unique<RTree>();
    CHECK(_rtree);
    return;
  }

  // no seeding possible without these two columns
  const auto cd_bounds =
      cat.getMetadataForColumn(td->tableId, geoColumnBaseName + "_bounds");
  const auto cd_render_group =
      cat.getMetadataForColumn(td->tableId, geoColumnBaseName + "_render_group");
  if (!cd_bounds || !cd_render_group) {
    throw std::runtime_error("RenderGroupAnalyzer: Table " + tableName +
                             " doesn't have bounds or render_group columns!");
  }

  // and validate their types
  if (cd_bounds->columnType.get_type() != kARRAY ||
      cd_bounds->columnType.get_subtype() != kDOUBLE) {
    throw std::runtime_error("RenderGroupAnalyzer: Table " + tableName +
                             " bounds column is wrong type!");
  }
  if (cd_render_group->columnType.get_type() != kINT) {
    throw std::runtime_error("RenderGroupAnalyzer: Table " + tableName +
                             " render_group column is wrong type!");
  }

  // get chunk accessor table
  auto chunkAccessorTable = getChunkAccessorTable(
      cat, td, {geoColumnBaseName + "_bounds", geoColumnBaseName + "_render_group"});
  const auto table_count = std::get<0>(chunkAccessorTable.back());

  if (DEBUG_RENDER_GROUP_ANALYZER) {
    LOG(INFO) << "DEBUG: Scanning existing table geo column set '" << geoColumnBaseName
              << "'";
  }

  std::vector<Node> nodes;
  try {
    nodes.resize(table_count);
  } catch (const std::exception& e) {
    throw std::runtime_error("RenderGroupAnalyzer failed to reserve memory for " +
                             std::to_string(table_count) + " rows");
  }

  for (size_t row = 0; row < table_count; row++) {
    ArrayDatum ad;
    VarlenDatum vd;
    bool is_end;

    // get ChunkIters and fragment row offset
    size_t rowOffset = 0;
    auto& chunkIters = getChunkItersAndRowOffset(chunkAccessorTable, row, rowOffset);
    auto& boundsChunkIter = chunkIters[0];
    auto& renderGroupChunkIter = chunkIters[1];

    // get bounds values
    ChunkIter_get_nth(&boundsChunkIter, row - rowOffset, &ad, &is_end);
    CHECK(!is_end);
    CHECK(ad.pointer);
    int numBounds = (int)(ad.length / sizeof(double));
    CHECK(numBounds == 4);

    // convert to bounding box
    double* bounds = reinterpret_cast<double*>(ad.pointer);
    BoundingBox bounding_box;
    boost::geometry::assign_inverse(bounding_box);
    boost::geometry::expand(bounding_box, Point(bounds[0], bounds[1]));
    boost::geometry::expand(bounding_box, Point(bounds[2], bounds[3]));

    // get render group
    ChunkIter_get_nth(&renderGroupChunkIter, row - rowOffset, false, &vd, &is_end);
    CHECK(!is_end);
    CHECK(vd.pointer);
    int renderGroup = *reinterpret_cast<int32_t*>(vd.pointer);

    // skip rows with invalid render groups (e.g. EMPTY geometry)
    if (renderGroup < 0) {
      continue;
    }

    // store
    nodes[row] = std::make_pair(bounding_box, renderGroup);

    // how many render groups do we have now?
    if (renderGroup >= _numRenderGroups) {
      _numRenderGroups = renderGroup + 1;
    }

    if (DEBUG_RENDER_GROUP_ANALYZER) {
      LOG(INFO) << "DEBUG:   Existing row " << row << " has Render Group " << renderGroup;
    }
  }

  // bulk-load the tree
  auto bulk_load_timer = timer_start();
  _rtree = std::make_unique<RTree>(nodes);
  CHECK(_rtree);
  LOG(INFO) << "Scanning render groups of poly column '" << geoColumnBaseName
            << "' of table '" << tableName << "' took " << timer_stop(seedTimer) << "ms ("
            << timer_stop(bulk_load_timer) << " ms for tree)";

  if (DEBUG_RENDER_GROUP_ANALYZER) {
    LOG(INFO) << "DEBUG: Done! Now have " << _numRenderGroups << " Render Groups";
  }
}

Here is the call graph for this function:

Member Data Documentation

int import_export::RenderGroupAnalyzer::_numRenderGroups

private

Definition at line 727 of file Importer.h.

Referenced by insertBoundsAndReturnRenderGroup(), and seedFromExistingTableContents().

std::unique_ptr<RTree> import_export::RenderGroupAnalyzer::_rtree

private

Definition at line 725 of file Importer.h.

Referenced by insertBoundsAndReturnRenderGroup(), and seedFromExistingTableContents().

std::mutex import_export::RenderGroupAnalyzer::_rtreeMutex

private

Definition at line 726 of file Importer.h.

Referenced by insertBoundsAndReturnRenderGroup().

---

The documentation for this class was generated from the following files:

• /home/jenkins-slave/workspace/core-os-doxygen/ImportExport/Importer.h
• /home/jenkins-slave/workspace/core-os-doxygen/ImportExport/Importer.cpp