OmniSciDB  85c2d10cdc
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Fragmenter_Namespace::InsertDataLoader Struct Reference

#include <InsertDataLoader.h>

+ Collaboration diagram for Fragmenter_Namespace::InsertDataLoader:

Classes

struct  DistributedConnector
 

Public Member Functions

 InsertDataLoader (DistributedConnector &connector)
 
void insertData (const Catalog_Namespace::SessionInfo &session_info, InsertData &insert_data)
 

Private Member Functions

void moveToNextLeaf ()
 

Private Attributes

size_t leaf_count_
 
size_t current_leaf_index_
 
DistributedConnectorconnector_
 

Detailed Description

Definition at line 25 of file InsertDataLoader.h.

Constructor & Destructor Documentation

Fragmenter_Namespace::InsertDataLoader::InsertDataLoader ( DistributedConnector connector)
inline

Definition at line 35 of file InsertDataLoader.h.

36  : leaf_count_(connector.leafCount())
38  , connector_(connector){};

Member Function Documentation

void Fragmenter_Namespace::InsertDataLoader::insertData ( const Catalog_Namespace::SessionInfo session_info,
InsertData insert_data 
)

Definition at line 310 of file InsertDataLoader.cpp.

References cat(), CHECK, Fragmenter_Namespace::computeRowIndicesOfShards(), connector_, Fragmenter_Namespace::copyDataOfShard(), current_leaf_index_, Catalog_Namespace::SessionInfo::getCatalog(), Fragmenter_Namespace::InsertDataLoader::DistributedConnector::insertDataToLeaf(), Fragmenter_Namespace::InsertDataLoader::DistributedConnector::leafCount(), moveToNextLeaf(), and Fragmenter_Namespace::InsertData::tableId.

Referenced by Parser::InsertIntoTableAsSelectStmt::populateData().

311  {
312  const auto& cat = session_info.getCatalog();
313  const auto* td = cat.getMetadataForTable(insert_data.tableId);
314 
315  CHECK(td);
316  if (td->nShards == 0) {
317  connector_.insertDataToLeaf(session_info, current_leaf_index_, insert_data);
318  } else {
319  // we have a sharded target table, start spreading to physical tables
320  auto rowIndicesOfShards =
322 
323  auto insertShardData =
324  [this, &session_info, &insert_data, &cat, &td, &rowIndicesOfShards](
325  size_t shardId) {
326  const auto shard_tables = cat.getPhysicalTablesDescriptors(td);
327  auto stardTableIdx = shardId % td->nShards;
328  auto shardLeafIdx = shardId / td->nShards;
329 
330  const auto& rowIndicesOfShard = rowIndicesOfShards[shardId];
331  ShardDataOwner shardDataOwner;
332 
333  InsertData shardData = copyDataOfShard(
334  cat, shardDataOwner, insert_data, stardTableIdx, rowIndicesOfShard);
335  connector_.insertDataToLeaf(session_info, shardLeafIdx, shardData);
336  };
337 
338  std::vector<std::future<void>> worker_threads;
339  for (size_t shardId = 0; shardId < rowIndicesOfShards.size(); shardId++) {
340  if (rowIndicesOfShards[shardId].size() > 0) {
341  worker_threads.push_back(
342  std::async(std::launch::async, insertShardData, shardId));
343  }
344  }
345  for (auto& child : worker_threads) {
346  child.wait();
347  }
348  for (auto& child : worker_threads) {
349  child.get();
350  }
351  }
352 
353  moveToNextLeaf();
354 }
std::vector< std::vector< size_t > > computeRowIndicesOfShards(size_t shard_count, size_t leaf_count, size_t row_count, SRC *src, bool duplicated_key_value)
std::string cat(Ts &&...args)
virtual void insertDataToLeaf(const Catalog_Namespace::SessionInfo &parent_session_info, const size_t leaf_idx, Fragmenter_Namespace::InsertData &insert_data)=0
Catalog & getCatalog() const
Definition: SessionInfo.h:66
#define CHECK(condition)
Definition: Logger.h:197
InsertData copyDataOfShard(const Catalog_Namespace::Catalog &cat, ShardDataOwner &dataOwner, InsertData &insert_data, int shardTableIndex, const std::vector< size_t > &rowIndices)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Fragmenter_Namespace::InsertDataLoader::moveToNextLeaf ( )
inlineprivate

Definition at line 44 of file InsertDataLoader.h.

References current_leaf_index_, and leaf_count_.

Referenced by insertData().

+ Here is the caller graph for this function:

Member Data Documentation

DistributedConnector& Fragmenter_Namespace::InsertDataLoader::connector_
private

Definition at line 53 of file InsertDataLoader.h.

Referenced by insertData().

size_t Fragmenter_Namespace::InsertDataLoader::current_leaf_index_
private

Definition at line 52 of file InsertDataLoader.h.

Referenced by insertData(), and moveToNextLeaf().

size_t Fragmenter_Namespace::InsertDataLoader::leaf_count_
private

Definition at line 51 of file InsertDataLoader.h.

Referenced by moveToNextLeaf().


The documentation for this struct was generated from the following files: