OmniSciDB  91042dcc5b
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
InputMetadata.h File Reference
+ Include dependency graph for InputMetadata.h:
+ This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Classes

struct  InputTableInfo
 
class  InputTableInfoCache
 

Namespaces

 Catalog_Namespace
 

Typedefs

using TemporaryTables = std::unordered_map< int, const ResultSetPtr & >
 

Functions

ChunkMetadataMap synthesize_metadata (const ResultSet *rows)
 
size_t get_frag_count_of_table (const int table_id, Executor *executor)
 
std::vector< InputTableInfoget_table_infos (const std::vector< InputDescriptor > &input_descs, Executor *executor)
 
std::vector< InputTableInfoget_table_infos (const RelAlgExecutionUnit &ra_exe_unit, Executor *executor)
 
Fragmenter_Namespace::TableInfo build_table_info (const std::vector< const TableDescriptor * > &shard_tables)
 

Typedef Documentation

using TemporaryTables = std::unordered_map<int, const ResultSetPtr&>

Definition at line 31 of file InputMetadata.h.

Function Documentation

Fragmenter_Namespace::TableInfo build_table_info ( const std::vector< const TableDescriptor * > &  shard_tables)

Definition at line 42 of file InputMetadata.cpp.

References CHECK, Fragmenter_Namespace::TableInfo::fragments, and Fragmenter_Namespace::TableInfo::setPhysicalNumTuples().

Referenced by InputTableInfoCache::getTableInfo().

43  {
44  size_t total_number_of_tuples{0};
45  Fragmenter_Namespace::TableInfo table_info_all_shards;
46  for (const TableDescriptor* shard_table : shard_tables) {
47  CHECK(shard_table->fragmenter);
48  const auto& shard_metainfo = shard_table->fragmenter->getFragmentsForQuery();
49  total_number_of_tuples += shard_metainfo.getPhysicalNumTuples();
50  table_info_all_shards.fragments.reserve(table_info_all_shards.fragments.size() +
51  shard_metainfo.fragments.size());
52  table_info_all_shards.fragments.insert(table_info_all_shards.fragments.end(),
53  shard_metainfo.fragments.begin(),
54  shard_metainfo.fragments.end());
55  }
56  table_info_all_shards.setPhysicalNumTuples(total_number_of_tuples);
57  return table_info_all_shards;
58 }
std::vector< FragmentInfo > fragments
Definition: Fragmenter.h:169
#define CHECK(condition)
Definition: Logger.h:211
void setPhysicalNumTuples(const size_t physNumTuples)
Definition: Fragmenter.h:164

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

size_t get_frag_count_of_table ( const int  table_id,
Executor executor 
)

Definition at line 417 of file InputMetadata.cpp.

References CHECK, and CHECK_GE.

Referenced by RelAlgExecutor::getOuterFragmentCount().

417  {
418  const auto temporary_tables = executor->getTemporaryTables();
419  CHECK(temporary_tables);
420  auto it = temporary_tables->find(table_id);
421  if (it != temporary_tables->end()) {
422  CHECK_GE(int(0), table_id);
423  return size_t(1);
424  } else {
425  const auto table_info = executor->getTableInfo(table_id);
426  return table_info.fragments.size();
427  }
428 }
#define CHECK_GE(x, y)
Definition: Logger.h:224
#define CHECK(condition)
Definition: Logger.h:211

+ Here is the caller graph for this function:

std::vector<InputTableInfo> get_table_infos ( const std::vector< InputDescriptor > &  input_descs,
Executor executor 
)

Definition at line 430 of file InputMetadata.cpp.

References anonymous_namespace{InputMetadata.cpp}::collect_table_infos().

Referenced by RelAlgExecutor::computeWindow(), RelAlgExecutor::createCompoundWorkUnit(), RelAlgExecutor::createProjectWorkUnit(), RelAlgExecutor::createTableFunctionWorkUnit(), RelAlgExecutor::createUnionWorkUnit(), RelAlgExecutor::executeDelete(), RelAlgExecutor::executeTableFunction(), RelAlgExecutor::executeUpdate(), RelAlgExecutor::executeWorkUnit(), get_table_infos(), TableOptimizer::getDeletedColumnStats(), RelAlgExecutor::getFilteredCountAll(), RelAlgExecutor::getFilterSelectivity(), RelAlgExecutor::getNDVEstimation(), RelAlgExecutor::handleOutOfMemoryRetry(), TableOptimizer::recomputeColumnMetadata(), and RelAlgExecutor::selectFiltersToBePushedDown().

432  {
433  std::vector<InputTableInfo> table_infos;
434  collect_table_infos(table_infos, input_descs, executor);
435  return table_infos;
436 }
void collect_table_infos(std::vector< InputTableInfo > &table_infos, const std::vector< InputDescriptor > &input_descs, Executor *executor)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::vector<InputTableInfo> get_table_infos ( const RelAlgExecutionUnit ra_exe_unit,
Executor executor 
)

Definition at line 438 of file InputMetadata.cpp.

References anonymous_namespace{InputMetadata.cpp}::collect_table_infos(), get_table_infos(), INJECT_TIMER, and RelAlgExecutionUnit::input_descs.

439  {
441  std::vector<InputTableInfo> table_infos;
442  collect_table_infos(table_infos, ra_exe_unit.input_descs, executor);
443  return table_infos;
444 }
std::vector< InputDescriptor > input_descs
#define INJECT_TIMER(DESC)
Definition: measure.h:93
std::vector< InputTableInfo > get_table_infos(const std::vector< InputDescriptor > &input_descs, Executor *executor)
void collect_table_infos(std::vector< InputTableInfo > &table_infos, const std::vector< InputDescriptor > &input_descs, Executor *executor)

+ Here is the call graph for this function:

ChunkMetadataMap synthesize_metadata ( const ResultSet rows)

Definition at line 307 of file InputMetadata.cpp.

References threading_serial::async(), CHECK, CHECK_LT, cpu_threads(), Encoder::Create(), DEBUG_TIMER, i, inline_fp_null_val(), inline_int_null_val(), kDOUBLE, kFLOAT, synthesize_metadata_table_function(), TableFunction, result_set::use_parallel_algorithms(), and anonymous_namespace{InputMetadata.cpp}::uses_int_meta().

Referenced by Fragmenter_Namespace::FragmentInfo::getChunkMetadataMap().

307  {
308  auto timer = DEBUG_TIMER(__func__);
309  rows->moveToBegin();
310  if (rows->getQueryMemDesc().getQueryDescriptionType() ==
313  }
314  std::vector<std::vector<std::unique_ptr<Encoder>>> dummy_encoders;
315  const size_t worker_count =
317  for (size_t worker_idx = 0; worker_idx < worker_count; ++worker_idx) {
318  dummy_encoders.emplace_back();
319  for (size_t i = 0; i < rows->colCount(); ++i) {
320  const auto& col_ti = rows->getColType(i);
321  dummy_encoders.back().emplace_back(Encoder::Create(nullptr, col_ti));
322  }
323  }
324  const auto do_work = [rows](const std::vector<TargetValue>& crt_row,
325  std::vector<std::unique_ptr<Encoder>>& dummy_encoders) {
326  for (size_t i = 0; i < rows->colCount(); ++i) {
327  const auto& col_ti = rows->getColType(i);
328  const auto& col_val = crt_row[i];
329  const auto scalar_col_val = boost::get<ScalarTargetValue>(&col_val);
330  CHECK(scalar_col_val);
331  if (uses_int_meta(col_ti)) {
332  const auto i64_p = boost::get<int64_t>(scalar_col_val);
333  CHECK(i64_p);
334  dummy_encoders[i]->updateStats(*i64_p, *i64_p == inline_int_null_val(col_ti));
335  } else if (col_ti.is_fp()) {
336  switch (col_ti.get_type()) {
337  case kFLOAT: {
338  const auto float_p = boost::get<float>(scalar_col_val);
339  CHECK(float_p);
340  dummy_encoders[i]->updateStats(*float_p,
341  *float_p == inline_fp_null_val(col_ti));
342  break;
343  }
344  case kDOUBLE: {
345  const auto double_p = boost::get<double>(scalar_col_val);
346  CHECK(double_p);
347  dummy_encoders[i]->updateStats(*double_p,
348  *double_p == inline_fp_null_val(col_ti));
349  break;
350  }
351  default:
352  CHECK(false);
353  }
354  } else {
355  throw std::runtime_error(col_ti.get_type_name() +
356  " is not supported in temporary table.");
357  }
358  }
359  };
361  const size_t worker_count = cpu_threads();
362  std::vector<std::future<void>> compute_stats_threads;
363  const auto entry_count = rows->entryCount();
364  for (size_t i = 0,
365  start_entry = 0,
366  stride = (entry_count + worker_count - 1) / worker_count;
367  i < worker_count && start_entry < entry_count;
368  ++i, start_entry += stride) {
369  const auto end_entry = std::min(start_entry + stride, entry_count);
370  compute_stats_threads.push_back(std::async(
372  [rows, &do_work, &dummy_encoders](
373  const size_t start, const size_t end, const size_t worker_idx) {
374  for (size_t i = start; i < end; ++i) {
375  const auto crt_row = rows->getRowAtNoTranslations(i);
376  if (!crt_row.empty()) {
377  do_work(crt_row, dummy_encoders[worker_idx]);
378  }
379  }
380  },
381  start_entry,
382  end_entry,
383  i));
384  }
385  for (auto& child : compute_stats_threads) {
386  child.wait();
387  }
388  for (auto& child : compute_stats_threads) {
389  child.get();
390  }
391  } else {
392  while (true) {
393  auto crt_row = rows->getNextRow(false, false);
394  if (crt_row.empty()) {
395  break;
396  }
397  do_work(crt_row, dummy_encoders[0]);
398  }
399  rows->moveToBegin();
400  }
401  ChunkMetadataMap metadata_map;
402  for (size_t worker_idx = 1; worker_idx < worker_count; ++worker_idx) {
403  CHECK_LT(worker_idx, dummy_encoders.size());
404  const auto& worker_encoders = dummy_encoders[worker_idx];
405  for (size_t i = 0; i < rows->colCount(); ++i) {
406  dummy_encoders[0][i]->reduceStats(*worker_encoders[i]);
407  }
408  }
409  for (size_t i = 0; i < rows->colCount(); ++i) {
410  const auto it_ok =
411  metadata_map.emplace(i, dummy_encoders[0][i]->getMetadata(rows->getColType(i)));
412  CHECK(it_ok.second);
413  }
414  return metadata_map;
415 }
ChunkMetadataMap synthesize_metadata_table_function(const ResultSet *rows)
static Encoder * Create(Data_Namespace::AbstractBuffer *buffer, const SQLTypeInfo sqlType)
Definition: Encoder.cpp:26
double inline_fp_null_val(const SQL_TYPE_INFO &ti)
std::map< int, std::shared_ptr< ChunkMetadata >> ChunkMetadataMap
bool use_parallel_algorithms(const ResultSet &rows)
Definition: ResultSet.cpp:1354
future< Result > async(Fn &&fn, Args &&...args)
bool uses_int_meta(const SQLTypeInfo &col_ti)
#define CHECK_LT(x, y)
Definition: Logger.h:221
#define CHECK(condition)
Definition: Logger.h:211
#define DEBUG_TIMER(name)
Definition: Logger.h:358
int64_t inline_int_null_val(const SQL_TYPE_INFO &ti)
int cpu_threads()
Definition: thread_count.h:24

+ Here is the call graph for this function:

+ Here is the caller graph for this function: