OmniSciDB  95562058bd
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
InputMetadata.cpp File Reference
#include "InputMetadata.h"
#include "Execute.h"
#include "../Fragmenter/Fragmenter.h"
#include <future>
+ Include dependency graph for InputMetadata.cpp:

Go to the source code of this file.

Namespaces

 

Functions

Fragmenter_Namespace::TableInfo anonymous_namespace{InputMetadata.cpp}::copy_table_info (const Fragmenter_Namespace::TableInfo &table_info)
 
Fragmenter_Namespace::TableInfo anonymous_namespace{InputMetadata.cpp}::build_table_info (const std::vector< const TableDescriptor * > &shard_tables)
 
bool anonymous_namespace{InputMetadata.cpp}::uses_int_meta (const SQLTypeInfo &col_ti)
 
Fragmenter_Namespace::TableInfo anonymous_namespace{InputMetadata.cpp}::synthesize_table_info (const ResultSetPtr &rows)
 
void anonymous_namespace{InputMetadata.cpp}::collect_table_infos (std::vector< InputTableInfo > &table_infos, const std::vector< InputDescriptor > &input_descs, Executor *executor)
 
ChunkMetadataMap synthesize_metadata (const ResultSet *rows)
 
size_t get_frag_count_of_table (const int table_id, Executor *executor)
 
std::vector< InputTableInfoget_table_infos (const std::vector< InputDescriptor > &input_descs, Executor *executor)
 
std::vector< InputTableInfoget_table_infos (const RelAlgExecutionUnit &ra_exe_unit, Executor *executor)
 

Function Documentation

size_t get_frag_count_of_table ( const int  table_id,
Executor executor 
)

Definition at line 239 of file InputMetadata.cpp.

References CHECK, and CHECK_GE.

Referenced by RelAlgExecutor::getOuterFragmentCount().

239  {
240  const auto temporary_tables = executor->getTemporaryTables();
241  CHECK(temporary_tables);
242  auto it = temporary_tables->find(table_id);
243  if (it != temporary_tables->end()) {
244  CHECK_GE(int(0), table_id);
245  return size_t(1);
246  } else {
247  const auto table_info = executor->getTableInfo(table_id);
248  return table_info.fragments.size();
249  }
250 }
#define CHECK_GE(x, y)
Definition: Logger.h:210
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the caller graph for this function:

std::vector<InputTableInfo> get_table_infos ( const std::vector< InputDescriptor > &  input_descs,
Executor executor 
)

Definition at line 252 of file InputMetadata.cpp.

References anonymous_namespace{InputMetadata.cpp}::collect_table_infos().

Referenced by RelAlgExecutor::computeWindow(), RelAlgExecutor::createCompoundWorkUnit(), RelAlgExecutor::createProjectWorkUnit(), RelAlgExecutor::createTableFunctionWorkUnit(), RelAlgExecutor::createUnionWorkUnit(), RelAlgExecutor::executeDelete(), RelAlgExecutor::executeTableFunction(), RelAlgExecutor::executeUpdate(), RelAlgExecutor::executeWorkUnit(), get_table_infos(), RelAlgExecutor::getFilteredCountAll(), RelAlgExecutor::getFilterSelectivity(), RelAlgExecutor::getNDVEstimation(), RelAlgExecutor::handleOutOfMemoryRetry(), TableOptimizer::recomputeMetadata(), and RelAlgExecutor::selectFiltersToBePushedDown().

254  {
255  std::vector<InputTableInfo> table_infos;
256  collect_table_infos(table_infos, input_descs, executor);
257  return table_infos;
258 }
void collect_table_infos(std::vector< InputTableInfo > &table_infos, const std::vector< InputDescriptor > &input_descs, Executor *executor)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::vector<InputTableInfo> get_table_infos ( const RelAlgExecutionUnit ra_exe_unit,
Executor executor 
)

Definition at line 260 of file InputMetadata.cpp.

References anonymous_namespace{InputMetadata.cpp}::collect_table_infos(), get_table_infos(), INJECT_TIMER, and RelAlgExecutionUnit::input_descs.

261  {
263  std::vector<InputTableInfo> table_infos;
264  collect_table_infos(table_infos, ra_exe_unit.input_descs, executor);
265  return table_infos;
266 }
std::vector< InputDescriptor > input_descs
#define INJECT_TIMER(DESC)
Definition: measure.h:93
std::vector< InputTableInfo > get_table_infos(const std::vector< InputDescriptor > &input_descs, Executor *executor)
void collect_table_infos(std::vector< InputTableInfo > &table_infos, const std::vector< InputDescriptor > &input_descs, Executor *executor)

+ Here is the call graph for this function:

ChunkMetadataMap synthesize_metadata ( const ResultSet rows)

Definition at line 135 of file InputMetadata.cpp.

References CHECK, CHECK_LT, cpu_threads(), Encoder::Create(), inline_fp_null_val(), inline_int_null_val(), kDOUBLE, kFLOAT, use_parallel_algorithms(), and anonymous_namespace{InputMetadata.cpp}::uses_int_meta().

Referenced by Fragmenter_Namespace::FragmentInfo::getChunkMetadataMap().

135  {
136  rows->moveToBegin();
137  std::vector<std::vector<std::unique_ptr<Encoder>>> dummy_encoders;
138  const size_t worker_count = use_parallel_algorithms(*rows) ? cpu_threads() : 1;
139  for (size_t worker_idx = 0; worker_idx < worker_count; ++worker_idx) {
140  dummy_encoders.emplace_back();
141  for (size_t i = 0; i < rows->colCount(); ++i) {
142  const auto& col_ti = rows->getColType(i);
143  dummy_encoders.back().emplace_back(Encoder::Create(nullptr, col_ti));
144  }
145  }
146  const auto do_work = [rows](const std::vector<TargetValue>& crt_row,
147  std::vector<std::unique_ptr<Encoder>>& dummy_encoders) {
148  for (size_t i = 0; i < rows->colCount(); ++i) {
149  const auto& col_ti = rows->getColType(i);
150  const auto& col_val = crt_row[i];
151  const auto scalar_col_val = boost::get<ScalarTargetValue>(&col_val);
152  CHECK(scalar_col_val);
153  if (uses_int_meta(col_ti)) {
154  const auto i64_p = boost::get<int64_t>(scalar_col_val);
155  CHECK(i64_p);
156  dummy_encoders[i]->updateStats(*i64_p, *i64_p == inline_int_null_val(col_ti));
157  } else if (col_ti.is_fp()) {
158  switch (col_ti.get_type()) {
159  case kFLOAT: {
160  const auto float_p = boost::get<float>(scalar_col_val);
161  CHECK(float_p);
162  dummy_encoders[i]->updateStats(*float_p,
163  *float_p == inline_fp_null_val(col_ti));
164  break;
165  }
166  case kDOUBLE: {
167  const auto double_p = boost::get<double>(scalar_col_val);
168  CHECK(double_p);
169  dummy_encoders[i]->updateStats(*double_p,
170  *double_p == inline_fp_null_val(col_ti));
171  break;
172  }
173  default:
174  CHECK(false);
175  }
176  } else {
177  throw std::runtime_error(col_ti.get_type_name() +
178  " is not supported in temporary table.");
179  }
180  }
181  };
182  if (use_parallel_algorithms(*rows)) {
183  const size_t worker_count = cpu_threads();
184  std::vector<std::future<void>> compute_stats_threads;
185  const auto entry_count = rows->entryCount();
186  for (size_t i = 0,
187  start_entry = 0,
188  stride = (entry_count + worker_count - 1) / worker_count;
189  i < worker_count && start_entry < entry_count;
190  ++i, start_entry += stride) {
191  const auto end_entry = std::min(start_entry + stride, entry_count);
192  compute_stats_threads.push_back(std::async(
193  std::launch::async,
194  [rows, &do_work, &dummy_encoders](
195  const size_t start, const size_t end, const size_t worker_idx) {
196  for (size_t i = start; i < end; ++i) {
197  const auto crt_row = rows->getRowAtNoTranslations(i);
198  if (!crt_row.empty()) {
199  do_work(crt_row, dummy_encoders[worker_idx]);
200  }
201  }
202  },
203  start_entry,
204  end_entry,
205  i));
206  }
207  for (auto& child : compute_stats_threads) {
208  child.wait();
209  }
210  for (auto& child : compute_stats_threads) {
211  child.get();
212  }
213  } else {
214  while (true) {
215  auto crt_row = rows->getNextRow(false, false);
216  if (crt_row.empty()) {
217  break;
218  }
219  do_work(crt_row, dummy_encoders[0]);
220  }
221  rows->moveToBegin();
222  }
223  ChunkMetadataMap metadata_map;
224  for (size_t worker_idx = 1; worker_idx < worker_count; ++worker_idx) {
225  CHECK_LT(worker_idx, dummy_encoders.size());
226  const auto& worker_encoders = dummy_encoders[worker_idx];
227  for (size_t i = 0; i < rows->colCount(); ++i) {
228  dummy_encoders[0][i]->reduceStats(*worker_encoders[i]);
229  }
230  }
231  for (size_t i = 0; i < rows->colCount(); ++i) {
232  const auto it_ok =
233  metadata_map.emplace(i, dummy_encoders[0][i]->getMetadata(rows->getColType(i)));
234  CHECK(it_ok.second);
235  }
236  return metadata_map;
237 }
bool use_parallel_algorithms(const ResultSet &rows)
Definition: ResultSet.cpp:1007
static Encoder * Create(Data_Namespace::AbstractBuffer *buffer, const SQLTypeInfo sqlType)
Definition: Encoder.cpp:26
double inline_fp_null_val(const SQL_TYPE_INFO &ti)
std::map< int, std::shared_ptr< ChunkMetadata >> ChunkMetadataMap
bool uses_int_meta(const SQLTypeInfo &col_ti)
#define CHECK_LT(x, y)
Definition: Logger.h:207
#define CHECK(condition)
Definition: Logger.h:197
int64_t inline_int_null_val(const SQL_TYPE_INFO &ti)
int cpu_threads()
Definition: thread_count.h:24

+ Here is the call graph for this function:

+ Here is the caller graph for this function: