InputMetadata.h File Reference

#include "QueryEngine/Descriptors/InputDescriptors.h"
#include "QueryEngine/RelAlgExecutionUnit.h"
#include <unordered_map>

Include dependency graph for InputMetadata.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Classes
struct	InputTableInfo
class	InputTableInfoCache

Namespaces
	Catalog_Namespace

Typedefs
using	TemporaryTables = std::unordered_map< int, const ResultSetPtr & >

Functions
ChunkMetadataMap	synthesize_metadata (const ResultSet *rows)
size_t	get_frag_count_of_table (const int table_id, Executor *executor)
std::vector< InputTableInfo >	get_table_infos (const std::vector< InputDescriptor > &input_descs, Executor *executor)
std::vector< InputTableInfo >	get_table_infos (const RelAlgExecutionUnit &ra_exe_unit, Executor *executor)
Fragmenter_Namespace::TableInfo	build_table_info (const std::vector< const TableDescriptor * > &shard_tables)

Typedef Documentation

using TemporaryTables = std::unordered_map<int, const ResultSetPtr&>

Definition at line 31 of file InputMetadata.h.

Function Documentation

Fragmenter_Namespace::TableInfo build_table_info

(

const std::vector< const TableDescriptor * > &

shard_tables

)

Definition at line 44 of file InputMetadata.cpp.

References CHECK, Fragmenter_Namespace::TableInfo::fragments, and Fragmenter_Namespace::TableInfo::setPhysicalNumTuples().

Referenced by InputTableInfoCache::getTableInfo().

                                                           {
  size_t total_number_of_tuples{0};
  Fragmenter_Namespace::TableInfo table_info_all_shards;
  for (const TableDescriptor* shard_table : shard_tables) {
    CHECK(shard_table->fragmenter);
    const auto& shard_metainfo = shard_table->fragmenter->getFragmentsForQuery();
    total_number_of_tuples += shard_metainfo.getPhysicalNumTuples();
    table_info_all_shards.fragments.reserve(table_info_all_shards.fragments.size() +
                                            shard_metainfo.fragments.size());
    table_info_all_shards.fragments.insert(table_info_all_shards.fragments.end(),
                                           shard_metainfo.fragments.begin(),
                                           shard_metainfo.fragments.end());
  }
  table_info_all_shards.setPhysicalNumTuples(total_number_of_tuples);
  return table_info_all_shards;
}

Here is the call graph for this function:

Here is the caller graph for this function:

size_t get_frag_count_of_table	(	const int	table_id,
		Executor *	executor
	)

Definition at line 430 of file InputMetadata.cpp.

References CHECK, and CHECK_GE.

Referenced by RelAlgExecutor::getOuterFragmentCount().

                                                                       {
  const auto temporary_tables = executor->getTemporaryTables();
  CHECK(temporary_tables);
  auto it = temporary_tables->find(table_id);
  if (it != temporary_tables->end()) {
    CHECK_GE(int(0), table_id);
    return size_t(1);
  } else {
    const auto table_info = executor->getTableInfo(table_id);
    return table_info.fragments.size();
  }
}

Here is the caller graph for this function:

std::vector<InputTableInfo> get_table_infos	(	const std::vector< InputDescriptor > &	input_descs,
		Executor *	executor
	)

Definition at line 443 of file InputMetadata.cpp.

References anonymous_namespace{InputMetadata.cpp}::collect_table_infos().

Referenced by RelAlgExecutor::computeWindow(), RelAlgExecutor::createAggregateWorkUnit(), RelAlgExecutor::createCompoundWorkUnit(), RelAlgExecutor::createFilterWorkUnit(), RelAlgExecutor::createProjectWorkUnit(), RelAlgExecutor::createTableFunctionWorkUnit(), RelAlgExecutor::createUnionWorkUnit(), RelAlgExecutor::executeDelete(), RelAlgExecutor::executeTableFunction(), RelAlgExecutor::executeUpdate(), RelAlgExecutor::executeWorkUnit(), get_table_infos(), TableOptimizer::getDeletedColumnStats(), RelAlgExecutor::getFilteredCountAll(), RelAlgExecutor::getFilterSelectivity(), RelAlgExecutor::getNDVEstimation(), RelAlgExecutor::handleOutOfMemoryRetry(), TableOptimizer::recomputeColumnMetadata(), and RelAlgExecutor::selectFiltersToBePushedDown().

                        {
  std::vector<InputTableInfo> table_infos;
  collect_table_infos(table_infos, input_descs, executor);
  return table_infos;
}

Here is the call graph for this function:

Here is the caller graph for this function:

std::vector<InputTableInfo> get_table_infos	(	const RelAlgExecutionUnit &	ra_exe_unit,
		Executor *	executor
	)

Definition at line 451 of file InputMetadata.cpp.

References anonymous_namespace{InputMetadata.cpp}::collect_table_infos(), get_table_infos(), INJECT_TIMER, and RelAlgExecutionUnit::input_descs.

                                                                {
  INJECT_TIMER(get_table_infos);
  std::vector<InputTableInfo> table_infos;
  collect_table_infos(table_infos, ra_exe_unit.input_descs, executor);
  return table_infos;
}

Here is the call graph for this function:

ChunkMetadataMap synthesize_metadata

(

const ResultSet *

rows

)

Definition at line 306 of file InputMetadata.cpp.

References threading_serial::async(), CHECK, CHECK_LT, cpu_threads(), Encoder::Create(), DEBUG_TIMER, inline_fp_null_val(), inline_int_null_val(), kDOUBLE, kFLOAT, synthesize_metadata_table_function(), TableFunction, result_set::use_parallel_algorithms(), and anonymous_namespace{InputMetadata.cpp}::uses_int_meta().

Referenced by Fragmenter_Namespace::FragmentInfo::getChunkMetadataMap().

                                                            {
  auto timer = DEBUG_TIMER(__func__);
  ChunkMetadataMap metadata_map;

  if (rows->definitelyHasNoRows()) {
    // resultset has no valid storage, so we fill dummy metadata and return early
    std::vector<std::unique_ptr<Encoder>> decoders;
    for (size_t i = 0; i < rows->colCount(); ++i) {
      decoders.emplace_back(Encoder::Create(nullptr, rows->getColType(i)));
      const auto it_ok =
          metadata_map.emplace(i, decoders.back()->getMetadata(rows->getColType(i)));
      CHECK(it_ok.second);
    }
    return metadata_map;
  }

  std::vector<std::vector<std::unique_ptr<Encoder>>> dummy_encoders;
  const size_t worker_count =
      result_set::use_parallel_algorithms(*rows) ? cpu_threads() : 1;
  for (size_t worker_idx = 0; worker_idx < worker_count; ++worker_idx) {
    dummy_encoders.emplace_back();
    for (size_t i = 0; i < rows->colCount(); ++i) {
      const auto& col_ti = rows->getColType(i);
      dummy_encoders.back().emplace_back(Encoder::Create(nullptr, col_ti));
    }
  }

  if (rows->getQueryMemDesc().getQueryDescriptionType() ==
      QueryDescriptionType::TableFunction) {
    return synthesize_metadata_table_function(rows);
  }
  rows->moveToBegin();
  const auto do_work = [rows](const std::vector<TargetValue>& crt_row,
                              std::vector<std::unique_ptr<Encoder>>& dummy_encoders) {
    for (size_t i = 0; i < rows->colCount(); ++i) {
      const auto& col_ti = rows->getColType(i);
      const auto& col_val = crt_row[i];
      const auto scalar_col_val = boost::get<ScalarTargetValue>(&col_val);
      CHECK(scalar_col_val);
      if (uses_int_meta(col_ti)) {
        const auto i64_p = boost::get<int64_t>(scalar_col_val);
        CHECK(i64_p);
        dummy_encoders[i]->updateStats(*i64_p, *i64_p == inline_int_null_val(col_ti));
      } else if (col_ti.is_fp()) {
        switch (col_ti.get_type()) {
          case kFLOAT: {
            const auto float_p = boost::get<float>(scalar_col_val);
            CHECK(float_p);
            dummy_encoders[i]->updateStats(*float_p,
                                           *float_p == inline_fp_null_val(col_ti));
            break;
          }
          case kDOUBLE: {
            const auto double_p = boost::get<double>(scalar_col_val);
            CHECK(double_p);
            dummy_encoders[i]->updateStats(*double_p,
                                           *double_p == inline_fp_null_val(col_ti));
            break;
          }
          default:
            CHECK(false);
        }
      } else {
        throw std::runtime_error(col_ti.get_type_name() +
                                 " is not supported in temporary table.");
      }
    }
  };
  if (result_set::use_parallel_algorithms(*rows)) {
    const size_t worker_count = cpu_threads();
    std::vector<std::future<void>> compute_stats_threads;
    const auto entry_count = rows->entryCount();
    for (size_t i = 0,
                start_entry = 0,
                stride = (entry_count + worker_count - 1) / worker_count;
         i < worker_count && start_entry < entry_count;
         ++i, start_entry += stride) {
      const auto end_entry = std::min(start_entry + stride, entry_count);
      compute_stats_threads.push_back(std::async(
          std::launch::async,
          [rows, &do_work, &dummy_encoders](
              const size_t start, const size_t end, const size_t worker_idx) {
            for (size_t i = start; i < end; ++i) {
              const auto crt_row = rows->getRowAtNoTranslations(i);
              if (!crt_row.empty()) {
                do_work(crt_row, dummy_encoders[worker_idx]);
              }
            }
          },
          start_entry,
          end_entry,
          i));
    }
    for (auto& child : compute_stats_threads) {
      child.wait();
    }
    for (auto& child : compute_stats_threads) {
      child.get();
    }
  } else {
    while (true) {
      auto crt_row = rows->getNextRow(false, false);
      if (crt_row.empty()) {
        break;
      }
      do_work(crt_row, dummy_encoders[0]);
    }
  }
  rows->moveToBegin();
  for (size_t worker_idx = 1; worker_idx < worker_count; ++worker_idx) {
    CHECK_LT(worker_idx, dummy_encoders.size());
    const auto& worker_encoders = dummy_encoders[worker_idx];
    for (size_t i = 0; i < rows->colCount(); ++i) {
      dummy_encoders[0][i]->reduceStats(*worker_encoders[i]);
    }
  }
  for (size_t i = 0; i < rows->colCount(); ++i) {
    const auto it_ok =
        metadata_map.emplace(i, dummy_encoders[0][i]->getMetadata(rows->getColType(i)));
    CHECK(it_ok.second);
  }
  return metadata_map;
}

Here is the call graph for this function:

Here is the caller graph for this function: