OmniSciDB  5ade3759e0
Execute.cpp File Reference
#include "Execute.h"
#include "AggregateUtils.h"
#include "BaselineJoinHashTable.h"
#include "CodeGenerator.h"
#include "ColumnFetcher.h"
#include "Descriptors/QueryCompilationDescriptor.h"
#include "Descriptors/QueryFragmentDescriptor.h"
#include "DynamicWatchdog.h"
#include "EquiJoinCondition.h"
#include "ErrorHandling.h"
#include "ExpressionRewrite.h"
#include "ExternalCacheInvalidators.h"
#include "GpuMemUtils.h"
#include "InPlaceSort.h"
#include "JsonAccessors.h"
#include "OutputBufferInitialization.h"
#include "OverlapsJoinHashTable.h"
#include "QueryRewrite.h"
#include "QueryTemplateGenerator.h"
#include "ResultSetReductionJIT.h"
#include "RuntimeFunctions.h"
#include "SpeculativeTopN.h"
#include "CudaMgr/CudaMgr.h"
#include "DataMgr/BufferMgr/BufferMgr.h"
#include "Parser/ParserNode.h"
#include "Shared/ExperimentalTypeUtilities.h"
#include "Shared/MapDParameters.h"
#include "Shared/TypedDataAccessors.h"
#include "Shared/checked_alloc.h"
#include "Shared/measure.h"
#include "Shared/scope.h"
#include "Shared/shard_key.h"
#include "AggregatedColRange.h"
#include "StringDictionaryGenerations.h"
#include <llvm/Transforms/Utils/BasicBlockUtils.h>
#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/path.hpp>
#include <future>
#include <memory>
#include <numeric>
#include <set>
#include <thread>
#include "ArrayOps.cpp"
#include "DateAdd.cpp"
#include "StringFunctions.cpp"
+ Include dependency graph for Execute.cpp:

Go to the source code of this file.

Classes

class  anonymous_namespace{Execute.cpp}::OutVecOwner
 
class  AggregateReductionEgress< META_TYPE_CLASS >
 
class  AggregateReductionEgress< Experimental::MetaTypeClass< Experimental::Geometry > >
 

Namespaces

 anonymous_namespace{Execute.cpp}
 
 Importer_NS
 

Macros

#define EXECUTE_INCLUDE
 

Functions

ResultSetPtr anonymous_namespace{Execute.cpp}::get_merged_result (std::vector< std::pair< ResultSetPtr, std::vector< size_t >>> &results_per_device)
 
std::unordered_set< int > get_available_gpus (const Catalog_Namespace::Catalog &cat)
 
size_t get_context_count (const ExecutorDeviceType device_type, const size_t cpu_count, const size_t gpu_count)
 
size_t anonymous_namespace{Execute.cpp}::compute_buffer_entry_guess (const std::vector< InputTableInfo > &query_infos)
 
std::string anonymous_namespace{Execute.cpp}::get_table_name (const InputDescriptor &input_desc, const Catalog_Namespace::Catalog &cat)
 
size_t anonymous_namespace{Execute.cpp}::getDeviceBasedScanLimit (const ExecutorDeviceType device_type, const int device_count)
 
void anonymous_namespace{Execute.cpp}::checkWorkUnitWatchdog (const RelAlgExecutionUnit &ra_exe_unit, const std::vector< InputTableInfo > &table_infos, const Catalog_Namespace::Catalog &cat, const ExecutorDeviceType device_type, const int device_count)
 
bool is_trivial_loop_join (const std::vector< InputTableInfo > &query_infos, const RelAlgExecutionUnit &ra_exe_unit)
 
RelAlgExecutionUnit anonymous_namespace{Execute.cpp}::replace_scan_limit (const RelAlgExecutionUnit &ra_exe_unit_in, const size_t new_scan_limit)
 
int64_t anonymous_namespace{Execute.cpp}::inline_null_val (const SQLTypeInfo &ti, const bool float_argument_input)
 
void anonymous_namespace{Execute.cpp}::fill_entries_for_empty_input (std::vector< TargetInfo > &target_infos, std::vector< int64_t > &entry, const std::vector< Analyzer::Expr *> &target_exprs, const QueryMemoryDescriptor &query_mem_desc)
 
ResultSetPtr anonymous_namespace{Execute.cpp}::build_row_for_empty_input (const std::vector< Analyzer::Expr *> &target_exprs_in, const QueryMemoryDescriptor &query_mem_desc, const ExecutorDeviceType device_type)
 
bool anonymous_namespace{Execute.cpp}::has_lazy_fetched_columns (const std::vector< ColumnLazyFetchInfo > &fetched_cols)
 
const ColumnDescriptoranonymous_namespace{Execute.cpp}::try_get_column_descriptor (const InputColDescriptor *col_desc, const Catalog_Namespace::Catalog &cat)
 
std::map< size_t, std::vector< uint64_t > > get_table_id_to_frag_offsets (const std::vector< InputDescriptor > &input_descs, const std::map< int, const TableFragments *> &all_tables_fragments)
 
bool anonymous_namespace{Execute.cpp}::check_rows_less_than_needed (const ResultSetPtr &results, const size_t scan_limit)
 
template<class T >
int64_t anonymous_namespace{Execute.cpp}::insert_one_dict_str (T *col_data, const std::string &columnName, const SQLTypeInfo &columnType, const Analyzer::Constant *col_cv, const Catalog_Namespace::Catalog &catalog)
 
template<class T >
int64_t anonymous_namespace{Execute.cpp}::insert_one_dict_str (T *col_data, const ColumnDescriptor *cd, const Analyzer::Constant *col_cv, const Catalog_Namespace::Catalog &catalog)
 
int8_t * Importer_NS::appendDatum (int8_t *buf, Datum d, const SQLTypeInfo &ti)
 
int64_t anonymous_namespace{Execute.cpp}::get_hpt_scaled_value (const int64_t &val, const int32_t &ldim, const int32_t &rdim)
 

Variables

bool g_enable_debug_timer {false}
 
bool g_enable_watchdog {false}
 
bool g_enable_dynamic_watchdog {false}
 
unsigned g_dynamic_watchdog_time_limit {10000}
 
bool g_allow_cpu_retry {true}
 
bool g_null_div_by_zero {false}
 
unsigned g_trivial_loop_join_threshold {1000}
 
bool g_from_table_reordering {true}
 
bool g_inner_join_fragment_skipping {true}
 
bool g_enable_smem_group_by
 
std::unique_ptr< llvm::Module > udf_gpu_module
 
std::unique_ptr< llvm::Module > udf_cpu_module
 
bool g_enable_filter_push_down {false}
 
float g_filter_push_down_low_frac {-1.0f}
 
float g_filter_push_down_high_frac {-1.0f}
 
size_t g_filter_push_down_passing_row_ubound {0}
 
bool g_enable_columnar_output {false}
 
bool g_enable_overlaps_hashjoin {false}
 
bool g_cache_string_hash {false}
 
size_t g_overlaps_max_table_size_bytes {1024 * 1024 * 1024}
 
bool g_strip_join_covered_quals {false}
 
size_t g_constrained_by_in_threshold {10}
 
size_t g_big_group_threshold {20000}
 
bool g_enable_window_functions {true}
 
size_t g_max_memory_allocation_size {2000000000}
 
size_t g_min_memory_allocation_size
 
bool g_enable_bump_allocator {false}
 
double g_bump_allocator_step_reduction {0.75}
 

Macro Definition Documentation

◆ EXECUTE_INCLUDE

#define EXECUTE_INCLUDE

Definition at line 2899 of file Execute.cpp.

Function Documentation

◆ get_available_gpus()

std::unordered_set<int> get_available_gpus ( const Catalog_Namespace::Catalog cat)

Definition at line 934 of file Execute.cpp.

References CHECK_GT, Data_Namespace::DataMgr::getCudaMgr(), Catalog_Namespace::Catalog::getDataMgr(), CudaMgr_Namespace::CudaMgr::getDeviceCount(), run-benchmark::gpu_count, and Data_Namespace::DataMgr::gpusPresent().

Referenced by Executor::executeWorkUnitImpl(), and is_unnest().

934  {
935  std::unordered_set<int> available_gpus;
936  if (cat.getDataMgr().gpusPresent()) {
938  CHECK_GT(gpu_count, 0);
939  for (int gpu_id = 0; gpu_id < gpu_count; ++gpu_id) {
940  available_gpus.insert(gpu_id);
941  }
942  }
943  return available_gpus;
944 }
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:177
int getDeviceCount() const
Definition: CudaMgr.h:81
#define CHECK_GT(x, y)
Definition: Logger.h:199
CudaMgr_Namespace::CudaMgr * getCudaMgr() const
Definition: DataMgr.h:116
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ get_context_count()

size_t get_context_count ( const ExecutorDeviceType  device_type,
const size_t  cpu_count,
const size_t  gpu_count 
)

Definition at line 946 of file Execute.cpp.

References GPU, and run-benchmark::gpu_count.

Referenced by Executor::executeWorkUnitImpl(), and is_unnest().

948  {
949  return device_type == ExecutorDeviceType::GPU ? gpu_count
950  : static_cast<size_t>(cpu_count);
951 }
+ Here is the caller graph for this function:

◆ get_table_id_to_frag_offsets()

std::map<size_t, std::vector<uint64_t> > get_table_id_to_frag_offsets ( const std::vector< InputDescriptor > &  input_descs,
const std::map< int, const TableFragments *> &  all_tables_fragments 
)

Definition at line 1803 of file Execute.cpp.

References CHECK.

Referenced by Executor::getRowCountAndOffsetForAllFrags().

1805  {
1806  std::map<size_t, std::vector<uint64_t>> tab_id_to_frag_offsets;
1807  for (auto& desc : input_descs) {
1808  const auto fragments_it = all_tables_fragments.find(desc.getTableId());
1809  CHECK(fragments_it != all_tables_fragments.end());
1810  const auto& fragments = *fragments_it->second;
1811  std::vector<uint64_t> frag_offsets(fragments.size(), 0);
1812  for (size_t i = 0, off = 0; i < fragments.size(); ++i) {
1813  frag_offsets[i] = off;
1814  off += fragments[i].getNumTuples();
1815  }
1816  tab_id_to_frag_offsets.insert(std::make_pair(desc.getTableId(), frag_offsets));
1817  }
1818  return tab_id_to_frag_offsets;
1819 }
#define CHECK(condition)
Definition: Logger.h:187
+ Here is the caller graph for this function:

◆ is_trivial_loop_join()

bool is_trivial_loop_join ( const std::vector< InputTableInfo > &  query_infos,
const RelAlgExecutionUnit ra_exe_unit 
)

Definition at line 1054 of file Execute.cpp.

References CHECK_NE, g_trivial_loop_join_threshold, and RelAlgExecutionUnit::input_descs.

Referenced by anonymous_namespace{IRCodegen.cpp}::check_if_loop_join_is_allowed(), and is_unnest().

1055  {
1056  if (ra_exe_unit.input_descs.size() < 2) {
1057  return false;
1058  }
1059 
1060  // We only support loop join at the end of folded joins
1061  // where ra_exe_unit.input_descs.size() > 2 for now.
1062  const auto inner_table_id = ra_exe_unit.input_descs.back().getTableId();
1063 
1064  ssize_t inner_table_idx = -1;
1065  for (size_t i = 0; i < query_infos.size(); ++i) {
1066  if (query_infos[i].table_id == inner_table_id) {
1067  inner_table_idx = i;
1068  break;
1069  }
1070  }
1071  CHECK_NE(ssize_t(-1), inner_table_idx);
1072  return query_infos[inner_table_idx].info.getNumTuples() <=
1074 }
unsigned g_trivial_loop_join_threshold
Definition: Execute.cpp:74
const std::vector< InputDescriptor > input_descs
#define CHECK_NE(x, y)
Definition: Logger.h:196
+ Here is the caller graph for this function:

Variable Documentation

◆ g_allow_cpu_retry

◆ g_big_group_threshold

size_t g_big_group_threshold {20000}

Definition at line 90 of file Execute.cpp.

Referenced by RelAlgExecutor::executeWorkUnit(), and TEST().

◆ g_bump_allocator_step_reduction

double g_bump_allocator_step_reduction {0.75}

Definition at line 97 of file Execute.cpp.

Referenced by create_dev_group_by_buffers().

◆ g_cache_string_hash

bool g_cache_string_hash {false}

◆ g_constrained_by_in_threshold

size_t g_constrained_by_in_threshold {10}

Definition at line 89 of file Execute.cpp.

Referenced by QueryRewriter::rewriteConstrainedByInImpl(), and TEST().

◆ g_dynamic_watchdog_time_limit

unsigned g_dynamic_watchdog_time_limit {10000}

◆ g_enable_bump_allocator

bool g_enable_bump_allocator {false}

◆ g_enable_columnar_output

◆ g_enable_debug_timer

◆ g_enable_dynamic_watchdog

◆ g_enable_filter_push_down

◆ g_enable_overlaps_hashjoin

bool g_enable_overlaps_hashjoin {false}

◆ g_enable_smem_group_by

bool g_enable_smem_group_by

Definition at line 26 of file QueryMemoryDescriptor.cpp.

◆ g_enable_watchdog

◆ g_enable_window_functions

bool g_enable_window_functions {true}

◆ g_filter_push_down_high_frac

float g_filter_push_down_high_frac {-1.0f}

◆ g_filter_push_down_low_frac

float g_filter_push_down_low_frac {-1.0f}

◆ g_filter_push_down_passing_row_ubound

size_t g_filter_push_down_passing_row_ubound {0}

◆ g_from_table_reordering

◆ g_inner_join_fragment_skipping

bool g_inner_join_fragment_skipping {true}

Definition at line 76 of file Execute.cpp.

Referenced by Executor::dispatchFragments(), and MapDProgramOptions::fillOptions().

◆ g_max_memory_allocation_size

◆ g_min_memory_allocation_size

◆ g_null_div_by_zero

bool g_null_div_by_zero {false}

◆ g_overlaps_max_table_size_bytes

size_t g_overlaps_max_table_size_bytes {1024 * 1024 * 1024}

◆ g_strip_join_covered_quals

bool g_strip_join_covered_quals {false}

◆ g_trivial_loop_join_threshold

unsigned g_trivial_loop_join_threshold {1000}

Definition at line 74 of file Execute.cpp.

Referenced by MapDProgramOptions::fillOptions(), is_trivial_loop_join(), and TEST().

◆ udf_cpu_module

std::unique_ptr<llvm::Module> udf_cpu_module

◆ udf_gpu_module

std::unique_ptr<llvm::Module> udf_gpu_module