OmniSciDB  b24e664e58
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Execute.cpp File Reference
#include "Execute.h"
#include "AggregateUtils.h"
#include "BaselineJoinHashTable.h"
#include "CodeGenerator.h"
#include "ColumnFetcher.h"
#include "Descriptors/QueryCompilationDescriptor.h"
#include "Descriptors/QueryFragmentDescriptor.h"
#include "DynamicWatchdog.h"
#include "EquiJoinCondition.h"
#include "ErrorHandling.h"
#include "ExpressionRewrite.h"
#include "ExternalCacheInvalidators.h"
#include "GpuMemUtils.h"
#include "InPlaceSort.h"
#include "JsonAccessors.h"
#include "OutputBufferInitialization.h"
#include "OverlapsJoinHashTable.h"
#include "QueryRewrite.h"
#include "QueryTemplateGenerator.h"
#include "ResultSetReductionJIT.h"
#include "RuntimeFunctions.h"
#include "SpeculativeTopN.h"
#include "TableFunctions/TableFunctionCompilationContext.h"
#include "TableFunctions/TableFunctionExecutionContext.h"
#include "CudaMgr/CudaMgr.h"
#include "DataMgr/BufferMgr/BufferMgr.h"
#include "Parser/ParserNode.h"
#include "Shared/ExperimentalTypeUtilities.h"
#include "Shared/MapDParameters.h"
#include "Shared/TypedDataAccessors.h"
#include "Shared/checked_alloc.h"
#include "Shared/measure.h"
#include "Shared/scope.h"
#include "Shared/shard_key.h"
#include "AggregatedColRange.h"
#include "StringDictionaryGenerations.h"
#include <llvm/Transforms/Utils/BasicBlockUtils.h>
#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/path.hpp>
#include <future>
#include <memory>
#include <numeric>
#include <set>
#include <thread>
#include "ArrayOps.cpp"
#include "DateAdd.cpp"
#include "StringFunctions.cpp"
+ Include dependency graph for Execute.cpp:

Go to the source code of this file.

Classes

class  anonymous_namespace{Execute.cpp}::OutVecOwner
 
class  AggregateReductionEgress< META_TYPE_CLASS >
 
class  AggregateReductionEgress< Experimental::MetaTypeClass< Experimental::Geometry > >
 

Namespaces

 anonymous_namespace{Execute.cpp}
 
 Importer_NS
 

Macros

#define EXECUTE_INCLUDE
 

Functions

 gpu_active_modules_device_mask_ (0x0)
 
 interrupted_ (false)
 
 cpu_code_cache_ (code_cache_size)
 
 gpu_code_cache_ (code_cache_size)
 
 render_manager_ (render_manager)
 
 block_size_x_ (block_size_x)
 
 grid_size_x_ (grid_size_x)
 
 debug_dir_ (debug_dir)
 
 debug_file_ (debug_file)
 
 db_id_ (db_id)
 
 catalog_ (nullptr)
 
 temporary_tables_ (nullptr)
 
 input_table_info_cache_ (this)
 
ResultSetPtr anonymous_namespace{Execute.cpp}::get_merged_result (std::vector< std::pair< ResultSetPtr, std::vector< size_t >>> &results_per_device)
 
std::unordered_set< int > get_available_gpus (const Catalog_Namespace::Catalog &cat)
 
size_t get_context_count (const ExecutorDeviceType device_type, const size_t cpu_count, const size_t gpu_count)
 
size_t anonymous_namespace{Execute.cpp}::compute_buffer_entry_guess (const std::vector< InputTableInfo > &query_infos)
 
std::string anonymous_namespace{Execute.cpp}::get_table_name (const InputDescriptor &input_desc, const Catalog_Namespace::Catalog &cat)
 
size_t anonymous_namespace{Execute.cpp}::getDeviceBasedScanLimit (const ExecutorDeviceType device_type, const int device_count)
 
void anonymous_namespace{Execute.cpp}::checkWorkUnitWatchdog (const RelAlgExecutionUnit &ra_exe_unit, const std::vector< InputTableInfo > &table_infos, const Catalog_Namespace::Catalog &cat, const ExecutorDeviceType device_type, const int device_count)
 
bool is_trivial_loop_join (const std::vector< InputTableInfo > &query_infos, const RelAlgExecutionUnit &ra_exe_unit)
 
RelAlgExecutionUnit anonymous_namespace{Execute.cpp}::replace_scan_limit (const RelAlgExecutionUnit &ra_exe_unit_in, const size_t new_scan_limit)
 
int64_t anonymous_namespace{Execute.cpp}::inline_null_val (const SQLTypeInfo &ti, const bool float_argument_input)
 
void anonymous_namespace{Execute.cpp}::fill_entries_for_empty_input (std::vector< TargetInfo > &target_infos, std::vector< int64_t > &entry, const std::vector< Analyzer::Expr * > &target_exprs, const QueryMemoryDescriptor &query_mem_desc)
 
ResultSetPtr anonymous_namespace{Execute.cpp}::build_row_for_empty_input (const std::vector< Analyzer::Expr * > &target_exprs_in, const QueryMemoryDescriptor &query_mem_desc, const ExecutorDeviceType device_type)
 
size_t anonymous_namespace{Execute.cpp}::permute_storage_columnar (const ResultSetStorage *input_storage, const QueryMemoryDescriptor &input_query_mem_desc, const ResultSetStorage *output_storage, size_t output_row_index, const QueryMemoryDescriptor &output_query_mem_desc, const std::vector< uint32_t > &top_permutation)
 
size_t anonymous_namespace{Execute.cpp}::permute_storage_row_wise (const ResultSetStorage *input_storage, const ResultSetStorage *output_storage, size_t output_row_index, const QueryMemoryDescriptor &output_query_mem_desc, const std::vector< uint32_t > &top_permutation)
 
bool anonymous_namespace{Execute.cpp}::has_lazy_fetched_columns (const std::vector< ColumnLazyFetchInfo > &fetched_cols)
 
const ColumnDescriptoranonymous_namespace{Execute.cpp}::try_get_column_descriptor (const InputColDescriptor *col_desc, const Catalog_Namespace::Catalog &cat)
 
std::map< size_t, std::vector
< uint64_t > > 
get_table_id_to_frag_offsets (const std::vector< InputDescriptor > &input_descs, const std::map< int, const TableFragments * > &all_tables_fragments)
 
bool anonymous_namespace{Execute.cpp}::check_rows_less_than_needed (const ResultSetPtr &results, const size_t scan_limit)
 
template<class T >
int64_t anonymous_namespace{Execute.cpp}::insert_one_dict_str (T *col_data, const std::string &columnName, const SQLTypeInfo &columnType, const Analyzer::Constant *col_cv, const Catalog_Namespace::Catalog &catalog)
 
template<class T >
int64_t anonymous_namespace{Execute.cpp}::insert_one_dict_str (T *col_data, const ColumnDescriptor *cd, const Analyzer::Constant *col_cv, const Catalog_Namespace::Catalog &catalog)
 
int8_t * Importer_NS::appendDatum (int8_t *buf, Datum d, const SQLTypeInfo &ti)
 
int64_t anonymous_namespace{Execute.cpp}::get_hpt_scaled_value (const int64_t &val, const int32_t &ldim, const int32_t &rdim)
 

Variables

bool g_enable_watchdog false
 
bool g_enable_dynamic_watchdog {false}
 
unsigned g_dynamic_watchdog_time_limit {10000}
 
bool g_allow_cpu_retry {true}
 
bool g_null_div_by_zero {false}
 
unsigned g_trivial_loop_join_threshold {1000}
 
bool g_from_table_reordering {true}
 
bool g_inner_join_fragment_skipping {true}
 
bool g_enable_smem_group_by
 
std::unique_ptr< llvm::Module > udf_gpu_module
 
std::unique_ptr< llvm::Module > udf_cpu_module
 
bool g_enable_filter_push_down {false}
 
float g_filter_push_down_low_frac {-1.0f}
 
float g_filter_push_down_high_frac {-1.0f}
 
size_t g_filter_push_down_passing_row_ubound {0}
 
bool g_enable_columnar_output {false}
 
bool g_enable_overlaps_hashjoin {false}
 
bool g_cache_string_hash {false}
 
size_t g_overlaps_max_table_size_bytes {1024 * 1024 * 1024}
 
bool g_strip_join_covered_quals {false}
 
size_t g_constrained_by_in_threshold {10}
 
size_t g_big_group_threshold {20000}
 
bool g_enable_window_functions {true}
 
bool g_enable_table_functions {false}
 
size_t g_max_memory_allocation_size {2000000000}
 
size_t g_min_memory_allocation_size
 
bool g_enable_bump_allocator {false}
 
double g_bump_allocator_step_reduction {0.75}
 
bool g_enable_direct_columnarization {true}
 
bool g_enable_experimental_string_functions
 

Macro Definition Documentation

#define EXECUTE_INCLUDE

Definition at line 3028 of file Execute.cpp.

Function Documentation

block_size_x_ ( block_size_x  )

Referenced by Executor::blockSize().

+ Here is the caller graph for this function:

cpu_code_cache_ ( code_cache_size  )

Referenced by Executor::optimizeAndCodegenCPU().

+ Here is the caller graph for this function:

db_id_ ( db_id  )
debug_dir_ ( debug_dir  )
debug_file_ ( debug_file  )
std::unordered_set<int> get_available_gpus ( const Catalog_Namespace::Catalog cat)

Definition at line 947 of file Execute.cpp.

References CHECK_GT, Data_Namespace::DataMgr::getCudaMgr(), Catalog_Namespace::Catalog::getDataMgr(), CudaMgr_Namespace::CudaMgr::getDeviceCount(), and Data_Namespace::DataMgr::gpusPresent().

Referenced by Executor::executeWorkUnitImpl().

947  {
948  std::unordered_set<int> available_gpus;
949  if (cat.getDataMgr().gpusPresent()) {
950  int gpu_count = cat.getDataMgr().getCudaMgr()->getDeviceCount();
951  CHECK_GT(gpu_count, 0);
952  for (int gpu_id = 0; gpu_id < gpu_count; ++gpu_id) {
953  available_gpus.insert(gpu_id);
954  }
955  }
956  return available_gpus;
957 }
CudaMgr_Namespace::CudaMgr * getCudaMgr() const
Definition: DataMgr.h:117
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:177
#define CHECK_GT(x, y)
Definition: Logger.h:202
int getDeviceCount() const
Definition: CudaMgr.h:93

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

size_t get_context_count ( const ExecutorDeviceType  device_type,
const size_t  cpu_count,
const size_t  gpu_count 
)

Definition at line 959 of file Execute.cpp.

References GPU.

Referenced by Executor::executeWorkUnitImpl().

961  {
962  return device_type == ExecutorDeviceType::GPU ? gpu_count
963  : static_cast<size_t>(cpu_count);
964 }

+ Here is the caller graph for this function:

std::map<size_t, std::vector<uint64_t> > get_table_id_to_frag_offsets ( const std::vector< InputDescriptor > &  input_descs,
const std::map< int, const TableFragments * > &  all_tables_fragments 
)

Definition at line 1929 of file Execute.cpp.

References CHECK().

Referenced by Executor::getRowCountAndOffsetForAllFrags().

1931  {
1932  std::map<size_t, std::vector<uint64_t>> tab_id_to_frag_offsets;
1933  for (auto& desc : input_descs) {
1934  const auto fragments_it = all_tables_fragments.find(desc.getTableId());
1935  CHECK(fragments_it != all_tables_fragments.end());
1936  const auto& fragments = *fragments_it->second;
1937  std::vector<uint64_t> frag_offsets(fragments.size(), 0);
1938  for (size_t i = 0, off = 0; i < fragments.size(); ++i) {
1939  frag_offsets[i] = off;
1940  off += fragments[i].getNumTuples();
1941  }
1942  tab_id_to_frag_offsets.insert(std::make_pair(desc.getTableId(), frag_offsets));
1943  }
1944  return tab_id_to_frag_offsets;
1945 }
CHECK(cgen_state)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

gpu_active_modules_device_mask_ ( 0x0  )

Referenced by Executor::interrupt(), Executor::registerActiveModule(), and Executor::unregisterActiveModule().

+ Here is the caller graph for this function:

gpu_code_cache_ ( code_cache_size  )
grid_size_x_ ( grid_size_x  )

Referenced by Executor::gridSize().

+ Here is the caller graph for this function:

input_table_info_cache_ ( this  )

Definition at line 125 of file Execute.cpp.

Referenced by Executor::clearMetaInfoCache(), and Executor::getTableInfo().

125 {}

+ Here is the caller graph for this function:

interrupted_ ( false  )

Referenced by Executor::executePlanWithGroupBy(), Executor::executePlanWithoutGroupBy(), Executor::executeWorkUnitImpl(), Executor::interrupt(), and Executor::resetInterrupt().

+ Here is the caller graph for this function:

bool is_trivial_loop_join ( const std::vector< InputTableInfo > &  query_infos,
const RelAlgExecutionUnit ra_exe_unit 
)

Definition at line 1067 of file Execute.cpp.

References CHECK_NE, g_trivial_loop_join_threshold, and RelAlgExecutionUnit::input_descs.

Referenced by anonymous_namespace{IRCodegen.cpp}::check_if_loop_join_is_allowed().

1068  {
1069  if (ra_exe_unit.input_descs.size() < 2) {
1070  return false;
1071  }
1072 
1073  // We only support loop join at the end of folded joins
1074  // where ra_exe_unit.input_descs.size() > 2 for now.
1075  const auto inner_table_id = ra_exe_unit.input_descs.back().getTableId();
1076 
1077  ssize_t inner_table_idx = -1;
1078  for (size_t i = 0; i < query_infos.size(); ++i) {
1079  if (query_infos[i].table_id == inner_table_id) {
1080  inner_table_idx = i;
1081  break;
1082  }
1083  }
1084  CHECK_NE(ssize_t(-1), inner_table_idx);
1085  return query_infos[inner_table_idx].info.getNumTuples() <=
1087 }
unsigned g_trivial_loop_join_threshold
Definition: Execute.cpp:76
const std::vector< InputDescriptor > input_descs
#define CHECK_NE(x, y)
Definition: Logger.h:199

+ Here is the caller graph for this function:

render_manager_ ( render_manager  )
temporary_tables_ ( nullptr  )

Referenced by Executor::getTemporaryTables().

+ Here is the caller graph for this function:

Variable Documentation

false

Definition at line 71 of file Execute.cpp.

size_t g_big_group_threshold {20000}

Definition at line 92 of file Execute.cpp.

Referenced by RelAlgExecutor::executeWorkUnit().

double g_bump_allocator_step_reduction {0.75}

Definition at line 100 of file Execute.cpp.

Referenced by create_dev_group_by_buffers().

size_t g_constrained_by_in_threshold {10}

Definition at line 91 of file Execute.cpp.

Referenced by QueryRewriter::rewriteConstrainedByInImpl().

unsigned g_dynamic_watchdog_time_limit {10000}
bool g_enable_bump_allocator {false}
bool g_enable_direct_columnarization {true}
bool g_enable_experimental_string_functions

Definition at line 40 of file RelAlgTranslator.cpp.

bool g_enable_overlaps_hashjoin {false}
bool g_enable_smem_group_by
bool g_enable_window_functions {true}
float g_filter_push_down_high_frac {-1.0f}
float g_filter_push_down_low_frac {-1.0f}
size_t g_filter_push_down_passing_row_ubound {0}
bool g_inner_join_fragment_skipping {true}

Definition at line 78 of file Execute.cpp.

Referenced by Executor::dispatchFragments(), and MapDProgramOptions::fillOptions().

size_t g_max_memory_allocation_size {2000000000}
size_t g_min_memory_allocation_size
Initial value:
{
256}

Definition at line 96 of file Execute.cpp.

Referenced by create_dev_group_by_buffers(), and MapDProgramOptions::fillAdvancedOptions().

bool g_null_div_by_zero {false}
size_t g_overlaps_max_table_size_bytes {1024 * 1024 * 1024}
bool g_strip_join_covered_quals {false}
unsigned g_trivial_loop_join_threshold {1000}

Definition at line 76 of file Execute.cpp.

Referenced by MapDProgramOptions::fillOptions(), and is_trivial_loop_join().

std::unique_ptr<llvm::Module> udf_cpu_module

Definition at line 67 of file NativeCodegen.cpp.

Referenced by Executor::inlineHoistedLiterals().

std::unique_ptr<llvm::Module> udf_gpu_module

Definition at line 27 of file NativeCodegen.cpp.