OmniSciDB  b28c0d5765
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Execute.cpp File Reference
#include "QueryEngine/Execute.h"
#include <llvm/Transforms/Utils/BasicBlockUtils.h>
#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/path.hpp>
#include <chrono>
#include <ctime>
#include <future>
#include <iostream>
#include <memory>
#include <mutex>
#include <numeric>
#include <set>
#include <thread>
#include "Catalog/Catalog.h"
#include "CudaMgr/CudaMgr.h"
#include "DataMgr/BufferMgr/BufferMgr.h"
#include "DataMgr/ForeignStorage/FsiChunkUtils.h"
#include "OSDependent/heavyai_path.h"
#include "Parser/ParserNode.h"
#include "QueryEngine/AggregateUtils.h"
#include "QueryEngine/AggregatedColRange.h"
#include "QueryEngine/CodeGenerator.h"
#include "QueryEngine/ColumnFetcher.h"
#include "QueryEngine/Descriptors/QueryCompilationDescriptor.h"
#include "QueryEngine/Descriptors/QueryFragmentDescriptor.h"
#include "QueryEngine/DynamicWatchdog.h"
#include "QueryEngine/EquiJoinCondition.h"
#include "QueryEngine/ErrorHandling.h"
#include "QueryEngine/ExpressionRewrite.h"
#include "QueryEngine/ExternalCacheInvalidators.h"
#include "QueryEngine/GpuMemUtils.h"
#include "QueryEngine/InPlaceSort.h"
#include "QueryEngine/JoinHashTable/BaselineJoinHashTable.h"
#include "QueryEngine/JoinHashTable/OverlapsJoinHashTable.h"
#include "QueryEngine/JsonAccessors.h"
#include "QueryEngine/OutputBufferInitialization.h"
#include "QueryEngine/QueryDispatchQueue.h"
#include "QueryEngine/QueryEngine.h"
#include "QueryEngine/QueryRewrite.h"
#include "QueryEngine/QueryTemplateGenerator.h"
#include "QueryEngine/ResultSetReductionJIT.h"
#include "QueryEngine/RuntimeFunctions.h"
#include "QueryEngine/SpeculativeTopN.h"
#include "QueryEngine/StringDictionaryGenerations.h"
#include "QueryEngine/TableFunctions/TableFunctionCompilationContext.h"
#include "QueryEngine/TableFunctions/TableFunctionExecutionContext.h"
#include "QueryEngine/Visitors/TransientStringLiteralsVisitor.h"
#include "Shared/SystemParameters.h"
#include "Shared/TypedDataAccessors.h"
#include "Shared/checked_alloc.h"
#include "Shared/measure.h"
#include "Shared/misc.h"
#include "Shared/scope.h"
#include "Shared/shard_key.h"
#include "Shared/threading.h"
#include "ArrayOps.cpp"
#include "DateAdd.cpp"
#include "GeoOps.cpp"
#include "RowFunctionOps.cpp"
#include "StringFunctions.cpp"
#include "TableFunctions/TableFunctionOps.cpp"

Go to the source code of this file.

Classes

struct  anonymous_namespace{Execute.cpp}::GetTargetInfo
 
class  anonymous_namespace{Execute.cpp}::OutVecOwner
 

Namespaces

 anonymous_namespace{Execute.cpp}
 
 foreign_storage
 

Macros

#define EXECUTE_INCLUDE
 

Functions

std::unique_ptr< llvm::Module > read_llvm_module_from_bc_file (const std::string &udf_ir_filename, llvm::LLVMContext &ctx)
 
std::unique_ptr< llvm::Module > read_llvm_module_from_ir_file (const std::string &udf_ir_filename, llvm::LLVMContext &ctx, bool is_gpu=false)
 
std::unique_ptr< llvm::Module > read_llvm_module_from_ir_string (const std::string &udf_ir_string, llvm::LLVMContext &ctx, bool is_gpu=false)
 
void anonymous_namespace{Execute.cpp}::prepare_string_dictionaries (const std::unordered_set< PhysicalInput > &phys_inputs, const Catalog_Namespace::Catalog &catalog)
 
bool anonymous_namespace{Execute.cpp}::is_empty_table (Fragmenter_Namespace::AbstractFragmenter *fragmenter)
 
void foreign_storage::populate_string_dictionary (const int32_t table_id, const int32_t col_id, const Catalog_Namespace::Catalog &catalog)
 
 block_size_x_ (block_size_x)
 
 grid_size_x_ (grid_size_x)
 
 max_gpu_slab_size_ (max_gpu_slab_size)
 
 debug_dir_ (debug_dir)
 
 debug_file_ (debug_file)
 
 catalog_ (nullptr)
 
 data_mgr_ (data_mgr)
 
 temporary_tables_ (nullptr)
 
 input_table_info_cache_ (this)
 
 thread_id_ (logger::thread_id())
 
ResultSetPtr anonymous_namespace{Execute.cpp}::get_merged_result (std::vector< std::pair< ResultSetPtr, std::vector< size_t >>> &results_per_device, std::vector< TargetInfo > const &targets)
 
ReductionCode anonymous_namespace{Execute.cpp}::get_reduction_code (const size_t executor_id, std::vector< std::pair< ResultSetPtr, std::vector< size_t >>> &results_per_device, int64_t *compilation_queue_time)
 
std::unordered_set< int > get_available_gpus (const Data_Namespace::DataMgr *data_mgr)
 
size_t get_context_count (const ExecutorDeviceType device_type, const size_t cpu_count, const size_t gpu_count)
 
size_t anonymous_namespace{Execute.cpp}::compute_buffer_entry_guess (const std::vector< InputTableInfo > &query_infos)
 
std::string anonymous_namespace{Execute.cpp}::get_table_name (const InputDescriptor &input_desc, const Catalog_Namespace::Catalog &cat)
 
size_t anonymous_namespace{Execute.cpp}::getDeviceBasedScanLimit (const ExecutorDeviceType device_type, const int device_count)
 
void anonymous_namespace{Execute.cpp}::checkWorkUnitWatchdog (const RelAlgExecutionUnit &ra_exe_unit, const std::vector< InputTableInfo > &table_infos, const Catalog_Namespace::Catalog &cat, const ExecutorDeviceType device_type, const int device_count)
 
size_t get_loop_join_size (const std::vector< InputTableInfo > &query_infos, const RelAlgExecutionUnit &ra_exe_unit)
 
template<typename T >
std::vector< std::string > anonymous_namespace{Execute.cpp}::expr_container_to_string (const T &expr_container)
 
template<>
std::vector< std::string > anonymous_namespace{Execute.cpp}::expr_container_to_string (const std::list< Analyzer::OrderEntry > &expr_container)
 
std::string anonymous_namespace{Execute.cpp}::sort_algorithm_to_string (const SortAlgorithm algorithm)
 
std::string ra_exec_unit_desc_for_caching (const RelAlgExecutionUnit &ra_exe_unit)
 
std::ostream & operator<< (std::ostream &os, const RelAlgExecutionUnit &ra_exe_unit)
 
RelAlgExecutionUnit anonymous_namespace{Execute.cpp}::replace_scan_limit (const RelAlgExecutionUnit &ra_exe_unit_in, const size_t new_scan_limit)
 
int64_t anonymous_namespace{Execute.cpp}::inline_null_val (const SQLTypeInfo &ti, const bool float_argument_input)
 
void anonymous_namespace{Execute.cpp}::fill_entries_for_empty_input (std::vector< TargetInfo > &target_infos, std::vector< int64_t > &entry, const std::vector< Analyzer::Expr * > &target_exprs, const QueryMemoryDescriptor &query_mem_desc)
 
ResultSetPtr anonymous_namespace{Execute.cpp}::build_row_for_empty_input (const std::vector< Analyzer::Expr * > &target_exprs_in, const QueryMemoryDescriptor &query_mem_desc, const ExecutorDeviceType device_type)
 
size_t anonymous_namespace{Execute.cpp}::permute_storage_columnar (const ResultSetStorage *input_storage, const QueryMemoryDescriptor &input_query_mem_desc, const ResultSetStorage *output_storage, size_t output_row_index, const QueryMemoryDescriptor &output_query_mem_desc, const std::vector< uint32_t > &top_permutation)
 
size_t anonymous_namespace{Execute.cpp}::permute_storage_row_wise (const ResultSetStorage *input_storage, const ResultSetStorage *output_storage, size_t output_row_index, const QueryMemoryDescriptor &output_query_mem_desc, const std::vector< uint32_t > &top_permutation)
 
bool anonymous_namespace{Execute.cpp}::has_lazy_fetched_columns (const std::vector< ColumnLazyFetchInfo > &fetched_cols)
 
const ColumnDescriptoranonymous_namespace{Execute.cpp}::try_get_column_descriptor (const InputColDescriptor *col_desc, const Catalog_Namespace::Catalog &cat)
 
std::map< size_t, std::vector
< uint64_t > > 
get_table_id_to_frag_offsets (const std::vector< InputDescriptor > &input_descs, const std::map< int, const TableFragments * > &all_tables_fragments)
 
std::ostream & operator<< (std::ostream &os, FetchResult const &fetch_result)
 
size_t anonymous_namespace{Execute.cpp}::get_selected_input_descs_index (int const table_id, std::vector< InputDescriptor > const &input_descs)
 
size_t anonymous_namespace{Execute.cpp}::get_selected_input_col_descs_index (int const table_id, std::list< std::shared_ptr< InputColDescriptor const >> const &input_col_descs)
 
std::list< std::shared_ptr
< const InputColDescriptor > > 
anonymous_namespace{Execute.cpp}::get_selected_input_col_descs (int const table_id, std::list< std::shared_ptr< InputColDescriptor const >> const &input_col_descs)
 
void anonymous_namespace{Execute.cpp}::set_mod_range (std::vector< int8_t const * > &frag_col_buffers, int8_t const *const ptr, size_t const local_col_id, size_t const N)
 
bool anonymous_namespace{Execute.cpp}::check_rows_less_than_needed (const ResultSetPtr &results, const size_t scan_limit)
 
void anonymous_namespace{Execute.cpp}::add_deleted_col_to_map (PlanState::DeletedColumnsMap &deleted_cols_map, const ColumnDescriptor *deleted_cd)
 
std::tuple< bool, int64_t,
int64_t > 
anonymous_namespace{Execute.cpp}::get_hpt_overflow_underflow_safe_scaled_values (const int64_t chunk_min, const int64_t chunk_max, const SQLTypeInfo &lhs_type, const SQLTypeInfo &rhs_type)
 

Variables

bool g_enable_watchdog false
 
bool g_enable_dynamic_watchdog {false}
 
size_t g_watchdog_none_encoded_string_translation_limit {1000000UL}
 
bool g_enable_cpu_sub_tasks {false}
 
size_t g_cpu_sub_task_size {500'000}
 
bool g_enable_filter_function {true}
 
unsigned g_dynamic_watchdog_time_limit {10000}
 
bool g_allow_cpu_retry {true}
 
bool g_allow_query_step_cpu_retry {true}
 
bool g_null_div_by_zero {false}
 
unsigned g_trivial_loop_join_threshold {1000}
 
bool g_from_table_reordering {true}
 
bool g_inner_join_fragment_skipping {true}
 
bool g_enable_smem_group_by
 
std::unique_ptr< llvm::Module > udf_gpu_module
 
std::unique_ptr< llvm::Module > udf_cpu_module
 
bool g_enable_filter_push_down {false}
 
float g_filter_push_down_low_frac {-1.0f}
 
float g_filter_push_down_high_frac {-1.0f}
 
size_t g_filter_push_down_passing_row_ubound {0}
 
bool g_enable_columnar_output {false}
 
bool g_enable_left_join_filter_hoisting {true}
 
bool g_optimize_row_initialization {true}
 
bool g_enable_overlaps_hashjoin {true}
 
bool g_enable_distance_rangejoin {true}
 
bool g_enable_hashjoin_many_to_many {false}
 
size_t g_overlaps_max_table_size_bytes {1024 * 1024 * 1024}
 
double g_overlaps_target_entries_per_bin {1.3}
 
bool g_strip_join_covered_quals {false}
 
size_t g_constrained_by_in_threshold {10}
 
size_t g_default_max_groups_buffer_entry_guess {16384}
 
size_t g_big_group_threshold {g_default_max_groups_buffer_entry_guess}
 
bool g_enable_window_functions {true}
 
bool g_enable_table_functions {true}
 
bool g_enable_dev_table_functions {false}
 
bool g_enable_geo_ops_on_uncompressed_coords {true}
 
bool g_enable_rf_prop_table_functions {true}
 
size_t g_max_memory_allocation_size {2000000000}
 
size_t g_min_memory_allocation_size
 
bool g_enable_bump_allocator {false}
 
double g_bump_allocator_step_reduction {0.75}
 
bool g_enable_direct_columnarization {true}
 
bool g_enable_string_functions
 
bool g_enable_lazy_fetch {true}
 
bool g_enable_runtime_query_interrupt {true}
 
bool g_enable_non_kernel_time_query_interrupt {true}
 
bool g_use_estimator_result_cache {true}
 
unsigned g_pending_query_interrupt_freq {1000}
 
double g_running_query_interrupt_freq {0.1}
 
size_t g_gpu_smem_threshold
 
bool g_enable_smem_grouped_non_count_agg
 
bool g_enable_smem_non_grouped_agg
 
bool g_is_test_env {false}
 
size_t g_enable_parallel_linearization
 
bool g_enable_data_recycler {true}
 
bool g_use_hashtable_cache {true}
 
bool g_use_query_resultset_cache {true}
 
bool g_use_chunk_metadata_cache {true}
 
bool g_allow_auto_resultset_caching {false}
 
bool g_allow_query_step_skipping {true}
 
size_t g_hashtable_cache_total_bytes {size_t(1) << 32}
 
size_t g_max_cacheable_hashtable_size_bytes {size_t(1) << 31}
 
size_t g_query_resultset_cache_total_bytes {size_t(1) << 32}
 
size_t g_max_cacheable_query_resultset_size_bytes {size_t(1) << 31}
 
size_t g_auto_resultset_caching_threshold {size_t(1) << 20}
 
bool g_optimize_cuda_block_and_grid_sizes {false}
 
size_t g_approx_quantile_buffer {1000}
 
size_t g_approx_quantile_centroids {300}
 
bool g_enable_automatic_ir_metadata {true}
 
size_t g_max_log_length {500}
 
bool g_cache_string_hash
 
 this
 

Macro Definition Documentation

#define EXECUTE_INCLUDE

Definition at line 3939 of file Execute.cpp.

Function Documentation

block_size_x_ ( block_size_x  )
data_mgr_ ( data_mgr  )

Referenced by BaselineHashTable::BaselineHashTable(), Executor::interrupt(), ResultSet::syncEstimatorBuffer(), BaselineHashTable::~BaselineHashTable(), and ResultSet::~ResultSet().

+ Here is the caller graph for this function:

debug_dir_ ( debug_dir  )
debug_file_ ( debug_file  )
std::unordered_set<int> get_available_gpus ( const Data_Namespace::DataMgr data_mgr)

Definition at line 1457 of file Execute.cpp.

References CHECK, CHECK_GT, Data_Namespace::DataMgr::getCudaMgr(), CudaMgr_Namespace::CudaMgr::getDeviceCount(), and Data_Namespace::DataMgr::gpusPresent().

Referenced by Executor::executeWorkUnitImpl().

1457  {
1458  CHECK(data_mgr);
1459  std::unordered_set<int> available_gpus;
1460  if (data_mgr->gpusPresent()) {
1461  CHECK(data_mgr->getCudaMgr());
1462  const int gpu_count = data_mgr->getCudaMgr()->getDeviceCount();
1463  CHECK_GT(gpu_count, 0);
1464  for (int gpu_id = 0; gpu_id < gpu_count; ++gpu_id) {
1465  available_gpus.insert(gpu_id);
1466  }
1467  }
1468  return available_gpus;
1469 }
CudaMgr_Namespace::CudaMgr * getCudaMgr() const
Definition: DataMgr.h:224
#define CHECK_GT(x, y)
Definition: Logger.h:234
int getDeviceCount() const
Definition: CudaMgr.h:87
#define CHECK(condition)
Definition: Logger.h:222
bool gpusPresent() const
Definition: DataMgr.h:218

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

size_t get_context_count ( const ExecutorDeviceType  device_type,
const size_t  cpu_count,
const size_t  gpu_count 
)

Definition at line 1471 of file Execute.cpp.

References GPU.

Referenced by Executor::executeWorkUnitImpl().

1473  {
1474  return device_type == ExecutorDeviceType::GPU ? gpu_count
1475  : static_cast<size_t>(cpu_count);
1476 }

+ Here is the caller graph for this function:

size_t get_loop_join_size ( const std::vector< InputTableInfo > &  query_infos,
const RelAlgExecutionUnit ra_exe_unit 
)

Definition at line 1579 of file Execute.cpp.

References CHECK, and RelAlgExecutionUnit::input_descs.

Referenced by anonymous_namespace{IRCodegen.cpp}::check_if_loop_join_is_allowed().

1580  {
1581  const auto inner_table_id = ra_exe_unit.input_descs.back().getTableId();
1582 
1583  std::optional<size_t> inner_table_idx;
1584  for (size_t i = 0; i < query_infos.size(); ++i) {
1585  if (query_infos[i].table_id == inner_table_id) {
1586  inner_table_idx = i;
1587  break;
1588  }
1589  }
1590  CHECK(inner_table_idx);
1591  return query_infos[*inner_table_idx].info.getNumTuples();
1592 }
std::vector< InputDescriptor > input_descs
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the caller graph for this function:

std::map<size_t, std::vector<uint64_t> > get_table_id_to_frag_offsets ( const std::vector< InputDescriptor > &  input_descs,
const std::map< int, const TableFragments * > &  all_tables_fragments 
)

Definition at line 2834 of file Execute.cpp.

References CHECK.

Referenced by Executor::getRowCountAndOffsetForAllFrags().

2836  {
2837  std::map<size_t, std::vector<uint64_t>> tab_id_to_frag_offsets;
2838  for (auto& desc : input_descs) {
2839  const auto fragments_it = all_tables_fragments.find(desc.getTableId());
2840  CHECK(fragments_it != all_tables_fragments.end());
2841  const auto& fragments = *fragments_it->second;
2842  std::vector<uint64_t> frag_offsets(fragments.size(), 0);
2843  for (size_t i = 0, off = 0; i < fragments.size(); ++i) {
2844  frag_offsets[i] = off;
2845  off += fragments[i].getNumTuples();
2846  }
2847  tab_id_to_frag_offsets.insert(std::make_pair(desc.getTableId(), frag_offsets));
2848  }
2849  return tab_id_to_frag_offsets;
2850 }
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the caller graph for this function:

grid_size_x_ ( grid_size_x  )
input_table_info_cache_ ( this  )
max_gpu_slab_size_ ( max_gpu_slab_size  )
std::ostream& operator<< ( std::ostream &  os,
const RelAlgExecutionUnit ra_exe_unit 
)

Definition at line 1685 of file Execute.cpp.

References RelAlgExecutionUnit::estimator, anonymous_namespace{Execute.cpp}::expr_container_to_string(), RelAlgExecutionUnit::groupby_exprs, RelAlgExecutionUnit::input_col_descs, join(), RelAlgExecutionUnit::join_quals, RelAlgExecutionUnit::quals, RelAlgExecutionUnit::query_plan_dag_hash, RelAlgExecutionUnit::scan_limit, RelAlgExecutionUnit::simple_quals, anonymous_namespace{Execute.cpp}::sort_algorithm_to_string(), RelAlgExecutionUnit::sort_info, RelAlgExecutionUnit::target_exprs, to_string(), toString(), RelAlgExecutionUnit::union_all, and RelAlgExecutionUnit::use_bump_allocator.

1685  {
1686  os << "\n\tExtracted Query Plan Dag Hash: " << ra_exe_unit.query_plan_dag_hash;
1687  os << "\n\tTable/Col/Levels: ";
1688  for (const auto& input_col_desc : ra_exe_unit.input_col_descs) {
1689  const auto& scan_desc = input_col_desc->getScanDesc();
1690  os << "(" << scan_desc.getTableId() << ", " << input_col_desc->getColId() << ", "
1691  << scan_desc.getNestLevel() << ") ";
1692  }
1693  if (!ra_exe_unit.simple_quals.empty()) {
1694  os << "\n\tSimple Quals: "
1696  ", ");
1697  }
1698  if (!ra_exe_unit.quals.empty()) {
1699  os << "\n\tQuals: "
1700  << boost::algorithm::join(expr_container_to_string(ra_exe_unit.quals), ", ");
1701  }
1702  if (!ra_exe_unit.join_quals.empty()) {
1703  os << "\n\tJoin Quals: ";
1704  for (size_t i = 0; i < ra_exe_unit.join_quals.size(); i++) {
1705  const auto& join_condition = ra_exe_unit.join_quals[i];
1706  os << "\t\t" << std::to_string(i) << " " << ::toString(join_condition.type);
1707  os << boost::algorithm::join(expr_container_to_string(join_condition.quals), ", ");
1708  }
1709  }
1710  if (!ra_exe_unit.groupby_exprs.empty()) {
1711  os << "\n\tGroup By: "
1713  ", ");
1714  }
1715  os << "\n\tProjected targets: "
1717  os << "\n\tHas Estimator: " << ::toString(ra_exe_unit.estimator == nullptr);
1718  os << "\n\tSort Info: ";
1719  const auto& sort_info = ra_exe_unit.sort_info;
1720  os << "\n\t Order Entries: "
1721  << boost::algorithm::join(expr_container_to_string(sort_info.order_entries), ", ");
1722  os << "\n\t Algorithm: " << sort_algorithm_to_string(sort_info.algorithm);
1723  os << "\n\t Limit: " << std::to_string(sort_info.limit);
1724  os << "\n\t Offset: " << std::to_string(sort_info.offset);
1725  os << "\n\tScan Limit: " << std::to_string(ra_exe_unit.scan_limit);
1726  os << "\n\tBump Allocator: " << ::toString(ra_exe_unit.use_bump_allocator);
1727  if (ra_exe_unit.union_all) {
1728  os << "\n\tUnion: " << std::string(*ra_exe_unit.union_all ? "UNION ALL" : "UNION");
1729  }
1730  return os;
1731 }
std::vector< Analyzer::Expr * > target_exprs
QueryPlanHash query_plan_dag_hash
const std::optional< bool > union_all
std::string join(T const &container, std::string const &delim)
const std::list< std::shared_ptr< Analyzer::Expr > > groupby_exprs
std::string to_string(char const *&&v)
const JoinQualsPerNestingLevel join_quals
std::vector< std::string > expr_container_to_string(const T &expr_container)
Definition: Execute.cpp:1597
std::string toString(const ExecutorDeviceType &device_type)
const std::shared_ptr< Analyzer::Estimator > estimator
std::string sort_algorithm_to_string(const SortAlgorithm algorithm)
Definition: Execute.cpp:1619
std::list< std::shared_ptr< Analyzer::Expr > > quals
std::list< std::shared_ptr< const InputColDescriptor > > input_col_descs
std::list< std::shared_ptr< Analyzer::Expr > > simple_quals

+ Here is the call graph for this function:

std::ostream& operator<< ( std::ostream &  os,
FetchResult const &  fetch_result 
)

Definition at line 2938 of file Execute.cpp.

References FetchResult::col_buffers, FetchResult::frag_offsets, FetchResult::num_rows, and shared::printContainer().

2938  {
2939  return os << "col_buffers" << shared::printContainer(fetch_result.col_buffers)
2940  << " num_rows" << shared::printContainer(fetch_result.num_rows)
2941  << " frag_offsets" << shared::printContainer(fetch_result.frag_offsets);
2942 }
PrintContainer< CONTAINER > printContainer(CONTAINER &container)
Definition: misc.h:107

+ Here is the call graph for this function:

std::string ra_exec_unit_desc_for_caching ( const RelAlgExecutionUnit ra_exe_unit)

Definition at line 1634 of file Execute.cpp.

References RelAlgExecutionUnit::estimator, RelAlgExecutionUnit::groupby_exprs, RelAlgExecutionUnit::input_col_descs, RelAlgExecutionUnit::join_quals, RelAlgExecutionUnit::quals, RelAlgExecutionUnit::scan_limit, RelAlgExecutionUnit::simple_quals, RelAlgExecutionUnit::target_exprs, to_string(), and toString().

Referenced by RelAlgExecutor::executeWorkUnit().

1634  {
1635  // todo(yoonmin): replace a cache key as a DAG representation of a query plan
1636  // instead of ra_exec_unit description if possible
1637  std::ostringstream os;
1638  for (const auto& input_col_desc : ra_exe_unit.input_col_descs) {
1639  const auto& scan_desc = input_col_desc->getScanDesc();
1640  os << scan_desc.getTableId() << "," << input_col_desc->getColId() << ","
1641  << scan_desc.getNestLevel();
1642  }
1643  if (!ra_exe_unit.simple_quals.empty()) {
1644  for (const auto& qual : ra_exe_unit.simple_quals) {
1645  if (qual) {
1646  os << qual->toString() << ",";
1647  }
1648  }
1649  }
1650  if (!ra_exe_unit.quals.empty()) {
1651  for (const auto& qual : ra_exe_unit.quals) {
1652  if (qual) {
1653  os << qual->toString() << ",";
1654  }
1655  }
1656  }
1657  if (!ra_exe_unit.join_quals.empty()) {
1658  for (size_t i = 0; i < ra_exe_unit.join_quals.size(); i++) {
1659  const auto& join_condition = ra_exe_unit.join_quals[i];
1660  os << std::to_string(i) << ::toString(join_condition.type);
1661  for (const auto& qual : join_condition.quals) {
1662  if (qual) {
1663  os << qual->toString() << ",";
1664  }
1665  }
1666  }
1667  }
1668  if (!ra_exe_unit.groupby_exprs.empty()) {
1669  for (const auto& qual : ra_exe_unit.groupby_exprs) {
1670  if (qual) {
1671  os << qual->toString() << ",";
1672  }
1673  }
1674  }
1675  for (const auto& expr : ra_exe_unit.target_exprs) {
1676  if (expr) {
1677  os << expr->toString() << ",";
1678  }
1679  }
1680  os << ::toString(ra_exe_unit.estimator == nullptr);
1681  os << std::to_string(ra_exe_unit.scan_limit);
1682  return os.str();
1683 }
std::vector< Analyzer::Expr * > target_exprs
const std::list< std::shared_ptr< Analyzer::Expr > > groupby_exprs
std::string to_string(char const *&&v)
const JoinQualsPerNestingLevel join_quals
std::string toString(const ExecutorDeviceType &device_type)
const std::shared_ptr< Analyzer::Estimator > estimator
std::list< std::shared_ptr< Analyzer::Expr > > quals
std::list< std::shared_ptr< const InputColDescriptor > > input_col_descs
std::list< std::shared_ptr< Analyzer::Expr > > simple_quals

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::unique_ptr<llvm::Module> read_llvm_module_from_bc_file ( const std::string &  udf_ir_filename,
llvm::LLVMContext &  ctx 
)

Definition at line 1518 of file NativeCodegen.cpp.

Referenced by Executor::update_extension_modules().

1520  {
1521  llvm::SMDiagnostic err;
1522 
1523  auto buffer_or_error = llvm::MemoryBuffer::getFile(bc_filename);
1524  CHECK(!buffer_or_error.getError()) << "bc_filename=" << bc_filename;
1525  llvm::MemoryBuffer* buffer = buffer_or_error.get().get();
1526 
1527  auto owner = llvm::parseBitcodeFile(buffer->getMemBufferRef(), context);
1528  CHECK(!owner.takeError());
1529  CHECK(owner->get());
1530  return std::move(owner.get());
1531 }
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the caller graph for this function:

std::unique_ptr<llvm::Module> read_llvm_module_from_ir_file ( const std::string &  udf_ir_filename,
llvm::LLVMContext &  ctx,
bool  is_gpu = false 
)

Definition at line 1533 of file NativeCodegen.cpp.

Referenced by Executor::update_extension_modules().

1536  {
1537  llvm::SMDiagnostic parse_error;
1538 
1539  llvm::StringRef file_name_arg(udf_ir_filename);
1540 
1541  auto owner = llvm::parseIRFile(file_name_arg, parse_error, ctx);
1542  if (!owner) {
1543  throw_parseIR_error(parse_error, udf_ir_filename, is_gpu);
1544  }
1545 
1546  if (is_gpu) {
1547  llvm::Triple gpu_triple(owner->getTargetTriple());
1548  if (!gpu_triple.isNVPTX()) {
1549  LOG(WARNING)
1550  << "Expected triple nvptx64-nvidia-cuda for NVVM IR of loadtime UDFs but got "
1551  << gpu_triple.str() << ". Disabling the NVVM IR module.";
1552  return std::unique_ptr<llvm::Module>();
1553  }
1554  }
1555  return owner;
1556 }
#define LOG(tag)
Definition: Logger.h:216
void throw_parseIR_error(const llvm::SMDiagnostic &parse_error, std::string src="", const bool is_gpu=false)

+ Here is the caller graph for this function:

std::unique_ptr<llvm::Module> read_llvm_module_from_ir_string ( const std::string &  udf_ir_string,
llvm::LLVMContext &  ctx,
bool  is_gpu = false 
)

Definition at line 1558 of file NativeCodegen.cpp.

Referenced by Executor::update_extension_modules().

1561  {
1562  llvm::SMDiagnostic parse_error;
1563 
1564  auto buf = std::make_unique<llvm::MemoryBufferRef>(udf_ir_string,
1565  "Runtime UDF/UDTF LLVM/NVVM IR");
1566 
1567  auto owner = llvm::parseIR(*buf, parse_error, ctx);
1568  if (!owner) {
1569  LOG(IR) << "read_llvm_module_from_ir_string:\n"
1570  << udf_ir_string << "\nEnd of LLVM/NVVM IR";
1571  throw_parseIR_error(parse_error, "", /* is_gpu= */ is_gpu);
1572  }
1573 
1574  if (is_gpu) {
1575  llvm::Triple gpu_triple(owner->getTargetTriple());
1576  if (!gpu_triple.isNVPTX()) {
1577  LOG(IR) << "read_llvm_module_from_ir_string:\n"
1578  << udf_ir_string << "\nEnd of NNVM IR";
1579  LOG(WARNING) << "Expected triple nvptx64-nvidia-cuda for NVVM IR but got "
1580  << gpu_triple.str()
1581  << ". Executing runtime UDF/UDTFs on GPU will be disabled.";
1582  return std::unique_ptr<llvm::Module>();
1583  ;
1584  }
1585  }
1586  return owner;
1587 }
#define LOG(tag)
Definition: Logger.h:216
void throw_parseIR_error(const llvm::SMDiagnostic &parse_error, std::string src="", const bool is_gpu=false)

+ Here is the caller graph for this function:

temporary_tables_ ( nullptr  )
thread_id_ ( logger::  thread_id())

Definition at line 264 of file Execute.cpp.

References Executor::initialize_extension_module_sources().

264  {
266  update_extension_modules();
267 }
static void initialize_extension_module_sources()
Definition: Execute.cpp:269

+ Here is the call graph for this function:

Variable Documentation

false

Definition at line 79 of file Execute.cpp.

bool g_allow_query_step_cpu_retry {true}
size_t g_approx_quantile_buffer {1000}
size_t g_approx_quantile_centroids {300}
size_t g_auto_resultset_caching_threshold {size_t(1) << 20}
size_t g_big_group_threshold {g_default_max_groups_buffer_entry_guess}

Definition at line 110 of file Execute.cpp.

Referenced by RelAlgExecutor::executeWorkUnit().

double g_bump_allocator_step_reduction {0.75}

Definition at line 121 of file Execute.cpp.

Referenced by create_dev_group_by_buffers().

bool g_cache_string_hash

Definition at line 52 of file StringDictionary.cpp.

size_t g_constrained_by_in_threshold {10}

Definition at line 108 of file Execute.cpp.

Referenced by QueryRewriter::rewriteConstrainedByInImpl().

size_t g_cpu_sub_task_size {500'000}
unsigned g_dynamic_watchdog_time_limit {10000}
bool g_enable_automatic_ir_metadata {true}
bool g_enable_bump_allocator {false}
bool g_enable_cpu_sub_tasks {false}

Definition at line 82 of file Execute.cpp.

Referenced by CommandLineOptions::fillAdvancedOptions(), and Executor::launchKernels().

bool g_enable_dev_table_functions {false}
bool g_enable_direct_columnarization {true}
bool g_enable_distance_rangejoin {true}
bool g_enable_filter_function {true}

Definition at line 84 of file Execute.cpp.

Referenced by CommandLineOptions::fillAdvancedOptions().

bool g_enable_geo_ops_on_uncompressed_coords {true}
bool g_enable_hashjoin_many_to_many {false}

Definition at line 104 of file Execute.cpp.

Referenced by CommandLineOptions::fillOptions(), and rewrite_overlaps_conjunction().

bool g_enable_lazy_fetch {true}
bool g_enable_left_join_filter_hoisting {true}
size_t g_enable_parallel_linearization
Initial value:
{
10000}

Definition at line 144 of file Execute.cpp.

Referenced by ColumnFetcher::linearizeVarLenArrayColFrags().

bool g_enable_rf_prop_table_functions {true}

Definition at line 115 of file Execute.cpp.

bool g_enable_smem_group_by
bool g_enable_smem_grouped_non_count_agg
Initial value:
{
true}

Definition at line 135 of file Execute.cpp.

Referenced by CommandLineOptions::fillAdvancedOptions().

bool g_enable_smem_non_grouped_agg
Initial value:
{
true}

Definition at line 138 of file Execute.cpp.

Referenced by CommandLineOptions::fillAdvancedOptions().

bool g_enable_string_functions

Definition at line 40 of file RelAlgTranslator.cpp.

bool g_enable_window_functions {true}
float g_filter_push_down_high_frac {-1.0f}
float g_filter_push_down_low_frac {-1.0f}
size_t g_filter_push_down_passing_row_ubound {0}
size_t g_gpu_smem_threshold
Initial value:
{
4096}

Definition at line 130 of file Execute.cpp.

Referenced by CommandLineOptions::fillAdvancedOptions().

size_t g_hashtable_cache_total_bytes {size_t(1) << 32}

Definition at line 152 of file Execute.cpp.

Referenced by CommandLineOptions::parse_command_line().

bool g_inner_join_fragment_skipping {true}

Definition at line 91 of file Execute.cpp.

Referenced by Executor::createKernels(), and CommandLineOptions::fillOptions().

size_t g_max_cacheable_hashtable_size_bytes {size_t(1) << 31}

Definition at line 153 of file Execute.cpp.

Referenced by CommandLineOptions::parse_command_line().

size_t g_max_cacheable_query_resultset_size_bytes {size_t(1) << 31}
size_t g_max_memory_allocation_size {2000000000}
size_t g_min_memory_allocation_size
Initial value:
{
256}

Definition at line 117 of file Execute.cpp.

Referenced by create_dev_group_by_buffers(), and CommandLineOptions::fillAdvancedOptions().

bool g_null_div_by_zero {false}
bool g_optimize_cuda_block_and_grid_sizes {false}
bool g_optimize_row_initialization {true}
size_t g_overlaps_max_table_size_bytes {1024 * 1024 * 1024}
double g_overlaps_target_entries_per_bin {1.3}
unsigned g_pending_query_interrupt_freq {1000}
size_t g_query_resultset_cache_total_bytes {size_t(1) << 32}
double g_running_query_interrupt_freq {0.1}
bool g_strip_join_covered_quals {false}
bool g_use_estimator_result_cache {true}
size_t g_watchdog_none_encoded_string_translation_limit {1000000UL}
std::unique_ptr<llvm::Module> udf_cpu_module
std::unique_ptr<llvm::Module> udf_gpu_module