#include "AggregatedColRange.h"
#include "BufferCompaction.h"
#include "CartesianProduct.h"
#include "CgenState.h"
#include "CodeCache.h"
#include "DateTimeUtils.h"
#include "Descriptors/QueryFragmentDescriptor.h"
#include "GroupByAndAggregate.h"
#include "JoinHashTable.h"
#include "LoopControlFlow/JoinLoop.h"
#include "NvidiaKernel.h"
#include "PlanState.h"
#include "RelAlgExecutionUnit.h"
#include "RelAlgTranslator.h"
#include "StringDictionaryGenerations.h"
#include "TableGenerations.h"
#include "TargetMetaInfo.h"
#include "WindowContext.h"
#include "../Chunk/Chunk.h"
#include "../Fragmenter/InsertOrderFragmenter.h"
#include "../Planner/Planner.h"
#include "../Shared/Logger.h"
#include "../Shared/MapDParameters.h"
#include "../Shared/measure.h"
#include "../Shared/thread_count.h"
#include "../StringDictionary/LruCache.hpp"
#include "../StringDictionary/StringDictionary.h"
#include "../StringDictionary/StringDictionaryProxy.h"
#include <llvm/IR/Function.h>
#include <llvm/IR/Value.h>
#include <llvm/Linker/Linker.h>
#include <llvm/Transforms/Utils/ValueMapper.h>
#include <rapidjson/document.h>
#include <unistd.h>
#include <algorithm>
#include <condition_variable>
#include <cstddef>
#include <cstdlib>
#include <deque>
#include <functional>
#include <limits>
#include <map>
#include <mutex>
#include <stack>
#include <unordered_map>
#include <unordered_set>

Include dependency graph for Execute.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Classes
class	WatchdogException

class	CompilationRetryNoLazyFetch

class	CompilationRetryNewScanLimit

class	TooManyLiterals

class	CompilationRetryNoCompaction

class	QueryMustRunOnCpu

class	SringConstInResultSet

struct	std::hash< std::vector< int > >

struct	std::hash< std::pair< int, int > >

class	UpdateLogForFragment

class	Executor

struct	Executor::CompilationResult

struct	Executor::FetchResult

class	Executor::ExecutionDispatch

struct	Executor::JoinHashTableOrError

struct	Executor::GroupColLLVMValue

class	Executor::FetchCacheAnchor

Namespaces
	std

Typedefs
using	QueryCompilationDescriptorOwned = std::unique_ptr< QueryCompilationDescriptor >

using	QueryMemoryDescriptorOwned = std::unique_ptr< QueryMemoryDescriptor >

using	RowDataProvider = Fragmenter_Namespace::RowDataProvider

using	LLVMValueVector = std::vector< llvm::Value * >

Functions
void	read_udf_gpu_module (const std::string &udf_ir_filename)

void	read_udf_cpu_module (const std::string &udf_ir_filename)

bool	is_udf_module_present (bool cpu_only=false)

void	read_rt_udf_gpu_module (const std::string &udf_ir)

void	read_rt_udf_cpu_module (const std::string &udf_ir)

bool	is_rt_udf_module_present (bool cpu_only=false)

llvm::Value *	get_arg_by_name (llvm::Function *func, const std::string &name)

uint32_t	log2_bytes (const uint32_t bytes)

const ColumnDescriptor *	get_column_descriptor (const int col_id, const int table_id, const Catalog_Namespace::Catalog &cat)

const Analyzer::Expr *	extract_cast_arg (const Analyzer::Expr *expr)

std::string	numeric_type_name (const SQLTypeInfo &ti)

const ColumnDescriptor *	get_column_descriptor_maybe (const int col_id, const int table_id, const Catalog_Namespace::Catalog &cat)

const ResultSetPtr &	get_temporary_table (const TemporaryTables *temporary_tables, const int table_id)

const SQLTypeInfo	get_column_type (const int col_id, const int table_id, const ColumnDescriptor cd, const TemporaryTables temporary_tables)

template<typename PtrTy >
const ColumnarResults *	rows_to_columnar_results (std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner, const PtrTy &result, const int number)

std::vector< Analyzer::Expr * >	get_exprs_not_owned (const std::vector< std::shared_ptr< Analyzer::Expr >> &exprs)

const ColumnarResults *	columnarize_result (std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner, const ResultSetPtr &result, const int frag_id)

std::string	get_null_check_suffix (const SQLTypeInfo &lhs_ti, const SQLTypeInfo &rhs_ti)

bool	is_unnest (const Analyzer::Expr *expr)

bool	is_trivial_loop_join (const std::vector< InputTableInfo > &query_infos, const RelAlgExecutionUnit &ra_exe_unit)

std::unordered_set< int >	get_available_gpus (const Catalog_Namespace::Catalog &cat)

size_t	get_context_count (const ExecutorDeviceType device_type, const size_t cpu_count, const size_t gpu_count)

void	register_buffer_with_executor_rsm (int64_t exec, int8_t *buffer)

const Analyzer::Expr *	remove_cast_to_int (const Analyzer::Expr *expr)

Variables
bool	g_enable_watchdog

bool	g_enable_dynamic_watchdog

unsigned	g_dynamic_watchdog_time_limit

unsigned	g_trivial_loop_join_threshold

bool	g_from_table_reordering

bool	g_enable_filter_push_down

bool	g_allow_cpu_retry

bool	g_null_div_by_zero

bool	g_bigint_count

bool	g_inner_join_fragment_skipping

float	g_filter_push_down_low_frac

float	g_filter_push_down_high_frac

size_t	g_filter_push_down_passing_row_ubound

bool	g_enable_columnar_output

bool	g_enable_overlaps_hashjoin

size_t	g_overlaps_max_table_size_bytes

bool	g_strip_join_covered_quals

size_t	g_constrained_by_in_threshold

size_t	g_big_group_threshold

bool	g_enable_window_functions

bool	g_enable_table_functions

size_t	g_max_memory_allocation_size

double	g_bump_allocator_step_reduction

bool	g_enable_direct_columnarization

Typedef Documentation

using LLVMValueVector = std::vector<llvm::Value*>

Definition at line 326 of file Execute.h.

using QueryCompilationDescriptorOwned = std::unique_ptr<QueryCompilationDescriptor>

Definition at line 96 of file Execute.h.

using QueryMemoryDescriptorOwned = std::unique_ptr<QueryMemoryDescriptor>

Definition at line 98 of file Execute.h.

using RowDataProvider = Fragmenter_Namespace::RowDataProvider

Definition at line 289 of file Execute.h.

Function Documentation

const ColumnarResults* columnarize_result	(	std::shared_ptr< RowSetMemoryOwner >	row_set_mem_owner,
		const ResultSetPtr &	result,
		const int	frag_id
	)

inline

Definition at line 223 of file Execute.h.

References CHECK_EQ, INJECT_TIMER, and rows_to_columnar_results().

Referenced by ColumnFetcher::getOneColumnFragment(), and ColumnFetcher::getResultSetColumn().

                        {
   INJECT_TIMER(columnarize_result);
   CHECK_EQ(0, frag_id);
   return rows_to_columnar_results(row_set_mem_owner, result, result->colCount());
 }

Here is the call graph for this function:

Here is the caller graph for this function:

const Analyzer::Expr* extract_cast_arg ( const Analyzer::Expr * expr )

inline

Definition at line 152 of file Execute.h.

References kCAST.

Referenced by CodeGenerator::codegen(), CodeGenerator::createInValuesBitmap(), and anonymous_namespace{ExpressionRewrite.cpp}::OrToInVisitor::visitBinOper().

                                                                       {
   const auto cast_expr = dynamic_cast<const Analyzer::UOper*>(expr);
   if (!cast_expr || cast_expr->get_optype() != kCAST) {
     return expr;
   }
   return cast_expr->get_operand();
 }

Here is the caller graph for this function:

llvm::Value* get_arg_by_name	(	llvm::Function *	func,
		const std::string &	name
	)

inline

Definition at line 117 of file Execute.h.

References CHECK().

Referenced by GroupByAndAggregate::codegen(), JoinHashTable::codegenHashTableLoad(), CodeGenerator::codegenHoistedConstantsLoads(), GroupByAndAggregate::codegenOutputSlot(), CodeGenerator::codegenRowId(), GroupByAndAggregate::getAdditionalLiteral(), and Executor::preloadFragOffsets().

                                                                              {
   for (auto& arg : func->args()) {
     if (arg.getName() == name) {
       return &arg;
     }
   }
   CHECK(false);
   return nullptr;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::unordered_set<int> get_available_gpus ( const Catalog_Namespace::Catalog & cat )

Definition at line 947 of file Execute.cpp.

References CHECK_GT, Data_Namespace::DataMgr::getCudaMgr(), Catalog_Namespace::Catalog::getDataMgr(), CudaMgr_Namespace::CudaMgr::getDeviceCount(), and Data_Namespace::DataMgr::gpusPresent().

Referenced by Executor::executeWorkUnitImpl().

                                                                               {
   std::unordered_set<int> available_gpus;
   if (cat.getDataMgr().gpusPresent()) {
     int gpu_count = cat.getDataMgr().getCudaMgr()->getDeviceCount();
     CHECK_GT(gpu_count, 0);
     for (int gpu_id = 0; gpu_id < gpu_count; ++gpu_id) {
       available_gpus.insert(gpu_id);
     }
   }
   return available_gpus;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

const ColumnDescriptor* get_column_descriptor	(	const int	col_id,
		const int	table_id,
		const Catalog_Namespace::Catalog &	cat
	)

inline

Definition at line 142 of file Execute.h.

References CHECK(), CHECK_GT, and Catalog_Namespace::Catalog::getMetadataForColumn().

Referenced by CodeGenerator::codegenColVar(), Executor::executeSimpleInsert(), get_column_descriptor_maybe(), Executor::getColLazyFetchInfo(), ColumnFetcher::getOneTableColumnFragment(), PlanState::isLazyFetchColumn(), RelAlgExecutor::isRowidLookup(), and Executor::skipFragment().

                                          {
   CHECK_GT(table_id, 0);
   const auto col_desc = cat.getMetadataForColumn(table_id, col_id);
   CHECK(col_desc);
   return col_desc;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

const ColumnDescriptor* get_column_descriptor_maybe	(	const int	col_id,
		const int	table_id,
		const Catalog_Namespace::Catalog &	cat
	)

inline

Definition at line 171 of file Execute.h.

References CHECK(), and get_column_descriptor().

Referenced by anonymous_namespace{GroupByAndAggregate.cpp}::expr_is_rowid(), OverlapsJoinHashTable::fetchColumnsForDevice(), BaselineJoinHashTable::fetchColumnsForDevice(), Executor::getColumnDescriptor(), ColumnFetcher::getOneColumnFragment(), Executor::getPhysicalColumnDescriptor(), needs_dictionary_translation(), normalize_column_pair(), JoinHashTable::reifyOneToManyForDevice(), JoinHashTable::reifyOneToOneForDevice(), anonymous_namespace{QueryMemoryDescriptor.cpp}::target_expr_proj_indices(), and anonymous_namespace{Execute.cpp}::try_get_column_descriptor().

                                          {
   CHECK(table_id);
   return table_id > 0 ? get_column_descriptor(col_id, table_id, cat) : nullptr;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

const SQLTypeInfo get_column_type	(	const int	col_id,
		const int	table_id,
		const ColumnDescriptor *	cd,
		const TemporaryTables *	temporary_tables
	)

inline

Definition at line 187 of file Execute.h.

References CHECK(), CHECK_EQ, ColumnDescriptor::columnId, ColumnDescriptor::columnType, get_temporary_table(), and ColumnDescriptor::tableId.

Referenced by ColumnFetcher::getOneTableColumnFragment(), needs_dictionary_translation(), and normalize_column_pair().

                                                                                   {
   CHECK(cd || temporary_tables);
   if (cd) {
     CHECK_EQ(col_id, cd->columnId);
     CHECK_EQ(table_id, cd->tableId);
     return cd->columnType;
   }
   const auto& temp = get_temporary_table(temporary_tables, table_id);
   return temp->getColType(col_id);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

size_t get_context_count	(	const ExecutorDeviceType	device_type,
		const size_t	cpu_count,
		const size_t	gpu_count
	)

Definition at line 959 of file Execute.cpp.

References GPU.

Referenced by Executor::executeWorkUnitImpl().

                                                  {
   return device_type == ExecutorDeviceType::GPU ? gpu_count
                                                 : static_cast<size_t>(cpu_count);
 }

Here is the caller graph for this function:

std::vector<Analyzer::Expr*> get_exprs_not_owned ( const std::vector< std::shared_ptr< Analyzer::Expr >> & exprs )

inline

Definition at line 214 of file Execute.h.

Referenced by RelAlgExecutor::createFilterWorkUnit(), RelAlgExecutor::createModifyCompoundWorkUnit(), RelAlgExecutor::createModifyProjectWorkUnit(), RelAlgExecutor::createProjectWorkUnit(), RelAlgExecutor::createTableFunctionWorkUnit(), and anonymous_namespace{RelAlgExecutor.cpp}::get_inputs_meta().

                                                          {
   std::vector<Analyzer::Expr*> exprs_not_owned;
   for (const auto expr : exprs) {
     exprs_not_owned.push_back(expr.get());
   }
   return exprs_not_owned;
 }

Here is the caller graph for this function:

std::string get_null_check_suffix	(	const SQLTypeInfo &	lhs_ti,
		const SQLTypeInfo &	rhs_ti
	)

inline

Definition at line 1074 of file Execute.h.

References CHECK(), and SQLTypeInfoCore< TYPE_FACET_PACK >::get_notnull().

Referenced by CodeGenerator::codegenCmp(), CodeGenerator::codegenDeciDiv(), CodeGenerator::codegenFpArith(), CodeGenerator::codegenIntArith(), and CodeGenerator::codegenStrCmp().

                                                                     {
   if (lhs_ti.get_notnull() && rhs_ti.get_notnull()) {
     return "";
   }
   std::string null_check_suffix{"_nullable"};
   if (lhs_ti.get_notnull()) {
     CHECK(!rhs_ti.get_notnull());
     null_check_suffix += "_rhs";
   } else if (rhs_ti.get_notnull()) {
     CHECK(!lhs_ti.get_notnull());
     null_check_suffix += "_lhs";
   }
   return null_check_suffix;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

const ResultSetPtr& get_temporary_table	(	const TemporaryTables *	temporary_tables,
		const int	table_id
	)

inline

Definition at line 179 of file Execute.h.

References CHECK(), and CHECK_LT.

Referenced by RelAlgExecutor::executeDeleteViaProject(), RelAlgExecutor::executeProject(), RelAlgExecutor::executeUpdateViaProject(), get_column_type(), ColumnFetcher::getOneColumnFragment(), and ColumnFetcher::getResultSetColumn().

                                                                    {
   CHECK_LT(table_id, 0);
   const auto it = temporary_tables->find(table_id);
   CHECK(it != temporary_tables->end());
   return it->second;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool is_rt_udf_module_present ( bool cpu_only = false )

Definition at line 1222 of file NativeCodegen.cpp.

                                              {
   return (cpu_only || rt_udf_gpu_module != nullptr) && (rt_udf_cpu_module != nullptr);
 }

bool is_trivial_loop_join	(	const std::vector< InputTableInfo > &	query_infos,
		const RelAlgExecutionUnit &	ra_exe_unit
	)

Definition at line 1067 of file Execute.cpp.

References CHECK_NE, g_trivial_loop_join_threshold, and RelAlgExecutionUnit::input_descs.

Referenced by anonymous_namespace{IRCodegen.cpp}::check_if_loop_join_is_allowed().

                                                                   {
   if (ra_exe_unit.input_descs.size() < 2) {
     return false;
   }
 
   // We only support loop join at the end of folded joins
   // where ra_exe_unit.input_descs.size() > 2 for now.
   const auto inner_table_id = ra_exe_unit.input_descs.back().getTableId();
 
   ssize_t inner_table_idx = -1;
   for (size_t i = 0; i < query_infos.size(); ++i) {
     if (query_infos[i].table_id == inner_table_id) {
       inner_table_idx = i;
       break;
     }
   }
   CHECK_NE(ssize_t(-1), inner_table_idx);
   return query_infos[inner_table_idx].info.getNumTuples() <=
          g_trivial_loop_join_threshold;
 }

Here is the caller graph for this function:

bool is_udf_module_present ( bool cpu_only = false )

Definition at line 1218 of file NativeCodegen.cpp.

                                           {
   return (cpu_only || udf_gpu_module != nullptr) && (udf_cpu_module != nullptr);
 }

bool is_unnest ( const Analyzer::Expr * expr )

inline

Definition at line 1090 of file Execute.h.

References Analyzer::UOper::get_optype(), and kUNNEST.

Referenced by CodeGenerator::codegen(), CodeGenerator::codegenCmp(), and anonymous_namespace{CalciteAdapter.cpp}::handle_logical_aggregate().

                                                 {
   return dynamic_cast<const Analyzer::UOper*>(expr) &&
          static_cast<const Analyzer::UOper*>(expr)->get_optype() == kUNNEST;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

uint32_t log2_bytes ( const uint32_t bytes )

inline

Definition at line 127 of file Execute.h.

Referenced by CodeGenerator::codegen(), GroupByAndAggregate::codegenAggArg(), CodeGenerator::codegenFunctionOperCastArgs(), and Executor::groupByColumnCodegen().

                                                  {
   switch (bytes) {
     case 1:
       return 0;
     case 2:
       return 1;
     case 4:
       return 2;
     case 8:
       return 3;
     default:
       abort();
   }
 }

Here is the caller graph for this function:

std::string numeric_type_name ( const SQLTypeInfo & ti )

inline

Definition at line 160 of file Execute.h.

                                                           {
   CHECK(ti.is_integer() || ti.is_decimal() || ti.is_boolean() || ti.is_time() ||
         ti.is_fp() || (ti.is_string() && ti.get_compression() == kENCODING_DICT) ||
         ti.is_timeinterval());
   if (ti.is_integer() || ti.is_decimal() || ti.is_boolean() || ti.is_time() ||
       ti.is_string() || ti.is_timeinterval()) {
     return "int" + std::to_string(ti.get_logical_size() * 8) + "_t";
   }
   return ti.get_type() == kDOUBLE ? "double" : "float";
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void read_rt_udf_cpu_module ( const std::string & udf_ir )

Definition at line 1267 of file NativeCodegen.cpp.

Referenced by MapDHandler::register_runtime_extension_functions().

                                                             {
   llvm::SMDiagnostic parse_error;
 
   auto buf =
       std::make_unique<llvm::MemoryBufferRef>(udf_ir_string, "Runtime UDF for CPU");
 
   rt_udf_cpu_module = llvm::parseIR(*buf, parse_error, getGlobalLLVMContext());
   if (!rt_udf_cpu_module) {
     throw_parseIR_error(parse_error);
   }
 }

Here is the caller graph for this function:

void read_rt_udf_gpu_module ( const std::string & udf_ir )

Definition at line 1255 of file NativeCodegen.cpp.

Referenced by MapDHandler::register_runtime_extension_functions().

                                                             {
   llvm::SMDiagnostic parse_error;
 
   auto buf =
       std::make_unique<llvm::MemoryBufferRef>(udf_ir_string, "Runtime UDF for GPU");
 
   rt_udf_gpu_module = llvm::parseIR(*buf, parse_error, getGlobalLLVMContext());
   if (!rt_udf_gpu_module) {
     throw_parseIR_error(parse_error);
   }
 }

Here is the caller graph for this function:

void read_udf_cpu_module ( const std::string & udf_ir_filename )

Definition at line 1244 of file NativeCodegen.cpp.

Referenced by UdfCompiler::readCpuCompiledModule().

                                                            {
   llvm::SMDiagnostic parse_error;
 
   llvm::StringRef file_name_arg(udf_ir_filename);
 
   udf_cpu_module = llvm::parseIRFile(file_name_arg, parse_error, getGlobalLLVMContext());
   if (!udf_cpu_module) {
     throw_parseIR_error(parse_error, udf_ir_filename);
   }
 }

Here is the caller graph for this function:

void read_udf_gpu_module ( const std::string & udf_ir_filename )

Definition at line 1233 of file NativeCodegen.cpp.

Referenced by UdfCompiler::readGpuCompiledModule().

                                                            {
   llvm::SMDiagnostic parse_error;
 
   llvm::StringRef file_name_arg(udf_ir_filename);
 
   udf_gpu_module = llvm::parseIRFile(file_name_arg, parse_error, getGlobalLLVMContext());
   if (!udf_gpu_module) {
     throw_parseIR_error(parse_error, udf_ir_filename);
   }
 }

Here is the caller graph for this function:

void register_buffer_with_executor_rsm	(	int64_t	exec,
		int8_t *	buffer
	)

Definition at line 102 of file ExtensionsIR.cpp.

                                                                                 {
   Executor* exec_ptr = reinterpret_cast<Executor*>(exec);
   if (buffer != nullptr) {
     exec_ptr->getRowSetMemoryOwner()->addVarlenBuffer(buffer);
   }
 }

const Analyzer::Expr* remove_cast_to_int ( const Analyzer::Expr * expr )

Definition at line 527 of file ColumnIR.cpp.

References Analyzer::Expr::get_type_info(), and kCAST.

Referenced by anonymous_namespace{DateTimePlusRewrite.cpp}::get_dt_field(), and CodeGenerator::hashJoinLhs().

                                                                  {
   const auto uoper = dynamic_cast<const Analyzer::UOper*>(expr);
   if (!uoper || uoper->get_optype() != kCAST) {
     return nullptr;
   }
   const auto& target_ti = uoper->get_type_info();
   if (!target_ti.is_integer()) {
     return nullptr;
   }
   return uoper->get_operand();
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename PtrTy >

const ColumnarResults* rows_to_columnar_results	(	std::shared_ptr< RowSetMemoryOwner >	row_set_mem_owner,
		const PtrTy &	result,
		const int	number
	)

inline

Definition at line 202 of file Execute.h.

References get_logical_type_info().

Referenced by columnarize_result().

                       {
   std::vector<SQLTypeInfo> col_types;
   for (size_t i = 0; i < result->colCount(); ++i) {
     col_types.push_back(get_logical_type_info(result->getColType(i)));
   }
   return new ColumnarResults(row_set_mem_owner, *result, number, col_types);
 }