#include <Execute.h>

Collaboration diagram for Executor:

Classes
class	CgenStateManager

struct	ExecutorMutexHolder

class	FetchCacheAnchor

struct	GroupColLLVMValue

struct	JoinHashTableOrError

Public Types
enum	ExtModuleKinds { ExtModuleKinds::template_module, ExtModuleKinds::udf_cpu_module, ExtModuleKinds::udf_gpu_module, ExtModuleKinds::rt_udf_cpu_module, ExtModuleKinds::rt_udf_gpu_module, ExtModuleKinds::rt_geos_module, ExtModuleKinds::rt_libdevice_module }

using	ExecutorId = size_t

using	CachedCardinality = std::pair< bool, size_t >

Public Member Functions
	Executor (const ExecutorId id, Data_Namespace::DataMgr *data_mgr, const size_t block_size_x, const size_t grid_size_x, const size_t max_gpu_slab_size, const std::string &debug_dir, const std::string &debug_file)

void	clearCaches (bool runtime_only=false)

std::string	dumpCache () const

void	reset (bool discard_runtime_modules_only=false)

const std::unique_ptr < llvm::Module > &	get_rt_module () const

const std::unique_ptr < llvm::Module > &	get_udf_module (bool is_gpu=false) const

const std::unique_ptr < llvm::Module > &	get_rt_udf_module (bool is_gpu=false) const

const std::unique_ptr < llvm::Module > &	get_geos_module () const

const std::unique_ptr < llvm::Module > &	get_libdevice_module () const

bool	has_rt_module () const

bool	has_udf_module (bool is_gpu=false) const

bool	has_rt_udf_module (bool is_gpu=false) const

bool	has_geos_module () const

bool	has_libdevice_module () const

const TemporaryTables *	getTemporaryTables ()

StringDictionaryProxy *	getStringDictionaryProxy (const int dict_id, const bool with_generation) const

StringDictionaryProxy *	getStringDictionaryProxy (const int dictId, const std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner, const bool with_generation) const

const StringDictionaryProxy::IdMap *	getStringProxyTranslationMap (const int source_dict_id, const int dest_dict_id, const RowSetMemoryOwner::StringTranslationType translation_type, const std::vector< StringOps_Namespace::StringOpInfo > &string_op_infos, std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner, const bool with_generation) const

const StringDictionaryProxy::IdMap *	getJoinIntersectionStringProxyTranslationMap (const StringDictionaryProxy source_proxy, StringDictionaryProxy dest_proxy, const std::vector< StringOps_Namespace::StringOpInfo > &source_string_op_infos, const std::vector< StringOps_Namespace::StringOpInfo > &dest_source_string_op_infos, std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner) const

const StringDictionaryProxy::TranslationMap < Datum > *	getStringProxyNumericTranslationMap (const int source_dict_id, const std::vector< StringOps_Namespace::StringOpInfo > &string_op_infos, std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner, const bool with_generation) const

bool	isCPUOnly () const

bool	isArchMaxwell (const ExecutorDeviceType dt) const

bool	containsLeftDeepOuterJoin () const

const ColumnDescriptor *	getColumnDescriptor (const Analyzer::ColumnVar *) const

const ColumnDescriptor *	getPhysicalColumnDescriptor (const Analyzer::ColumnVar *, int) const

const Catalog_Namespace::Catalog *	getCatalog () const

void	setCatalog (const Catalog_Namespace::Catalog *catalog)

Data_Namespace::DataMgr *	getDataMgr () const

const std::shared_ptr < RowSetMemoryOwner >	getRowSetMemoryOwner () const

const TemporaryTables *	getTemporaryTables () const

Fragmenter_Namespace::TableInfo	getTableInfo (const int table_id) const

const TableGeneration &	getTableGeneration (const int table_id) const

ExpressionRange	getColRange (const PhysicalInput &) const

size_t	getNumBytesForFetchedRow (const std::set< int > &table_ids_to_fetch) const

bool	hasLazyFetchColumns (const std::vector< Analyzer::Expr * > &target_exprs) const

std::vector< ColumnLazyFetchInfo >	getColLazyFetchInfo (const std::vector< Analyzer::Expr * > &target_exprs) const

void	interrupt (const QuerySessionId &query_session="", const QuerySessionId &interrupt_session="")

void	resetInterrupt ()

void	enableRuntimeQueryInterrupt (const double runtime_query_check_freq, const unsigned pending_query_check_freq) const

int8_t	warpSize () const

unsigned	gridSize () const

void	setGridSize (unsigned grid_size)

void	resetGridSize ()

unsigned	numBlocksPerMP () const

unsigned	blockSize () const

void	setBlockSize (unsigned block_size)

void	resetBlockSize ()

size_t	maxGpuSlabSize () const

ResultSetPtr	executeWorkUnit (size_t &max_groups_buffer_entry_guess, const bool is_agg, const std::vector< InputTableInfo > &, const RelAlgExecutionUnit &, const CompilationOptions &, const ExecutionOptions &options, const Catalog_Namespace::Catalog &, RenderInfo *render_info, const bool has_cardinality_estimation, ColumnCacheMap &column_cache)

TableUpdateMetadata	executeUpdate (const RelAlgExecutionUnit &ra_exe_unit, const std::vector< InputTableInfo > &table_infos, const TableDescriptor *updated_table_desc, const CompilationOptions &co, const ExecutionOptions &eo, Catalog_Namespace::Catalog &cat, std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner, const UpdateLogForFragment::Callback &cb, const bool is_agg)

void	addTransientStringLiterals (const RelAlgExecutionUnit &ra_exe_unit, const std::shared_ptr< RowSetMemoryOwner > &row_set_mem_owner)

int	deviceCount (const ExecutorDeviceType) const

void	setupCaching (const std::unordered_set< PhysicalInput > &phys_inputs, const std::unordered_set< int > &phys_table_ids)

void	setColRangeCache (const AggregatedColRange &aggregated_col_range)

ExecutorId	getExecutorId () const

QuerySessionId &	getCurrentQuerySession (heavyai::shared_lock< heavyai::shared_mutex > &read_lock)

QuerySessionStatus::QueryStatus	getQuerySessionStatus (const QuerySessionId &candidate_query_session, heavyai::shared_lock< heavyai::shared_mutex > &read_lock)

bool	checkCurrentQuerySession (const std::string &candidate_query_session, heavyai::shared_lock< heavyai::shared_mutex > &read_lock)

void	invalidateRunningQuerySession (heavyai::unique_lock< heavyai::shared_mutex > &write_lock)

bool	addToQuerySessionList (const QuerySessionId &query_session, const std::string &query_str, const std::string &submitted, const size_t executor_id, const QuerySessionStatus::QueryStatus query_status, heavyai::unique_lock< heavyai::shared_mutex > &write_lock)

bool	removeFromQuerySessionList (const QuerySessionId &query_session, const std::string &submitted_time_str, heavyai::unique_lock< heavyai::shared_mutex > &write_lock)

void	setQuerySessionAsInterrupted (const QuerySessionId &query_session, heavyai::unique_lock< heavyai::shared_mutex > &write_lock)

bool	checkIsQuerySessionInterrupted (const std::string &query_session, heavyai::shared_lock< heavyai::shared_mutex > &read_lock)

bool	checkIsQuerySessionEnrolled (const QuerySessionId &query_session, heavyai::shared_lock< heavyai::shared_mutex > &read_lock)

bool	updateQuerySessionStatusWithLock (const QuerySessionId &query_session, const std::string &submitted_time_str, const QuerySessionStatus::QueryStatus updated_query_status, heavyai::unique_lock< heavyai::shared_mutex > &write_lock)

bool	updateQuerySessionExecutorAssignment (const QuerySessionId &query_session, const std::string &submitted_time_str, const size_t executor_id, heavyai::unique_lock< heavyai::shared_mutex > &write_lock)

std::vector< QuerySessionStatus >	getQuerySessionInfo (const QuerySessionId &query_session, heavyai::shared_lock< heavyai::shared_mutex > &read_lock)

heavyai::shared_mutex &	getSessionLock ()

CurrentQueryStatus	attachExecutorToQuerySession (const QuerySessionId &query_session_id, const std::string &query_str, const std::string &query_submitted_time)

void	checkPendingQueryStatus (const QuerySessionId &query_session)

void	clearQuerySessionStatus (const QuerySessionId &query_session, const std::string &submitted_time_str)

void	updateQuerySessionStatus (const QuerySessionId &query_session, const std::string &submitted_time_str, const QuerySessionStatus::QueryStatus new_query_status)

void	enrollQuerySession (const QuerySessionId &query_session, const std::string &query_str, const std::string &submitted_time_str, const size_t executor_id, const QuerySessionStatus::QueryStatus query_session_status)

size_t	getNumCurentSessionsEnrolled () const

const std::vector< size_t >	getExecutorIdsRunningQuery (const QuerySessionId &interrupt_session) const

bool	checkNonKernelTimeInterrupted () const

void	registerExtractedQueryPlanDag (const QueryPlanDAG &query_plan_dag)

const QueryPlanDAG	getLatestQueryPlanDagExtracted () const

void	addToCardinalityCache (const std::string &cache_key, const size_t cache_value)

CachedCardinality	getCachedCardinality (const std::string &cache_key)

heavyai::shared_mutex &	getDataRecyclerLock ()

QueryPlanDagCache &	getQueryPlanDagCache ()

ResultSetRecyclerHolder &	getRecultSetRecyclerHolder ()

CgenState *	getCgenStatePtr () const

PlanState *	getPlanStatePtr () const

llvm::LLVMContext &	getContext ()

void	update_extension_modules (bool update_runtime_modules_only=false)

Static Public Member Functions
static void	clearExternalCaches (bool for_update, const TableDescriptor *td, const int current_db_id)

template<typename F >
static void	registerExtensionFunctions (F register_extension_functions)

static std::shared_ptr< Executor >	getExecutor (const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())

static void	nukeCacheOfExecutors ()

static void	clearMemory (const Data_Namespace::MemoryLevel memory_level)

static size_t	getArenaBlockSize ()

static void	addUdfIrToModule (const std::string &udf_ir_filename, const bool is_cuda_ir)

static void	initialize_extension_module_sources ()

static void	registerActiveModule (void *module, const int device_id)

static void	unregisterActiveModule (const int device_id)

static std::pair< int64_t, int32_t >	reduceResults (const SQLAgg agg, const SQLTypeInfo &ti, const int64_t agg_init_val, const int8_t out_byte_width, const int64_t *out_vec, const size_t out_vec_sz, const bool is_group_by, const bool float_argument_input)

static void	update_after_registration (bool update_runtime_modules_only=false)

Public Attributes
std::mutex	compilation_mutex_

const logger::ThreadId	thread_id_

Static Public Attributes
static const ExecutorId	UNITARY_EXECUTOR_ID = 0

static const ExecutorId	INVALID_EXECUTOR_ID = SIZE_MAX

static std::map < ExtModuleKinds, std::string >	extension_module_sources

static const size_t	high_scan_limit

static const int32_t	ERR_DIV_BY_ZERO {1}

static const int32_t	ERR_OUT_OF_GPU_MEM {2}

static const int32_t	ERR_OUT_OF_SLOTS {3}

static const int32_t	ERR_UNSUPPORTED_SELF_JOIN {4}

static const int32_t	ERR_OUT_OF_RENDER_MEM {5}

static const int32_t	ERR_OUT_OF_CPU_MEM {6}

static const int32_t	ERR_OVERFLOW_OR_UNDERFLOW {7}

static const int32_t	ERR_OUT_OF_TIME {9}

static const int32_t	ERR_INTERRUPTED {10}

static const int32_t	ERR_COLUMNAR_CONVERSION_NOT_SUPPORTED {11}

static const int32_t	ERR_TOO_MANY_LITERALS {12}

static const int32_t	ERR_STRING_CONST_IN_RESULTSET {13}

static const int32_t	ERR_STREAMING_TOP_N_NOT_SUPPORTED_IN_RENDER_QUERY {14}

static const int32_t	ERR_SINGLE_VALUE_FOUND_MULTIPLE_VALUES {15}

static const int32_t	ERR_GEOS {16}

static const int32_t	ERR_WIDTH_BUCKET_INVALID_ARGUMENT {17}

static std::mutex	register_runtime_extension_functions_mutex_

static std::mutex	kernel_mutex_

Private Types
using	PerFragmentCallBack = std::function< void(ResultSetPtr, const Fragmenter_Namespace::FragmentInfo &)>

Private Member Functions
void	clearMetaInfoCache ()

int	deviceCountForMemoryLevel (const Data_Namespace::MemoryLevel memory_level) const

llvm::Value *	codegenWindowFunction (const size_t target_index, const CompilationOptions &co)

llvm::Value *	codegenWindowFunctionAggregate (const CompilationOptions &co)

llvm::BasicBlock *	codegenWindowResetStateControlFlow ()

void	codegenWindowFunctionStateInit (llvm::Value *aggregate_state)

llvm::Value *	codegenWindowFunctionAggregateCalls (llvm::Value *aggregate_state, const CompilationOptions &co)

llvm::Value *	codegenWindowFunctionOnFrame (const CompilationOptions &co)

llvm::Value *	codegenCurrentPartitionIndex (const WindowFunctionContext window_func_context, llvm::Value current_row_pos_lv)

llvm::Value *	codegenFrameBoundExpr (const Analyzer::WindowFunction window_func, const Analyzer::WindowFrame frame_bound, CodeGenerator &code_generator, const CompilationOptions &co)

std::pair< llvm::Value , llvm::Value >	codegenWindowFrameBound (WindowFunctionContext window_func_context, const Analyzer::WindowFrame frame_start_bound, const Analyzer::WindowFrame frame_end_bound, llvm::Value current_row_pos_lv, llvm::Value current_partition_start_offset_lv, llvm::Value order_key_buf_ptr_lv, llvm::Value order_key_col_null_val_lv, llvm::Value frame_start_bound_expr_lv, llvm::Value frame_end_bound_expr_lv, llvm::Value num_elem_current_partition_lv, llvm::Value target_partition_rowid_ptr_lv, llvm::Value target_partition_sorted_rowid_ptr_lv, llvm::Value null_start_pos_lv, llvm::Value null_end_pos_lv, CodeGenerator &code_generator)

void	codegenWindowAvgEpilogue (llvm::Value crt_val, llvm::Value window_func_null_val, llvm::Value *multiplicity_lv)

llvm::Value *	codegenAggregateWindowState ()

llvm::Value *	aggregateWindowStatePtr ()

CudaMgr_Namespace::CudaMgr *	cudaMgr () const

bool	isArchPascalOrLater (const ExecutorDeviceType dt) const

bool	needFetchAllFragments (const InputColDescriptor &col_desc, const RelAlgExecutionUnit &ra_exe_unit, const FragmentsList &selected_fragments) const

bool	needLinearizeAllFragments (const ColumnDescriptor *cd, const InputColDescriptor &inner_col_desc, const RelAlgExecutionUnit &ra_exe_unit, const FragmentsList &selected_fragments, const Data_Namespace::MemoryLevel memory_level) const

void	executeWorkUnitPerFragment (const RelAlgExecutionUnit &ra_exe_unit, const InputTableInfo &table_info, const CompilationOptions &co, const ExecutionOptions &eo, const Catalog_Namespace::Catalog &cat, PerFragmentCallBack &cb, const std::set< size_t > &fragment_indexes_param)
	Compiles and dispatches a work unit per fragment processing results with the per fragment callback. Currently used for computing metrics over fragments (metadata). More...

ResultSetPtr	executeExplain (const QueryCompilationDescriptor &)

ResultSetPtr	executeTableFunction (const TableFunctionExecutionUnit exe_unit, const std::vector< InputTableInfo > &table_infos, const CompilationOptions &co, const ExecutionOptions &eo, const Catalog_Namespace::Catalog &cat)
	Compiles and dispatches a table function; that is, a function that takes as input one or more columns and returns a ResultSet, which can be parsed by subsequent execution steps. More...

ExecutorDeviceType	getDeviceTypeForTargets (const RelAlgExecutionUnit &ra_exe_unit, const ExecutorDeviceType requested_device_type)

ResultSetPtr	collectAllDeviceResults (SharedKernelContext &shared_context, const RelAlgExecutionUnit &ra_exe_unit, const QueryMemoryDescriptor &query_mem_desc, const ExecutorDeviceType device_type, std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner)

ResultSetPtr	collectAllDeviceShardedTopResults (SharedKernelContext &shared_context, const RelAlgExecutionUnit &ra_exe_unit) const

std::unordered_map< int, const Analyzer::BinOper * >	getInnerTabIdToJoinCond () const

std::vector< std::unique_ptr < ExecutionKernel > >	createKernels (SharedKernelContext &shared_context, const RelAlgExecutionUnit &ra_exe_unit, ColumnFetcher &column_fetcher, const std::vector< InputTableInfo > &table_infos, const ExecutionOptions &eo, const bool is_agg, const bool allow_single_frag_table_opt, const size_t context_count, const QueryCompilationDescriptor &query_comp_desc, const QueryMemoryDescriptor &query_mem_desc, RenderInfo *render_info, std::unordered_set< int > &available_gpus, int &available_cpus)

void	launchKernels (SharedKernelContext &shared_context, std::vector< std::unique_ptr< ExecutionKernel >> &&kernels, const ExecutorDeviceType device_type)

std::vector< size_t >	getTableFragmentIndices (const RelAlgExecutionUnit &ra_exe_unit, const ExecutorDeviceType device_type, const size_t table_idx, const size_t outer_frag_idx, std::map< int, const TableFragments * > &selected_tables_fragments, const std::unordered_map< int, const Analyzer::BinOper * > &inner_table_id_to_join_condition)

bool	skipFragmentPair (const Fragmenter_Namespace::FragmentInfo &outer_fragment_info, const Fragmenter_Namespace::FragmentInfo &inner_fragment_info, const int inner_table_id, const std::unordered_map< int, const Analyzer::BinOper * > &inner_table_id_to_join_condition, const RelAlgExecutionUnit &ra_exe_unit, const ExecutorDeviceType device_type)

FetchResult	fetchChunks (const ColumnFetcher &, const RelAlgExecutionUnit &ra_exe_unit, const int device_id, const Data_Namespace::MemoryLevel, const std::map< int, const TableFragments * > &, const FragmentsList &selected_fragments, const Catalog_Namespace::Catalog &, std::list< ChunkIter > &, std::list< std::shared_ptr< Chunk_NS::Chunk >> &, DeviceAllocator *device_allocator, const size_t thread_idx, const bool allow_runtime_interrupt)

FetchResult	fetchUnionChunks (const ColumnFetcher &, const RelAlgExecutionUnit &ra_exe_unit, const int device_id, const Data_Namespace::MemoryLevel, const std::map< int, const TableFragments * > &, const FragmentsList &selected_fragments, const Catalog_Namespace::Catalog &, std::list< ChunkIter > &, std::list< std::shared_ptr< Chunk_NS::Chunk >> &, DeviceAllocator *device_allocator, const size_t thread_idx, const bool allow_runtime_interrupt)

std::pair< std::vector < std::vector< int64_t > >, std::vector< std::vector < uint64_t > > >	getRowCountAndOffsetForAllFrags (const RelAlgExecutionUnit &ra_exe_unit, const CartesianProduct< std::vector< std::vector< size_t >>> &frag_ids_crossjoin, const std::vector< InputDescriptor > &input_descs, const std::map< int, const TableFragments * > &all_tables_fragments)

void	buildSelectedFragsMapping (std::vector< std::vector< size_t >> &selected_fragments_crossjoin, std::vector< size_t > &local_col_to_frag_pos, const std::list< std::shared_ptr< const InputColDescriptor >> &col_global_ids, const FragmentsList &selected_fragments, const RelAlgExecutionUnit &ra_exe_unit)

void	buildSelectedFragsMappingForUnion (std::vector< std::vector< size_t >> &selected_fragments_crossjoin, const FragmentsList &selected_fragments, const RelAlgExecutionUnit &ra_exe_unit)

std::vector< size_t >	getFragmentCount (const FragmentsList &selected_fragments, const size_t scan_idx, const RelAlgExecutionUnit &ra_exe_unit)

int32_t	executePlanWithGroupBy (const RelAlgExecutionUnit &ra_exe_unit, const CompilationResult &, const bool hoist_literals, ResultSetPtr results, const ExecutorDeviceType device_type, std::vector< std::vector< const int8_t >> &col_buffers, const std::vector< size_t > outer_tab_frag_ids, QueryExecutionContext , const std::vector< std::vector< int64_t >> &num_rows, const std::vector< std::vector< uint64_t >> &frag_offsets, Data_Namespace::DataMgr , const int device_id, const int outer_table_id, const int64_t limit, const uint32_t start_rowid, const uint32_t num_tables, const bool allow_runtime_interrupt, RenderInfo *render_info, const int64_t rows_to_process=-1)

int32_t	executePlanWithoutGroupBy (const RelAlgExecutionUnit &ra_exe_unit, const CompilationResult &, const bool hoist_literals, ResultSetPtr results, const std::vector< Analyzer::Expr > &target_exprs, const ExecutorDeviceType device_type, std::vector< std::vector< const int8_t * >> &col_buffers, QueryExecutionContext query_exe_context, const std::vector< std::vector< int64_t >> &num_rows, const std::vector< std::vector< uint64_t >> &frag_offsets, Data_Namespace::DataMgr data_mgr, const int device_id, const uint32_t start_rowid, const uint32_t num_tables, const bool allow_runtime_interrupt, RenderInfo *render_info, const int64_t rows_to_process=-1)

ResultSetPtr	resultsUnion (SharedKernelContext &shared_context, const RelAlgExecutionUnit &ra_exe_unit)

std::vector< int8_t * >	getJoinHashTablePtrs (const ExecutorDeviceType device_type, const int device_id)

ResultSetPtr	reduceMultiDeviceResults (const RelAlgExecutionUnit &, std::vector< std::pair< ResultSetPtr, std::vector< size_t >>> &all_fragment_results, std::shared_ptr< RowSetMemoryOwner >, const QueryMemoryDescriptor &) const

ResultSetPtr	reduceMultiDeviceResultSets (std::vector< std::pair< ResultSetPtr, std::vector< size_t >>> &all_fragment_results, std::shared_ptr< RowSetMemoryOwner >, const QueryMemoryDescriptor &) const

ResultSetPtr	reduceSpeculativeTopN (const RelAlgExecutionUnit &, std::vector< std::pair< ResultSetPtr, std::vector< size_t >>> &all_fragment_results, std::shared_ptr< RowSetMemoryOwner >, const QueryMemoryDescriptor &) const

ResultSetPtr	executeWorkUnitImpl (size_t &max_groups_buffer_entry_guess, const bool is_agg, const bool allow_single_frag_table_opt, const std::vector< InputTableInfo > &, const RelAlgExecutionUnit &, const CompilationOptions &, const ExecutionOptions &options, const Catalog_Namespace::Catalog &, std::shared_ptr< RowSetMemoryOwner >, RenderInfo *render_info, const bool has_cardinality_estimation, ColumnCacheMap &column_cache)

std::vector< llvm::Value * >	inlineHoistedLiterals ()

void	AutoTrackBuffersInRuntimeIR ()

std::tuple< CompilationResult, std::unique_ptr < QueryMemoryDescriptor > >	compileWorkUnit (const std::vector< InputTableInfo > &query_infos, const PlanState::DeletedColumnsMap &deleted_cols_map, const RelAlgExecutionUnit &ra_exe_unit, const CompilationOptions &co, const ExecutionOptions &eo, const CudaMgr_Namespace::CudaMgr cuda_mgr, const bool allow_lazy_fetch, std::shared_ptr< RowSetMemoryOwner >, const size_t max_groups_buffer_entry_count, const int8_t crt_min_byte_width, const bool has_cardinality_estimation, ColumnCacheMap &column_cache, RenderInfo render_info=nullptr)

llvm::BasicBlock *	codegenSkipDeletedOuterTableRow (const RelAlgExecutionUnit &ra_exe_unit, const CompilationOptions &co)

std::vector< JoinLoop >	buildJoinLoops (RelAlgExecutionUnit &ra_exe_unit, const CompilationOptions &co, const ExecutionOptions &eo, const std::vector< InputTableInfo > &query_infos, ColumnCacheMap &column_cache)

JoinLoop::HoistedFiltersCallback	buildHoistLeftHandSideFiltersCb (const RelAlgExecutionUnit &ra_exe_unit, const size_t level_idx, const int inner_table_id, const CompilationOptions &co)

std::function< llvm::Value (const std::vector < llvm::Value > &, llvm::Value *)>	buildIsDeletedCb (const RelAlgExecutionUnit &ra_exe_unit, const size_t level_idx, const CompilationOptions &co)

std::shared_ptr< HashJoin >	buildCurrentLevelHashTable (const JoinCondition &current_level_join_conditions, size_t level_idx, RelAlgExecutionUnit &ra_exe_unit, const CompilationOptions &co, const std::vector< InputTableInfo > &query_infos, ColumnCacheMap &column_cache, std::vector< std::string > &fail_reasons)

void	redeclareFilterFunction ()

llvm::Value *	addJoinLoopIterator (const std::vector< llvm::Value * > &prev_iters, const size_t level_idx)

void	codegenJoinLoops (const std::vector< JoinLoop > &join_loops, const RelAlgExecutionUnit &ra_exe_unit, GroupByAndAggregate &group_by_and_aggregate, llvm::Function query_func, llvm::BasicBlock entry_bb, QueryMemoryDescriptor &query_mem_desc, const CompilationOptions &co, const ExecutionOptions &eo)

bool	compileBody (const RelAlgExecutionUnit &ra_exe_unit, GroupByAndAggregate &group_by_and_aggregate, QueryMemoryDescriptor &query_mem_desc, const CompilationOptions &co, const GpuSharedMemoryContext &gpu_smem_context={})

void	createErrorCheckControlFlow (llvm::Function *query_func, bool run_with_dynamic_watchdog, bool run_with_allowing_runtime_interrupt, const std::vector< JoinLoop > &join_loops, ExecutorDeviceType device_type, const std::vector< InputTableInfo > &input_table_infos)

void	insertErrorCodeChecker (llvm::Function *query_func, bool hoist_literals, bool allow_runtime_query_interrupt)

void	preloadFragOffsets (const std::vector< InputDescriptor > &input_descs, const std::vector< InputTableInfo > &query_infos)

JoinHashTableOrError	buildHashTableForQualifier (const std::shared_ptr< Analyzer::BinOper > &qual_bin_oper, const std::vector< InputTableInfo > &query_infos, const MemoryLevel memory_level, const JoinType join_type, const HashType preferred_hash_type, ColumnCacheMap &column_cache, const HashTableBuildDagMap &hashtable_build_dag_map, const RegisteredQueryHint &query_hint, const TableIdToNodeMap &table_id_to_node_map)

void	nukeOldState (const bool allow_lazy_fetch, const std::vector< InputTableInfo > &query_infos, const PlanState::DeletedColumnsMap &deleted_cols_map, const RelAlgExecutionUnit *ra_exe_unit)

std::shared_ptr < CompilationContext >	optimizeAndCodegenCPU (llvm::Function , llvm::Function , const std::unordered_set< llvm::Function * > &, const CompilationOptions &)

std::shared_ptr < CompilationContext >	optimizeAndCodegenGPU (llvm::Function , llvm::Function , std::unordered_set< llvm::Function * > &, const bool no_inline, const CudaMgr_Namespace::CudaMgr *cuda_mgr, const bool is_gpu_smem_used, const CompilationOptions &)

std::string	generatePTX (const std::string &) const

void	initializeNVPTXBackend () const

int64_t	deviceCycles (int milliseconds) const

GroupColLLVMValue	groupByColumnCodegen (Analyzer::Expr group_by_col, const size_t col_width, const CompilationOptions &, const bool translate_null_val, const int64_t translated_null_val, DiamondCodegen &, std::stack< llvm::BasicBlock > &, const bool thread_mem_shared)

llvm::Value *	castToFP (llvm::Value *, SQLTypeInfo const &from_ti, SQLTypeInfo const &to_ti)

llvm::Value *	castToIntPtrTyIn (llvm::Value *val, const size_t bit_width)

std::tuple < RelAlgExecutionUnit, PlanState::DeletedColumnsMap >	addDeletedColumn (const RelAlgExecutionUnit &ra_exe_unit, const CompilationOptions &co)

bool	isFragmentFullyDeleted (const int table_id, const Fragmenter_Namespace::FragmentInfo &fragment)

FragmentSkipStatus	canSkipFragmentForFpQual (const Analyzer::BinOper comp_expr, const Analyzer::ColumnVar lhs_col, const Fragmenter_Namespace::FragmentInfo &fragment, const Analyzer::Constant *rhs_const) const

std::pair< bool, int64_t >	skipFragment (const InputDescriptor &table_desc, const Fragmenter_Namespace::FragmentInfo &frag_info, const std::list< std::shared_ptr< Analyzer::Expr >> &simple_quals, const std::vector< uint64_t > &frag_offsets, const size_t frag_idx)

std::pair< bool, int64_t >	skipFragmentInnerJoins (const InputDescriptor &table_desc, const RelAlgExecutionUnit &ra_exe_unit, const Fragmenter_Namespace::FragmentInfo &fragment, const std::vector< uint64_t > &frag_offsets, const size_t frag_idx)

AggregatedColRange	computeColRangesCache (const std::unordered_set< PhysicalInput > &phys_inputs)

StringDictionaryGenerations	computeStringDictionaryGenerations (const std::unordered_set< PhysicalInput > &phys_inputs)

TableGenerations	computeTableGenerations (std::unordered_set< int > phys_table_ids)

std::vector< int8_t >	serializeLiterals (const std::unordered_map< int, CgenState::LiteralValues > &literals, const int device_id)

const std::unique_ptr < llvm::Module > &	get_extension_module (ExtModuleKinds kind) const

bool	has_extension_module (ExtModuleKinds kind) const

llvm::Value *	spillDoubleElement (llvm::Value elem_val, llvm::Type elem_ty)

ExecutorMutexHolder	acquireExecuteMutex ()

Static Private Member Functions
static size_t	align (const size_t off_in, const size_t alignment)

Private Attributes
const ExecutorId	executor_id_

std::unique_ptr < llvm::LLVMContext >	context_

std::unique_ptr< CgenState >	cgen_state_

std::map< ExtModuleKinds, std::unique_ptr< llvm::Module > >	extension_modules_

std::unique_ptr< PlanState >	plan_state_

std::shared_ptr < RowSetMemoryOwner >	row_set_mem_owner_

std::mutex	gpu_exec_mutex_ [max_gpu_count]

std::atomic< bool >	interrupted_

std::mutex	str_dict_mutex_

std::unique_ptr < llvm::TargetMachine >	nvptx_target_machine_

unsigned	block_size_x_

unsigned	grid_size_x_

const size_t	max_gpu_slab_size_

const std::string	debug_dir_

const std::string	debug_file_

const Catalog_Namespace::Catalog *	catalog_

Data_Namespace::DataMgr *	data_mgr_

const TemporaryTables *	temporary_tables_

TableIdToNodeMap	table_id_to_node_map_

int64_t	kernel_queue_time_ms_ = 0

int64_t	compilation_queue_time_ms_ = 0

std::unique_ptr < WindowProjectNodeContext >	window_project_node_context_owned_

WindowFunctionContext *	active_window_function_ {nullptr}

InputTableInfoCache	input_table_info_cache_

AggregatedColRange	agg_col_range_cache_

TableGenerations	table_generations_

QuerySessionId	current_query_session_

Static Private Attributes
static const int	max_gpu_count {16}

static std::mutex	gpu_active_modules_mutex_

static uint32_t	gpu_active_modules_device_mask_ {0x0}

static void *	gpu_active_modules_ [max_gpu_count]

static const size_t	baseline_threshold

static heavyai::shared_mutex	executor_session_mutex_

static InterruptFlagMap	queries_interrupt_flag_

static QuerySessionMap	queries_session_map_

static std::map< int, std::shared_ptr< Executor > >	executors_

static heavyai::shared_mutex	execute_mutex_

static heavyai::shared_mutex	executors_cache_mutex_

static QueryPlanDagCache	query_plan_dag_cache_

static heavyai::shared_mutex	recycler_mutex_

static std::unordered_map < std::string, size_t >	cardinality_cache_

static ResultSetRecyclerHolder	resultset_recycler_holder_

static QueryPlanDAG	latest_query_plan_extracted_ {EMPTY_QUERY_PLAN}

Friends
class	BaselineJoinHashTable

class	CodeGenerator

class	ColumnFetcher

struct	DiamondCodegen

class	ExecutionKernel

class	KernelSubtask

class	HashJoin

class	OverlapsJoinHashTable

class	RangeJoinHashTable

class	GroupByAndAggregate

class	QueryCompilationDescriptor

class	QueryMemoryDescriptor

class	QueryMemoryInitializer

class	QueryFragmentDescriptor

class	QueryExecutionContext

class	ResultSet

class	InValuesBitmap

class	StringDictionaryTranslationMgr

class	LeafAggregator

class	PerfectJoinHashTable

class	QueryRewriter

class	PendingExecutionClosure

class	RelAlgExecutor

class	TableOptimizer

class	TableFunctionCompilationContext

class	TableFunctionExecutionContext

struct	TargetExprCodegenBuilder

struct	TargetExprCodegen

class	WindowProjectNodeContext

Detailed Description

Definition at line 368 of file Execute.h.

Member Typedef Documentation

using Executor::CachedCardinality = std::pair<bool, size_t>

Definition at line 1209 of file Execute.h.

using Executor::ExecutorId = size_t

Definition at line 375 of file Execute.h.

using Executor::PerFragmentCallBack = std::function<void(ResultSetPtr, const Fragmenter_Namespace::FragmentInfo&)>

private

Definition at line 739 of file Execute.h.

Member Enumeration Documentation

enum Executor::ExtModuleKinds

strong

Enumerator
template_module
udf_cpu_module
udf_gpu_module
rt_udf_cpu_module
rt_udf_gpu_module
rt_geos_module
rt_libdevice_module

Definition at line 469 of file Execute.h.

                             {
     template_module,     // RuntimeFunctions.bc
     udf_cpu_module,      // Load-time UDFs for CPU execution
     udf_gpu_module,      // Load-time UDFs for GPU execution
     rt_udf_cpu_module,   // Run-time UDF/UDTFs for CPU execution
     rt_udf_gpu_module,   // Run-time UDF/UDTFs for GPU execution
     rt_geos_module,      // geos functions
     rt_libdevice_module  // math library functions for GPU execution
   };

Constructor & Destructor Documentation

Executor::Executor	(	const ExecutorId	id,
		Data_Namespace::DataMgr *	data_mgr,
		const size_t	block_size_x,
		const size_t	grid_size_x,
		const size_t	max_gpu_slab_size,
		const std::string &	debug_dir,
		const std::string &	debug_file
	)

Definition at line 244 of file Execute.cpp.

     : executor_id_(executor_id)
     , context_(new llvm::LLVMContext())
     , cgen_state_(new CgenState({}, false, this))

Member Function Documentation

ExecutorMutexHolder Executor::acquireExecuteMutex ( )

inlineprivate

Definition at line 1357 of file Execute.h.

References execute_mutex_, executor_id_, Executor::ExecutorMutexHolder::shared_lock, Executor::ExecutorMutexHolder::unique_lock, and UNITARY_EXECUTOR_ID.

                                                    {
     ExecutorMutexHolder ret;
     if (executor_id_ == Executor::UNITARY_EXECUTOR_ID) {
       // Only one unitary executor can run at a time
       ret.unique_lock = heavyai::unique_lock<heavyai::shared_mutex>(execute_mutex_);
     } else {
       ret.shared_lock = heavyai::shared_lock<heavyai::shared_mutex>(execute_mutex_);
     }
     return ret;
   }

std::tuple< RelAlgExecutionUnit, PlanState::DeletedColumnsMap > Executor::addDeletedColumn	(	const RelAlgExecutionUnit &	ra_exe_unit,
		const CompilationOptions &	co
	)

private

Definition at line 3958 of file Execute.cpp.

References anonymous_namespace{Execute.cpp}::add_deleted_col_to_map(), catalog_, CHECK, CompilationOptions::filter_on_deleted_column, Catalog_Namespace::Catalog::getDeletedColumnIfRowsDeleted(), Catalog_Namespace::Catalog::getMetadataForTable(), and TABLE.

Referenced by executeWorkUnitImpl(), and executeWorkUnitPerFragment().

                                   {
   if (!co.filter_on_deleted_column) {
     return std::make_tuple(ra_exe_unit, PlanState::DeletedColumnsMap{});
   }
   auto ra_exe_unit_with_deleted = ra_exe_unit;
   PlanState::DeletedColumnsMap deleted_cols_map;
   for (const auto& input_table : ra_exe_unit_with_deleted.input_descs) {
     if (input_table.getSourceType() != InputSourceType::TABLE) {
       continue;
     }
     const auto td = catalog_->getMetadataForTable(input_table.getTableId());
     CHECK(td);
     const auto deleted_cd = catalog_->getDeletedColumnIfRowsDeleted(td);
     if (!deleted_cd) {
       continue;
     }
     CHECK(deleted_cd->columnType.is_boolean());
     // check deleted column is not already present
     bool found = false;
     for (const auto& input_col : ra_exe_unit_with_deleted.input_col_descs) {
       if (input_col.get()->getColId() == deleted_cd->columnId &&
           input_col.get()->getScanDesc().getTableId() == deleted_cd->tableId &&
           input_col.get()->getScanDesc().getNestLevel() == input_table.getNestLevel()) {
         found = true;
         add_deleted_col_to_map(deleted_cols_map, deleted_cd);
         break;
       }
     }
     if (!found) {
       // add deleted column
       ra_exe_unit_with_deleted.input_col_descs.emplace_back(new InputColDescriptor(
           deleted_cd->columnId, deleted_cd->tableId, input_table.getNestLevel()));
       add_deleted_col_to_map(deleted_cols_map, deleted_cd);
     }
   }
   return std::make_tuple(ra_exe_unit_with_deleted, deleted_cols_map);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

llvm::Value * Executor::addJoinLoopIterator	(	const std::vector< llvm::Value * > &	prev_iters,
		const size_t	level_idx
	)

private

Definition at line 1094 of file IRCodegen.cpp.

References AUTOMATIC_IR_METADATA, and CHECK.

                                                                    {
   AUTOMATIC_IR_METADATA(cgen_state_.get());
   // Iterators are added for loop-outer joins when the head of the loop is generated,
   // then once again when the body if generated. Allow this instead of special handling
   // of call sites.
   const auto it = cgen_state_->scan_idx_to_hash_pos_.find(level_idx);
   if (it != cgen_state_->scan_idx_to_hash_pos_.end()) {
     return it->second;
   }
   CHECK(!prev_iters.empty());
   llvm::Value* matching_row_index = prev_iters.back();
   const auto it_ok =
       cgen_state_->scan_idx_to_hash_pos_.emplace(level_idx, matching_row_index);
   CHECK(it_ok.second);
   return matching_row_index;
 }

void Executor::addToCardinalityCache	(	const std::string &	cache_key,
		const size_t	cache_value
	)

Definition at line 4760 of file Execute.cpp.

References cardinality_cache_, g_use_estimator_result_cache, recycler_mutex_, and VLOG.

                                                                {
   if (g_use_estimator_result_cache) {
     heavyai::unique_lock<heavyai::shared_mutex> lock(recycler_mutex_);
     cardinality_cache_[cache_key] = cache_value;
     VLOG(1) << "Put estimated cardinality to the cache";
   }
 }

bool Executor::addToQuerySessionList	(	const QuerySessionId &	query_session,
		const std::string &	query_str,
		const std::string &	submitted,
		const size_t	executor_id,
		const QuerySessionStatus::QueryStatus	query_status,
		heavyai::unique_lock< heavyai::shared_mutex > &	write_lock
	)

Definition at line 4591 of file Execute.cpp.

References queries_interrupt_flag_, and queries_session_map_.

Referenced by enrollQuerySession().

                                                          {
   // an internal API that enrolls the query session into the Executor's session map
   if (queries_session_map_.count(query_session)) {
     if (queries_session_map_.at(query_session).count(submitted_time_str)) {
       queries_session_map_.at(query_session).erase(submitted_time_str);
       queries_session_map_.at(query_session)
           .emplace(submitted_time_str,
                    QuerySessionStatus(query_session,
                                       executor_id,
                                       query_str,
                                       submitted_time_str,
                                       query_status));
     } else {
       queries_session_map_.at(query_session)
           .emplace(submitted_time_str,
                    QuerySessionStatus(query_session,
                                       executor_id,
                                       query_str,
                                       submitted_time_str,
                                       query_status));
     }
   } else {
     std::map<std::string, QuerySessionStatus> executor_per_query_map;
     executor_per_query_map.emplace(
         submitted_time_str,
         QuerySessionStatus(
             query_session, executor_id, query_str, submitted_time_str, query_status));
     queries_session_map_.emplace(query_session, executor_per_query_map);
   }
   return queries_interrupt_flag_.emplace(query_session, false).second;
 }

Here is the caller graph for this function:

void Executor::addTransientStringLiterals	(	const RelAlgExecutionUnit &	ra_exe_unit,
		const std::shared_ptr< RowSetMemoryOwner > &	row_set_mem_owner
	)

Definition at line 2163 of file Execute.cpp.

References CHECK, getStringDictionaryProxy(), RelAlgExecutionUnit::groupby_exprs, kENCODING_DICT, kSAMPLE, kSINGLE_VALUE, RelAlgExecutionUnit::quals, RelAlgExecutionUnit::simple_quals, RelAlgExecutionUnit::target_exprs, RelAlgExecutionUnit::target_exprs_union, and ScalarExprVisitor< T >::visit().

                                                                {
   TransientDictIdVisitor dict_id_visitor;
 
   auto visit_expr =
       [this, &dict_id_visitor, &row_set_mem_owner](const Analyzer::Expr* expr) {
         if (!expr) {
           return;
         }
         const auto dict_id = dict_id_visitor.visit(expr);
         if (dict_id >= 0) {
           auto sdp = getStringDictionaryProxy(dict_id, row_set_mem_owner, true);
           CHECK(sdp);
           TransientStringLiteralsVisitor visitor(sdp, this);
           visitor.visit(expr);
         }
       };
 
   for (const auto& group_expr : ra_exe_unit.groupby_exprs) {
     visit_expr(group_expr.get());
   }
 
   for (const auto& group_expr : ra_exe_unit.quals) {
     visit_expr(group_expr.get());
   }
 
   for (const auto& group_expr : ra_exe_unit.simple_quals) {
     visit_expr(group_expr.get());
   }
 
   const auto visit_target_expr = [&](const Analyzer::Expr* target_expr) {
     const auto& target_type = target_expr->get_type_info();
     if (!target_type.is_string() || target_type.get_compression() == kENCODING_DICT) {
       const auto agg_expr = dynamic_cast<const Analyzer::AggExpr*>(target_expr);
       if (agg_expr) {
         if (agg_expr->get_aggtype() == kSINGLE_VALUE ||
             agg_expr->get_aggtype() == kSAMPLE) {
           visit_expr(agg_expr->get_arg());
         }
       } else {
         visit_expr(target_expr);
       }
     }
   };
   const auto& target_exprs = ra_exe_unit.target_exprs;
   std::for_each(target_exprs.begin(), target_exprs.end(), visit_target_expr);
   const auto& target_exprs_union = ra_exe_unit.target_exprs_union;
   std::for_each(target_exprs_union.begin(), target_exprs_union.end(), visit_target_expr);
 }

Here is the call graph for this function:

void Executor::addUdfIrToModule	(	const std::string &	udf_ir_filename,
		const bool	is_cuda_ir
	)

static

Definition at line 1857 of file NativeCodegen.cpp.

Referenced by DBHandler::initialize().

                                                        {
   Executor::extension_module_sources[is_cuda_ir
                                          ? Executor::ExtModuleKinds::udf_gpu_module
                                          : Executor::ExtModuleKinds::udf_cpu_module] =
       udf_ir_filename;
 }

Here is the caller graph for this function:

llvm::Value * Executor::aggregateWindowStatePtr ( )

private

Definition at line 211 of file WindowFunctionIR.cpp.

References AUTOMATIC_IR_METADATA, anonymous_namespace{WindowFunctionIR.cpp}::get_adjusted_window_type_info(), get_int_type(), WindowProjectNodeContext::getActiveWindowFunctionContext(), and kFLOAT.

                                              {
   AUTOMATIC_IR_METADATA(cgen_state_.get());
   const auto window_func_context =
       WindowProjectNodeContext::getActiveWindowFunctionContext(this);
   const auto window_func = window_func_context->getWindowFunction();
   const auto arg_ti = get_adjusted_window_type_info(window_func);
   llvm::Type* aggregate_state_type =
       arg_ti.get_type() == kFLOAT
           ? llvm::PointerType::get(get_int_type(32, cgen_state_->context_), 0)
           : llvm::PointerType::get(get_int_type(64, cgen_state_->context_), 0);
   const auto aggregate_state_i64 = cgen_state_->llInt(
       reinterpret_cast<const int64_t>(window_func_context->aggregateState()));
   return cgen_state_->ir_builder_.CreateIntToPtr(aggregate_state_i64,
                                                  aggregate_state_type);
 }

Here is the call graph for this function:

static size_t Executor::align	(	const size_t	off_in,
		const size_t	alignment
	)

inlinestaticprivate

Definition at line 1234 of file Execute.h.

Referenced by serializeLiterals().

                                                                    {
     size_t off = off_in;
     if (off % alignment != 0) {
       off += (alignment - off % alignment);
     }
     return off;
   }

Here is the caller graph for this function:

CurrentQueryStatus Executor::attachExecutorToQuerySession	(	const QuerySessionId &	query_session_id,
		const std::string &	query_str,
		const std::string &	query_submitted_time
	)

Definition at line 4489 of file Execute.cpp.

References executor_id_, executor_session_mutex_, updateQuerySessionExecutorAssignment(), and updateQuerySessionStatusWithLock().

                                            {
   if (!query_session_id.empty()) {
     // if session is valid, do update 1) the exact executor id and 2) query status
     heavyai::unique_lock<heavyai::shared_mutex> write_lock(executor_session_mutex_);
     updateQuerySessionExecutorAssignment(
         query_session_id, query_submitted_time, executor_id_, write_lock);
     updateQuerySessionStatusWithLock(query_session_id,
                                      query_submitted_time,
                                      QuerySessionStatus::QueryStatus::PENDING_EXECUTOR,
                                      write_lock);
   }
   return {query_session_id, query_str};
 }

Here is the call graph for this function:

void Executor::AutoTrackBuffersInRuntimeIR ( )

private

Definition at line 2200 of file NativeCodegen.cpp.

References CHECK.

                                            {
   llvm::Module* M = cgen_state_->module_;
   if (M->getFunction("allocate_varlen_buffer") == nullptr)
     return;
 
   // read metadata
   bool should_track = false;
   auto* flag = M->getModuleFlag("manage_memory_buffer");
   if (auto* cnt = llvm::mdconst::extract_or_null<llvm::ConstantInt>(flag)) {
     if (cnt->getZExtValue() == 1) {
       should_track = true;
     }
   }
 
   if (!should_track) {
     // metadata is not present
     return;
   }
 
   LOG(INFO) << "Found 'manage_memory_buffer' metadata.";
   llvm::SmallVector<llvm::CallInst*, 4> calls_to_analyze;
 
   for (llvm::Function& F : *M) {
     for (llvm::BasicBlock& BB : F) {
       for (llvm::Instruction& I : BB) {
         if (llvm::CallInst* CI = llvm::dyn_cast<llvm::CallInst>(&I)) {
           // Keep track of calls to "allocate_varlen_buffer" for later processing
           llvm::Function* called = CI->getCalledFunction();
           if (called) {
             if (called->getName() == "allocate_varlen_buffer") {
               calls_to_analyze.push_back(CI);
             }
           }
         }
       }
     }
   }
 
   // for each call to "allocate_varlen_buffer", check if there's a corresponding
   // call to "register_buffer_with_executor_rsm". If not, add a call to it
   llvm::IRBuilder<> Builder(cgen_state_->context_);
   auto i64 = get_int_type(64, cgen_state_->context_);
   auto i8p = get_int_ptr_type(8, cgen_state_->context_);
   auto void_ = llvm::Type::getVoidTy(cgen_state_->context_);
   llvm::FunctionType* fnty = llvm::FunctionType::get(void_, {i64, i8p}, false);
   llvm::FunctionCallee register_buffer_fn =
       M->getOrInsertFunction("register_buffer_with_executor_rsm", fnty, {});
 
   int64_t executor_addr = reinterpret_cast<int64_t>(this);
   for (llvm::CallInst* CI : calls_to_analyze) {
     bool found = false;
     // for each user of the function, check if its a callinst
     // and if the callinst is calling "register_buffer_with_executor_rsm"
     // if no such instruction exist, add one registering the buffer
     for (llvm::User* U : CI->users()) {
       if (llvm::CallInst* call = llvm::dyn_cast<llvm::CallInst>(U)) {
         if (call->getCalledFunction() and
             call->getCalledFunction()->getName() == "register_buffer_with_executor_rsm") {
           found = true;
           break;
         }
       }
     }
     if (!found) {
       Builder.SetInsertPoint(CI->getNextNode());
       Builder.CreateCall(register_buffer_fn,
                          {ll_int(executor_addr, cgen_state_->context_), CI});
     }
   }
 }

unsigned Executor::blockSize ( ) const

Definition at line 3850 of file Execute.cpp.

References block_size_x_, CHECK, data_mgr_, CudaMgr_Namespace::CudaMgr::getAllDeviceProperties(), and Data_Namespace::DataMgr::getCudaMgr().

Referenced by collectAllDeviceShardedTopResults(), executePlanWithGroupBy(), executePlanWithoutGroupBy(), executeTableFunction(), executeWorkUnitImpl(), reduceMultiDeviceResults(), reduceMultiDeviceResultSets(), and resultsUnion().

                                    {
   CHECK(data_mgr_);
   const auto cuda_mgr = data_mgr_->getCudaMgr();
   if (!cuda_mgr) {
     return 0;
   }
   const auto& dev_props = cuda_mgr->getAllDeviceProperties();
   return block_size_x_ ? block_size_x_ : dev_props.front().maxThreadsPerBlock;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::shared_ptr< HashJoin > Executor::buildCurrentLevelHashTable	(	const JoinCondition &	current_level_join_conditions,
		size_t	level_idx,
		RelAlgExecutionUnit &	ra_exe_unit,
		const CompilationOptions &	co,
		const std::vector< InputTableInfo > &	query_infos,
		ColumnCacheMap &	column_cache,
		std::vector< std::string > &	fail_reasons
	)

private

Definition at line 935 of file IRCodegen.cpp.

References anonymous_namespace{IRCodegen.cpp}::add_qualifier_to_execution_unit(), AUTOMATIC_IR_METADATA, anonymous_namespace{IRCodegen.cpp}::check_valid_join_qual(), Data_Namespace::CPU_LEVEL, CompilationOptions::device_type, Executor::JoinHashTableOrError::fail_reason, GPU, Data_Namespace::GPU_LEVEL, Executor::JoinHashTableOrError::hash_table, RelAlgExecutionUnit::hash_table_build_plan_dag, IS_EQUIVALENCE, LEFT, OneToOne, JoinCondition::quals, RelAlgExecutionUnit::query_hint, RelAlgExecutionUnit::table_id_to_node_map, JoinCondition::type, and VLOG.

                                         {
   AUTOMATIC_IR_METADATA(cgen_state_.get());
   std::shared_ptr<HashJoin> current_level_hash_table;
   auto handleNonHashtableQual = [&ra_exe_unit, &level_idx, this](
                                     JoinType join_type,
                                     std::shared_ptr<Analyzer::Expr> qual) {
     if (join_type == JoinType::LEFT) {
       plan_state_->addNonHashtableQualForLeftJoin(level_idx, qual);
     } else {
       add_qualifier_to_execution_unit(ra_exe_unit, qual);
     }
   };
   for (const auto& join_qual : current_level_join_conditions.quals) {
     auto qual_bin_oper = std::dynamic_pointer_cast<Analyzer::BinOper>(join_qual);
     if (current_level_hash_table || !qual_bin_oper ||
         !IS_EQUIVALENCE(qual_bin_oper->get_optype())) {
       handleNonHashtableQual(current_level_join_conditions.type, join_qual);
       if (!current_level_hash_table) {
         fail_reasons.emplace_back("No equijoin expression found");
       }
       continue;
     }
     check_valid_join_qual(qual_bin_oper);
     JoinHashTableOrError hash_table_or_error;
     if (!current_level_hash_table) {
       hash_table_or_error = buildHashTableForQualifier(
           qual_bin_oper,
           query_infos,
           co.device_type == ExecutorDeviceType::GPU ? MemoryLevel::GPU_LEVEL
                                                     : MemoryLevel::CPU_LEVEL,
           current_level_join_conditions.type,
           HashType::OneToOne,
           column_cache,
           ra_exe_unit.hash_table_build_plan_dag,
           ra_exe_unit.query_hint,
           ra_exe_unit.table_id_to_node_map);
       current_level_hash_table = hash_table_or_error.hash_table;
     }
     if (hash_table_or_error.hash_table) {
       plan_state_->join_info_.join_hash_tables_.push_back(hash_table_or_error.hash_table);
       plan_state_->join_info_.equi_join_tautologies_.push_back(qual_bin_oper);
     } else {
       fail_reasons.push_back(hash_table_or_error.fail_reason);
       if (!current_level_hash_table) {
         VLOG(2) << "Building a hashtable based on a qual " << qual_bin_oper->toString()
                 << " fails: " << hash_table_or_error.fail_reason;
       }
       handleNonHashtableQual(current_level_join_conditions.type, qual_bin_oper);
     }
   }
   return current_level_hash_table;
 }

Here is the call graph for this function:

Executor::JoinHashTableOrError Executor::buildHashTableForQualifier	(	const std::shared_ptr< Analyzer::BinOper > &	qual_bin_oper,
		const std::vector< InputTableInfo > &	query_infos,
		const MemoryLevel	memory_level,
		const JoinType	join_type,
		const HashType	preferred_hash_type,
		ColumnCacheMap &	column_cache,
		const HashTableBuildDagMap &	hashtable_build_dag_map,
		const RegisteredQueryHint &	query_hint,
		const TableIdToNodeMap &	table_id_to_node_map
	)

private

Definition at line 3794 of file Execute.cpp.

References deviceCountForMemoryLevel(), ERR_INTERRUPTED, g_enable_dynamic_watchdog, g_enable_overlaps_hashjoin, HashJoin::getInstance(), and interrupted_.

                                                   {
   if (!g_enable_overlaps_hashjoin && qual_bin_oper->is_overlaps_oper()) {
     return {nullptr, "Overlaps hash join disabled, attempting to fall back to loop join"};
   }
   if (g_enable_dynamic_watchdog && interrupted_.load()) {
     throw QueryExecutionError(ERR_INTERRUPTED);
   }
   try {
     auto tbl = HashJoin::getInstance(qual_bin_oper,
                                      query_infos,
                                      memory_level,
                                      join_type,
                                      preferred_hash_type,
                                      deviceCountForMemoryLevel(memory_level),
                                      column_cache,
                                      this,
                                      hashtable_build_dag_map,
                                      query_hint,
                                      table_id_to_node_map);
     return {tbl, ""};
   } catch (const HashJoinFail& e) {
     return {nullptr, e.what()};
   }
 }

Here is the call graph for this function:

JoinLoop::HoistedFiltersCallback Executor::buildHoistLeftHandSideFiltersCb	(	const RelAlgExecutionUnit &	ra_exe_unit,
		const size_t	level_idx,
		const int	inner_table_id,
		const CompilationOptions &	co
	)

private

Definition at line 768 of file IRCodegen.cpp.

References AUTOMATIC_IR_METADATA, CHECK, CodeGenerator::codegen(), g_enable_left_join_filter_hoisting, RelAlgExecutionUnit::join_quals, LEFT, RelAlgExecutionUnit::quals, RelAlgExecutionUnit::simple_quals, CodeGenerator::toBool(), and VLOG.

                                   {
   if (!g_enable_left_join_filter_hoisting) {
     return nullptr;
   }
 
   const auto& current_level_join_conditions = ra_exe_unit.join_quals[level_idx];
   if (level_idx == 0 && current_level_join_conditions.type == JoinType::LEFT) {
     const auto& condition = current_level_join_conditions.quals.front();
     const auto bin_oper = dynamic_cast<const Analyzer::BinOper*>(condition.get());
     CHECK(bin_oper) << condition->toString();
     const auto rhs =
         dynamic_cast<const Analyzer::ColumnVar*>(bin_oper->get_right_operand());
     const auto lhs =
         dynamic_cast<const Analyzer::ColumnVar*>(bin_oper->get_left_operand());
     if (lhs && rhs && lhs->get_table_id() != rhs->get_table_id()) {
       const Analyzer::ColumnVar* selected_lhs{nullptr};
       // grab the left hand side column -- this is somewhat similar to normalize column
       // pair, and a better solution may be to hoist that function out of the join
       // framework and normalize columns at the top of build join loops
       if (lhs->get_table_id() == inner_table_id) {
         selected_lhs = rhs;
       } else if (rhs->get_table_id() == inner_table_id) {
         selected_lhs = lhs;
       }
       if (selected_lhs) {
         std::list<std::shared_ptr<Analyzer::Expr>> hoisted_quals;
         // get all LHS-only filters
         auto should_hoist_qual = [&hoisted_quals](const auto& qual, const int table_id) {
           CHECK(qual);
 
           ExprTableIdVisitor visitor;
           const auto table_ids = visitor.visit(qual.get());
           if (table_ids.size() == 1 && table_ids.find(table_id) != table_ids.end()) {
             hoisted_quals.push_back(qual);
           }
         };
         for (const auto& qual : ra_exe_unit.simple_quals) {
           should_hoist_qual(qual, selected_lhs->get_table_id());
         }
         for (const auto& qual : ra_exe_unit.quals) {
           should_hoist_qual(qual, selected_lhs->get_table_id());
         }
 
         // build the filters callback and return it
         if (!hoisted_quals.empty()) {
           return [this, hoisted_quals, co](llvm::BasicBlock* true_bb,
                                            llvm::BasicBlock* exit_bb,
                                            const std::string& loop_name,
                                            llvm::Function* parent_func,
                                            CgenState* cgen_state) -> llvm::BasicBlock* {
             // make sure we have quals to hoist
             bool has_quals_to_hoist = false;
             for (const auto& qual : hoisted_quals) {
               // check to see if the filter was previously hoisted. if all filters were
               // previously hoisted, this callback becomes a noop
               if (plan_state_->hoisted_filters_.count(qual) == 0) {
                 has_quals_to_hoist = true;
                 break;
               }
             }
 
             if (!has_quals_to_hoist) {
               return nullptr;
             }
 
             AUTOMATIC_IR_METADATA(cgen_state);
 
             llvm::IRBuilder<>& builder = cgen_state->ir_builder_;
             auto& context = builder.getContext();
 
             const auto filter_bb =
                 llvm::BasicBlock::Create(context,
                                          "hoisted_left_join_filters_" + loop_name,
                                          parent_func,
                                          /*insert_before=*/true_bb);
             builder.SetInsertPoint(filter_bb);
 
             llvm::Value* filter_lv = cgen_state_->llBool(true);
             CodeGenerator code_generator(this);
             CHECK(plan_state_);
             for (const auto& qual : hoisted_quals) {
               if (plan_state_->hoisted_filters_.insert(qual).second) {
                 // qual was inserted into the hoisted filters map, which means we have not
                 // seen this qual before. Generate filter.
                 VLOG(1) << "Generating code for hoisted left hand side qualifier "
                         << qual->toString();
                 auto cond = code_generator.toBool(
                     code_generator.codegen(qual.get(), true, co).front());
                 filter_lv = builder.CreateAnd(filter_lv, cond);
               }
             }
             CHECK(filter_lv->getType()->isIntegerTy(1));
 
             builder.CreateCondBr(filter_lv, true_bb, exit_bb);
             return filter_bb;
           };
         }
       }
     }
   }
   return nullptr;
 }

Here is the call graph for this function:

std::function< llvm::Value (const std::vector< llvm::Value > &, llvm::Value *)> Executor::buildIsDeletedCb	(	const RelAlgExecutionUnit &	ra_exe_unit,
		const size_t	level_idx,
		const CompilationOptions &	co
	)

private

Definition at line 876 of file IRCodegen.cpp.

References AUTOMATIC_IR_METADATA, CHECK, CHECK_LT, CodeGenerator::codegen(), CompilationOptions::filter_on_deleted_column, RelAlgExecutionUnit::input_descs, TABLE, and CodeGenerator::toBool().

                                                          {
   AUTOMATIC_IR_METADATA(cgen_state_.get());
   if (!co.filter_on_deleted_column) {
     return nullptr;
   }
   CHECK_LT(level_idx + 1, ra_exe_unit.input_descs.size());
   const auto input_desc = ra_exe_unit.input_descs[level_idx + 1];
   if (input_desc.getSourceType() != InputSourceType::TABLE) {
     return nullptr;
   }
 
   const auto deleted_cd = plan_state_->getDeletedColForTable(input_desc.getTableId());
   if (!deleted_cd) {
     return nullptr;
   }
   CHECK(deleted_cd->columnType.is_boolean());
   const auto deleted_expr = makeExpr<Analyzer::ColumnVar>(deleted_cd->columnType,
                                                           input_desc.getTableId(),
                                                           deleted_cd->columnId,
                                                           input_desc.getNestLevel());
   return [this, deleted_expr, level_idx, &co](const std::vector<llvm::Value*>& prev_iters,
                                               llvm::Value* have_more_inner_rows) {
     const auto matching_row_index = addJoinLoopIterator(prev_iters, level_idx + 1);
     // Avoid fetching the deleted column from a position which is not valid.
     // An invalid position can be returned by a one to one hash lookup (negative)
     // or at the end of iteration over a set of matching values.
     llvm::Value* is_valid_it{nullptr};
     if (have_more_inner_rows) {
       is_valid_it = have_more_inner_rows;
     } else {
       is_valid_it = cgen_state_->ir_builder_.CreateICmp(
           llvm::ICmpInst::ICMP_SGE, matching_row_index, cgen_state_->llInt<int64_t>(0));
     }
     const auto it_valid_bb = llvm::BasicBlock::Create(
         cgen_state_->context_, "it_valid", cgen_state_->current_func_);
     const auto it_not_valid_bb = llvm::BasicBlock::Create(
         cgen_state_->context_, "it_not_valid", cgen_state_->current_func_);
     cgen_state_->ir_builder_.CreateCondBr(is_valid_it, it_valid_bb, it_not_valid_bb);
     const auto row_is_deleted_bb = llvm::BasicBlock::Create(
         cgen_state_->context_, "row_is_deleted", cgen_state_->current_func_);
     cgen_state_->ir_builder_.SetInsertPoint(it_valid_bb);
     CodeGenerator code_generator(this);
     const auto row_is_deleted = code_generator.toBool(
         code_generator.codegen(deleted_expr.get(), true, co).front());
     cgen_state_->ir_builder_.CreateBr(row_is_deleted_bb);
     cgen_state_->ir_builder_.SetInsertPoint(it_not_valid_bb);
     const auto row_is_deleted_default = cgen_state_->llBool(false);
     cgen_state_->ir_builder_.CreateBr(row_is_deleted_bb);
     cgen_state_->ir_builder_.SetInsertPoint(row_is_deleted_bb);
     auto row_is_deleted_or_default =
         cgen_state_->ir_builder_.CreatePHI(row_is_deleted->getType(), 2);
     row_is_deleted_or_default->addIncoming(row_is_deleted, it_valid_bb);
     row_is_deleted_or_default->addIncoming(row_is_deleted_default, it_not_valid_bb);
     return row_is_deleted_or_default;
   };
 }

Here is the call graph for this function:

std::vector< JoinLoop > Executor::buildJoinLoops	(	RelAlgExecutionUnit &	ra_exe_unit,
		const CompilationOptions &	co,
		const ExecutionOptions &	eo,
		const std::vector< InputTableInfo > &	query_infos,
		ColumnCacheMap &	column_cache
	)

private

Definition at line 523 of file IRCodegen.cpp.

References AUTOMATIC_IR_METADATA, CodeGenerator::cgen_state_, CHECK, CHECK_LT, CodeGenerator::codegen(), INJECT_TIMER, CgenState::ir_builder_, RelAlgExecutionUnit::join_quals, LEFT, PlanState::left_join_non_hashtable_quals_, CgenState::llBool(), MultiSet, OneToOne, CgenState::outer_join_match_found_per_level_, CodeGenerator::plan_state_, Set, Singleton, JoinLoopDomain::slot_lookup_result, CodeGenerator::toBool(), and JoinLoopDomain::values_buffer.

                                   {
   INJECT_TIMER(buildJoinLoops);
   AUTOMATIC_IR_METADATA(cgen_state_.get());
   std::vector<JoinLoop> join_loops;
   for (size_t level_idx = 0, current_hash_table_idx = 0;
        level_idx < ra_exe_unit.join_quals.size();
        ++level_idx) {
     const auto& current_level_join_conditions = ra_exe_unit.join_quals[level_idx];
     std::vector<std::string> fail_reasons;
     const auto current_level_hash_table =
         buildCurrentLevelHashTable(current_level_join_conditions,
                                    level_idx,
                                    ra_exe_unit,
                                    co,
                                    query_infos,
                                    column_cache,
                                    fail_reasons);
     const auto found_outer_join_matches_cb =
         [this, level_idx](llvm::Value* found_outer_join_matches) {
           CHECK_LT(level_idx, cgen_state_->outer_join_match_found_per_level_.size());
           CHECK(!cgen_state_->outer_join_match_found_per_level_[level_idx]);
           cgen_state_->outer_join_match_found_per_level_[level_idx] =
               found_outer_join_matches;
         };
     const auto is_deleted_cb = buildIsDeletedCb(ra_exe_unit, level_idx, co);
     auto rem_left_join_quals_it =
         plan_state_->left_join_non_hashtable_quals_.find(level_idx);
     bool has_remaining_left_join_quals =
         rem_left_join_quals_it != plan_state_->left_join_non_hashtable_quals_.end() &&
         !rem_left_join_quals_it->second.empty();
     const auto outer_join_condition_remaining_quals_cb =
         [this, level_idx, &co](const std::vector<llvm::Value*>& prev_iters) {
           // when we have multiple quals for the left join in the current join level
           // we first try to build a hashtable by using one of the possible qual,
           // and deal with remaining quals as extra join conditions
           FetchCacheAnchor anchor(cgen_state_.get());
           addJoinLoopIterator(prev_iters, level_idx + 1);
           llvm::Value* left_join_cond = cgen_state_->llBool(true);
           CodeGenerator code_generator(this);
           auto it = plan_state_->left_join_non_hashtable_quals_.find(level_idx);
           if (it != plan_state_->left_join_non_hashtable_quals_.end()) {
             for (auto expr : it->second) {
               left_join_cond = cgen_state_->ir_builder_.CreateAnd(
                   left_join_cond,
                   code_generator.toBool(
                       code_generator.codegen(expr.get(), true, co).front()));
             }
           }
           return left_join_cond;
         };
     if (current_level_hash_table) {
       const auto hoisted_filters_cb = buildHoistLeftHandSideFiltersCb(
           ra_exe_unit, level_idx, current_level_hash_table->getInnerTableId(), co);
       if (current_level_hash_table->getHashType() == HashType::OneToOne) {
         join_loops.emplace_back(
             /*kind=*/JoinLoopKind::Singleton,
             /*type=*/current_level_join_conditions.type,
             /*iteration_domain_codegen=*/
             [this, current_hash_table_idx, level_idx, current_level_hash_table, &co](
                 const std::vector<llvm::Value*>& prev_iters) {
               addJoinLoopIterator(prev_iters, level_idx);
               JoinLoopDomain domain{{0}};
               domain.slot_lookup_result =
                   current_level_hash_table->codegenSlot(co, current_hash_table_idx);
               return domain;
             },
             /*outer_condition_match=*/
             current_level_join_conditions.type == JoinType::LEFT &&
                     has_remaining_left_join_quals
                 ? std::function<llvm::Value*(const std::vector<llvm::Value*>&)>(
                       outer_join_condition_remaining_quals_cb)
                 : nullptr,
             /*found_outer_matches=*/current_level_join_conditions.type == JoinType::LEFT
                 ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
                 : nullptr,
             /*hoisted_filters=*/hoisted_filters_cb,
             /*is_deleted=*/is_deleted_cb,
             /*nested_loop_join=*/false);
       } else if (auto range_join_table =
                      dynamic_cast<RangeJoinHashTable*>(current_level_hash_table.get())) {
         join_loops.emplace_back(
             /* kind= */ JoinLoopKind::MultiSet,
             /* type= */ current_level_join_conditions.type,
             /* iteration_domain_codegen= */
             [this,
              range_join_table,
              current_hash_table_idx,
              level_idx,
              current_level_hash_table,
              &co](const std::vector<llvm::Value*>& prev_iters) {
               addJoinLoopIterator(prev_iters, level_idx);
               JoinLoopDomain domain{{0}};
               CHECK(!prev_iters.empty());
               const auto matching_set = range_join_table->codegenMatchingSetWithOffset(
                   co, current_hash_table_idx, prev_iters.back());
               domain.values_buffer = matching_set.elements;
               domain.element_count = matching_set.count;
               return domain;
             },
             /* outer_condition_match= */
             current_level_join_conditions.type == JoinType::LEFT
                 ? std::function<llvm::Value*(const std::vector<llvm::Value*>&)>(
                       outer_join_condition_remaining_quals_cb)
                 : nullptr,
             /* found_outer_matches= */
             current_level_join_conditions.type == JoinType::LEFT
                 ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
                 : nullptr,
             /* hoisted_filters= */ nullptr,  // <<! TODO
             /* is_deleted= */ is_deleted_cb,
             /*nested_loop_join=*/false);
       } else {
         join_loops.emplace_back(
             /*kind=*/JoinLoopKind::Set,
             /*type=*/current_level_join_conditions.type,
             /*iteration_domain_codegen=*/
             [this, current_hash_table_idx, level_idx, current_level_hash_table, &co](
                 const std::vector<llvm::Value*>& prev_iters) {
               addJoinLoopIterator(prev_iters, level_idx);
               JoinLoopDomain domain{{0}};
               const auto matching_set = current_level_hash_table->codegenMatchingSet(
                   co, current_hash_table_idx);
               domain.values_buffer = matching_set.elements;
               domain.element_count = matching_set.count;
               return domain;
             },
             /*outer_condition_match=*/
             current_level_join_conditions.type == JoinType::LEFT
                 ? std::function<llvm::Value*(const std::vector<llvm::Value*>&)>(
                       outer_join_condition_remaining_quals_cb)
                 : nullptr,
             /*found_outer_matches=*/current_level_join_conditions.type == JoinType::LEFT
                 ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
                 : nullptr,
             /*hoisted_filters=*/hoisted_filters_cb,
             /*is_deleted=*/is_deleted_cb,
             /*nested_loop_join=*/false);
       }
       ++current_hash_table_idx;
     } else {
       const auto fail_reasons_str = current_level_join_conditions.quals.empty()
                                         ? "No equijoin expression found"
                                         : boost::algorithm::join(fail_reasons, " | ");
       check_if_loop_join_is_allowed(
           ra_exe_unit, eo, query_infos, level_idx, fail_reasons_str);
       // Callback provided to the `JoinLoop` framework to evaluate the (outer) join
       // condition.
       VLOG(1) << "Unable to build hash table, falling back to loop join: "
               << fail_reasons_str;
       const auto outer_join_condition_cb =
           [this, level_idx, &co, &current_level_join_conditions](
               const std::vector<llvm::Value*>& prev_iters) {
             // The values generated for the match path don't dominate all uses
             // since on the non-match path nulls are generated. Reset the cache
             // once the condition is generated to avoid incorrect reuse.
             FetchCacheAnchor anchor(cgen_state_.get());
             addJoinLoopIterator(prev_iters, level_idx + 1);
             llvm::Value* left_join_cond = cgen_state_->llBool(true);
             CodeGenerator code_generator(this);
             for (auto expr : current_level_join_conditions.quals) {
               left_join_cond = cgen_state_->ir_builder_.CreateAnd(
                   left_join_cond,
                   code_generator.toBool(
                       code_generator.codegen(expr.get(), true, co).front()));
             }
             return left_join_cond;
           };
       join_loops.emplace_back(
           /*kind=*/JoinLoopKind::UpperBound,
           /*type=*/current_level_join_conditions.type,
           /*iteration_domain_codegen=*/
           [this, level_idx](const std::vector<llvm::Value*>& prev_iters) {
             addJoinLoopIterator(prev_iters, level_idx);
             JoinLoopDomain domain{{0}};
             auto* arg = get_arg_by_name(cgen_state_->row_func_, "num_rows_per_scan");
             const auto rows_per_scan_ptr = cgen_state_->ir_builder_.CreateGEP(
                 arg->getType()->getScalarType()->getPointerElementType(),
                 arg,
                 cgen_state_->llInt(int32_t(level_idx + 1)));
             domain.upper_bound = cgen_state_->ir_builder_.CreateLoad(
                 rows_per_scan_ptr->getType()->getPointerElementType(),
                 rows_per_scan_ptr,
                 "num_rows_per_scan");
             return domain;
           },
           /*outer_condition_match=*/
           current_level_join_conditions.type == JoinType::LEFT
               ? std::function<llvm::Value*(const std::vector<llvm::Value*>&)>(
                     outer_join_condition_cb)
               : nullptr,
           /*found_outer_matches=*/
           current_level_join_conditions.type == JoinType::LEFT
               ? std::function<void(llvm::Value*)>(found_outer_join_matches_cb)
               : nullptr,
           /*hoisted_filters=*/nullptr,
           /*is_deleted=*/is_deleted_cb,
           /*nested_loop_join=*/true);
     }
   }
   return join_loops;
 }

Here is the call graph for this function:

void Executor::buildSelectedFragsMapping	(	std::vector< std::vector< size_t >> &	selected_fragments_crossjoin,
		std::vector< size_t > &	local_col_to_frag_pos,
		const std::list< std::shared_ptr< const InputColDescriptor >> &	col_global_ids,
		const FragmentsList &	selected_fragments,
		const RelAlgExecutionUnit &	ra_exe_unit
	)

private

Definition at line 3269 of file Execute.cpp.

References CHECK, CHECK_EQ, CHECK_LT, getFragmentCount(), RelAlgExecutionUnit::input_descs, and plan_state_.

Referenced by fetchChunks().

                                             {
   local_col_to_frag_pos.resize(plan_state_->global_to_local_col_ids_.size());
   size_t frag_pos{0};
   const auto& input_descs = ra_exe_unit.input_descs;
   for (size_t scan_idx = 0; scan_idx < input_descs.size(); ++scan_idx) {
     const int table_id = input_descs[scan_idx].getTableId();
     CHECK_EQ(selected_fragments[scan_idx].table_id, table_id);
     selected_fragments_crossjoin.push_back(
         getFragmentCount(selected_fragments, scan_idx, ra_exe_unit));
     for (const auto& col_id : col_global_ids) {
       CHECK(col_id);
       const auto& input_desc = col_id->getScanDesc();
       if (input_desc.getTableId() != table_id ||
           input_desc.getNestLevel() != static_cast<int>(scan_idx)) {
         continue;
       }
       auto it = plan_state_->global_to_local_col_ids_.find(*col_id);
       CHECK(it != plan_state_->global_to_local_col_ids_.end());
       CHECK_LT(static_cast<size_t>(it->second),
                plan_state_->global_to_local_col_ids_.size());
       local_col_to_frag_pos[it->second] = frag_pos;
     }
     ++frag_pos;
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void Executor::buildSelectedFragsMappingForUnion	(	std::vector< std::vector< size_t >> &	selected_fragments_crossjoin,
		const FragmentsList &	selected_fragments,
		const RelAlgExecutionUnit &	ra_exe_unit
	)

private

Definition at line 3300 of file Execute.cpp.

References RelAlgExecutionUnit::input_descs.

Referenced by fetchUnionChunks().

                                             {
   const auto& input_descs = ra_exe_unit.input_descs;
   for (size_t scan_idx = 0; scan_idx < input_descs.size(); ++scan_idx) {
     // selected_fragments is set in assignFragsToKernelDispatch execution_kernel.fragments
     if (selected_fragments[0].table_id == input_descs[scan_idx].getTableId()) {
       selected_fragments_crossjoin.push_back({size_t(1)});
     }
   }
 }

Here is the caller graph for this function:

FragmentSkipStatus Executor::canSkipFragmentForFpQual	(	const Analyzer::BinOper *	comp_expr,
		const Analyzer::ColumnVar *	lhs_col,
		const Fragmenter_Namespace::FragmentInfo &	fragment,
		const Analyzer::Constant *	rhs_const
	)		const

private

Definition at line 4070 of file Execute.cpp.

References CHECK, extract_max_stat_fp_type(), extract_min_stat_fp_type(), Analyzer::ColumnVar::get_column_id(), Analyzer::Constant::get_constval(), Analyzer::BinOper::get_optype(), SQLTypeInfo::get_type(), Analyzer::Expr::get_type_info(), Fragmenter_Namespace::FragmentInfo::getChunkMetadataMap(), INVALID, kDOUBLE, kEQ, kFLOAT, kGE, kGT, kLE, kLT, NOT_SKIPPABLE, and SKIPPABLE.

Referenced by skipFragment().

                                              {
   const int col_id = lhs_col->get_column_id();
   auto chunk_meta_it = fragment.getChunkMetadataMap().find(col_id);
   if (chunk_meta_it == fragment.getChunkMetadataMap().end()) {
     return FragmentSkipStatus::NOT_SKIPPABLE;
   }
   double chunk_min{0.};
   double chunk_max{0.};
   const auto& chunk_type = lhs_col->get_type_info();
   chunk_min = extract_min_stat_fp_type(chunk_meta_it->second->chunkStats, chunk_type);
   chunk_max = extract_max_stat_fp_type(chunk_meta_it->second->chunkStats, chunk_type);
   if (chunk_min > chunk_max) {
     return FragmentSkipStatus::INVALID;
   }
 
   const auto datum_fp = rhs_const->get_constval();
   const auto rhs_type = rhs_const->get_type_info().get_type();
   CHECK(rhs_type == kFLOAT || rhs_type == kDOUBLE);
 
   // Do we need to codegen the constant like the integer path does?
   const auto rhs_val = rhs_type == kFLOAT ? datum_fp.floatval : datum_fp.doubleval;
 
   // Todo: dedup the following comparison code with the integer/timestamp path, it is
   // slightly tricky due to do cleanly as we do not have rowid on this path
   switch (comp_expr->get_optype()) {
     case kGE:
       if (chunk_max < rhs_val) {
         return FragmentSkipStatus::SKIPPABLE;
       }
       break;
     case kGT:
       if (chunk_max <= rhs_val) {
         return FragmentSkipStatus::SKIPPABLE;
       }
       break;
     case kLE:
       if (chunk_min > rhs_val) {
         return FragmentSkipStatus::SKIPPABLE;
       }
       break;
     case kLT:
       if (chunk_min >= rhs_val) {
         return FragmentSkipStatus::SKIPPABLE;
       }
       break;
     case kEQ:
       if (chunk_min > rhs_val || chunk_max < rhs_val) {
         return FragmentSkipStatus::SKIPPABLE;
       }
       break;
     default:
       break;
   }
   return FragmentSkipStatus::NOT_SKIPPABLE;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

llvm::Value * Executor::castToFP	(	llvm::Value *	value,
		SQLTypeInfo const &	from_ti,
		SQLTypeInfo const &	to_ti
	)

private

Definition at line 3885 of file Execute.cpp.

References AUTOMATIC_IR_METADATA, cgen_state_, exp_to_scale(), logger::FATAL, SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::is_fp(), SQLTypeInfo::is_number(), and LOG.

                                                           {
   AUTOMATIC_IR_METADATA(cgen_state_.get());
   if (value->getType()->isIntegerTy() && from_ti.is_number() && to_ti.is_fp() &&
       (!from_ti.is_fp() || from_ti.get_size() != to_ti.get_size())) {
     llvm::Type* fp_type{nullptr};
     switch (to_ti.get_size()) {
       case 4:
         fp_type = llvm::Type::getFloatTy(cgen_state_->context_);
         break;
       case 8:
         fp_type = llvm::Type::getDoubleTy(cgen_state_->context_);
         break;
       default:
         LOG(FATAL) << "Unsupported FP size: " << to_ti.get_size();
     }
     value = cgen_state_->ir_builder_.CreateSIToFP(value, fp_type);
     if (from_ti.get_scale()) {
       value = cgen_state_->ir_builder_.CreateFDiv(
           value,
           llvm::ConstantFP::get(value->getType(), exp_to_scale(from_ti.get_scale())));
     }
   }
   return value;
 }

Here is the call graph for this function:

llvm::Value * Executor::castToIntPtrTyIn	(	llvm::Value *	val,
		const size_t	bit_width
	)

private

Definition at line 3912 of file Execute.cpp.

References AUTOMATIC_IR_METADATA, cgen_state_, CHECK, CHECK_LT, and get_int_type().

                                                                            {
   AUTOMATIC_IR_METADATA(cgen_state_.get());
   CHECK(val->getType()->isPointerTy());
 
   const auto val_ptr_type = static_cast<llvm::PointerType*>(val->getType());
   const auto val_type = val_ptr_type->getPointerElementType();
   size_t val_width = 0;
   if (val_type->isIntegerTy()) {
     val_width = val_type->getIntegerBitWidth();
   } else {
     if (val_type->isFloatTy()) {
       val_width = 32;
     } else {
       CHECK(val_type->isDoubleTy());
       val_width = 64;
     }
   }
   CHECK_LT(size_t(0), val_width);
   if (bitWidth == val_width) {
     return val;
   }
   return cgen_state_->ir_builder_.CreateBitCast(
       val, llvm::PointerType::get(get_int_type(bitWidth, cgen_state_->context_), 0));
 }

Here is the call graph for this function:

bool Executor::checkCurrentQuerySession	(	const std::string &	candidate_query_session,
		heavyai::shared_lock< heavyai::shared_mutex > &	read_lock
	)

Definition at line 4462 of file Execute.cpp.

References current_query_session_.

                                                         {
   // if current_query_session is equal to the candidate_query_session,
   // or it is empty session we consider
   return !candidate_query_session.empty() &&
          (current_query_session_ == candidate_query_session);
 }

bool Executor::checkIsQuerySessionEnrolled	(	const QuerySessionId &	query_session,
		heavyai::shared_lock< heavyai::shared_mutex > &	read_lock
	)

Definition at line 4736 of file Execute.cpp.

References queries_session_map_.

Referenced by executeWorkUnitImpl().

                                                         {
   if (query_session.empty()) {
     return false;
   }
   return !query_session.empty() && queries_session_map_.count(query_session);
 }

Here is the caller graph for this function:

bool Executor::checkIsQuerySessionInterrupted	(	const std::string &	query_session,
		heavyai::shared_lock< heavyai::shared_mutex > &	read_lock
	)

Definition at line 4725 of file Execute.cpp.

References queries_interrupt_flag_.

Referenced by executePlanWithGroupBy(), executePlanWithoutGroupBy(), fetchChunks(), and fetchUnionChunks().

                                                         {
   if (query_session.empty()) {
     return false;
   }
   auto flag_it = queries_interrupt_flag_.find(query_session);
   return !query_session.empty() && flag_it != queries_interrupt_flag_.end() &&
          flag_it->second;
 }

Here is the caller graph for this function:

bool Executor::checkNonKernelTimeInterrupted ( ) const

Definition at line 4813 of file Execute.cpp.

References current_query_session_, executor_id_, executor_session_mutex_, queries_interrupt_flag_, and UNITARY_EXECUTOR_ID.

                                                    {
   // this function should be called within an executor which is assigned
   // to the specific query thread (that indicates we already enroll the session)
   // check whether this is called from non unitary executor
   if (executor_id_ == UNITARY_EXECUTOR_ID) {
     return false;
   };
   heavyai::shared_lock<heavyai::shared_mutex> session_read_lock(executor_session_mutex_);
   auto flag_it = queries_interrupt_flag_.find(current_query_session_);
   return !current_query_session_.empty() && flag_it != queries_interrupt_flag_.end() &&
          flag_it->second;
 }

void Executor::checkPendingQueryStatus ( const QuerySessionId & query_session )

Definition at line 4506 of file Execute.cpp.

References ERR_INTERRUPTED, executor_session_mutex_, queries_interrupt_flag_, queries_session_map_, and VLOG.

                                                                           {
   // check whether we are okay to execute the "pending" query
   // i.e., before running the query check if this query session is "ALREADY" interrupted
   heavyai::shared_lock<heavyai::shared_mutex> session_read_lock(executor_session_mutex_);
   if (query_session.empty()) {
     return;
   }
   if (queries_interrupt_flag_.find(query_session) == queries_interrupt_flag_.end()) {
     // something goes wrong since we assume this is caller's responsibility
     // (call this function only for enrolled query session)
     if (!queries_session_map_.count(query_session)) {
       VLOG(1) << "Interrupting pending query is not available since the query session is "
                  "not enrolled";
     } else {
       // here the query session is enrolled but the interrupt flag is not registered
       VLOG(1)
           << "Interrupting pending query is not available since its interrupt flag is "
              "not registered";
     }
     return;
   }
   if (queries_interrupt_flag_[query_session]) {
     throw QueryExecutionError(Executor::ERR_INTERRUPTED);
   }
 }

void Executor::clearCaches ( bool runtime_only = false )

static void Executor::clearExternalCaches	(	bool	for_update,
		const TableDescriptor *	td,
		const int	current_db_id
	)

inlinestatic

Definition at line 391 of file Execute.h.

References TableDescriptor::getTableChunkKey(), CacheInvalidator< CACHE_HOLDING_TYPES >::invalidateCaches(), and CacheInvalidator< CACHE_HOLDING_TYPES >::invalidateCachesByTable().

Referenced by Parser::InsertIntoTableAsSelectStmt::execute(), Parser::DropTableStmt::execute(), Parser::TruncateTableStmt::execute(), Parser::DropColumnStmt::execute(), Parser::CopyTableStmt::execute(), RelAlgExecutor::executeDelete(), and RelAlgExecutor::executeUpdate().

                                                            {
     bool clearEntireCache = true;
     if (td) {
       const auto& table_chunk_key_prefix = td->getTableChunkKey(current_db_id);
       if (!table_chunk_key_prefix.empty()) {
         auto table_key = boost::hash_value(table_chunk_key_prefix);
         ResultSetCacheInvalidator::invalidateCachesByTable(table_key);
         if (for_update) {
           UpdateTriggeredCacheInvalidator::invalidateCachesByTable(table_key);
         } else {
           DeleteTriggeredCacheInvalidator::invalidateCachesByTable(table_key);
         }
         clearEntireCache = false;
       }
     }
     if (clearEntireCache) {
       ResultSetCacheInvalidator::invalidateCaches();
       if (for_update) {
         UpdateTriggeredCacheInvalidator::invalidateCaches();
       } else {
         DeleteTriggeredCacheInvalidator::invalidateCaches();
       }
     }
   }

Here is the call graph for this function:

Here is the caller graph for this function:

void Executor::clearMemory ( const Data_Namespace::MemoryLevel memory_level )

static

Definition at line 501 of file Execute.cpp.

References Data_Namespace::DataMgr::clearMemory(), Data_Namespace::CPU_LEVEL, execute_mutex_, Catalog_Namespace::SysCatalog::getDataMgr(), Data_Namespace::GPU_LEVEL, Catalog_Namespace::SysCatalog::instance(), and CacheInvalidator< CACHE_HOLDING_TYPES >::invalidateCaches().

Referenced by DBHandler::clear_cpu_memory(), DBHandler::clear_gpu_memory(), QueryRunner::QueryRunner::clearCpuMemory(), and QueryRunner::QueryRunner::clearGpuMemory().

                                                                        {
   switch (memory_level) {
     case Data_Namespace::MemoryLevel::CPU_LEVEL:
     case Data_Namespace::MemoryLevel::GPU_LEVEL: {
       heavyai::unique_lock<heavyai::shared_mutex> flush_lock(
           execute_mutex_);  // Don't flush memory while queries are running
 
       if (memory_level == Data_Namespace::MemoryLevel::CPU_LEVEL) {
         // The hash table cache uses CPU memory not managed by the buffer manager. In the
         // future, we should manage these allocations with the buffer manager directly.
         // For now, assume the user wants to purge the hash table cache when they clear
         // CPU memory (currently used in ExecuteTest to lower memory pressure)
         JoinHashTableCacheInvalidator::invalidateCaches();
       }
       ResultSetCacheInvalidator::invalidateCaches();
       Catalog_Namespace::SysCatalog::instance().getDataMgr().clearMemory(memory_level);
       break;
     }
     default: {
       throw std::runtime_error(
           "Clearing memory levels other than the CPU level or GPU level is not "
           "supported.");
     }
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void Executor::clearMetaInfoCache ( )

private

Definition at line 788 of file Execute.cpp.

References agg_col_range_cache_, TableGenerations::clear(), AggregatedColRange::clear(), InputTableInfoCache::clear(), input_table_info_cache_, and table_generations_.

                                   {
   input_table_info_cache_.clear();
   agg_col_range_cache_.clear();
   table_generations_.clear();
 }

Here is the call graph for this function:

void Executor::clearQuerySessionStatus	(	const QuerySessionId &	query_session,
		const std::string &	submitted_time_str
	)

Definition at line 4532 of file Execute.cpp.

References current_query_session_, executor_session_mutex_, invalidateRunningQuerySession(), removeFromQuerySessionList(), and resetInterrupt().

                                                                             {
   heavyai::unique_lock<heavyai::shared_mutex> session_write_lock(executor_session_mutex_);
   // clear the interrupt-related info for a finished query
   if (query_session.empty()) {
     return;
   }
   removeFromQuerySessionList(query_session, submitted_time_str, session_write_lock);
   if (query_session.compare(current_query_session_) == 0) {
     invalidateRunningQuerySession(session_write_lock);
     resetInterrupt();
   }
 }

Here is the call graph for this function:

llvm::Value * Executor::codegenAggregateWindowState ( )

private

Definition at line 1336 of file WindowFunctionIR.cpp.

References AUTOMATIC_IR_METADATA, AVG, COUNT, anonymous_namespace{WindowFunctionIR.cpp}::get_adjusted_window_type_info(), get_int_type(), WindowProjectNodeContext::getActiveWindowFunctionContext(), Analyzer::WindowFunction::getKind(), kDECIMAL, kDOUBLE, and kFLOAT.

                                                  {
   AUTOMATIC_IR_METADATA(cgen_state_.get());
   const auto pi32_type =
       llvm::PointerType::get(get_int_type(32, cgen_state_->context_), 0);
   const auto pi64_type =
       llvm::PointerType::get(get_int_type(64, cgen_state_->context_), 0);
   const auto window_func_context =
       WindowProjectNodeContext::getActiveWindowFunctionContext(this);
   const Analyzer::WindowFunction* window_func = window_func_context->getWindowFunction();
   const auto window_func_ti = get_adjusted_window_type_info(window_func);
   const auto aggregate_state_type =
       window_func_ti.get_type() == kFLOAT ? pi32_type : pi64_type;
   auto aggregate_state = aggregateWindowStatePtr();
   if (window_func->getKind() == SqlWindowFunctionKind::AVG) {
     const auto aggregate_state_count_i64 = cgen_state_->llInt(
         reinterpret_cast<const int64_t>(window_func_context->aggregateStateCount()));
     auto aggregate_state_count = cgen_state_->ir_builder_.CreateIntToPtr(
         aggregate_state_count_i64, aggregate_state_type);
     const auto double_null_lv = cgen_state_->inlineFpNull(SQLTypeInfo(kDOUBLE));
     switch (window_func_ti.get_type()) {
       case kFLOAT: {
         return cgen_state_->emitCall(
             "load_avg_float", {aggregate_state, aggregate_state_count, double_null_lv});
       }
       case kDOUBLE: {
         return cgen_state_->emitCall(
             "load_avg_double", {aggregate_state, aggregate_state_count, double_null_lv});
       }
       case kDECIMAL: {
         return cgen_state_->emitCall(
             "load_avg_decimal",
             {aggregate_state,
              aggregate_state_count,
              double_null_lv,
              cgen_state_->llInt<int32_t>(window_func_ti.get_scale())});
       }
       default: {
         return cgen_state_->emitCall(
             "load_avg_int", {aggregate_state, aggregate_state_count, double_null_lv});
       }
     }
   }
   if (window_func->getKind() == SqlWindowFunctionKind::COUNT) {
     return cgen_state_->ir_builder_.CreateLoad(
         aggregate_state->getType()->getPointerElementType(), aggregate_state);
   }
   switch (window_func_ti.get_type()) {
     case kFLOAT: {
       return cgen_state_->emitCall("load_float", {aggregate_state});
     }
     case kDOUBLE: {
       return cgen_state_->emitCall("load_double", {aggregate_state});
     }
     default: {
       return cgen_state_->ir_builder_.CreateLoad(
           aggregate_state->getType()->getPointerElementType(), aggregate_state);
     }
   }
 }

Here is the call graph for this function:

llvm::Value * Executor::codegenCurrentPartitionIndex	(	const WindowFunctionContext *	window_func_context,
		llvm::Value *	current_row_pos_lv
	)

private

Definition at line 603 of file WindowFunctionIR.cpp.

References get_int_type(), WindowFunctionContext::partitionCount(), and WindowFunctionContext::partitionNumCountBuf().

                                    {
   const auto pi64_type =
       llvm::PointerType::get(get_int_type(64, cgen_state_->context_), 0);
   // given current row's pos, calculate the partition index that it belongs to
   auto partition_count_lv = cgen_state_->llInt(window_func_context->partitionCount());
   auto partition_num_count_buf_lv = cgen_state_->llInt(
       reinterpret_cast<int64_t>(window_func_context->partitionNumCountBuf()));
   auto partition_num_count_ptr_lv =
       cgen_state_->ir_builder_.CreateIntToPtr(partition_num_count_buf_lv, pi64_type);
   return cgen_state_->emitCall(
       "compute_int64_t_lower_bound",
       {partition_count_lv, current_row_pos_lv, partition_num_count_ptr_lv});
 }

Here is the call graph for this function:

llvm::Value * Executor::codegenFrameBoundExpr	(	const Analyzer::WindowFunction *	window_func,
		const Analyzer::WindowFrame *	frame_bound,
		CodeGenerator &	code_generator,
		const CompilationOptions &	co
	)

private

Definition at line 558 of file WindowFunctionIR.cpp.

References CHECK, CodeGenerator::codegen(), EXPR_FOLLOWING, EXPR_PRECEDING, g_cluster, SQLTypeInfo::get_size(), Analyzer::Expr::get_type_info(), Analyzer::WindowFrame::getBoundExpr(), Analyzer::WindowFunction::getOrderKeys(), Analyzer::WindowFunction::hasRangeModeFraming(), kBIGINT, kINT, and kSMALLINT.

                                                                            {
   auto needs_bound_expr_codegen = [](const Analyzer::WindowFrame* window_frame) {
     return window_frame->getBoundType() == SqlWindowFrameBoundType::EXPR_FOLLOWING ||
            window_frame->getBoundType() == SqlWindowFrameBoundType::EXPR_PRECEDING;
   };
   const auto order_col_ti = window_func->getOrderKeys().front()->get_type_info();
   auto encode_date_col_val = [&order_col_ti, this](llvm::Value* bound_expr_lv) {
     if (order_col_ti.get_comp_param() == 16) {
       return cgen_state_->emitCall(
           "fixed_width_date_encode_noinline",
           {bound_expr_lv,
            cgen_state_->castToTypeIn(cgen_state_->inlineIntNull(SQLTypeInfo(kSMALLINT)),
                                      32),
            cgen_state_->inlineIntNull(SQLTypeInfo(kBIGINT))});
     } else {
       return cgen_state_->emitCall("fixed_width_date_encode_noinline",
                                    {bound_expr_lv,
                                     cgen_state_->inlineIntNull(SQLTypeInfo(kINT)),
                                     cgen_state_->inlineIntNull(SQLTypeInfo(kBIGINT))});
     }
   };
   llvm::Value* bound_expr_lv{nullptr};
   if (needs_bound_expr_codegen(frame_bound)) {
     auto bound_expr_lvs = code_generator.codegen(frame_bound->getBoundExpr(), true, co);
     bound_expr_lv = bound_expr_lvs.front();
     if (order_col_ti.is_date() && window_func->hasRangeModeFraming()) {
       if (g_cluster) {
         throw std::runtime_error(
             "Range mode with date type ordering column is not supported yet.");
       }
       bound_expr_lv = encode_date_col_val(bound_expr_lv);
     }
     if (frame_bound->getBoundExpr()->get_type_info().get_size() != 8) {
       bound_expr_lv = cgen_state_->castToTypeIn(bound_expr_lv, 64);
     }
   } else {
     bound_expr_lv = cgen_state_->llInt((int64_t)-1);
   }
   CHECK(bound_expr_lv);
   return bound_expr_lv;
 }

Here is the call graph for this function:

void Executor::codegenJoinLoops	(	const std::vector< JoinLoop > &	join_loops,
		const RelAlgExecutionUnit &	ra_exe_unit,
		GroupByAndAggregate &	group_by_and_aggregate,
		llvm::Function *	query_func,
		llvm::BasicBlock *	entry_bb,
		QueryMemoryDescriptor &	query_mem_desc,
		const CompilationOptions &	co,
		const ExecutionOptions &	eo
	)

private

Definition at line 1112 of file IRCodegen.cpp.

References ExecutionOptions::allow_runtime_query_interrupt, AUTOMATIC_IR_METADATA, CHECK, CHECK_EQ, JoinLoop::codegen(), CompilationOptions::device_type, JoinLoopDomain::element_count, get_int_array_type(), get_int_type(), INNER, MultiSet, CodeGenerator::posArg(), GroupByAndAggregate::query_infos_, query_mem_desc, Set, and ExecutionOptions::with_dynamic_watchdog.

                                                             {
   AUTOMATIC_IR_METADATA(cgen_state_.get());
   const auto exit_bb =
       llvm::BasicBlock::Create(cgen_state_->context_, "exit", cgen_state_->current_func_);
   cgen_state_->ir_builder_.SetInsertPoint(exit_bb);
   cgen_state_->ir_builder_.CreateRet(cgen_state_->llInt<int32_t>(0));
   cgen_state_->ir_builder_.SetInsertPoint(entry_bb);
   CodeGenerator code_generator(this);
 
   llvm::BasicBlock* loops_entry_bb{nullptr};
   auto has_range_join =
       std::any_of(join_loops.begin(), join_loops.end(), [](const auto& join_loop) {
         return join_loop.kind() == JoinLoopKind::MultiSet;
       });
   if (has_range_join) {
     CHECK_EQ(join_loops.size(), size_t(1));
     const auto element_count =
         llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_), 9);
 
     auto compute_packed_offset = [](const int32_t x, const int32_t y) -> uint64_t {
       const uint64_t y_shifted = static_cast<uint64_t>(y) << 32;
       return y_shifted | static_cast<uint32_t>(x);
     };
 
     const auto values_arr = std::vector<llvm::Constant*>{
         llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_), 0),
         llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_),
                                compute_packed_offset(0, 1)),
         llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_),
                                compute_packed_offset(0, -1)),
         llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_),
                                compute_packed_offset(1, 0)),
         llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_),
                                compute_packed_offset(1, 1)),
         llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_),
                                compute_packed_offset(1, -1)),
         llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_),
                                compute_packed_offset(-1, 0)),
         llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_),
                                compute_packed_offset(-1, 1)),
         llvm::ConstantInt::get(get_int_type(64, cgen_state_->context_),
                                compute_packed_offset(-1, -1))};
 
     const auto constant_values_array = llvm::ConstantArray::get(
         get_int_array_type(64, 9, cgen_state_->context_), values_arr);
     CHECK(cgen_state_->module_);
     const auto values =
         new llvm::GlobalVariable(*cgen_state_->module_,
                                  get_int_array_type(64, 9, cgen_state_->context_),
                                  true,
                                  llvm::GlobalValue::LinkageTypes::InternalLinkage,
                                  constant_values_array);
     JoinLoop join_loop(
         JoinLoopKind::Set,
         JoinType::INNER,
         [element_count, values](const std::vector<llvm::Value*>& v) {
           JoinLoopDomain domain{{0}};
           domain.element_count = element_count;
           domain.values_buffer = values;
           return domain;
         },
         nullptr,
         nullptr,
         nullptr,
         nullptr,
         "range_key_loop");
 
     loops_entry_bb = JoinLoop::codegen(
         {join_loop},
         [this,
          query_func,
          &query_mem_desc,
          &co,
          &eo,
          &group_by_and_aggregate,
          &join_loops,
          &ra_exe_unit](const std::vector<llvm::Value*>& prev_iters) {
           auto& builder = cgen_state_->ir_builder_;
 
           auto body_exit_bb =
               llvm::BasicBlock::Create(cgen_state_->context_,
                                        "range_key_inner_body_exit",
                                        builder.GetInsertBlock()->getParent());
 
           auto range_key_body_bb =
               llvm::BasicBlock::Create(cgen_state_->context_,
                                        "range_key_loop_body",
                                        builder.GetInsertBlock()->getParent());
           builder.SetInsertPoint(range_key_body_bb);
 
           const auto body_loops_entry_bb = JoinLoop::codegen(
               join_loops,
               [this,
                query_func,
                &query_mem_desc,
                &co,
                &eo,
                &group_by_and_aggregate,
                &join_loops,
                &ra_exe_unit](const std::vector<llvm::Value*>& prev_iters) {
                 addJoinLoopIterator(prev_iters, join_loops.size());
                 auto& builder = cgen_state_->ir_builder_;
                 const auto loop_body_bb =
                     llvm::BasicBlock::Create(builder.getContext(),
                                              "loop_body",
                                              builder.GetInsertBlock()->getParent());
                 builder.SetInsertPoint(loop_body_bb);
                 const bool can_return_error =
                     compileBody(ra_exe_unit, group_by_and_aggregate, query_mem_desc, co);
                 if (can_return_error || cgen_state_->needs_error_check_ ||
                     eo.with_dynamic_watchdog || eo.allow_runtime_query_interrupt) {
                   createErrorCheckControlFlow(query_func,
                                               eo.with_dynamic_watchdog,
                                               eo.allow_runtime_query_interrupt,
                                               join_loops,
                                               co.device_type,
                                               group_by_and_aggregate.query_infos_);
                 }
                 return loop_body_bb;
               },
               prev_iters.back(),
               body_exit_bb,
               cgen_state_.get());
 
           builder.SetInsertPoint(range_key_body_bb);
           cgen_state_->ir_builder_.CreateBr(body_loops_entry_bb);
 
           builder.SetInsertPoint(body_exit_bb);
           return range_key_body_bb;
         },
         code_generator.posArg(nullptr),
         exit_bb,
         cgen_state_.get());
   } else {
     loops_entry_bb = JoinLoop::codegen(
         join_loops,
         /*body_codegen=*/
         [this,
          query_func,
          &query_mem_desc,
          &co,
          &eo,
          &group_by_and_aggregate,
          &join_loops,
          &ra_exe_unit](const std::vector<llvm::Value*>& prev_iters) {
           AUTOMATIC_IR_METADATA(cgen_state_.get());
           addJoinLoopIterator(prev_iters, join_loops.size());
           auto& builder = cgen_state_->ir_builder_;
           const auto loop_body_bb = llvm::BasicBlock::Create(
               builder.getContext(), "loop_body", builder.GetInsertBlock()->getParent());
           builder.SetInsertPoint(loop_body_bb);
           const bool can_return_error =
               compileBody(ra_exe_unit, group_by_and_aggregate, query_mem_desc, co);
           if (can_return_error || cgen_state_->needs_error_check_ ||
               eo.with_dynamic_watchdog || eo.allow_runtime_query_interrupt) {
             createErrorCheckControlFlow(query_func,
                                         eo.with_dynamic_watchdog,
                                         eo.allow_runtime_query_interrupt,
                                         join_loops,
                                         co.device_type,
                                         group_by_and_aggregate.query_infos_);
           }
           return loop_body_bb;
         },
         /*outer_iter=*/code_generator.posArg(nullptr),
         exit_bb,
         cgen_state_.get());
   }
   CHECK(loops_entry_bb);
   cgen_state_->ir_builder_.SetInsertPoint(entry_bb);
   cgen_state_->ir_builder_.CreateBr(loops_entry_bb);
 }

Here is the call graph for this function:

llvm::BasicBlock * Executor::codegenSkipDeletedOuterTableRow	(	const RelAlgExecutionUnit &	ra_exe_unit,
		const CompilationOptions &	co
	)

private

Definition at line 3177 of file NativeCodegen.cpp.

                                   {
   AUTOMATIC_IR_METADATA(cgen_state_.get());
   if (!co.filter_on_deleted_column) {
     return nullptr;
   }
   CHECK(!ra_exe_unit.input_descs.empty());
   const auto& outer_input_desc = ra_exe_unit.input_descs[0];
   if (outer_input_desc.getSourceType() != InputSourceType::TABLE) {
     return nullptr;
   }
   const auto deleted_cd =
       plan_state_->getDeletedColForTable(outer_input_desc.getTableId());
   if (!deleted_cd) {
     return nullptr;
   }
   CHECK(deleted_cd->columnType.is_boolean());
   const auto deleted_expr =
       makeExpr<Analyzer::ColumnVar>(deleted_cd->columnType,
                                     outer_input_desc.getTableId(),
                                     deleted_cd->columnId,
                                     outer_input_desc.getNestLevel());
   CodeGenerator code_generator(this);
   const auto is_deleted =
       code_generator.toBool(code_generator.codegen(deleted_expr.get(), true, co).front());
   const auto is_deleted_bb = llvm::BasicBlock::Create(
       cgen_state_->context_, "is_deleted", cgen_state_->row_func_);
   llvm::BasicBlock* bb = llvm::BasicBlock::Create(
       cgen_state_->context_, "is_not_deleted", cgen_state_->row_func_);
   cgen_state_->ir_builder_.CreateCondBr(is_deleted, is_deleted_bb, bb);
   cgen_state_->ir_builder_.SetInsertPoint(is_deleted_bb);
   cgen_state_->ir_builder_.CreateRet(cgen_state_->llInt<int32_t>(0));
   cgen_state_->ir_builder_.SetInsertPoint(bb);
   return bb;
 }

void Executor::codegenWindowAvgEpilogue	(	llvm::Value *	crt_val,
		llvm::Value *	window_func_null_val,
		llvm::Value *	multiplicity_lv
	)

private

Definition at line 1299 of file WindowFunctionIR.cpp.

References AUTOMATIC_IR_METADATA, anonymous_namespace{WindowFunctionIR.cpp}::get_adjusted_window_type_info(), get_int_type(), WindowProjectNodeContext::getActiveWindowFunctionContext(), kDOUBLE, and kFLOAT.

                                                                     {
   AUTOMATIC_IR_METADATA(cgen_state_.get());
   const auto window_func_context =
       WindowProjectNodeContext::getActiveWindowFunctionContext(this);
   const auto window_func = window_func_context->getWindowFunction();
   const auto window_func_ti = get_adjusted_window_type_info(window_func);
   const auto pi32_type =
       llvm::PointerType::get(get_int_type(32, cgen_state_->context_), 0);
   const auto pi64_type =
       llvm::PointerType::get(get_int_type(64, cgen_state_->context_), 0);
   const auto aggregate_state_type =
       window_func_ti.get_type() == kFLOAT ? pi32_type : pi64_type;
   const auto aggregate_state_count_i64 = cgen_state_->llInt(
       reinterpret_cast<const int64_t>(window_func_context->aggregateStateCount()));
   auto aggregate_state_count = cgen_state_->ir_builder_.CreateIntToPtr(
       aggregate_state_count_i64, aggregate_state_type);
   std::string agg_count_func_name = "agg_count";
   switch (window_func_ti.get_type()) {
     case kFLOAT: {
       agg_count_func_name += "_float";
       break;
     }
     case kDOUBLE: {
       agg_count_func_name += "_double";
       break;
     }
     default: {
       break;
     }
   }
   agg_count_func_name += "_skip_val";
   cgen_state_->emitCall(agg_count_func_name,
                         {aggregate_state_count, crt_val, window_func_null_val});
 }

Here is the call graph for this function:

std::pair< llvm::Value , llvm::Value > Executor::codegenWindowFrameBound	(	WindowFunctionContext *	window_func_context,
		const Analyzer::WindowFrame *	frame_start_bound,
		const Analyzer::WindowFrame *	frame_end_bound,
		llvm::Value *	current_row_pos_lv,
		llvm::Value *	current_partition_start_offset_lv,
		llvm::Value *	order_key_buf_ptr_lv,
		llvm::Value *	order_key_col_null_val_lv,
		llvm::Value *	frame_start_bound_expr_lv,
		llvm::Value *	frame_end_bound_expr_lv,
		llvm::Value *	num_elem_current_partition_lv,
		llvm::Value *	target_partition_rowid_ptr_lv,
		llvm::Value *	target_partition_sorted_rowid_ptr_lv,
		llvm::Value *	null_start_pos_lv,
		llvm::Value *	null_end_pos_lv,
		CodeGenerator &	code_generator
	)

private

Definition at line 619 of file WindowFunctionIR.cpp.

References CHECK, CodeGenerator::codegenWindowPosition(), CURRENT_ROW, EXPR_FOLLOWING, EXPR_PRECEDING, anonymous_namespace{WindowFunctionIR.cpp}::get_col_type_name_by_size(), get_int_type(), Analyzer::Expr::get_type_info(), Analyzer::WindowFrame::getBoundExpr(), Analyzer::WindowFrame::getBoundType(), WindowFunctionContext::getOrderKeyColumnBuffers(), WindowFunctionContext::getOrderKeyColumnBufferTypes(), WindowFunctionContext::getWindowFunction(), SQLTypeInfo::is_date(), SQLTypeInfo::is_timestamp(), UNBOUNDED_FOLLOWING, and UNBOUNDED_PRECEDING.

                                    {
   const auto window_func = window_func_context->getWindowFunction();
   CHECK(window_func);
   std::string order_col_type_name{""};
   llvm::Value* current_col_value_lv{nullptr};
   llvm::Value* frame_start_bound_lv{nullptr};
   llvm::Value* frame_end_bound_lv{nullptr};
 
   if (window_func->hasRangeModeFraming()) {
     CHECK(window_func_context->getOrderKeyColumnBuffers().size() == 1);
     CHECK(window_func->getOrderKeys().size() == 1UL);
     CHECK(window_func_context->getOrderKeyColumnBuffers().size() == 1UL);
     const auto order_key_ti = window_func->getOrderKeys().front()->get_type_info();
     const auto order_key_size = order_key_ti.get_size();
     size_t order_key_size_in_byte = order_key_size * 8;
     order_col_type_name = get_col_type_name_by_size(
         order_key_size,
         window_func_context->getOrderKeyColumnBufferTypes().front().is_fp());
 
     // load column value of the current row (of ordering column)
     auto rowid_in_partition_lv =
         code_generator.codegenWindowPosition(window_func_context, current_row_pos_lv);
     auto current_col_value_ptr_lv = cgen_state_->ir_builder_.CreateGEP(
         get_int_type(order_key_size_in_byte, cgen_state_->context_),
         order_key_buf_ptr_lv,
         rowid_in_partition_lv);
     current_col_value_lv = cgen_state_->ir_builder_.CreateLoad(
         current_col_value_ptr_lv->getType()->getPointerElementType(),
         current_col_value_ptr_lv,
         "current_col_value");
   }
 
   // compute frame start depending on the bound type
   if (frame_start_bound->getBoundType() == SqlWindowFrameBoundType::UNBOUNDED_PRECEDING) {
     // frame starts at the first row of the partition
     frame_start_bound_lv = cgen_state_->llInt((int64_t)0);
   } else if (frame_start_bound->getBoundType() ==
              SqlWindowFrameBoundType::EXPR_PRECEDING) {
     // frame starts at the position before X rows of the current row
     CHECK(frame_start_bound_expr_lv);
     if (window_func->hasRowModeFraming()) {
       frame_start_bound_lv = cgen_state_->emitCall("compute_row_mode_start_index_sub",
                                                    {current_row_pos_lv,
                                                     current_partition_start_offset_lv,
                                                     frame_start_bound_expr_lv});
     } else {
       CHECK(window_func->hasRangeModeFraming());
       if (frame_start_bound->getBoundExpr()->get_type_info().is_date() ||
           frame_start_bound->getBoundExpr()->get_type_info().is_timestamp()) {
         std::string lower_bound_func_name{"compute_"};
         lower_bound_func_name.append(order_col_type_name);
         lower_bound_func_name.append("_lower_bound_from_ordered_index_for_timeinterval");
         frame_start_bound_lv = cgen_state_->emitCall(
             lower_bound_func_name,
             {num_elem_current_partition_lv,
              frame_start_bound_expr_lv,
              order_key_buf_ptr_lv,
              target_partition_rowid_ptr_lv,
              target_partition_sorted_rowid_ptr_lv,
              cgen_state_->castToTypeIn(order_key_col_null_val_lv, 64),
              null_start_pos_lv,
              null_end_pos_lv});
       } else {
         std::string lower_bound_func_name{"range_mode_"};
         lower_bound_func_name.append(order_col_type_name);
         lower_bound_func_name.append("_sub_frame_lower_bound");
         frame_start_bound_lv =
             cgen_state_->emitCall(lower_bound_func_name,
                                   {num_elem_current_partition_lv,
                                    current_col_value_lv,
                                    order_key_buf_ptr_lv,
                                    target_partition_rowid_ptr_lv,
                                    target_partition_sorted_rowid_ptr_lv,
                                    frame_start_bound_expr_lv,
                                    order_key_col_null_val_lv,
                                    null_start_pos_lv,
                                    null_end_pos_lv});
       }
     }
   } else if (frame_start_bound->getBoundType() == SqlWindowFrameBoundType::CURRENT_ROW) {
     // frame start at the current row
     if (window_func->hasRowModeFraming()) {
       frame_start_bound_lv = cgen_state_->emitCall("compute_row_mode_start_index_sub",
                                                    {current_row_pos_lv,
                                                     current_partition_start_offset_lv,
                                                     cgen_state_->llInt(((int64_t)0))});
     } else {
       CHECK(window_func->hasRangeModeFraming());
       std::string lower_bound_func_name{"compute_"};
       lower_bound_func_name.append(order_col_type_name);
       lower_bound_func_name.append("_lower_bound_from_ordered_index");
       frame_start_bound_lv = cgen_state_->emitCall(lower_bound_func_name,
                                                    {num_elem_current_partition_lv,
                                                     current_col_value_lv,
                                                     order_key_buf_ptr_lv,
                                                     target_partition_rowid_ptr_lv,
                                                     target_partition_sorted_rowid_ptr_lv,
                                                     order_key_col_null_val_lv,
                                                     null_start_pos_lv,
                                                     null_end_pos_lv});
     }
   } else if (frame_start_bound->getBoundType() ==
              SqlWindowFrameBoundType::EXPR_FOLLOWING) {
     // frame start at the position after X rows of the current row
     CHECK(frame_start_bound_expr_lv);
     if (window_func->hasRowModeFraming()) {
       frame_start_bound_lv = cgen_state_->emitCall("compute_row_mode_start_index_add",
                                                    {current_row_pos_lv,
                                                     current_partition_start_offset_lv,
                                                     frame_start_bound_expr_lv,
                                                     num_elem_current_partition_lv});
     } else {
       CHECK(window_func->hasRangeModeFraming());
       if (frame_start_bound->getBoundExpr()->get_type_info().is_date() ||
           frame_start_bound->getBoundExpr()->get_type_info().is_timestamp()) {
         std::string lower_bound_func_name{"compute_"};
         lower_bound_func_name.append(order_col_type_name);
         lower_bound_func_name.append("_lower_bound_from_ordered_index_for_timeinterval");
         frame_start_bound_lv = cgen_state_->emitCall(
             lower_bound_func_name,
             {num_elem_current_partition_lv,
              frame_start_bound_expr_lv,
              order_key_buf_ptr_lv,
              target_partition_rowid_ptr_lv,
              target_partition_sorted_rowid_ptr_lv,
              cgen_state_->castToTypeIn(order_key_col_null_val_lv, 64),
              null_start_pos_lv,
              null_end_pos_lv});
       } else {
         std::string lower_bound_func_name{"range_mode_"};
         lower_bound_func_name.append(order_col_type_name);
         lower_bound_func_name.append("_add_frame_lower_bound");
         frame_start_bound_lv =
             cgen_state_->emitCall(lower_bound_func_name,
                                   {num_elem_current_partition_lv,
                                    current_col_value_lv,
                                    order_key_buf_ptr_lv,
                                    target_partition_rowid_ptr_lv,
                                    target_partition_sorted_rowid_ptr_lv,
                                    frame_start_bound_expr_lv,
                                    order_key_col_null_val_lv,
                                    null_start_pos_lv,
                                    null_end_pos_lv});
       }
     }
   } else {
     CHECK(false) << "frame start cannot be UNBOUNDED FOLLOWING";
   }
 
   // compute frame end
   if (frame_end_bound->getBoundType() == SqlWindowFrameBoundType::UNBOUNDED_PRECEDING) {
     // frame ends at the first row of the partition
     CHECK(false) << "frame end cannot be UNBOUNDED PRECEDING";
   } else if (frame_end_bound->getBoundType() == SqlWindowFrameBoundType::EXPR_PRECEDING) {
     // frame ends at the position X rows before the current row
     CHECK(frame_end_bound_expr_lv);
     if (window_func->hasRowModeFraming()) {
       frame_end_bound_lv = cgen_state_->emitCall("compute_row_mode_end_index_sub",
                                                  {current_row_pos_lv,
                                                   current_partition_start_offset_lv,
                                                   frame_end_bound_expr_lv});
     } else {
       CHECK(window_func->hasRangeModeFraming());
       if (frame_end_bound->getBoundExpr()->get_type_info().is_date() ||
           frame_end_bound->getBoundExpr()->get_type_info().is_timestamp()) {
         std::string upper_bound_func_name{"compute_"};
         upper_bound_func_name.append(order_col_type_name);
         upper_bound_func_name.append("_upper_bound_from_ordered_index_for_timeinterval");
         frame_end_bound_lv = cgen_state_->emitCall(
             upper_bound_func_name,
             {num_elem_current_partition_lv,
              frame_end_bound_expr_lv,
              order_key_buf_ptr_lv,
              target_partition_rowid_ptr_lv,
              target_partition_sorted_rowid_ptr_lv,
              cgen_state_->castToTypeIn(order_key_col_null_val_lv, 64),
              null_start_pos_lv,
              null_end_pos_lv});
       } else {
         std::string upper_bound_func_name{"range_mode_"};
         upper_bound_func_name.append(order_col_type_name);
         upper_bound_func_name.append("_sub_frame_upper_bound");
         frame_end_bound_lv = cgen_state_->emitCall(upper_bound_func_name,
                                                    {num_elem_current_partition_lv,
                                                     current_col_value_lv,
                                                     order_key_buf_ptr_lv,
                                                     target_partition_rowid_ptr_lv,
                                                     target_partition_sorted_rowid_ptr_lv,
                                                     frame_end_bound_expr_lv,
                                                     order_key_col_null_val_lv,
                                                     null_start_pos_lv,
                                                     null_end_pos_lv});
       }
     }
   } else if (frame_end_bound->getBoundType() == SqlWindowFrameBoundType::CURRENT_ROW) {
     // frame ends at the current row
     if (window_func->hasRowModeFraming()) {
       frame_end_bound_lv = cgen_state_->emitCall("compute_row_mode_end_index_sub",
                                                  {current_row_pos_lv,
                                                   current_partition_start_offset_lv,
                                                   cgen_state_->llInt((int64_t)0)});
     } else {
       CHECK(window_func->hasRangeModeFraming());
       std::string upper_bound_func_name{"compute_"};
       upper_bound_func_name.append(order_col_type_name);
       upper_bound_func_name.append("_upper_bound_from_ordered_index");
       frame_end_bound_lv = cgen_state_->emitCall(upper_bound_func_name,
                                                  {num_elem_current_partition_lv,
                                                   current_col_value_lv,
                                                   order_key_buf_ptr_lv,
                                                   target_partition_rowid_ptr_lv,
                                                   target_partition_sorted_rowid_ptr_lv,
                                                   order_key_col_null_val_lv,
                                                   null_start_pos_lv,
                                                   null_end_pos_lv});
     }
   } else if (frame_end_bound->getBoundType() == SqlWindowFrameBoundType::EXPR_FOLLOWING) {
     // frame ends at the position X rows after the current row
     CHECK(frame_end_bound_expr_lv);
     if (window_func->hasRowModeFraming()) {
       frame_end_bound_lv = cgen_state_->emitCall("compute_row_mode_end_index_add",
                                                  {current_row_pos_lv,
                                                   current_partition_start_offset_lv,
                                                   frame_end_bound_expr_lv,
                                                   num_elem_current_partition_lv});
     } else {
       CHECK(window_func->hasRangeModeFraming());
       if (frame_end_bound->getBoundExpr()->get_type_info().is_date() ||
           frame_end_bound->getBoundExpr()->get_type_info().is_timestamp()) {
         std::string upper_bound_func_name{"compute_"};
         upper_bound_func_name.append(order_col_type_name);
         upper_bound_func_name.append("_upper_bound_from_ordered_index_for_timeinterval");
         frame_end_bound_lv = cgen_state_->emitCall(
             upper_bound_func_name,
             {num_elem_current_partition_lv,
              frame_end_bound_expr_lv,
              order_key_buf_ptr_lv,
              target_partition_rowid_ptr_lv,
              target_partition_sorted_rowid_ptr_lv,
              cgen_state_->castToTypeIn(order_key_col_null_val_lv, 64),
              null_start_pos_lv,
              null_end_pos_lv});
       } else {
         std::string upper_bound_func_name{"range_mode_"};
         upper_bound_func_name.append(order_col_type_name);
         upper_bound_func_name.append("_add_frame_upper_bound");
         frame_end_bound_lv = cgen_state_->emitCall(upper_bound_func_name,
                                                    {num_elem_current_partition_lv,
                                                     current_col_value_lv,
                                                     order_key_buf_ptr_lv,
                                                     target_partition_rowid_ptr_lv,
                                                     target_partition_sorted_rowid_ptr_lv,
                                                     frame_end_bound_expr_lv,
                                                     order_key_col_null_val_lv,
                                                     null_start_pos_lv,
                                                     null_end_pos_lv});
       }
     }
   } else {
     // frame ends at the last row of the partition
     CHECK(frame_end_bound->getBoundType() ==
           SqlWindowFrameBoundType::UNBOUNDED_FOLLOWING);
     frame_end_bound_lv = num_elem_current_partition_lv;
   }
   CHECK(frame_start_bound_lv);
   CHECK(frame_end_bound_lv);
   return std::make_pair(frame_start_bound_lv, frame_end_bound_lv);
 }

Here is the call graph for this function:

llvm::Value * Executor::codegenWindowFunction	(	const size_t	target_index,
		const CompilationOptions &	co
	)

private

Definition at line 21 of file WindowFunctionIR.cpp.

References WindowProjectNodeContext::activateWindowFunctionContext(), run_benchmark_import::args, AUTOMATIC_IR_METADATA, AVG, CHECK, CHECK_EQ, COUNT, CUME_DIST, DENSE_RANK, logger::FATAL, FIRST_VALUE, WindowProjectNodeContext::get(), WindowFunctionContext::getWindowFunction(), LAG, LAG_IN_FRAME, LAST_VALUE, LEAD, LEAD_IN_FRAME, LOG, MAX, MIN, NTILE, PERCENT_RANK, RANK, ROW_NUMBER, and SUM.

                                                                            {
   AUTOMATIC_IR_METADATA(cgen_state_.get());
   CodeGenerator code_generator(this);
 
   const auto window_func_context =
       WindowProjectNodeContext::get(this)->activateWindowFunctionContext(this,
                                                                          target_index);
   const auto window_func = window_func_context->getWindowFunction();
   switch (window_func->getKind()) {
     case SqlWindowFunctionKind::ROW_NUMBER:
     case SqlWindowFunctionKind::RANK:
     case SqlWindowFunctionKind::DENSE_RANK:
     case SqlWindowFunctionKind::NTILE: {
       // they are always evaluated on the entire partition
       return code_generator.codegenWindowPosition(window_func_context,
                                                   code_generator.posArg(nullptr));
     }
     case SqlWindowFunctionKind::PERCENT_RANK:
     case SqlWindowFunctionKind::CUME_DIST: {
       // they are always evaluated on the entire partition
       return cgen_state_->emitCall("percent_window_func",
                                    {cgen_state_->llInt(reinterpret_cast<const int64_t>(
                                         window_func_context->output())),
                                     code_generator.posArg(nullptr)});
     }
     case SqlWindowFunctionKind::LAG:
     case SqlWindowFunctionKind::LEAD:
     case SqlWindowFunctionKind::FIRST_VALUE:
     case SqlWindowFunctionKind::LAST_VALUE: {
       // they are always evaluated on the current frame
       CHECK(WindowProjectNodeContext::get(this));
       const auto& args = window_func->getArgs();
       CHECK(!args.empty());
       const auto arg_lvs = code_generator.codegen(args.front().get(), true, co);
       CHECK_EQ(arg_lvs.size(), size_t(1));
       return arg_lvs.front();
     }
     case SqlWindowFunctionKind::AVG:
     case SqlWindowFunctionKind::MIN:
     case SqlWindowFunctionKind::MAX:
     case SqlWindowFunctionKind::SUM:
     case SqlWindowFunctionKind::COUNT: {
       // they are always evaluated on the current frame
       return codegenWindowFunctionAggregate(co);
     }
     case SqlWindowFunctionKind::LEAD_IN_FRAME:
     case SqlWindowFunctionKind::LAG_IN_FRAME: {
       return codegenWindowFunctionOnFrame(co);
     }
     default: {
       LOG(FATAL) << "Invalid window function kind";
     }
   }
   return nullptr;
 }

Here is the call graph for this function:

llvm::Value * Executor::codegenWindowFunctionAggregate ( const CompilationOptions & co )

private

Definition at line 227 of file WindowFunctionIR.cpp.

References AUTOMATIC_IR_METADATA, AVG, CHECK, WindowProjectNodeContext::get(), get_int_type(), and WindowProjectNodeContext::getActiveWindowFunctionContext().

                                                                                 {
   AUTOMATIC_IR_METADATA(cgen_state_.get());
   const auto reset_state_false_bb = codegenWindowResetStateControlFlow();
   auto aggregate_state = aggregateWindowStatePtr();
   llvm::Value* aggregate_state_count = nullptr;
   const auto window_func_context =
       WindowProjectNodeContext::getActiveWindowFunctionContext(this);
   const auto window_func = window_func_context->getWindowFunction();
   if (window_func->getKind() == SqlWindowFunctionKind::AVG) {
     const auto aggregate_state_count_i64 = cgen_state_->llInt(
         reinterpret_cast<const int64_t>(window_func_context->aggregateStateCount()));
     const auto pi64_type =
         llvm::PointerType::get(get_int_type(64, cgen_state_->context_), 0);
     aggregate_state_count =
         cgen_state_->ir_builder_.CreateIntToPtr(aggregate_state_count_i64, pi64_type);
   }
   codegenWindowFunctionStateInit(aggregate_state);
   if (window_func->getKind() == SqlWindowFunctionKind::AVG) {
     const auto count_zero = cgen_state_->llInt(int64_t(0));
     cgen_state_->emitCall("agg_id", {aggregate_state_count, count_zero});
   }
   cgen_state_->ir_builder_.CreateBr(reset_state_false_bb);
   cgen_state_->ir_builder_.SetInsertPoint(reset_state_false_bb);
   CHECK(WindowProjectNodeContext::get(this));
   return codegenWindowFunctionAggregateCalls(aggregate_state, co);
 }

Here is the call graph for this function:

llvm::Value * Executor::codegenWindowFunctionAggregateCalls	(	llvm::Value *	aggregate_state,
		const CompilationOptions &	co
	)

private

Definition at line 903 of file WindowFunctionIR.cpp.

References run_benchmark_import::args, AUTOMATIC_IR_METADATA, AVG, CHECK, CHECK_EQ, CodeGenerator::codegen(), CodeGenerator::codegenCastBetweenIntTypes(), COUNT, anonymous_namespace{WindowFunctionIR.cpp}::get_adjusted_window_type_info(), anonymous_namespace{WindowFunctionIR.cpp}::get_col_type_name_by_size(), get_int_type(), anonymous_namespace{WindowFunctionIR.cpp}::get_window_agg_name(), WindowProjectNodeContext::getActiveWindowFunctionContext(), kBIGINT, kDATE, kDECIMAL, kDOUBLE, kFLOAT, kINT, kNUMERIC, kSMALLINT, kTIME, kTIMESTAMP, kTINYINT, MAX, MIN, CodeGenerator::posArg(), and SUM.

                                                                                          {
   AUTOMATIC_IR_METADATA(cgen_state_.get());
   const auto window_func_context =
       WindowProjectNodeContext::getActiveWindowFunctionContext(this);
   const auto window_func = window_func_context->getWindowFunction();
   const auto window_func_ti = get_adjusted_window_type_info(window_func);
   const auto window_func_null_val =
       window_func_ti.is_fp()
           ? cgen_state_->inlineFpNull(window_func_ti)
           : cgen_state_->castToTypeIn(cgen_state_->inlineIntNull(window_func_ti), 64);
   const auto& args = window_func->getArgs();
   llvm::Value* crt_val;
   CodeGenerator code_generator(this);
   if (args.empty()) {
     CHECK(window_func->getKind() == SqlWindowFunctionKind::COUNT);
     crt_val = cgen_state_->llInt(int64_t(1));
   } else {
     const auto arg_lvs = code_generator.codegen(args.front().get(), true, co);
     CHECK_EQ(arg_lvs.size(), size_t(1));
     if (window_func->getKind() == SqlWindowFunctionKind::SUM && !window_func_ti.is_fp()) {
       crt_val = code_generator.codegenCastBetweenIntTypes(
           arg_lvs.front(), args.front()->get_type_info(), window_func_ti, false);
     } else {
       crt_val = window_func_ti.get_type() == kFLOAT
                     ? arg_lvs.front()
                     : cgen_state_->castToTypeIn(arg_lvs.front(), 64);
     }
   }
   if (window_func_context->needsToBuildAggregateTree()) {
     // compute an aggregated value for each row of the window frame by using segment tree
     // when constructing a window context, we build a necessary segment tree for it
     // and use the tree array (so called `aggregate tree`) to query the aggregated value
     // of the specific window frame
     // we fall back to the non-framing window func evaluation logic if an input
     // of the window function can be an empty one
     const auto pi64_type =
         llvm::PointerType::get(get_int_type(64, cgen_state_->context_), 0);
     const auto ppi64_type = llvm::PointerType::get(pi64_type, 0);
     const auto pi32_type =
         llvm::PointerType::get(get_int_type(32, cgen_state_->context_), 0);
 
     // codegen frame bound expr if necessary
     const auto frame_start_bound = window_func->getFrameStartBound();
     const auto frame_end_bound = window_func->getFrameEndBound();
     auto frame_start_bound_expr_lv =
         codegenFrameBoundExpr(window_func, frame_start_bound, code_generator, co);
     auto frame_end_bound_expr_lv =
         codegenFrameBoundExpr(window_func, frame_end_bound, code_generator, co);
     CHECK(frame_start_bound_expr_lv);
     CHECK(frame_end_bound_expr_lv);
 
     // compute aggregated value over the computed frame range
     auto current_row_pos_lv = code_generator.posArg(nullptr);
     auto partition_index_lv =
         codegenCurrentPartitionIndex(window_func_context, current_row_pos_lv);
 
     // ordering column buffer
     const auto target_col_ti = window_func->getArgs().front()->get_type_info();
     const auto target_col_size = target_col_ti.get_size();
     const auto col_type_name =
         get_col_type_name_by_size(target_col_size, target_col_ti.is_fp());
 
     // partial sum of # elems of partitions
     auto partition_start_offset_buf_lv = cgen_state_->llInt(
         reinterpret_cast<int64_t>(window_func_context->partitionStartOffset()));
     auto partition_start_offset_ptr_lv =
         cgen_state_->ir_builder_.CreateIntToPtr(partition_start_offset_buf_lv, pi64_type);
 
     // get start offset of the current partition
     auto current_partition_start_offset_ptr_lv =
         cgen_state_->ir_builder_.CreateGEP(get_int_type(64, cgen_state_->context_),
                                            partition_start_offset_ptr_lv,
                                            partition_index_lv);
     auto current_partition_start_offset_lv = cgen_state_->ir_builder_.CreateLoad(
         current_partition_start_offset_ptr_lv->getType()->getPointerElementType(),
         current_partition_start_offset_ptr_lv);
 
     // row_id buf of the current partition
     const auto partition_rowid_buf_lv =
         cgen_state_->llInt(reinterpret_cast<int64_t>(window_func_context->payload()));
     const auto partition_rowid_ptr_lv =
         cgen_state_->ir_builder_.CreateIntToPtr(partition_rowid_buf_lv, pi32_type);
     auto target_partition_rowid_ptr_lv =
         cgen_state_->ir_builder_.CreateGEP(get_int_type(32, cgen_state_->context_),
                                            partition_rowid_ptr_lv,
                                            current_partition_start_offset_lv);
 
     // row_id buf of ordered current partition
     const auto sorted_rowid_lv = cgen_state_->llInt(
         reinterpret_cast<int64_t>(window_func_context->sortedPartition()));
     const auto sorted_rowid_ptr_lv =
         cgen_state_->ir_builder_.CreateIntToPtr(sorted_rowid_lv, pi64_type);
     auto target_partition_sorted_rowid_ptr_lv =
         cgen_state_->ir_builder_.CreateGEP(get_int_type(64, cgen_state_->context_),
                                            sorted_rowid_ptr_lv,
                                            current_partition_start_offset_lv);
 
     // # elems per partition
     const auto partition_count_buf =
         cgen_state_->llInt(reinterpret_cast<int64_t>(window_func_context->counts()));
     auto partition_count_buf_ptr_lv =
         cgen_state_->ir_builder_.CreateIntToPtr(partition_count_buf, pi32_type);
 
     // # elems of the given partition
     const auto num_elem_current_partition_ptr =
         cgen_state_->ir_builder_.CreateGEP(get_int_type(32, cgen_state_->context_),
                                            partition_count_buf_ptr_lv,
                                            partition_index_lv);
     auto num_elem_current_partition_lv = cgen_state_->castToTypeIn(
         cgen_state_->ir_builder_.CreateLoad(
             num_elem_current_partition_ptr->getType()->getPointerElementType(),
             num_elem_current_partition_ptr),
         64);
 
     const auto order_key_ti = window_func->getOrderKeys().front()->get_type_info();
     const auto order_key_size = order_key_ti.get_size();
     const auto order_col_type_name = get_col_type_name_by_size(
         order_key_size,
         window_func_context->getOrderKeyColumnBufferTypes().front().is_fp());
     size_t order_key_size_in_byte = order_key_size * 8;
 
     const auto order_key_buf_type = llvm::PointerType::get(
         get_int_type(order_key_size_in_byte, cgen_state_->context_), 0);
     const auto order_key_buf = cgen_state_->llInt(reinterpret_cast<int64_t>(
         window_func_context->getOrderKeyColumnBuffers().front()));
     auto order_key_buf_ptr_lv =
         cgen_state_->ir_builder_.CreateIntToPtr(order_key_buf, order_key_buf_type);
 
     // null value of the ordering column
     const auto order_key_buf_ti =
         window_func_context->getOrderKeyColumnBufferTypes().front();
     llvm::Value* order_key_col_null_val_lv{nullptr};
     switch (order_key_buf_ti.get_type()) {
       case kDATE:
       case kTIME:
       case kTIMESTAMP: {
         switch (order_key_buf_ti.get_size()) {
           case 1: {
             order_key_col_null_val_lv =
                 cgen_state_->inlineNull(SQLTypeInfo(SQLTypes::kTINYINT));
             break;
           }
           case 2: {
             order_key_col_null_val_lv =
                 cgen_state_->inlineNull(SQLTypeInfo(SQLTypes::kSMALLINT));
             break;
           }
           case 4: {
             order_key_col_null_val_lv =
                 cgen_state_->inlineNull(SQLTypeInfo(SQLTypes::kINT));
             break;
           }
           case 8: {
             order_key_col_null_val_lv =
                 cgen_state_->inlineNull(SQLTypeInfo(SQLTypes::kBIGINT));
             break;
           }
           default:
             break;
         }
         break;
       }
       default: {
         order_key_col_null_val_lv = cgen_state_->inlineNull(order_key_buf_ti);
         break;
       }
     }
 
     // null range of the aggregate tree
     const auto null_start_pos_buf = cgen_state_->llInt(
         reinterpret_cast<int64_t>(window_func_context->getNullValueStartPos()));
     const auto null_start_pos_buf_ptr =
         cgen_state_->ir_builder_.CreateIntToPtr(null_start_pos_buf, pi64_type);
     const auto null_start_pos_ptr =
         cgen_state_->ir_builder_.CreateGEP(get_int_type(64, cgen_state_->context_),
                                            null_start_pos_buf_ptr,
                                            partition_index_lv);
     auto null_start_pos_lv = cgen_state_->ir_builder_.CreateLoad(
         null_start_pos_ptr->getType()->getPointerElementType(),
         null_start_pos_ptr,
         "null_start_pos");
     const auto null_end_pos_buf = cgen_state_->llInt(
         reinterpret_cast<int64_t>(window_func_context->getNullValueEndPos()));
     const auto null_end_pos_buf_ptr =
         cgen_state_->ir_builder_.CreateIntToPtr(null_end_pos_buf, pi64_type);
     const auto null_end_pos_ptr =
         cgen_state_->ir_builder_.CreateGEP(get_int_type(64, cgen_state_->context_),
                                            null_end_pos_buf_ptr,
                                            partition_index_lv);
     auto null_end_pos_lv = cgen_state_->ir_builder_.CreateLoad(
         null_end_pos_ptr->getType()->getPointerElementType(),
         null_end_pos_ptr,
         "null_end_pos");
 
     llvm::Value* frame_start_bound_lv{nullptr};
     llvm::Value* frame_end_bound_lv{nullptr};
     std::tie(frame_start_bound_lv, frame_end_bound_lv) =
         codegenWindowFrameBound(window_func_context,
                                 frame_start_bound,
                                 frame_end_bound,
                                 current_row_pos_lv,
                                 current_partition_start_offset_lv,
                                 order_key_buf_ptr_lv,
                                 order_key_col_null_val_lv,
                                 frame_start_bound_expr_lv,
                                 frame_end_bound_expr_lv,
                                 num_elem_current_partition_lv,
                                 target_partition_rowid_ptr_lv,
                                 target_partition_sorted_rowid_ptr_lv,
                                 null_start_pos_lv,
                                 null_end_pos_lv,
                                 code_generator);
     CHECK(frame_start_bound_lv);
     CHECK(frame_end_bound_lv);
 
     // codegen to send a query with frame bound to aggregate tree searcher
     llvm::Value* aggregation_trees_lv{nullptr};
     llvm::Value* invalid_val_lv{nullptr};
     llvm::Value* null_val_lv{nullptr};
     std::string aggregation_tree_search_func_name{"search_"};
     std::string aggregation_tree_getter_func_name{"get_"};
 
     // prepare null values and aggregate_tree getter and searcher depending on
     // a type of the ordering column
     auto agg_expr_ti = args.front()->get_type_info();
     switch (agg_expr_ti.get_type()) {
       case SQLTypes::kTINYINT:
       case SQLTypes::kSMALLINT:
       case SQLTypes::kINT:
       case SQLTypes::kBIGINT:
       case SQLTypes::kNUMERIC:
       case SQLTypes::kDECIMAL: {
         if (window_func->getKind() == SqlWindowFunctionKind::MIN) {
           invalid_val_lv = cgen_state_->llInt(std::numeric_limits<int64_t>::max());
         } else if (window_func->getKind() == SqlWindowFunctionKind::MAX) {
           invalid_val_lv = cgen_state_->llInt(std::numeric_limits<int64_t>::lowest());
         } else {
           invalid_val_lv = cgen_state_->llInt((int64_t)0);
         }
         null_val_lv = cgen_state_->llInt(inline_int_null_value<int64_t>());
         aggregation_tree_search_func_name += "int64_t";
         aggregation_tree_getter_func_name += "integer";
         break;
       }
       case SQLTypes::kFLOAT:
       case SQLTypes::kDOUBLE: {
         if (window_func->getKind() == SqlWindowFunctionKind::MIN) {
           invalid_val_lv = cgen_state_->llFp(std::numeric_limits<double>::max());
         } else if (window_func->getKind() == SqlWindowFunctionKind::MAX) {
           invalid_val_lv = cgen_state_->llFp(std::numeric_limits<double>::lowest());
         } else {
           invalid_val_lv = cgen_state_->llFp((double)0);
         }
         null_val_lv = cgen_state_->inlineFpNull(SQLTypeInfo(kDOUBLE));
         aggregation_tree_search_func_name += "double";
         aggregation_tree_getter_func_name += "double";
         break;
       }
       default: {
         CHECK(false);
         break;
       }
     }
 
     // derived aggregation has a different code path
     if (window_func->getKind() == SqlWindowFunctionKind::AVG) {
       aggregation_tree_search_func_name += "_derived";
       aggregation_tree_getter_func_name += "_derived";
     }
 
     // get a buffer holding aggregate trees for each partition
     if (agg_expr_ti.is_integer() || agg_expr_ti.is_decimal()) {
       if (window_func->getKind() == SqlWindowFunctionKind::AVG) {
         aggregation_trees_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
             window_func_context->getDerivedAggregationTreesForIntegerTypeWindowExpr()));
       } else {
         aggregation_trees_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
             window_func_context->getAggregationTreesForIntegerTypeWindowExpr()));
       }
     } else if (agg_expr_ti.is_fp()) {
       if (window_func->getKind() == SqlWindowFunctionKind::AVG) {
         aggregation_trees_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
             window_func_context->getDerivedAggregationTreesForDoubleTypeWindowExpr()));
       } else {
         aggregation_trees_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
             window_func_context->getAggregationTreesForDoubleTypeWindowExpr()));
       }
     }
 
     CHECK(aggregation_trees_lv);
     CHECK(invalid_val_lv);
     aggregation_tree_search_func_name += "_aggregation_tree";
     aggregation_tree_getter_func_name += "_aggregation_tree";
 
     // get the aggregate tree of the current partition from a window context
     auto aggregation_trees_ptr =
         cgen_state_->ir_builder_.CreateIntToPtr(aggregation_trees_lv, ppi64_type);
     auto target_aggregation_tree_lv = cgen_state_->emitCall(
         aggregation_tree_getter_func_name, {aggregation_trees_ptr, partition_index_lv});
 
     // a depth of segment tree
     const auto tree_depth_buf = cgen_state_->llInt(
         reinterpret_cast<int64_t>(window_func_context->getAggregateTreeDepth()));
     const auto tree_depth_buf_ptr =
         cgen_state_->ir_builder_.CreateIntToPtr(tree_depth_buf, pi64_type);
     const auto current_partition_tree_depth_buf_ptr = cgen_state_->ir_builder_.CreateGEP(
         get_int_type(64, cgen_state_->context_), tree_depth_buf_ptr, partition_index_lv);
     const auto current_partition_tree_depth_lv = cgen_state_->ir_builder_.CreateLoad(
         current_partition_tree_depth_buf_ptr->getType()->getPointerElementType(),
         current_partition_tree_depth_buf_ptr);
 
     // a fanout of the current partition's segment tree
     const auto aggregation_tree_fanout_lv = cgen_state_->llInt(
         static_cast<int64_t>(window_func_context->getAggregateTreeFanout()));
 
     // agg_type
     const auto agg_type_lv =
         cgen_state_->llInt(static_cast<int32_t>(window_func->getKind()));
 
     // send a query to the aggregate tree with the frame range:
     // `frame_start_bound_lv` ~ `frame_end_bound_lv`
     auto res_lv =
         cgen_state_->emitCall(aggregation_tree_search_func_name,
                               {target_aggregation_tree_lv,
                                frame_start_bound_lv,
                                frame_end_bound_lv,
                                current_partition_tree_depth_lv,
                                aggregation_tree_fanout_lv,
                                cgen_state_->llBool(agg_expr_ti.is_decimal()),
                                cgen_state_->llInt((int64_t)agg_expr_ti.get_scale()),
                                invalid_val_lv,
                                null_val_lv,
                                agg_type_lv});
 
     // handling returned null value if exists
     std::string null_handler_func_name{"handle_null_val_"};
     std::vector<llvm::Value*> null_handler_args{res_lv, null_val_lv};
 
     // determine null_handling function's name
     if (window_func->getKind() == SqlWindowFunctionKind::AVG) {
       // average aggregate function returns a value as a double
       // (and our search* function also returns a double)
       if (agg_expr_ti.is_fp()) {
         // fp type: double null value
         null_handler_func_name += "double_double";
       } else {
         // non-fp type: int64_t null type
         null_handler_func_name += "double_int64_t";
       }
     } else if (agg_expr_ti.is_fp()) {
       // fp type: double null value
       null_handler_func_name += "double_double";
     } else {
       // non-fp type: int64_t null type
       null_handler_func_name += "int64_t_int64_t";
     }
     null_handler_func_name += "_window_framing_agg";
 
     // prepare null_val
     if (window_func->getKind() == SqlWindowFunctionKind::COUNT) {
       if (agg_expr_ti.is_fp()) {
         null_handler_args.push_back(cgen_state_->llFp((double)0));
       } else {
         null_handler_args.push_back(cgen_state_->llInt((int64_t)0));
       }
     } else if (window_func->getKind() == SqlWindowFunctionKind::AVG) {
       null_handler_args.push_back(cgen_state_->inlineFpNull(SQLTypeInfo(kDOUBLE)));
     } else {
       null_handler_args.push_back(cgen_state_->castToTypeIn(window_func_null_val, 64));
     }
     res_lv = cgen_state_->emitCall(null_handler_func_name, null_handler_args);
 
     // when AGG_TYPE is double, we get a double type return value we expect an integer
     // type value for the count aggregation
     if (window_func->getKind() == SqlWindowFunctionKind::COUNT && agg_expr_ti.is_fp()) {
       return cgen_state_->ir_builder_.CreateFPToSI(
           res_lv, get_int_type(64, cgen_state_->context_));
     }
     return res_lv;
   } else {
     llvm::Value* multiplicity_lv = nullptr;
     const auto agg_name = get_window_agg_name(window_func->getKind(), window_func_ti);
     if (args.empty()) {
       cgen_state_->emitCall(agg_name, {aggregate_state, crt_val});
     } else {
       cgen_state_->emitCall(agg_name + "_skip_val",
                             {aggregate_state, crt_val, window_func_null_val});
     }
     if (window_func->getKind() == SqlWindowFunctionKind::AVG) {
       codegenWindowAvgEpilogue(crt_val, window_func_null_val, multiplicity_lv);
     }
     return codegenAggregateWindowState();
   }
 }

Here is the call graph for this function:

llvm::Value * Executor::codegenWindowFunctionOnFrame ( const CompilationOptions & co )

private

Definition at line 333 of file WindowFunctionIR.cpp.

References run_benchmark_import::args, AUTOMATIC_IR_METADATA, CHECK, anonymous_namespace{WindowFunctionIR.cpp}::get_col_type_name_by_size(), get_fp_type(), get_int_type(), anonymous_namespace{WindowFunctionIR.cpp}::get_null_value_by_size(), anonymous_namespace{WindowFunctionIR.cpp}::get_null_value_by_size_with_encoding(), WindowProjectNodeContext::getActiveWindowFunctionContext(), kENCODING_DATE_IN_DAYS, kSecsPerDay, LAG_IN_FRAME, and LEAD_IN_FRAME.

                                                                               {
   AUTOMATIC_IR_METADATA(cgen_state_.get());
   const auto window_func_context =
       WindowProjectNodeContext::getActiveWindowFunctionContext(this);
   const auto window_func = window_func_context->getWindowFunction();
   const auto window_func_kind = window_func->getKind();
   const auto& args = window_func->getArgs();
   CHECK(args.size() >= 1 && args.size() <= 3);
   CodeGenerator code_generator(this);
   const auto offset_lv =
       cgen_state_->castToTypeIn(code_generator.codegen(args[1].get(), true, co)[0], 64);
 
   // codegen frame bound expr if necessary
   const auto frame_start_bound = window_func->getFrameStartBound();
   const auto frame_end_bound = window_func->getFrameEndBound();
   auto frame_start_bound_expr_lv =
       codegenFrameBoundExpr(window_func, frame_start_bound, code_generator, co);
   auto frame_end_bound_expr_lv =
       codegenFrameBoundExpr(window_func, frame_end_bound, code_generator, co);
   CHECK(frame_start_bound_expr_lv);
   CHECK(frame_end_bound_expr_lv);
 
   auto current_row_pos_lv = code_generator.posArg(nullptr);
   auto partition_index_lv =
       codegenCurrentPartitionIndex(window_func_context, current_row_pos_lv);
 
   llvm::Value* res_lv{nullptr};
   // currently, we only support below two window functions on frame
   // todo (yonnmin): remove this when supporting more window functions on frame
   CHECK(window_func_kind == SqlWindowFunctionKind::LEAD_IN_FRAME ||
         window_func_kind == SqlWindowFunctionKind::LAG_IN_FRAME);
   const auto pi32_type =
       llvm::PointerType::get(get_int_type(32, cgen_state_->context_), 0);
   const auto pi64_type =
       llvm::PointerType::get(get_int_type(64, cgen_state_->context_), 0);
   bool is_lag_in_frame = window_func_kind == SqlWindowFunctionKind::LAG_IN_FRAME;
 
   // ordering column buffer
   const auto target_col_ti = window_func->getArgs().front()->get_type_info();
   const auto target_col_size = target_col_ti.get_size();
   const auto target_col_type_name =
       get_col_type_name_by_size(target_col_size, target_col_ti.is_fp());
   const auto target_col_logical_type_name = get_col_type_name_by_size(
       window_func->get_type_info().get_size(), window_func->get_type_info().is_fp());
 
   // when target_column is fixed encoded, we store the actual column value by
   // considering it, but our resultset analyzer only considers the type without encoding
   // scheme so we handle them separately
   auto logical_null_val_lv =
       get_null_value_by_size(cgen_state_.get(), window_func->get_type_info());
   auto target_col_null_val_lv =
       get_null_value_by_size_with_encoding(cgen_state_.get(), target_col_ti);
   size_t target_col_size_in_byte = target_col_size * 8;
   llvm::Type* col_buf_ptr_type =
       target_col_ti.is_fp()
           ? get_fp_type(target_col_size_in_byte, cgen_state_->context_)
           : get_int_type(target_col_size_in_byte, cgen_state_->context_);
   auto col_buf_type = llvm::PointerType::get(col_buf_ptr_type, 0);
   auto target_col_buf_ptr_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
       window_func_context->getColumnBufferForWindowFunctionExpressions().front()));
   auto target_col_buf_lv =
       cgen_state_->ir_builder_.CreateIntToPtr(target_col_buf_ptr_lv, col_buf_type);
 
   // partial sum of # elems of partitions
   auto partition_start_offset_buf_lv = cgen_state_->llInt(
       reinterpret_cast<int64_t>(window_func_context->partitionStartOffset()));
   auto partition_start_offset_ptr_lv =
       cgen_state_->ir_builder_.CreateIntToPtr(partition_start_offset_buf_lv, pi64_type);
 
   // get start offset of the current partition
   auto current_partition_start_offset_ptr_lv =
       cgen_state_->ir_builder_.CreateGEP(get_int_type(64, cgen_state_->context_),
                                          partition_start_offset_ptr_lv,
                                          partition_index_lv);
   auto current_partition_start_offset_lv = cgen_state_->ir_builder_.CreateLoad(
       current_partition_start_offset_ptr_lv->getType()->getPointerElementType(),
       current_partition_start_offset_ptr_lv);
 
   // row_id buf of the current partition
   const auto partition_rowid_buf_lv =
       cgen_state_->llInt(reinterpret_cast<int64_t>(window_func_context->payload()));
   const auto partition_rowid_ptr_lv =
       cgen_state_->ir_builder_.CreateIntToPtr(partition_rowid_buf_lv, pi32_type);
   auto target_partition_rowid_ptr_lv =
       cgen_state_->ir_builder_.CreateGEP(get_int_type(32, cgen_state_->context_),
                                          partition_rowid_ptr_lv,
                                          current_partition_start_offset_lv);
 
   // row_id buf of ordered current partition
   const auto sorted_rowid_lv = cgen_state_->llInt(
       reinterpret_cast<int64_t>(window_func_context->sortedPartition()));
   const auto sorted_rowid_ptr_lv =
       cgen_state_->ir_builder_.CreateIntToPtr(sorted_rowid_lv, pi64_type);
   auto target_partition_sorted_rowid_ptr_lv =
       cgen_state_->ir_builder_.CreateGEP(get_int_type(64, cgen_state_->context_),
                                          sorted_rowid_ptr_lv,
                                          current_partition_start_offset_lv);
 
   // # elems per partition
   const auto partition_count_buf =
       cgen_state_->llInt(reinterpret_cast<int64_t>(window_func_context->counts()));
   auto partition_count_buf_ptr_lv =
       cgen_state_->ir_builder_.CreateIntToPtr(partition_count_buf, pi32_type);
 
   // # elems of the given partition
   const auto num_elem_current_partition_ptr =
       cgen_state_->ir_builder_.CreateGEP(get_int_type(32, cgen_state_->context_),
                                          partition_count_buf_ptr_lv,
                                          partition_index_lv);
   auto num_elem_current_partition_lv = cgen_state_->castToTypeIn(
       cgen_state_->ir_builder_.CreateLoad(
           num_elem_current_partition_ptr->getType()->getPointerElementType(),
           num_elem_current_partition_ptr),
       64);
 
   const auto order_key_ti = window_func->getOrderKeys().front()->get_type_info();
   const auto order_key_size = order_key_ti.get_size();
   const auto order_col_type_name = get_col_type_name_by_size(
       order_key_size,
       window_func_context->getOrderKeyColumnBufferTypes().front().is_fp());
   size_t order_key_size_in_byte = order_key_size * 8;
 
   const auto order_key_buf_type = llvm::PointerType::get(
       get_int_type(order_key_size_in_byte, cgen_state_->context_), 0);
   const auto order_key_buf = cgen_state_->llInt(
       reinterpret_cast<int64_t>(window_func_context->getOrderKeyColumnBuffers().front()));
   auto order_key_buf_ptr_lv =
       cgen_state_->ir_builder_.CreateIntToPtr(order_key_buf, order_key_buf_type);
 
   // null value of the ordering column
   const auto order_key_buf_ti =
       window_func_context->getOrderKeyColumnBufferTypes().front();
   auto order_key_col_null_val_lv =
       get_null_value_by_size_with_encoding(cgen_state_.get(), order_key_buf_ti);
 
   // null range of the aggregate tree
   const auto null_start_pos_buf = cgen_state_->llInt(
       reinterpret_cast<int64_t>(window_func_context->getNullValueStartPos()));
   const auto null_start_pos_buf_ptr =
       cgen_state_->ir_builder_.CreateIntToPtr(null_start_pos_buf, pi64_type);
   const auto null_start_pos_ptr =
       cgen_state_->ir_builder_.CreateGEP(get_int_type(64, cgen_state_->context_),
                                          null_start_pos_buf_ptr,
                                          partition_index_lv);
   auto null_start_pos_lv = cgen_state_->ir_builder_.CreateLoad(
       null_start_pos_ptr->getType()->getPointerElementType(),
       null_start_pos_ptr,
       "null_start_pos");
   const auto null_end_pos_buf = cgen_state_->llInt(
       reinterpret_cast<int64_t>(window_func_context->getNullValueEndPos()));
   const auto null_end_pos_buf_ptr =
       cgen_state_->ir_builder_.CreateIntToPtr(null_end_pos_buf, pi64_type);
   const auto null_end_pos_ptr = cgen_state_->ir_builder_.CreateGEP(
       get_int_type(64, cgen_state_->context_), null_end_pos_buf_ptr, partition_index_lv);
   auto null_end_pos_lv = cgen_state_->ir_builder_.CreateLoad(
       null_end_pos_ptr->getType()->getPointerElementType(),
       null_end_pos_ptr,
       "null_end_pos");
 
   std::string compute_row_idx_on_frame_func_name = "compute_";
   compute_row_idx_on_frame_func_name += order_col_type_name + "_current_row_idx_in_frame";
   auto cur_row_idx_in_frame_lv =
       cgen_state_->emitCall(compute_row_idx_on_frame_func_name,
                             {num_elem_current_partition_lv,
                              current_row_pos_lv,
                              order_key_buf_ptr_lv,
                              target_partition_rowid_ptr_lv,
                              target_partition_sorted_rowid_ptr_lv,
                              order_key_col_null_val_lv,
                              null_start_pos_lv,
                              null_end_pos_lv});
 
   llvm::Value* frame_start_bound_lv{nullptr};
   llvm::Value* frame_end_bound_lv{nullptr};
   std::tie(frame_start_bound_lv, frame_end_bound_lv) =
       codegenWindowFrameBound(window_func_context,
                               frame_start_bound,
                               frame_end_bound,
                               cur_row_idx_in_frame_lv,
                               cgen_state_->llInt((int64_t)0),
                               order_key_buf_ptr_lv,
                               order_key_col_null_val_lv,
                               frame_start_bound_expr_lv,
                               frame_end_bound_expr_lv,
                               num_elem_current_partition_lv,
                               target_partition_rowid_ptr_lv,
                               target_partition_sorted_rowid_ptr_lv,
                               null_start_pos_lv,
                               null_end_pos_lv,
                               code_generator);
   CHECK(frame_start_bound_lv);
   CHECK(frame_end_bound_lv);
 
   llvm::Value* modified_cur_row_idx_in_frame_lv{nullptr};
   if (is_lag_in_frame) {
     modified_cur_row_idx_in_frame_lv =
         cgen_state_->ir_builder_.CreateSub(cur_row_idx_in_frame_lv, offset_lv);
   } else {
     modified_cur_row_idx_in_frame_lv =
         cgen_state_->ir_builder_.CreateAdd(cur_row_idx_in_frame_lv, offset_lv);
   }
   CHECK(modified_cur_row_idx_in_frame_lv);
 
   std::string target_func_name = "get_";
   target_func_name += target_col_type_name + "_value_";
   target_func_name += target_col_logical_type_name + "_type_";
   target_func_name += "in_frame";
   res_lv = cgen_state_->emitCall(target_func_name,
                                  {modified_cur_row_idx_in_frame_lv,
                                   frame_start_bound_lv,
                                   frame_end_bound_lv,
                                   target_col_buf_lv,
                                   target_partition_rowid_ptr_lv,
                                   target_partition_sorted_rowid_ptr_lv,
                                   logical_null_val_lv,
                                   target_col_null_val_lv});
   if (target_col_ti.get_compression() == kENCODING_DATE_IN_DAYS) {
     res_lv = cgen_state_->emitCall(
         "encode_date",
         {res_lv, logical_null_val_lv, cgen_state_->llInt((int64_t)kSecsPerDay)});
   }
   CHECK(res_lv);
   return res_lv;
 }

Here is the call graph for this function:

void Executor::codegenWindowFunctionStateInit ( llvm::Value * aggregate_state )

private

Definition at line 283 of file WindowFunctionIR.cpp.

References AUTOMATIC_IR_METADATA, COUNT, anonymous_namespace{WindowFunctionIR.cpp}::get_adjusted_window_type_info(), get_int_type(), WindowProjectNodeContext::getActiveWindowFunctionContext(), kDOUBLE, and kFLOAT.

                                                                         {
   AUTOMATIC_IR_METADATA(cgen_state_.get());
   const auto window_func_context =
       WindowProjectNodeContext::getActiveWindowFunctionContext(this);
   const auto window_func = window_func_context->getWindowFunction();
   const auto window_func_ti = get_adjusted_window_type_info(window_func);
   const auto window_func_null_val =
       window_func_ti.is_fp()
           ? cgen_state_->inlineFpNull(window_func_ti)
           : cgen_state_->castToTypeIn(cgen_state_->inlineIntNull(window_func_ti), 64);
   llvm::Value* window_func_init_val;
   if (window_func_context->getWindowFunction()->getKind() ==
       SqlWindowFunctionKind::COUNT) {
     switch (window_func_ti.get_type()) {
       case kFLOAT: {
         window_func_init_val = cgen_state_->llFp(float(0));
         break;
       }
       case kDOUBLE: {
         window_func_init_val = cgen_state_->llFp(double(0));
         break;
       }
       default: {
         window_func_init_val = cgen_state_->llInt(int64_t(0));
         break;
       }
     }
   } else {
     window_func_init_val = window_func_null_val;
   }
   const auto pi32_type =
       llvm::PointerType::get(get_int_type(32, cgen_state_->context_), 0);
   switch (window_func_ti.get_type()) {
     case kDOUBLE: {
       cgen_state_->emitCall("agg_id_double", {aggregate_state, window_func_init_val});
       break;
     }
     case kFLOAT: {
       aggregate_state =
           cgen_state_->ir_builder_.CreateBitCast(aggregate_state, pi32_type);
       cgen_state_->emitCall("agg_id_float", {aggregate_state, window_func_init_val});
       break;
     }
     default: {
       cgen_state_->emitCall("agg_id", {aggregate_state, window_func_init_val});
       break;
     }
   }
 }

Here is the call graph for this function:

llvm::BasicBlock * Executor::codegenWindowResetStateControlFlow ( )

private

Definition at line 254 of file WindowFunctionIR.cpp.

References AUTOMATIC_IR_METADATA, WindowProjectNodeContext::getActiveWindowFunctionContext(), CodeGenerator::posArg(), and CodeGenerator::toBool().

                                                              {
   AUTOMATIC_IR_METADATA(cgen_state_.get());
   const auto window_func_context =
       WindowProjectNodeContext::getActiveWindowFunctionContext(this);
   const auto bitset = cgen_state_->llInt(
       reinterpret_cast<const int64_t>(window_func_context->partitionStart()));
   const auto min_val = cgen_state_->llInt(int64_t(0));
   const auto max_val = cgen_state_->llInt(window_func_context->elementCount() - 1);
   const auto null_val = cgen_state_->llInt(inline_int_null_value<int64_t>());
   const auto null_bool_val = cgen_state_->llInt<int8_t>(inline_int_null_value<int8_t>());
   CodeGenerator code_generator(this);
   const auto reset_state =
       code_generator.toBool(cgen_state_->emitCall("bit_is_set",
                                                   {bitset,
                                                    code_generator.posArg(nullptr),
                                                    min_val,
                                                    max_val,
                                                    null_val,
                                                    null_bool_val}));
   const auto reset_state_true_bb = llvm::BasicBlock::Create(
       cgen_state_->context_, "reset_state.true", cgen_state_->current_func_);
   const auto reset_state_false_bb = llvm::BasicBlock::Create(
       cgen_state_->context_, "reset_state.false", cgen_state_->current_func_);
   cgen_state_->ir_builder_.CreateCondBr(
       reset_state, reset_state_true_bb, reset_state_false_bb);
   cgen_state_->ir_builder_.SetInsertPoint(reset_state_true_bb);
   return reset_state_false_bb;
 }

Here is the call graph for this function:

ResultSetPtr Executor::collectAllDeviceResults	(	SharedKernelContext &	shared_context,
		const RelAlgExecutionUnit &	ra_exe_unit,
		const QueryMemoryDescriptor &	query_mem_desc,
		const ExecutorDeviceType	device_type,
		std::shared_ptr< RowSetMemoryOwner >	row_set_mem_owner
	)

private

Definition at line 2347 of file Execute.cpp.

References anonymous_namespace{Execute.cpp}::build_row_for_empty_input(), catalog_, collectAllDeviceShardedTopResults(), DEBUG_TIMER, SharedKernelContext::getFragmentResults(), QueryMemoryDescriptor::getQueryDescriptionType(), GPU, NonGroupedAggregate, reduceMultiDeviceResults(), reduceSpeculativeTopN(), GroupByAndAggregate::shard_count_for_top_groups(), RelAlgExecutionUnit::target_exprs, and use_speculative_top_n().

Referenced by executeWorkUnitImpl().

                                                         {
   auto timer = DEBUG_TIMER(__func__);
   auto& result_per_device = shared_context.getFragmentResults();
   if (result_per_device.empty() && query_mem_desc.getQueryDescriptionType() ==
                                        QueryDescriptionType::NonGroupedAggregate) {
     return build_row_for_empty_input(
         ra_exe_unit.target_exprs, query_mem_desc, device_type);
   }
   if (use_speculative_top_n(ra_exe_unit, query_mem_desc)) {
     try {
       return reduceSpeculativeTopN(
           ra_exe_unit, result_per_device, row_set_mem_owner, query_mem_desc);
     } catch (const std::bad_alloc&) {
       throw SpeculativeTopNFailed("Failed during multi-device reduction.");
     }
   }
   const auto shard_count =
       device_type == ExecutorDeviceType::GPU
           ? GroupByAndAggregate::shard_count_for_top_groups(ra_exe_unit, *catalog_)
           : 0;
 
   if (shard_count && !result_per_device.empty()) {
     return collectAllDeviceShardedTopResults(shared_context, ra_exe_unit);
   }
   return reduceMultiDeviceResults(
       ra_exe_unit, result_per_device, row_set_mem_owner, query_mem_desc);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

ResultSetPtr Executor::collectAllDeviceShardedTopResults	(	SharedKernelContext &	shared_context,
		const RelAlgExecutionUnit &	ra_exe_unit
	)		const

private

Definition at line 2462 of file Execute.cpp.

References blockSize(), catalog_, CHECK, CHECK_EQ, CHECK_LE, SharedKernelContext::getFragmentResults(), gridSize(), SortInfo::limit, SortInfo::offset, SortInfo::order_entries, anonymous_namespace{Execute.cpp}::permute_storage_columnar(), anonymous_namespace{Execute.cpp}::permute_storage_row_wise(), run_benchmark_import::result, and RelAlgExecutionUnit::sort_info.

Referenced by collectAllDeviceResults().

                                                   {
   auto& result_per_device = shared_context.getFragmentResults();
   const auto first_result_set = result_per_device.front().first;
   CHECK(first_result_set);
   auto top_query_mem_desc = first_result_set->getQueryMemDesc();
   CHECK(!top_query_mem_desc.hasInterleavedBinsOnGpu());
   const auto top_n = ra_exe_unit.sort_info.limit + ra_exe_unit.sort_info.offset;
   top_query_mem_desc.setEntryCount(0);
   for (auto& result : result_per_device) {
     const auto result_set = result.first;
     CHECK(result_set);
     result_set->sort(ra_exe_unit.sort_info.order_entries, top_n, this);
     size_t new_entry_cnt = top_query_mem_desc.getEntryCount() + result_set->rowCount();
     top_query_mem_desc.setEntryCount(new_entry_cnt);
   }
   auto top_result_set = std::make_shared<ResultSet>(first_result_set->getTargetInfos(),
                                                     first_result_set->getDeviceType(),
                                                     top_query_mem_desc,
                                                     first_result_set->getRowSetMemOwner(),
                                                     catalog_,
                                                     blockSize(),
                                                     gridSize());
   auto top_storage = top_result_set->allocateStorage();
   size_t top_output_row_idx{0};
   for (auto& result : result_per_device) {
     const auto result_set = result.first;
     CHECK(result_set);
     const auto& top_permutation = result_set->getPermutationBuffer();
     CHECK_LE(top_permutation.size(), top_n);
     if (top_query_mem_desc.didOutputColumnar()) {
       top_output_row_idx = permute_storage_columnar(result_set->getStorage(),
                                                     result_set->getQueryMemDesc(),
                                                     top_storage,
                                                     top_output_row_idx,
                                                     top_query_mem_desc,
                                                     top_permutation);
     } else {
       top_output_row_idx = permute_storage_row_wise(result_set->getStorage(),
                                                     top_storage,
                                                     top_output_row_idx,
                                                     top_query_mem_desc,
                                                     top_permutation);
     }
   }
   CHECK_EQ(top_output_row_idx, top_query_mem_desc.getEntryCount());
   return top_result_set;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool Executor::compileBody	(	const RelAlgExecutionUnit &	ra_exe_unit,
		GroupByAndAggregate &	group_by_and_aggregate,
		QueryMemoryDescriptor &	query_mem_desc,
		const CompilationOptions &	co,
		const GpuSharedMemoryContext &	gpu_smem_context = `{}`
	)

private

Definition at line 3214 of file NativeCodegen.cpp.

                                                                            {
   AUTOMATIC_IR_METADATA(cgen_state_.get());
 
   // Switch the code generation into a separate filter function if enabled.
   // Note that accesses to function arguments are still codegenned from the
   // row function's arguments, then later automatically forwarded and
   // remapped into filter function arguments by redeclareFilterFunction().
   cgen_state_->row_func_bb_ = cgen_state_->ir_builder_.GetInsertBlock();
   llvm::Value* loop_done{nullptr};
   std::unique_ptr<Executor::FetchCacheAnchor> fetch_cache_anchor;
   if (cgen_state_->filter_func_) {
     if (cgen_state_->row_func_bb_->getName() == "loop_body") {
       auto row_func_entry_bb = &cgen_state_->row_func_->getEntryBlock();
       cgen_state_->ir_builder_.SetInsertPoint(row_func_entry_bb,
                                               row_func_entry_bb->begin());
       loop_done = cgen_state_->ir_builder_.CreateAlloca(
           get_int_type(1, cgen_state_->context_), nullptr, "loop_done");
       cgen_state_->ir_builder_.SetInsertPoint(cgen_state_->row_func_bb_);
       cgen_state_->ir_builder_.CreateStore(cgen_state_->llBool(true), loop_done);
     }
     cgen_state_->ir_builder_.SetInsertPoint(cgen_state_->filter_func_bb_);
     cgen_state_->current_func_ = cgen_state_->filter_func_;
     fetch_cache_anchor = std::make_unique<Executor::FetchCacheAnchor>(cgen_state_.get());
   }
 
   // generate the code for the filter
   std::vector<Analyzer::Expr*> primary_quals;
   std::vector<Analyzer::Expr*> deferred_quals;
   bool short_circuited = CodeGenerator::prioritizeQuals(
       ra_exe_unit, primary_quals, deferred_quals, plan_state_->hoisted_filters_);
   if (short_circuited) {
     VLOG(1) << "Prioritized " << std::to_string(primary_quals.size()) << " quals, "
             << "short-circuited and deferred " << std::to_string(deferred_quals.size())
             << " quals";
   }
   llvm::Value* filter_lv = cgen_state_->llBool(true);
   CodeGenerator code_generator(this);
   for (auto expr : primary_quals) {
     // Generate the filter for primary quals
     auto cond = code_generator.toBool(code_generator.codegen(expr, true, co).front());
     filter_lv = cgen_state_->ir_builder_.CreateAnd(filter_lv, cond);
   }
   CHECK(filter_lv->getType()->isIntegerTy(1));
   llvm::BasicBlock* sc_false{nullptr};
   if (!deferred_quals.empty()) {
     auto sc_true = llvm::BasicBlock::Create(
         cgen_state_->context_, "sc_true", cgen_state_->current_func_);
     sc_false = llvm::BasicBlock::Create(
         cgen_state_->context_, "sc_false", cgen_state_->current_func_);
     cgen_state_->ir_builder_.CreateCondBr(filter_lv, sc_true, sc_false);
     cgen_state_->ir_builder_.SetInsertPoint(sc_false);
     if (ra_exe_unit.join_quals.empty()) {
       cgen_state_->ir_builder_.CreateRet(cgen_state_->llInt(int32_t(0)));
     }
     cgen_state_->ir_builder_.SetInsertPoint(sc_true);
     filter_lv = cgen_state_->llBool(true);
   }
   for (auto expr : deferred_quals) {
     filter_lv = cgen_state_->ir_builder_.CreateAnd(
         filter_lv, code_generator.toBool(code_generator.codegen(expr, true, co).front()));
   }
 
   CHECK(filter_lv->getType()->isIntegerTy(1));
   auto ret = group_by_and_aggregate.codegen(
       filter_lv, sc_false, query_mem_desc, co, gpu_smem_context);
 
   // Switch the code generation back to the row function if a filter
   // function was enabled.
   if (cgen_state_->filter_func_) {
     if (cgen_state_->row_func_bb_->getName() == "loop_body") {
       cgen_state_->ir_builder_.CreateStore(cgen_state_->llBool(false), loop_done);
       cgen_state_->ir_builder_.CreateRet(cgen_state_->llInt<int32_t>(0));
     }
 
     cgen_state_->ir_builder_.SetInsertPoint(cgen_state_->row_func_bb_);
     cgen_state_->current_func_ = cgen_state_->row_func_;
     cgen_state_->filter_func_call_ =
         cgen_state_->ir_builder_.CreateCall(cgen_state_->filter_func_, {});
 
     // Create real filter function declaration after placeholder call
     // is emitted.
     redeclareFilterFunction();
 
     if (cgen_state_->row_func_bb_->getName() == "loop_body") {
       auto loop_done_true = llvm::BasicBlock::Create(
           cgen_state_->context_, "loop_done_true", cgen_state_->row_func_);
       auto loop_done_false = llvm::BasicBlock::Create(
           cgen_state_->context_, "loop_done_false", cgen_state_->row_func_);
       auto loop_done_flag = cgen_state_->ir_builder_.CreateLoad(
           loop_done->getType()->getPointerElementType(), loop_done);
       cgen_state_->ir_builder_.CreateCondBr(
           loop_done_flag, loop_done_true, loop_done_false);
       cgen_state_->ir_builder_.SetInsertPoint(loop_done_true);
       cgen_state_->ir_builder_.CreateRet(cgen_state_->filter_func_call_);
       cgen_state_->ir_builder_.SetInsertPoint(loop_done_false);
     } else {
       cgen_state_->ir_builder_.CreateRet(cgen_state_->filter_func_call_);
     }
   }
   return ret;
 }

std::tuple< CompilationResult, std::unique_ptr< QueryMemoryDescriptor > > Executor::compileWorkUnit	(	const std::vector< InputTableInfo > &	query_infos,
		const PlanState::DeletedColumnsMap &	deleted_cols_map,
		const RelAlgExecutionUnit &	ra_exe_unit,
		const CompilationOptions &	co,
		const ExecutionOptions &	eo,
		const CudaMgr_Namespace::CudaMgr *	cuda_mgr,
		const bool	allow_lazy_fetch,
		std::shared_ptr< RowSetMemoryOwner >	row_set_mem_owner,
		const size_t	max_groups_buffer_entry_count,
		const int8_t	crt_min_byte_width,
		const bool	has_cardinality_estimation,
		ColumnCacheMap &	column_cache,
		RenderInfo *	render_info = `nullptr`
	)

private

Definition at line 2668 of file NativeCodegen.cpp.

                                                    {
   auto timer = DEBUG_TIMER(__func__);
 
   if (co.device_type == ExecutorDeviceType::GPU) {
     if (!cuda_mgr) {
       throw QueryMustRunOnCpu();
     }
   }
 
 #ifndef NDEBUG
   static std::uint64_t counter = 0;
   ++counter;
   VLOG(1) << "CODEGEN #" << counter << ":";
   LOG(IR) << "CODEGEN #" << counter << ":";
   LOG(PTX) << "CODEGEN #" << counter << ":";
   LOG(ASM) << "CODEGEN #" << counter << ":";
 #endif
 
   // cgenstate_manager uses RAII pattern to manage the live time of
   // CgenState instances.
   Executor::CgenStateManager cgenstate_manager(*this,
                                                allow_lazy_fetch,
                                                query_infos,
                                                deleted_cols_map,
                                                &ra_exe_unit);  // locks compilation_mutex
 
   addTransientStringLiterals(ra_exe_unit, row_set_mem_owner);
 
   GroupByAndAggregate group_by_and_aggregate(
       this,
       co.device_type,
       ra_exe_unit,
       query_infos,
       row_set_mem_owner,
       has_cardinality_estimation ? std::optional<int64_t>(max_groups_buffer_entry_guess)
                                  : std::nullopt);
   auto query_mem_desc =
       group_by_and_aggregate.initQueryMemoryDescriptor(eo.allow_multifrag,
                                                        max_groups_buffer_entry_guess,
                                                        crt_min_byte_width,
                                                        render_info,
                                                        eo.output_columnar_hint);
 
   if (query_mem_desc->getQueryDescriptionType() ==
           QueryDescriptionType::GroupByBaselineHash &&
       !has_cardinality_estimation && (!render_info || !render_info->isInSitu()) &&
       !eo.just_explain) {
     const auto col_range_info = group_by_and_aggregate.getColRangeInfo();
     throw CardinalityEstimationRequired(col_range_info.max - col_range_info.min);
   }
 
   const bool output_columnar = query_mem_desc->didOutputColumnar();
   const bool gpu_shared_mem_optimization =
       is_gpu_shared_mem_supported(query_mem_desc.get(),
                                   ra_exe_unit,
                                   cuda_mgr,
                                   co.device_type,
                                   cuda_mgr ? this->blockSize() : 1,
                                   cuda_mgr ? this->numBlocksPerMP() : 1);
   if (gpu_shared_mem_optimization) {
     // disable interleaved bins optimization on the GPU
     query_mem_desc->setHasInterleavedBinsOnGpu(false);
     LOG(DEBUG1) << "GPU shared memory is used for the " +
                        query_mem_desc->queryDescTypeToString() + " query(" +
                        std::to_string(get_shared_memory_size(gpu_shared_mem_optimization,
                                                              query_mem_desc.get())) +
                        " out of " + std::to_string(g_gpu_smem_threshold) + " bytes).";
   }
 
   const GpuSharedMemoryContext gpu_smem_context(
       get_shared_memory_size(gpu_shared_mem_optimization, query_mem_desc.get()));
 
   if (co.device_type == ExecutorDeviceType::GPU) {
     const size_t num_count_distinct_descs =
         query_mem_desc->getCountDistinctDescriptorsSize();
     for (size_t i = 0; i < num_count_distinct_descs; i++) {
       const auto& count_distinct_descriptor =
           query_mem_desc->getCountDistinctDescriptor(i);
       if (count_distinct_descriptor.impl_type_ == CountDistinctImplType::UnorderedSet ||
           (count_distinct_descriptor.impl_type_ != CountDistinctImplType::Invalid &&
            !co.hoist_literals)) {
         throw QueryMustRunOnCpu();
       }
     }
 
     // we currently do not support varlen projection based on baseline groupby when
     // 1) target table is multi-fragmented and 2) multiple gpus are involved for query
     // processing in this case, we punt the query to cpu to avoid server crash
     for (const auto expr : ra_exe_unit.target_exprs) {
       if (auto gby_expr = dynamic_cast<Analyzer::AggExpr*>(expr)) {
         bool has_multiple_gpus = cuda_mgr ? cuda_mgr->getDeviceCount() > 1 : false;
         if (gby_expr->get_aggtype() == SQLAgg::kSAMPLE && has_multiple_gpus &&
             !g_leaf_count) {
           std::set<const Analyzer::ColumnVar*,
                    bool (*)(const Analyzer::ColumnVar*, const Analyzer::ColumnVar*)>
               colvar_set(Analyzer::ColumnVar::colvar_comp);
           gby_expr->collect_column_var(colvar_set, true);
           for (const auto cv : colvar_set) {
             if (cv->get_type_info().is_varlen()) {
               const auto tbl_id = cv->get_table_id();
               std::for_each(query_infos.begin(),
                             query_infos.end(),
                             [tbl_id](const InputTableInfo& input_table_info) {
                               if (input_table_info.table_id == tbl_id &&
                                   input_table_info.info.fragments.size() > 1) {
                                 throw QueryMustRunOnCpu();
                               }
                             });
             }
           }
         }
       }
     }
   }
 
   // Read the module template and target either CPU or GPU
   // by binding the stream position functions to the right implementation:
   // stride access for GPU, contiguous for CPU
   CHECK(cgen_state_->module_ == nullptr);
   cgen_state_->set_module_shallow_copy(get_rt_module(), /*always_clone=*/true);
 
   auto is_gpu = co.device_type == ExecutorDeviceType::GPU;
   if (is_gpu) {
     cgen_state_->module_->setDataLayout(get_gpu_data_layout());
     cgen_state_->module_->setTargetTriple(get_gpu_target_triple_string());
   }
   if (has_udf_module(/*is_gpu=*/is_gpu)) {
     CodeGenerator::link_udf_module(
         get_udf_module(/*is_gpu=*/is_gpu), *cgen_state_->module_, cgen_state_.get());
   }
   if (has_rt_udf_module(/*is_gpu=*/is_gpu)) {
     CodeGenerator::link_udf_module(
         get_rt_udf_module(/*is_gpu=*/is_gpu), *cgen_state_->module_, cgen_state_.get());
   }
 
   AUTOMATIC_IR_METADATA(cgen_state_.get());
 
   auto agg_fnames =
       get_agg_fnames(ra_exe_unit.target_exprs, !ra_exe_unit.groupby_exprs.empty());
 
   const auto agg_slot_count = ra_exe_unit.estimator ? size_t(1) : agg_fnames.size();
 
   const bool is_group_by{query_mem_desc->isGroupBy()};
   auto [query_func, row_func_call] = is_group_by
                                          ? query_group_by_template(cgen_state_->module_,
                                                                    co.hoist_literals,
                                                                    *query_mem_desc,
                                                                    co.device_type,
                                                                    ra_exe_unit.scan_limit,
                                                                    gpu_smem_context)
                                          : query_template(cgen_state_->module_,
                                                           agg_slot_count,
                                                           co.hoist_literals,
                                                           !!ra_exe_unit.estimator,
                                                           gpu_smem_context);
   bind_pos_placeholders("pos_start", true, query_func, cgen_state_->module_);
   bind_pos_placeholders("group_buff_idx", false, query_func, cgen_state_->module_);
   bind_pos_placeholders("pos_step", false, query_func, cgen_state_->module_);
 
   cgen_state_->query_func_ = query_func;
   cgen_state_->row_func_call_ = row_func_call;
   cgen_state_->query_func_entry_ir_builder_.SetInsertPoint(
       &query_func->getEntryBlock().front());
 
   // Generate the function signature and column head fetches s.t.
   // double indirection isn't needed in the inner loop
   auto& fetch_bb = query_func->front();
   llvm::IRBuilder<> fetch_ir_builder(&fetch_bb);
   fetch_ir_builder.SetInsertPoint(&*fetch_bb.begin());
   auto col_heads = generate_column_heads_load(ra_exe_unit.input_col_descs.size(),
                                               query_func->args().begin(),
                                               fetch_ir_builder,
                                               cgen_state_->context_);
   CHECK_EQ(ra_exe_unit.input_col_descs.size(), col_heads.size());
 
   cgen_state_->row_func_ = create_row_function(ra_exe_unit.input_col_descs.size(),
                                                is_group_by ? 0 : agg_slot_count,
                                                co.hoist_literals,
                                                cgen_state_->module_,
                                                cgen_state_->context_);
   CHECK(cgen_state_->row_func_);
   cgen_state_->row_func_bb_ =
       llvm::BasicBlock::Create(cgen_state_->context_, "entry", cgen_state_->row_func_);
 
   if (g_enable_filter_function) {
     auto filter_func_ft =
         llvm::FunctionType::get(get_int_type(32, cgen_state_->context_), {}, false);
     cgen_state_->filter_func_ = llvm::Function::Create(filter_func_ft,
                                                        llvm::Function::ExternalLinkage,
                                                        "filter_func",
                                                        cgen_state_->module_);
     CHECK(cgen_state_->filter_func_);
     cgen_state_->filter_func_bb_ = llvm::BasicBlock::Create(
         cgen_state_->context_, "entry", cgen_state_->filter_func_);
   }
 
   cgen_state_->current_func_ = cgen_state_->row_func_;
   cgen_state_->ir_builder_.SetInsertPoint(cgen_state_->row_func_bb_);
 
   preloadFragOffsets(ra_exe_unit.input_descs, query_infos);
   RelAlgExecutionUnit body_execution_unit = ra_exe_unit;
   const auto join_loops =
       buildJoinLoops(body_execution_unit, co, eo, query_infos, column_cache);
 
   plan_state_->allocateLocalColumnIds(ra_exe_unit.input_col_descs);
   for (auto& simple_qual : ra_exe_unit.simple_quals) {
     plan_state_->addSimpleQual(simple_qual);
   }
   const auto is_not_deleted_bb = codegenSkipDeletedOuterTableRow(ra_exe_unit, co);
   if (is_not_deleted_bb) {
     cgen_state_->row_func_bb_ = is_not_deleted_bb;
   }
   if (!join_loops.empty()) {
     codegenJoinLoops(join_loops,
                      body_execution_unit,
                      group_by_and_aggregate,
                      query_func,
                      cgen_state_->row_func_bb_,
                      *(query_mem_desc.get()),
                      co,
                      eo);
   } else {
     const bool can_return_error = compileBody(
         ra_exe_unit, group_by_and_aggregate, *query_mem_desc, co, gpu_smem_context);
     if (can_return_error || cgen_state_->needs_error_check_ || eo.with_dynamic_watchdog ||
         eo.allow_runtime_query_interrupt) {
       createErrorCheckControlFlow(query_func,
                                   eo.with_dynamic_watchdog,
                                   eo.allow_runtime_query_interrupt,
                                   join_loops,
                                   co.device_type,
                                   group_by_and_aggregate.query_infos_);
     }
   }
   std::vector<llvm::Value*> hoisted_literals;
 
   if (co.hoist_literals) {
     VLOG(1) << "number of hoisted literals: "
             << cgen_state_->query_func_literal_loads_.size()
             << " / literal buffer usage: " << cgen_state_->getLiteralBufferUsage(0)
             << " bytes";
   }
 
   if (co.hoist_literals && !cgen_state_->query_func_literal_loads_.empty()) {
     // we have some hoisted literals...
     hoisted_literals = inlineHoistedLiterals();
   }
 
   // replace the row func placeholder call with the call to the actual row func
   std::vector<llvm::Value*> row_func_args;
   for (size_t i = 0; i < cgen_state_->row_func_call_->getNumOperands() - 1; ++i) {
     row_func_args.push_back(cgen_state_->row_func_call_->getArgOperand(i));
   }
   row_func_args.insert(row_func_args.end(), col_heads.begin(), col_heads.end());
   row_func_args.push_back(get_arg_by_name(query_func, "join_hash_tables"));
   // push hoisted literals arguments, if any
   row_func_args.insert(
       row_func_args.end(), hoisted_literals.begin(), hoisted_literals.end());
   llvm::ReplaceInstWithInst(
       cgen_state_->row_func_call_,
       llvm::CallInst::Create(cgen_state_->row_func_, row_func_args, ""));
 
   // replace the filter func placeholder call with the call to the actual filter func
   if (cgen_state_->filter_func_) {
     std::vector<llvm::Value*> filter_func_args;
     for (auto arg_it = cgen_state_->filter_func_args_.begin();
          arg_it != cgen_state_->filter_func_args_.end();
          ++arg_it) {
       filter_func_args.push_back(arg_it->first);
     }
     llvm::ReplaceInstWithInst(
         cgen_state_->filter_func_call_,
         llvm::CallInst::Create(cgen_state_->filter_func_, filter_func_args, ""));
   }
 
   // Aggregate
   plan_state_->init_agg_vals_ =
       init_agg_val_vec(ra_exe_unit.target_exprs, ra_exe_unit.quals, *query_mem_desc);
 
   /*
    * If we have decided to use GPU shared memory (decision is not made here), then
    * we generate proper code for extra components that it needs (buffer initialization and
    * gpu reduction from shared memory to global memory). We then replace these functions
    * into the already compiled query_func (replacing two placeholders, write_back_nop and
    * init_smem_nop). The rest of the code should be as before (row_func, etc.).
    */
   if (gpu_smem_context.isSharedMemoryUsed()) {
     if (query_mem_desc->getQueryDescriptionType() ==
         QueryDescriptionType::GroupByPerfectHash) {
       GpuSharedMemCodeBuilder gpu_smem_code(
           cgen_state_->module_,
           cgen_state_->context_,
           *query_mem_desc,
           target_exprs_to_infos(ra_exe_unit.target_exprs, *query_mem_desc),
           plan_state_->init_agg_vals_,
           executor_id_);
       gpu_smem_code.codegen();
       gpu_smem_code.injectFunctionsInto(query_func);
 
       // helper functions are used for caching purposes later
       cgen_state_->helper_functions_.push_back(gpu_smem_code.getReductionFunction());
       cgen_state_->helper_functions_.push_back(gpu_smem_code.getInitFunction());
       LOG(IR) << gpu_smem_code.toString();
     }
   }
 
   auto multifrag_query_func = cgen_state_->module_->getFunction(
       "multifrag_query" + std::string(co.hoist_literals ? "_hoisted_literals" : ""));
   CHECK(multifrag_query_func);
 
   if (co.device_type == ExecutorDeviceType::GPU && eo.allow_multifrag) {
     insertErrorCodeChecker(
         multifrag_query_func, co.hoist_literals, eo.allow_runtime_query_interrupt);
   }
 
   bind_query(query_func,
              "query_stub" + std::string(co.hoist_literals ? "_hoisted_literals" : ""),
              multifrag_query_func,
              cgen_state_->module_);
 
   std::vector<llvm::Function*> root_funcs{query_func, cgen_state_->row_func_};
   if (cgen_state_->filter_func_) {
     root_funcs.push_back(cgen_state_->filter_func_);
   }
   auto live_funcs = CodeGenerator::markDeadRuntimeFuncs(
       *cgen_state_->module_, root_funcs, {multifrag_query_func});
 
   // Always inline the row function and the filter function.
   // We don't want register spills in the inner loops.
   // LLVM seems to correctly free up alloca instructions
   // in these functions even when they are inlined.
   mark_function_always_inline(cgen_state_->row_func_);
   if (cgen_state_->filter_func_) {
     mark_function_always_inline(cgen_state_->filter_func_);
   }
 
 #ifndef NDEBUG
   // Add helpful metadata to the LLVM IR for debugging.
   AUTOMATIC_IR_METADATA_DONE();
 #endif
 
   // Serialize the important LLVM IR functions to text for SQL EXPLAIN.
   std::string llvm_ir;
   if (eo.just_explain) {
     if (co.explain_type == ExecutorExplainType::Optimized) {
 #ifdef WITH_JIT_DEBUG
       throw std::runtime_error(
           "Explain optimized not available when JIT runtime debug symbols are enabled");
 

Classes

Public Types

Public Member Functions

Static Public Member Functions

Public Attributes

Static Public Attributes

Private Types

Private Member Functions

Static Private Member Functions

Private Attributes

Static Private Attributes

Friends

Detailed Description

Member Typedef Documentation

Member Enumeration Documentation

Constructor & Destructor Documentation

Member Function Documentation