#include <DBHandler.h>

Inheritance diagram for DBHandler:

Collaboration diagram for DBHandler:

Classes
struct	DeferredCopyFromSessions

struct	DeferredCopyFromState

Public Member Functions
	DBHandler (const std::vector< LeafHostInfo > &db_leaves, const std::vector< LeafHostInfo > &string_leaves, const std::string &base_data_path, const bool allow_multifrag, const bool jit_debug, const bool intel_jit_profile, const bool read_only, const bool allow_loop_joins, const bool enable_rendering, const bool renderer_use_ppll_polys, const bool renderer_prefer_igpu, const unsigned renderer_vulkan_timeout_ms, const bool renderer_use_parallel_executors, const bool enable_auto_clear_render_mem, const int render_oom_retry_threshold, const size_t render_mem_bytes, const size_t max_concurrent_render_sessions, const size_t reserved_gpu_mem, const bool render_compositor_use_last_gpu, const size_t num_reader_threads, const AuthMetadata &authMetadata, SystemParameters &system_parameters, const bool legacy_syntax, const int idle_session_duration, const int max_session_duration, const std::string &udf_filename, const std::string &clang_path, const std::vector< std::string > &clang_options, const File_Namespace::DiskCacheConfig &disk_cache_config, const bool is_new_db)

void	initialize (const bool is_new_db)

	~DBHandler () override

void	krb5_connect (TKrb5Session &session, const std::string &token, const std::string &dbname) override

void	connect (TSessionId &session, const std::string &username, const std::string &passwd, const std::string &dbname) override

void	disconnect (const TSessionId &session) override

void	switch_database (const TSessionId &session, const std::string &dbname) override

void	clone_session (TSessionId &session2, const TSessionId &session1) override

void	get_server_status (TServerStatus &_return, const TSessionId &session) override

void	get_status (std::vector< TServerStatus > &_return, const TSessionId &session) override

void	get_hardware_info (TClusterHardwareInfo &_return, const TSessionId &session) override

bool	hasTableAccessPrivileges (const TableDescriptor *td, const Catalog_Namespace::SessionInfo &session_info)

void	get_tables (std::vector< std::string > &_return, const TSessionId &session) override

void	get_tables_for_database (std::vector< std::string > &_return, const TSessionId &session, const std::string &database_name) override

void	get_physical_tables (std::vector< std::string > &_return, const TSessionId &session) override

void	get_views (std::vector< std::string > &_return, const TSessionId &session) override

void	get_tables_meta (std::vector< TTableMeta > &_return, const TSessionId &session) override

void	get_table_details (TTableDetails &_return, const TSessionId &session, const std::string &table_name) override

void	get_table_details_for_database (TTableDetails &_return, const TSessionId &session, const std::string &table_name, const std::string &database_name) override

void	get_internal_table_details (TTableDetails &_return, const TSessionId &session, const std::string &table_name, const bool include_system_columns) override

void	get_internal_table_details_for_database (TTableDetails &_return, const TSessionId &session, const std::string &table_name, const std::string &database_name) override

void	get_users (std::vector< std::string > &_return, const TSessionId &session) override

void	get_databases (std::vector< TDBInfo > &_return, const TSessionId &session) override

void	get_version (std::string &_return) override

void	start_heap_profile (const TSessionId &session) override

void	stop_heap_profile (const TSessionId &session) override

void	get_heap_profile (std::string &_return, const TSessionId &session) override

void	get_memory (std::vector< TNodeMemoryInfo > &_return, const TSessionId &session, const std::string &memory_level) override

void	clear_cpu_memory (const TSessionId &session) override

void	clear_gpu_memory (const TSessionId &session) override

void	clearRenderMemory (const TSessionId &session)

void	set_cur_session (const TSessionId &parent_session, const TSessionId &leaf_session, const std::string &start_time_str, const std::string &label, bool for_running_query_kernel) override

void	invalidate_cur_session (const TSessionId &parent_session, const TSessionId &leaf_session, const std::string &start_time_str, const std::string &label, bool for_running_query_kernel) override

void	set_table_epoch (const TSessionId &session, const int db_id, const int table_id, const int new_epoch) override

void	set_table_epoch_by_name (const TSessionId &session, const std::string &table_name, const int new_epoch) override

int32_t	get_table_epoch (const TSessionId &session, const int32_t db_id, const int32_t table_id) override

int32_t	get_table_epoch_by_name (const TSessionId &session, const std::string &table_name) override

void	get_table_epochs (std::vector< TTableEpochInfo > &_return, const TSessionId &session, const int32_t db_id, const int32_t table_id) override

void	set_table_epochs (const TSessionId &session, const int32_t db_id, const std::vector< TTableEpochInfo > &table_epochs) override

void	get_session_info (TSessionInfo &_return, const TSessionId &session) override

void	set_leaf_info (const TSessionId &session, const TLeafInfo &info) override

void	sql_execute (ExecutionResult &_return, const TSessionId &session, const std::string &query, const bool column_format, const int32_t first_n, const int32_t at_most_n, lockmgr::LockedTableDescriptors &locks)

void	sql_execute (TQueryResult &_return, const TSessionId &session, const std::string &query, const bool column_format, const std::string &nonce, const int32_t first_n, const int32_t at_most_n) override

void	get_completion_hints (std::vector< TCompletionHint > &hints, const TSessionId &session, const std::string &sql, const int cursor) override

void	sql_execute_df (TDataFrame &_return, const TSessionId &session, const std::string &query, const TDeviceType::type device_type, const int32_t device_id, const int32_t first_n, const TArrowTransport::type transport_method) override

void	sql_execute_gdf (TDataFrame &_return, const TSessionId &session, const std::string &query, const int32_t device_id, const int32_t first_n) override

void	deallocate_df (const TSessionId &session, const TDataFrame &df, const TDeviceType::type device_type, const int32_t device_id) override

void	interrupt (const TSessionId &query_session, const TSessionId &interrupt_session) override

void	sql_validate (TRowDescriptor &_return, const TSessionId &session, const std::string &query) override

TExecuteMode::type	getExecutionMode (const TSessionId &session)

void	set_execution_mode (const TSessionId &session, const TExecuteMode::type mode) override

void	render_vega (TRenderResult &_return, const TSessionId &session, const int64_t widget_id, const std::string &vega_json, const int32_t compression_level, const std::string &nonce) override

void	get_result_row_for_pixel (TPixelTableRowResult &_return, const TSessionId &session, const int64_t widget_id, const TPixel &pixel, const std::map< std::string, std::vector< std::string >> &table_col_names, const bool column_format, const int32_t pixel_radius, const std::string &nonce) override

int32_t	create_custom_expression (const TSessionId &session, const TCustomExpression &custom_expression) override

void	get_custom_expressions (std::vector< TCustomExpression > &_return, const TSessionId &session) override

void	update_custom_expression (const TSessionId &session, const int32_t id, const std::string &expression_json) override

void	delete_custom_expressions (const TSessionId &session, const std::vector< int32_t > &custom_expression_ids, const bool do_soft_delete) override

void	get_dashboard (TDashboard &_return, const TSessionId &session, const int32_t dashboard_id) override

void	get_dashboards (std::vector< TDashboard > &_return, const TSessionId &session) override

int32_t	create_dashboard (const TSessionId &session, const std::string &dashboard_name, const std::string &dashboard_state, const std::string &image_hash, const std::string &dashboard_metadata) override

void	replace_dashboard (const TSessionId &session, const int32_t dashboard_id, const std::string &dashboard_name, const std::string &dashboard_owner, const std::string &dashboard_state, const std::string &image_hash, const std::string &dashboard_metadata) override

void	delete_dashboard (const TSessionId &session, const int32_t dashboard_id) override

void	share_dashboards (const TSessionId &session, const std::vector< int32_t > &dashboard_ids, const std::vector< std::string > &groups, const TDashboardPermissions &permissions) override

void	delete_dashboards (const TSessionId &session, const std::vector< int32_t > &dashboard_ids) override

void	share_dashboard (const TSessionId &session, const int32_t dashboard_id, const std::vector< std::string > &groups, const std::vector< std::string > &objects, const TDashboardPermissions &permissions, const bool grant_role) override

void	unshare_dashboards (const TSessionId &session, const std::vector< int32_t > &dashboard_ids, const std::vector< std::string > &groups, const TDashboardPermissions &permissions) override

void	unshare_dashboard (const TSessionId &session, const int32_t dashboard_id, const std::vector< std::string > &groups, const std::vector< std::string > &objects, const TDashboardPermissions &permissions) override

void	get_dashboard_grantees (std::vector< TDashboardGrantees > &_return, const TSessionId &session, const int32_t dashboard_id) override

void	get_link_view (TFrontendView &_return, const TSessionId &session, const std::string &link) override

void	create_link (std::string &_return, const TSessionId &session, const std::string &view_state, const std::string &view_metadata) override

void	load_table_binary (const TSessionId &session, const std::string &table_name, const std::vector< TRow > &rows, const std::vector< std::string > &column_names) override

void	load_table_binary_columnar (const TSessionId &session, const std::string &table_name, const std::vector< TColumn > &cols, const std::vector< std::string > &column_names) override

void	load_table_binary_columnar_polys (const TSessionId &session, const std::string &table_name, const std::vector< TColumn > &cols, const std::vector< std::string > &column_names, const bool assign_render_groups) override

void	load_table_binary_arrow (const TSessionId &session, const std::string &table_name, const std::string &arrow_stream, const bool use_column_names) override

void	load_table (const TSessionId &session, const std::string &table_name, const std::vector< TStringRow > &rows, const std::vector< std::string > &column_names) override

void	detect_column_types (TDetectResult &_return, const TSessionId &session, const std::string &file_name, const TCopyParams &copy_params) override

void	create_table (const TSessionId &session, const std::string &table_name, const TRowDescriptor &row_desc, const TCreateParams &create_params) override

void	import_table (const TSessionId &session, const std::string &table_name, const std::string &file_name, const TCopyParams &copy_params) override

void	import_geo_table (const TSessionId &session, const std::string &table_name, const std::string &file_name, const TCopyParams &copy_params, const TRowDescriptor &row_desc, const TCreateParams &create_params) override

void	import_table_status (TImportStatus &_return, const TSessionId &session, const std::string &import_id) override

void	get_first_geo_file_in_archive (std::string &_return, const TSessionId &session, const std::string &archive_path, const TCopyParams &copy_params) override

void	get_all_files_in_archive (std::vector< std::string > &_return, const TSessionId &session, const std::string &archive_path, const TCopyParams &copy_params) override

void	get_layers_in_geo_file (std::vector< TGeoFileLayerInfo > &_return, const TSessionId &session, const std::string &file_name, const TCopyParams &copy_params) override

int64_t	query_get_outer_fragment_count (const TSessionId &session, const std::string &select_query) override

void	check_table_consistency (TTableMeta &_return, const TSessionId &session, const int32_t table_id) override

void	start_query (TPendingQuery &_return, const TSessionId &leaf_session, const TSessionId &parent_session, const std::string &serialized_rel_alg_dag, const std::string &start_time_str, const bool just_explain, const std::vector< int64_t > &outer_fragment_indices) override

void	execute_query_step (TStepResult &_return, const TPendingQuery &pending_query, const TSubqueryId subquery_id, const std::string &start_time_str) override

void	broadcast_serialized_rows (const TSerializedRows &serialized_rows, const TRowDescriptor &row_desc, const TQueryId query_id, const TSubqueryId subquery_id, const bool is_final_subquery_result) override

void	start_render_query (TPendingRenderQuery &_return, const TSessionId &session, const int64_t widget_id, const int16_t node_idx, const std::string &vega_json) override

void	execute_next_render_step (TRenderStepResult &_return, const TPendingRenderQuery &pending_render, const TRenderAggDataMap &merged_data) override

void	insert_data (const TSessionId &session, const TInsertData &insert_data) override

void	insert_chunks (const TSessionId &session, const TInsertChunks &insert_chunks) override

void	checkpoint (const TSessionId &session, const int32_t table_id) override

void	get_roles (std::vector< std::string > &_return, const TSessionId &session) override

bool	has_role (const TSessionId &sessionId, const std::string &granteeName, const std::string &roleName) override

bool	has_object_privilege (const TSessionId &sessionId, const std::string &granteeName, const std::string &objectName, const TDBObjectType::type object_type, const TDBObjectPermissions &permissions) override

void	get_db_objects_for_grantee (std::vector< TDBObject > &_return, const TSessionId &session, const std::string &roleName) override

void	get_db_object_privs (std::vector< TDBObject > &_return, const TSessionId &session, const std::string &objectName, const TDBObjectType::type type) override

void	get_all_roles_for_user (std::vector< std::string > &_return, const TSessionId &session, const std::string &granteeName) override

void	get_all_effective_roles_for_user (std::vector< std::string > &_return, const TSessionId &session, const std::string &granteeName) override

std::vector< std::string >	get_valid_groups (const TSessionId &session, int32_t dashboard_id, std::vector< std::string > groups)

void	set_license_key (TLicenseInfo &_return, const TSessionId &session, const std::string &key, const std::string &nonce) override

void	get_license_claims (TLicenseInfo &_return, const TSessionId &session, const std::string &nonce) override

void	get_device_parameters (std::map< std::string, std::string > &_return, const TSessionId &session) override

void	register_runtime_extension_functions (const TSessionId &session, const std::vector< TUserDefinedFunction > &udfs, const std::vector< TUserDefinedTableFunction > &udtfs, const std::map< std::string, std::string > &device_ir_map) override

void	get_function_names (std::vector< std::string > &_return, const TSessionId &session) override

void	get_runtime_function_names (std::vector< std::string > &_return, const TSessionId &session) override

void	get_function_details (std::vector< TUserDefinedFunction > &_return, const TSessionId &session, const std::vector< std::string > &udf_names) override

void	get_table_function_names (std::vector< std::string > &_return, const TSessionId &session) override

void	get_runtime_table_function_names (std::vector< std::string > &_return, const TSessionId &session) override

void	get_table_function_details (std::vector< TUserDefinedTableFunction > &_return, const TSessionId &session, const std::vector< std::string > &udtf_names) override

void	shutdown ()

void	emergency_shutdown ()

TSessionId	getInvalidSessionId () const

void	internal_connect (TSessionId &session, const std::string &username, const std::string &dbname)

bool	isAggregator () const

bool	checkInMemorySystemTableQuery (const std::unordered_set< shared::TableKey > &tables_selected_from) const

template<typename... ARGS>
std::shared_ptr < query_state::QueryState >	create_query_state (ARGS &&...args)

Catalog_Namespace::SessionInfo	get_session_copy (const TSessionId &session_id)

void	get_tables_meta_impl (std::vector< TTableMeta > &_return, QueryStateProxy query_state_proxy, const Catalog_Namespace::SessionInfo &session_info, const bool with_table_locks=true)

void	resizeDispatchQueue (size_t queue_size)

Static Public Member Functions
static size_t	max_bytes_for_thrift ()

Public Attributes
std::shared_ptr < Data_Namespace::DataMgr >	data_mgr_

LeafAggregator	leaf_aggregator_

std::vector< LeafHostInfo >	db_leaves_

std::vector< LeafHostInfo >	string_leaves_

const std::string	base_data_path_

boost::filesystem::path	import_path_

ExecutorDeviceType	executor_device_type_

std::default_random_engine	random_gen_

std::uniform_int_distribution < int64_t >	session_id_dist_

const bool	jit_debug_

const bool	intel_jit_profile_

bool	allow_multifrag_

const bool	read_only_

const bool	allow_loop_joins_

bool	cpu_mode_only_

heavyai::shared_mutex	sessions_mutex_

std::mutex	render_mutex_

int64_t	start_time_

const AuthMetadata &	authMetadata_

SystemParameters &	system_parameters_

std::shared_ptr< QueryEngine >	query_engine_

std::unique_ptr< RenderHandler >	render_handler_

std::unique_ptr < HeavyDBAggHandler >	agg_handler_

std::unique_ptr < HeavyDBLeafHandler >	leaf_handler_

std::shared_ptr< Calcite >	calcite_

const bool	legacy_syntax_

std::unique_ptr < QueryDispatchQueue >	dispatch_queue_

Protected Member Functions
std::shared_ptr < Catalog_Namespace::SessionInfo >	get_session_ptr (const TSessionId &session_id)

ConnectionInfo	getConnectionInfo () const

Private Types
enum	AssignRenderGroupsMode { AssignRenderGroupsMode::kNone, AssignRenderGroupsMode::kAssign, AssignRenderGroupsMode::kCleanUp }

using	RenderGroupAssignmentColumnMap = std::unordered_map< std::string, std::unique_ptr< import_export::RenderGroupAnalyzer >>

using	RenderGroupAssignmentTableMap = std::unordered_map< std::string, RenderGroupAssignmentColumnMap >

using	RenderGroupAnalyzerSessionMap = std::unordered_map< TSessionId, RenderGroupAssignmentTableMap >

Private Member Functions
std::shared_ptr < Catalog_Namespace::SessionInfo >	create_new_session (TSessionId &session, const std::string &dbname, const Catalog_Namespace::UserMetadata &user_meta, std::shared_ptr< Catalog_Namespace::Catalog > cat)

void	connect_impl (TSessionId &session, const std::string &passwd, const std::string &dbname, const Catalog_Namespace::UserMetadata &user_meta, std::shared_ptr< Catalog_Namespace::Catalog > cat, query_state::StdLog &stdlog)

void	disconnect_impl (Catalog_Namespace::SessionInfoPtr &session_ptr)

void	check_table_load_privileges (const Catalog_Namespace::SessionInfo &session_info, const std::string &table_name)

void	get_tables_impl (std::vector< std::string > &table_names, const Catalog_Namespace::SessionInfo &, const GetTablesType get_tables_type, const std::string &database_name={})

void	get_table_details_impl (TTableDetails &_return, query_state::StdLog &stdlog, const std::string &table_name, const bool get_system, const bool get_physical, const std::string &database_name={})

void	getAllRolesForUserImpl (std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr, std::vector< std::string > &roles, const std::string &granteeName, bool effective)

void	check_read_only (const std::string &str)

void	validateGroups (const std::vector< std::string > &groups)

void	validateDashboardIdsForSharing (const Catalog_Namespace::SessionInfo &session_info, const std::vector< int32_t > &dashboard_ids)

void	shareOrUnshareDashboards (const TSessionId &session, const std::vector< int32_t > &dashboard_ids, const std::vector< std::string > &groups, const TDashboardPermissions &permissions, const bool do_share)

std::pair< TPlanResult, lockmgr::LockedTableDescriptors >	parse_to_ra (QueryStateProxy, const std::string &query_str, const std::vector< TFilterPushDownInfo > &filter_push_down_info, const bool acquire_locks, const SystemParameters &system_parameters, bool check_privileges=true)

void	sql_execute_local (TQueryResult &_return, const QueryStateProxy &query_state_proxy, const std::shared_ptr< Catalog_Namespace::SessionInfo > session_ptr, const std::string &query_str, const bool column_format, const std::string &nonce, const int32_t first_n, const int32_t at_most_n, const bool use_calcite)

int64_t	process_deferred_copy_from (const TSessionId &session_id)

void	sql_execute_impl (ExecutionResult &_return, QueryStateProxy, const bool column_format, const ExecutorDeviceType executor_device_type, const int32_t first_n, const int32_t at_most_n, const bool use_calcite, lockmgr::LockedTableDescriptors &locks)

bool	user_can_access_table (const Catalog_Namespace::SessionInfo &, const TableDescriptor *td, const AccessPrivileges acess_priv)

void	execute_distributed_copy_statement (Parser::CopyTableStmt *, const Catalog_Namespace::SessionInfo &session_info)

TPlanResult	processCalciteRequest (QueryStateProxy, const std::shared_ptr< Catalog_Namespace::Catalog > &cat, const std::string &query_str, const std::vector< TFilterPushDownInfo > &filter_push_down_info, const SystemParameters &system_parameters, const bool check_privileges)

TQueryResult	validate_rel_alg (const std::string &query_ra, QueryStateProxy)

void	dispatch_query_task (std::shared_ptr< QueryDispatchQueue::Task > query_task, const bool is_update_delete)

std::vector< PushedDownFilterInfo >	execute_rel_alg (ExecutionResult &_return, QueryStateProxy, const std::string &query_ra, const bool column_format, const ExecutorDeviceType executor_device_type, const int32_t first_n, const int32_t at_most_n, const bool just_validate, const bool find_push_down_candidates, const ExplainInfo &explain_info, const std::optional< size_t > executor_index=std::nullopt) const

void	execute_rel_alg_with_filter_push_down (ExecutionResult &_return, QueryStateProxy, std::string &query_ra, const bool column_format, const ExecutorDeviceType executor_device_type, const int32_t first_n, const int32_t at_most_n, const bool just_explain, const bool just_calcite_explain, const std::vector< PushedDownFilterInfo > &filter_push_down_requests)

void	executeDdl (TQueryResult &_return, const std::string &query_ra, std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)

void	executeDdl (ExecutionResult &_return, const std::string &query_ra, std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)

TColumnType	populateThriftColumnType (const Catalog_Namespace::Catalog cat, const ColumnDescriptor cd)

TRowDescriptor	fixup_row_descriptor (const TRowDescriptor &row_desc, const Catalog_Namespace::Catalog &cat)

void	set_execution_mode_nolock (Catalog_Namespace::SessionInfo *session_ptr, const TExecuteMode::type mode)

char	unescape_char (std::string str)

import_export::CopyParams	thrift_to_copyparams (const TCopyParams &cp)

TCopyParams	copyparams_to_thrift (const import_export::CopyParams &cp)

void	check_geospatial_files (const boost::filesystem::path file_path, const import_export::CopyParams &copy_params)

void	render_rel_alg (TRenderResult &_return, const std::string &query_ra, const std::string &query_str, const Catalog_Namespace::SessionInfo &session_info, const std::string &render_type, const bool is_projection_query)

TColumnType	create_geo_column (const TDatumType::type type, const std::string &name, const bool is_array)

void	convertResultSet (ExecutionResult &result, const Catalog_Namespace::SessionInfo &session_info, const std::string &query_state_str, TQueryResult &_return)

std::vector< TargetMetaInfo >	getTargetMetaInfo (const std::vector< std::shared_ptr< Analyzer::TargetEntry >> &targets) const

std::vector< std::string >	getTargetNames (const std::vector< TargetMetaInfo > &targets) const

std::vector< std::string >	getTargetNames (const std::vector< std::shared_ptr< Analyzer::TargetEntry >> &targets) const

void	get_completion_hints_unsorted (std::vector< TCompletionHint > &hints, std::vector< std::string > &visible_tables, query_state::StdLog &stdlog, const std::string &sql, const int cursor)

void	get_token_based_completions (std::vector< TCompletionHint > &hints, query_state::StdLog &stdlog, std::vector< std::string > &visible_tables, const std::string &sql, const int cursor)

std::unordered_map < std::string, std::unordered_set < std::string > >	fill_column_names_by_table (std::vector< std::string > &table_names, query_state::StdLog &stdlog)

TDashboard	get_dashboard_impl (const std::shared_ptr< Catalog_Namespace::SessionInfo const > &session_ptr, Catalog_Namespace::UserMetadata &user_meta, const DashboardDescriptor *dash, const bool populate_state=true)

std::unordered_set< std::string >	get_uc_compatible_table_names_by_column (const std::unordered_set< std::string > &uc_column_names, std::vector< std::string > &table_names, query_state::StdLog &stdlog)

std::unique_ptr < lockmgr::AbstractLockContainer < const TableDescriptor * > >	prepare_loader_generic (const Catalog_Namespace::SessionInfo &session_info, const std::string &table_name, size_t num_cols, std::unique_ptr< import_export::Loader > loader, std::vector< std::unique_ptr< import_export::TypedImportBuffer >> import_buffers, const std::vector< std::string > &column_names, std::string load_type)

void	fillGeoColumns (const TSessionId &session, const Catalog_Namespace::Catalog &catalog, std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &import_buffers, const ColumnDescriptor *cd, size_t &col_idx, size_t num_rows, const std::string &table_name, bool assign_render_groups)

void	fillMissingBuffers (const TSessionId &session, const Catalog_Namespace::Catalog &catalog, std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &import_buffers, const std::list< const ColumnDescriptor * > &cds, const std::vector< int > &desc_id_to_column_id, size_t num_rows, const std::string &table_name, bool assign_render_groups)

Catalog_Namespace::SessionInfoPtr	findCalciteSession (TSessionId const &) const

void	check_and_invalidate_sessions (Parser::DDLStmt *ddl)

std::string const	createInMemoryCalciteSession (const std::shared_ptr< Catalog_Namespace::Catalog > &catalog_ptr)

void	removeInMemoryCalciteSession (const std::string &session_id)

ExecutionResult	getUserSessions (std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)

ExecutionResult	getQueries (std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)

void	get_queries_info (std::vector< TQueryInfo > &_return, const TSessionId &session) override

void	interruptQuery (const Catalog_Namespace::SessionInfo &session_info, const std::string &target_session)

void	alterSystemClear (const std::string &sesson_id, ExecutionResult &result, const std::string &cache_type, int64_t &execution_time_ms)

void	alterSession (const std::string &sesson_id, ExecutionResult &result, const std::pair< std::string, std::string > &session_parameter, int64_t &execution_time_ms)

void	loadTableBinaryColumnarInternal (const TSessionId &session, const std::string &table_name, const std::vector< TColumn > &cols, const std::vector< std::string > &column_names, const AssignRenderGroupsMode assign_render_groups_mode)

TRole::type	getServerRole () const

void	importGeoTableGlobFilterSort (const TSessionId &session, const std::string &table_name, const std::string &file_name, const import_export::CopyParams &copy_params, const TRowDescriptor &row_desc, const TCreateParams &create_params)

void	importGeoTableSingle (const TSessionId &session, const std::string &table_name, const std::string &file_name, const import_export::CopyParams &copy_params, const TRowDescriptor &row_desc, const TCreateParams &create_params)

void	resetSessionsStore ()

Static Private Member Functions
static void	value_to_thrift_column (const TargetValue &tv, const SQLTypeInfo &ti, TColumn &column)

static TDatum	value_to_thrift (const TargetValue &tv, const SQLTypeInfo &ti)

static void	convertData (TQueryResult &_return, ExecutionResult &result, const QueryStateProxy &query_state_proxy, const bool column_format, const int32_t first_n, const int32_t at_most_n)

static void	convertExplain (TQueryResult &_return, const ResultSet &results, const bool column_format)

static void	convertResult (TQueryResult &_return, const ResultSet &results, const bool column_format)

static void	convertRows (TQueryResult &_return, QueryStateProxy query_state_proxy, const std::vector< TargetMetaInfo > &targets, const ResultSet &results, const bool column_format, const int32_t first_n, const int32_t at_most_n)

static void	createSimpleResult (TQueryResult &_return, const ResultSet &results, const bool column_format, const std::string label)

static bool	has_database_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permissions)

static bool	has_table_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permission)

static bool	has_dashboard_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permissions)

static bool	has_view_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permissions)

static bool	has_server_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permissions)

Private Attributes
std::atomic< bool >	initialized_ {false}

query_state::QueryStates	query_states_

std::unordered_map < std::string, Catalog_Namespace::SessionInfoPtr >	calcite_sessions_

heavyai::shared_mutex	calcite_sessions_mtx_

bool	super_user_rights_

const int	idle_session_duration_

const int	max_session_duration_

const bool	enable_rendering_

const bool	renderer_use_ppll_polys_

const bool	renderer_prefer_igpu_

const unsigned	renderer_vulkan_timeout_

const bool	renderer_use_parallel_executors_

const bool	enable_auto_clear_render_mem_

const int	render_oom_retry_threshold_

const size_t	max_concurrent_render_sessions_

const size_t	reserved_gpu_mem_

const bool	render_compositor_use_last_gpu_

const size_t	render_mem_bytes_

const size_t	num_reader_threads_

const File_Namespace::DiskCacheConfig &	disk_cache_config_

const std::string &	udf_filename_

const std::string &	clang_path_

const std::vector< std::string > &	clang_options_

int32_t	max_num_sessions_ {-1}

std::unique_ptr < Catalog_Namespace::SessionsStore >	sessions_store_

DeferredCopyFromSessions	deferred_copy_from_sessions

std::mutex	handle_to_dev_ptr_mutex_

std::unordered_map < std::string, std::string >	ipc_handle_to_dev_ptr_

std::map< const std::string, const PermissionFuncPtr >	permissionFuncMap_

RenderGroupAnalyzerSessionMap	render_group_assignment_map_

std::mutex	render_group_assignment_mutex_

heavyai::shared_mutex	custom_expressions_mutex_

Friends
class	RenderHandler::Impl

class	HeavyDBAggHandler

class	HeavyDBLeafHandler

void	run_warmup_queries (std::shared_ptr< DBHandler > handler, std::string base_path, std::string query_file_path)

Detailed Description

Definition at line 166 of file DBHandler.h.

Member Typedef Documentation

using DBHandler::RenderGroupAnalyzerSessionMap = std::unordered_map<TSessionId, RenderGroupAssignmentTableMap>

private

Definition at line 1082 of file DBHandler.h.

using DBHandler::RenderGroupAssignmentColumnMap = std::unordered_map<std::string, std::unique_ptr<import_export::RenderGroupAnalyzer>>

private

Definition at line 1078 of file DBHandler.h.

using DBHandler::RenderGroupAssignmentTableMap = std::unordered_map<std::string, RenderGroupAssignmentColumnMap>

private

Definition at line 1080 of file DBHandler.h.

Member Enumeration Documentation

enum DBHandler::AssignRenderGroupsMode

strongprivate

Enumerator
kNone
kAssign
kCleanUp

Definition at line 1065 of file DBHandler.h.

1065 { kNone, kAssign, kCleanUp };

import_export::RasterPointType::kNone

Constructor & Destructor Documentation

DBHandler::DBHandler	(	const std::vector< LeafHostInfo > &	db_leaves,
		const std::vector< LeafHostInfo > &	string_leaves,
		const std::string &	base_data_path,
		const bool	allow_multifrag,
		const bool	jit_debug,
		const bool	intel_jit_profile,
		const bool	read_only,
		const bool	allow_loop_joins,
		const bool	enable_rendering,
		const bool	renderer_use_ppll_polys,
		const bool	renderer_prefer_igpu,
		const unsigned	renderer_vulkan_timeout_ms,
		const bool	renderer_use_parallel_executors,
		const bool	enable_auto_clear_render_mem,
		const int	render_oom_retry_threshold,
		const size_t	render_mem_bytes,
		const size_t	max_concurrent_render_sessions,
		const size_t	reserved_gpu_mem,
		const bool	render_compositor_use_last_gpu,
		const size_t	num_reader_threads,
		const AuthMetadata &	authMetadata,
		SystemParameters &	system_parameters,
		const bool	legacy_syntax,
		const int	idle_session_duration,
		const int	max_session_duration,
		const std::string &	udf_filename,
		const std::string &	clang_path,
		const std::vector< std::string > &	clang_options,
		const File_Namespace::DiskCacheConfig &	disk_cache_config,
		const bool	is_new_db
	)

Definition at line 165 of file DBHandler.cpp.

     : leaf_aggregator_(db_leaves)
     , db_leaves_(db_leaves)
     , string_leaves_(string_leaves)
     , base_data_path_(base_data_path)
     , random_gen_(std::random_device{}())
     , session_id_dist_(0, INT32_MAX)
     , jit_debug_(jit_debug)
     , intel_jit_profile_(intel_jit_profile)
     , allow_multifrag_(allow_multifrag)
     , read_only_(read_only)
     , allow_loop_joins_(allow_loop_joins)
     , authMetadata_(authMetadata)
     , system_parameters_(system_parameters)
     , legacy_syntax_(legacy_syntax)
     , dispatch_queue_(
           std::make_unique<QueryDispatchQueue>(system_parameters.num_executors))
     , super_user_rights_(false)
     , idle_session_duration_(idle_session_duration * 60)
     , max_session_duration_(max_session_duration * 60)
     , enable_rendering_(enable_rendering)
     , renderer_use_ppll_polys_(renderer_use_ppll_polys)
     , renderer_prefer_igpu_(renderer_prefer_igpu)
     , renderer_vulkan_timeout_(renderer_vulkan_timeout_ms)
     , renderer_use_parallel_executors_(renderer_use_parallel_executors)
     , enable_auto_clear_render_mem_(enable_auto_clear_render_mem)
     , render_oom_retry_threshold_(render_oom_retry_threshold)
     , max_concurrent_render_sessions_(max_concurrent_render_sessions)
     , reserved_gpu_mem_(reserved_gpu_mem)
     , render_compositor_use_last_gpu_(render_compositor_use_last_gpu)
     , render_mem_bytes_(render_mem_bytes)
     , num_reader_threads_(num_reader_threads)
 #ifdef ENABLE_GEOS
     , libgeos_so_filename_(libgeos_so_filename)
 #endif
     , disk_cache_config_(disk_cache_config)
     , udf_filename_(udf_filename)
     , clang_path_(clang_path)
     , clang_options_(clang_options)
     , max_num_sessions_(-1) {
   LOG(INFO) << "HeavyDB Server " << MAPD_RELEASE;
   initialize(is_new_db);
   resetSessionsStore();
 }

DBHandler::~DBHandler ( )

override

Definition at line 429 of file DBHandler.cpp.

References shutdown().

                       {
   shutdown();
 }

Here is the call graph for this function:

Member Function Documentation

void DBHandler::alterSession	(	const std::string &	sesson_id,
		ExecutionResult &	result,
		const std::pair< std::string, std::string > &	session_parameter,
		int64_t &	execution_time_ms
	)

private

Definition at line 7853 of file DBHandler.cpp.

References measure< TimeT >::execution(), set_execution_mode(), switch_database(), to_upper(), and run_benchmark_import::type.

Referenced by executeDdl().

                                                          {
   result = ExecutionResult();
   if (session_parameter.first == "EXECUTOR_DEVICE") {
     std::string parameter_value = to_upper(session_parameter.second);
     TExecuteMode::type executorType;
     if (parameter_value == "GPU") {
       executorType = TExecuteMode::type::GPU;
     } else if (parameter_value == "CPU") {
       executorType = TExecuteMode::type::CPU;
     } else {
       throw std::runtime_error("Cannot set the " + session_parameter.first + " to " +
                                session_parameter.second +
                                ". Valid options are CPU and GPU");
     }
     execution_time_ms =
         measure<>::execution([&]() { set_execution_mode(session_id, executorType); });
   } else if (session_parameter.first == "CURRENT_DATABASE") {
     execution_time_ms = measure<>::execution(
         [&]() { switch_database(session_id, session_parameter.second); });
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void DBHandler::alterSystemClear	(	const std::string &	sesson_id,
		ExecutionResult &	result,
		const std::string &	cache_type,
		int64_t &	execution_time_ms
	)

private

Definition at line 7837 of file DBHandler.cpp.

References clear_cpu_memory(), clear_gpu_memory(), clearRenderMemory(), measure< TimeT >::execution(), and to_upper().

Referenced by executeDdl().

                                                              {
   result = ExecutionResult();
   if (to_upper(cache_type) == "CPU") {
     execution_time_ms = measure<>::execution([&]() { clear_cpu_memory(session_id); });
   } else if (to_upper(cache_type) == "GPU") {
     execution_time_ms = measure<>::execution([&]() { clear_gpu_memory(session_id); });
   } else if (to_upper(cache_type) == "RENDER") {
     execution_time_ms = measure<>::execution([&]() { clearRenderMemory(session_id); });
   } else {
     throw std::runtime_error("Invalid cache type. Valid values are CPU,GPU or RENDER");
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void DBHandler::broadcast_serialized_rows	(	const TSerializedRows &	serialized_rows,
		const TRowDescriptor &	row_desc,
		const TQueryId	query_id,
		const TSubqueryId	subquery_id,
		const bool	is_final_subquery_result
	)

override

Definition at line 6899 of file DBHandler.cpp.

References measure< TimeT >::execution(), logger::INFO, leaf_handler_, LOG, and THROW_DB_EXCEPTION.

                                                                                {
   if (!leaf_handler_) {
     THROW_DB_EXCEPTION("Distributed support is disabled.");
   }
   LOG(INFO) << "BROADCAST-SERIALIZED-ROWS  id:" << query_id;
   auto time_ms = measure<>::execution([&]() {
     try {
       leaf_handler_->broadcast_serialized_rows(
           serialized_rows, row_desc, query_id, subquery_id, is_final_subquery_result);
     } catch (std::exception& e) {
       THROW_DB_EXCEPTION(e.what());
     }
   });
   LOG(INFO) << "BROADCAST-SERIALIZED-ROWS COMPLETED " << time_ms << "ms";
 }

Here is the call graph for this function:

void DBHandler::check_and_invalidate_sessions ( Parser::DDLStmt * ddl )

private

Definition at line 6240 of file DBHandler.cpp.

References sessions_store_.

                                                                 {
   if (const auto drop_db_stmt = dynamic_cast<Parser::DropDBStmt*>(ddl)) {
     sessions_store_->eraseByDB(*drop_db_stmt->getDatabaseName());
   } else if (const auto rename_db_stmt = dynamic_cast<Parser::RenameDBStmt*>(ddl)) {
     sessions_store_->eraseByDB(*rename_db_stmt->getPreviousDatabaseName());
   } else if (const auto drop_user_stmt = dynamic_cast<Parser::DropUserStmt*>(ddl)) {
     sessions_store_->eraseByUser(*drop_user_stmt->getUserName());
   } else if (const auto rename_user_stmt = dynamic_cast<Parser::RenameUserStmt*>(ddl)) {
     sessions_store_->eraseByUser(*rename_user_stmt->getOldUserName());
   }
 }

void DBHandler::check_geospatial_files	(	const boost::filesystem::path	file_path,
		const import_export::CopyParams &	copy_params
	)

private

Definition at line 4961 of file DBHandler.cpp.

References import_export::Importer::gdalFileExists().

Referenced by detect_column_types(), and importGeoTableSingle().

                                                                                    {
   const std::list<std::string> shp_ext{".shp", ".shx", ".dbf"};
   if (std::find(shp_ext.begin(),
                 shp_ext.end(),
                 boost::algorithm::to_lower_copy(file_path.extension().string())) !=
       shp_ext.end()) {
     for (auto ext : shp_ext) {
       auto aux_file = file_path;
       if (!import_export::Importer::gdalFileExists(
               aux_file.replace_extension(boost::algorithm::to_upper_copy(ext)).string(),
               copy_params) &&
           !import_export::Importer::gdalFileExists(
               aux_file.replace_extension(ext).string(), copy_params)) {
         throw std::runtime_error("required file for shapefile does not exist: " +
                                  aux_file.filename().string());
       }
     }
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void DBHandler::check_read_only ( const std::string & str )

private

Definition at line 433 of file DBHandler.cpp.

References read_only_, and THROW_DB_EXCEPTION.

Referenced by create_custom_expression(), create_dashboard(), create_table(), delete_custom_expressions(), delete_dashboards(), detect_column_types(), import_table(), importGeoTableSingle(), prepare_loader_generic(), replace_dashboard(), set_license_key(), shareOrUnshareDashboards(), sql_execute_impl(), and update_custom_expression().

                                                     {
   if (DBHandler::read_only_) {
     THROW_DB_EXCEPTION(str + " disabled: server running in read-only mode.");
   }
 }

Here is the caller graph for this function:

void DBHandler::check_table_consistency	(	TTableMeta &	_return,
		const TSessionId &	session,
		const int32_t	table_id
	)

override

Definition at line 6830 of file DBHandler.cpp.

References get_session_ptr(), leaf_handler_, heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, and THROW_DB_EXCEPTION.

                                                                 {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
   if (!leaf_handler_) {
     THROW_DB_EXCEPTION("Distributed support is disabled.");
   }
   try {
     leaf_handler_->check_table_consistency(_return, request_info.sessionId(), table_id);
   } catch (std::exception& e) {
     THROW_DB_EXCEPTION(e.what());
   }
 }

Here is the call graph for this function:

void DBHandler::check_table_load_privileges	(	const Catalog_Namespace::SessionInfo &	session_info,
		const std::string &	table_name
	)

private

Definition at line 5957 of file DBHandler.cpp.

References cat(), Catalog_Namespace::SessionInfo::get_currentUser(), Catalog_Namespace::SessionInfo::getCatalog(), AccessPrivileges::INSERT_INTO_TABLE, DBObject::loadKey(), DBObject::setPrivileges(), TableDBObjectType, and THROW_DB_EXCEPTION.

Referenced by import_table(), importGeoTableSingle(), and prepare_loader_generic().

                                  {
   auto user_metadata = session_info.get_currentUser();
   auto& cat = session_info.getCatalog();
   DBObject dbObject(table_name, TableDBObjectType);
   dbObject.loadKey(cat);
   dbObject.setPrivileges(AccessPrivileges::INSERT_INTO_TABLE);
   std::vector<DBObject> privObjects;
   privObjects.push_back(dbObject);
   if (!SysCatalog::instance().checkPrivileges(user_metadata, privObjects)) {
     THROW_DB_EXCEPTION("Violation of access privileges: user " +
                        user_metadata.userLoggable() +
                        " has no insert privileges for table " + table_name + ".");
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool DBHandler::checkInMemorySystemTableQuery ( const std::unordered_set< shared::TableKey > & tables_selected_from ) const

Definition at line 7970 of file DBHandler.cpp.

References CHECK, anonymous_namespace{DBHandler.cpp}::check_and_reset_in_memory_system_table(), Catalog_Namespace::SysCatalog::getCatalog(), Catalog_Namespace::SysCatalog::instance(), and shared::kInfoSchemaDbName.

                                                                        {
   bool is_in_memory_system_table_query{false};
   const auto info_schema_catalog =
       Catalog_Namespace::SysCatalog::instance().getCatalog(shared::kInfoSchemaDbName);
   if (info_schema_catalog) {
     for (const auto& table_key : selected_table_keys) {
       if (table_key.db_id == info_schema_catalog->getDatabaseId()) {
         auto td = info_schema_catalog->getMetadataForTable(table_key.table_id, false);
         CHECK(td);
         if (check_and_reset_in_memory_system_table(*info_schema_catalog, *td)) {
           is_in_memory_system_table_query = true;
         }
       }
     }
   }
   return is_in_memory_system_table_query;
 }

Here is the call graph for this function:

void DBHandler::checkpoint	(	const TSessionId &	session,
		const int32_t	table_id
	)

override

Definition at line 7145 of file DBHandler.cpp.

References cat(), get_session_ptr(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, and STDLOG.

                                                                                        {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
   auto session_ptr = stdlog.getConstSessionInfo();
   auto& cat = session_ptr->getCatalog();
   cat.checkpoint(table_id);
 }

Here is the call graph for this function:

void DBHandler::clear_cpu_memory ( const TSessionId & session )

override

Definition at line 2733 of file DBHandler.cpp.

References Executor::clearMemory(), Data_Namespace::CPU_LEVEL, get_session_ptr(), render_handler_, heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, and THROW_DB_EXCEPTION.

Referenced by alterSystemClear().

                                                                      {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
   auto session_ptr = stdlog.getConstSessionInfo();
   if (!session_ptr->get_currentUser().isSuper) {
     THROW_DB_EXCEPTION("Superuser privilege is required to run clear_cpu_memory");
   }
   try {
     Executor::clearMemory(Data_Namespace::MemoryLevel::CPU_LEVEL);
   } catch (const std::exception& e) {
     THROW_DB_EXCEPTION(e.what());
   }
   if (render_handler_) {
     render_handler_->clear_cpu_memory();
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void DBHandler::clear_gpu_memory ( const TSessionId & session )

override

Definition at line 2714 of file DBHandler.cpp.

References Executor::clearMemory(), get_session_ptr(), getConnectionInfo(), Data_Namespace::GPU_LEVEL, render_handler_, heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, THROW_DB_EXCEPTION, and toString().

Referenced by alterSystemClear().

                                                                      {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
   stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
   auto session_ptr = stdlog.getConstSessionInfo();
   if (!session_ptr->get_currentUser().isSuper) {
     THROW_DB_EXCEPTION("Superuser privilege is required to run clear_gpu_memory");
   }
   try {
     Executor::clearMemory(Data_Namespace::MemoryLevel::GPU_LEVEL);
   } catch (const std::exception& e) {
     THROW_DB_EXCEPTION(e.what());
   }
   if (render_handler_) {
     render_handler_->clear_gpu_memory();
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void DBHandler::clearRenderMemory ( const TSessionId & session )

Definition at line 2751 of file DBHandler.cpp.

References get_session_ptr(), getConnectionInfo(), render_handler_, heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, THROW_DB_EXCEPTION, and toString().

Referenced by alterSystemClear().

                                                                       {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
   stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
   auto session_ptr = stdlog.getConstSessionInfo();
   if (!session_ptr->get_currentUser().isSuper) {
     THROW_DB_EXCEPTION("Superuser privilege is required to run clear_render_memory");
   }
   if (render_handler_) {
     render_handler_->clear_cpu_memory();
     render_handler_->clear_gpu_memory();
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void DBHandler::clone_session	(	TSessionId &	session2,
		const TSessionId &	session1
	)

override

Definition at line 626 of file DBHandler.cpp.

References cat(), LeafAggregator::clone_session(), executor_device_type_, get_session_ptr(), getConnectionInfo(), logger::INFO, leaf_aggregator_, LeafAggregator::leafCount(), LOG, heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), sessions_store_, SET_REQUEST_ID, STDLOG, THROW_DB_EXCEPTION, toString(), and Catalog_Namespace::UserMetadata::userLoggable().

                                                                      {
   heavyai::RequestInfo const request_info(session1_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto session1_ptr = get_session_ptr(request_info.sessionId());
   auto stdlog = STDLOG(session1_ptr);
   stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
 
   try {
     const Catalog_Namespace::UserMetadata& user_meta = session1_ptr->get_currentUser();
     std::shared_ptr<Catalog> cat = session1_ptr->get_catalog_ptr();
     auto session2_ptr = sessions_store_->add(user_meta, cat, executor_device_type_);
     session2_id = session2_ptr->get_session_id();
     LOG(INFO) << "User " << user_meta.userLoggable() << " connected to database "
               << cat->name();
     if (leaf_aggregator_.leafCount() > 0) {
       leaf_aggregator_.clone_session(request_info.sessionId(), session2_id);
       return;
     }
   } catch (std::exception& e) {
     THROW_DB_EXCEPTION(e.what());
   }
 }

Here is the call graph for this function:

void DBHandler::connect	(	TSessionId &	session,
		const std::string &	username,
		const std::string &	passwd,
		const std::string &	dbname
	)

override

Definition at line 511 of file DBHandler.cpp.

References AccessPrivileges::ACCESS, cat(), connect_impl(), DatabaseDBObjectType, getConnectionInfo(), DBObject::loadKey(), logger::set_new_request_id(), DBObject::setPrivileges(), STDLOG, super_user_rights_, THROW_DB_EXCEPTION, toString(), and Catalog_Namespace::UserMetadata::userLoggable().

                                                  {
   logger::set_new_request_id();
   auto stdlog = STDLOG();  // session_info set by connect_impl()
   stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
   std::string username2 = username;  // login() may reset username given as argument
   std::string dbname2 = dbname;      // login() may reset dbname given as argument
   Catalog_Namespace::UserMetadata user_meta;
   std::shared_ptr<Catalog> cat = nullptr;
   try {
     cat = SysCatalog::instance().login(
         dbname2, username2, passwd, user_meta, !super_user_rights_);
   } catch (std::exception& e) {
     stdlog.appendNameValuePairs("user", username, "db", dbname, "exception", e.what());
     THROW_DB_EXCEPTION(e.what());
   }
 
   DBObject dbObject(dbname2, DatabaseDBObjectType);
   dbObject.loadKey(*cat);
   dbObject.setPrivileges(AccessPrivileges::ACCESS);
   std::vector<DBObject> dbObjects;
   dbObjects.push_back(dbObject);
   if (!SysCatalog::instance().checkPrivileges(user_meta, dbObjects)) {
     stdlog.appendNameValuePairs(
         "user", username, "db", dbname, "exception", "Missing Privileges");
     THROW_DB_EXCEPTION("Unauthorized Access: user " + user_meta.userLoggable() +
                        " is not allowed to access database " + dbname2 + ".");
   }
   connect_impl(session_id, passwd, dbname2, user_meta, cat, stdlog);
 
   // if pki auth session_id will come back encrypted with user pubkey
   SysCatalog::instance().check_for_session_encryption(passwd, session_id);
 }

Here is the call graph for this function:

void DBHandler::connect_impl	(	TSessionId &	session,
		const std::string &	passwd,
		const std::string &	dbname,
		const Catalog_Namespace::UserMetadata &	user_meta,
		std::shared_ptr< Catalog_Namespace::Catalog >	cat,
		query_state::StdLog &	stdlog
	)

private

Definition at line 547 of file DBHandler.cpp.

References query_state::StdLog::appendNameValuePairs(), executor_device_type_, getConnectionInfo(), query_state::StdLog::getConstSessionInfo(), logger::INFO, join(), LOG, sessions_store_, query_state::StdLog::setSessionInfo(), super_user_rights_, toString(), and Catalog_Namespace::UserMetadata::userLoggable().

Referenced by connect(), and internal_connect().

                                                         {
   // TODO(sy): Is there any reason to have dbname as a parameter
   // here when the cat parameter already provides cat->name()?
   // Should dbname and cat->name() ever differ?
   auto session_ptr = sessions_store_->add(user_meta, cat, executor_device_type_);
   session_id = session_ptr->get_session_id();
   LOG(INFO) << "User " << user_meta.userLoggable() << " connected to database " << dbname;
   stdlog.setSessionInfo(session_ptr);
   session_ptr->set_connection_info(getConnectionInfo().toString());
   if (!super_user_rights_) {  // no need to connect to leaf_aggregator_ at this time
     // while doing warmup
   }
   auto const roles =
       stdlog.getConstSessionInfo()->get_currentUser().isSuper
           ? std::vector<std::string>{{"super"}}
           : SysCatalog::instance().getRoles(
                 false, false, stdlog.getConstSessionInfo()->get_currentUser().userName);
   stdlog.appendNameValuePairs("roles", boost::algorithm::join(roles, ","));
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void DBHandler::convertData	(	TQueryResult &	_return,
		ExecutionResult &	result,
		const QueryStateProxy &	query_state_proxy,
		const bool	column_format,
		const int32_t	first_n,
		const int32_t	at_most_n
	)

staticprivate

Definition at line 1103 of file DBHandler.cpp.

References ExecutionResult::CalciteDdl, convertExplain(), convertResult(), convertRows(), ExecutionResult::empty(), ExecutionResult::Explanation, ExecutionResult::getExecutionTime(), ExecutionResult::getResultType(), ExecutionResult::getRows(), ExecutionResult::getTargetsMeta(), ExecutionResult::QueryResult, and ExecutionResult::SimpleResult.

Referenced by convertResultSet(), get_tables_meta_impl(), sql_execute_local(), and validate_rel_alg().

                                                      {
   _return.execution_time_ms += result.getExecutionTime();
   if (result.empty()) {
     return;
   }
 
   switch (result.getResultType()) {
     case ExecutionResult::QueryResult:
       convertRows(_return,
                   query_state_proxy,
                   result.getTargetsMeta(),
                   *result.getRows(),
                   column_format,
                   first_n,
                   at_most_n);
       break;
     case ExecutionResult::SimpleResult:
       convertResult(_return, *result.getRows(), true);
       break;
     case ExecutionResult::Explanation:
       convertExplain(_return, *result.getRows(), true);
       break;
     case ExecutionResult::CalciteDdl:
       convertRows(_return,
                   query_state_proxy,
                   result.getTargetsMeta(),
                   *result.getRows(),
                   column_format,
                   -1,
                   -1);
       break;
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void DBHandler::convertExplain	(	TQueryResult &	_return,
		const ResultSet &	results,
		const bool	column_format
	)

staticprivate

Definition at line 6212 of file DBHandler.cpp.

References createSimpleResult().

Referenced by convertData().

                                                          {
   createSimpleResult(_return, results, column_format, "Explanation");
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void DBHandler::convertResult	(	TQueryResult &	_return,
		const ResultSet &	results,
		const bool	column_format
	)

staticprivate

Definition at line 6218 of file DBHandler.cpp.

References createSimpleResult().

Referenced by convertData().

                                                         {
   createSimpleResult(_return, results, column_format, "Result");
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void DBHandler::convertResultSet	(	ExecutionResult &	result,
		const Catalog_Namespace::SessionInfo &	session_info,
		const std::string &	query_state_str,
		TQueryResult &	_return
	)

private

Definition at line 7554 of file DBHandler.cpp.

References CHECK, convertData(), create_query_state(), query_state::QueryState::createQueryStateProxy(), Catalog_Namespace::SessionInfo::get_session_id(), get_session_ptr(), and ExecutionResult::getDataPtr().

Referenced by executeDdl().

                                                         {
   // Stuff ResultSet into _return (which is a TQueryResult)
   // calls convertRows, but after some setup using session_info
 
   auto session_ptr = get_session_ptr(session_info.get_session_id());
   CHECK(session_ptr);
   auto qs = create_query_state(session_ptr, query_state_str);
   QueryStateProxy qsp = qs->createQueryStateProxy();
 
   // heavysql only accepts column format as being 'VALID",
   //   assume that heavydb should only return column format
   int32_t nRows = result.getDataPtr()->rowCount();
 
   convertData(_return,
               result,
               qsp,
               /*column_format=*/true,
               /*first_n=*/nRows,
               /*at_most_n=*/nRows);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void DBHandler::convertRows	(	TQueryResult &	_return,
		QueryStateProxy	query_state_proxy,
		const std::vector< TargetMetaInfo > &	targets,
		const ResultSet &	results,
		const bool	column_format,
		const int32_t	first_n,
		const int32_t	at_most_n
	)

staticprivate

Definition at line 6104 of file DBHandler.cpp.

References query_state::QueryStateProxy::createTimer(), ThriftSerializers::target_meta_infos_to_thrift(), THROW_DB_EXCEPTION, to_string(), value_to_thrift(), and value_to_thrift_column().

Referenced by convertData().

                                                      {
   query_state::Timer timer = query_state_proxy.createTimer(__func__);
   _return.row_set.row_desc = ThriftSerializers::target_meta_infos_to_thrift(targets);
   int32_t fetched{0};
   if (column_format) {
     _return.row_set.is_columnar = true;
     std::vector<TColumn> tcolumns(results.colCount());
     while (first_n == -1 || fetched < first_n) {
       const auto crt_row = results.getNextRow(true, true);
       if (crt_row.empty()) {
         break;
       }
       ++fetched;
       if (at_most_n >= 0 && fetched > at_most_n) {
         THROW_DB_EXCEPTION("The result contains more rows than the specified cap of " +
                            std::to_string(at_most_n));
       }
       for (size_t i = 0; i < results.colCount(); ++i) {
         const auto agg_result = crt_row[i];
         value_to_thrift_column(agg_result, targets[i].get_type_info(), tcolumns[i]);
       }
     }
     for (size_t i = 0; i < results.colCount(); ++i) {
       _return.row_set.columns.push_back(tcolumns[i]);
     }
   } else {
     _return.row_set.is_columnar = false;
     while (first_n == -1 || fetched < first_n) {
       const auto crt_row = results.getNextRow(true, true);
       if (crt_row.empty()) {
         break;
       }
       ++fetched;
       if (at_most_n >= 0 && fetched > at_most_n) {
         THROW_DB_EXCEPTION("The result contains more rows than the specified cap of " +
                            std::to_string(at_most_n));
       }
       TRow trow;
       trow.cols.reserve(results.colCount());
       for (size_t i = 0; i < results.colCount(); ++i) {
         const auto agg_result = crt_row[i];
         trow.cols.push_back(value_to_thrift(agg_result, targets[i].get_type_info()));
       }
       _return.row_set.rows.push_back(trow);
     }
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

TCopyParams DBHandler::copyparams_to_thrift ( const import_export::CopyParams & cp )

private

Definition at line 3840 of file DBHandler.cpp.

Referenced by detect_column_types(), and importGeoTableSingle().

                                                                              {
   TCopyParams copy_params;
   copy_params.delimiter = cp.delimiter;
   copy_params.null_str = cp.null_str;
   switch (cp.has_header) {
     case import_export::ImportHeaderRow::kAutoDetect:
       copy_params.has_header = TImportHeaderRow::AUTODETECT;
       break;
     case import_export::ImportHeaderRow::kNoHeader:
       copy_params.has_header = TImportHeaderRow::NO_HEADER;
       break;
     case import_export::ImportHeaderRow::kHasHeader:
       copy_params.has_header = TImportHeaderRow::HAS_HEADER;
       break;
     default:
       CHECK(false);
   }
   copy_params.quoted = cp.quoted;
   copy_params.quote = cp.quote;
   copy_params.escape = cp.escape;
   copy_params.line_delim = cp.line_delim;
   copy_params.array_delim = cp.array_delim;
   copy_params.array_begin = cp.array_begin;
   copy_params.array_end = cp.array_end;
   copy_params.threads = cp.threads;
   copy_params.s3_access_key = cp.s3_access_key;
   copy_params.s3_secret_key = cp.s3_secret_key;
   copy_params.s3_session_token = cp.s3_session_token;
   copy_params.s3_region = cp.s3_region;
   copy_params.s3_endpoint = cp.s3_endpoint;
   switch (cp.source_type) {
     case import_export::SourceType::kDelimitedFile:
       copy_params.source_type = TSourceType::DELIMITED_FILE;
       break;
     case import_export::SourceType::kGeoFile:
       copy_params.source_type = TSourceType::GEO_FILE;
       break;
     case import_export::SourceType::kParquetFile:
       copy_params.source_type = TSourceType::PARQUET_FILE;
       break;
     case import_export::SourceType::kRasterFile:
       copy_params.source_type = TSourceType::RASTER_FILE;
       break;
     case import_export::SourceType::kOdbc:
       copy_params.source_type = TSourceType::ODBC;
       break;
     default:
       CHECK(false);
   }
   switch (cp.geo_coords_encoding) {
     case kENCODING_GEOINT:
       copy_params.geo_coords_encoding = TEncodingType::GEOINT;
       break;
     default:
       copy_params.geo_coords_encoding = TEncodingType::NONE;
       break;
   }
   copy_params.geo_coords_comp_param = cp.geo_coords_comp_param;
   switch (cp.geo_coords_type) {
     case kGEOGRAPHY:
       copy_params.geo_coords_type = TDatumType::GEOGRAPHY;
       break;
     case kGEOMETRY:
       copy_params.geo_coords_type = TDatumType::GEOMETRY;
       break;
     default:
       CHECK(false);
   }
   copy_params.geo_coords_srid = cp.geo_coords_srid;
   copy_params.sanitize_column_names = cp.sanitize_column_names;
   copy_params.geo_layer_name = cp.geo_layer_name;
   copy_params.geo_assign_render_groups = cp.geo_assign_render_groups;
   copy_params.geo_explode_collections = cp.geo_explode_collections;
   copy_params.source_srid = cp.source_srid;
   switch (cp.raster_point_type) {
     case import_export::RasterPointType::kNone:
       copy_params.raster_point_type = TRasterPointType::NONE;
       break;
     case import_export::RasterPointType::kAuto:
       copy_params.raster_point_type = TRasterPointType::AUTO;
       break;
     case import_export::RasterPointType::kSmallInt:
       copy_params.raster_point_type = TRasterPointType::SMALLINT;
       break;
     case import_export::RasterPointType::kInt:
       copy_params.raster_point_type = TRasterPointType::INT;
       break;
     case import_export::RasterPointType::kFloat:
       copy_params.raster_point_type = TRasterPointType::FLOAT;
       break;
     case import_export::RasterPointType::kDouble:
       copy_params.raster_point_type = TRasterPointType::DOUBLE;
       break;
     case import_export::RasterPointType::kPoint:
       copy_params.raster_point_type = TRasterPointType::POINT;
       break;
     default:
       CHECK(false);
   }
   copy_params.raster_import_bands = cp.raster_import_bands;
   copy_params.raster_scanlines_per_thread = cp.raster_scanlines_per_thread;
   switch (cp.raster_point_transform) {
     case import_export::RasterPointTransform::kNone:
       copy_params.raster_point_transform = TRasterPointTransform::NONE;
       break;
     case import_export::RasterPointTransform::kAuto:
       copy_params.raster_point_transform = TRasterPointTransform::AUTO;
       break;
     case import_export::RasterPointTransform::kFile:
       copy_params.raster_point_transform = TRasterPointTransform::FILE;
       break;
     case import_export::RasterPointTransform::kWorld:
       copy_params.raster_point_transform = TRasterPointTransform::WORLD;
       break;
     default:
       CHECK(false);
   }
   copy_params.raster_point_compute_angle = cp.raster_point_compute_angle;
   copy_params.raster_import_dimensions = cp.raster_import_dimensions;
   copy_params.odbc_dsn = cp.dsn;
   copy_params.odbc_connection_string = cp.connection_string;
   copy_params.odbc_sql_select = cp.sql_select;
   copy_params.odbc_sql_order_by = cp.sql_order_by;
   copy_params.odbc_username = cp.username;
   copy_params.odbc_password = cp.password;
   copy_params.odbc_credential_string = cp.credential_string;
   copy_params.add_metadata_columns = cp.add_metadata_columns;
   copy_params.trim_spaces = cp.trim_spaces;
   return copy_params;
 }

Here is the caller graph for this function:

int32_t DBHandler::create_custom_expression	(	const TSessionId &	session,
		const TCustomExpression &	custom_expression
	)

override

Definition at line 4416 of file DBHandler.cpp.

References check_read_only(), anonymous_namespace{DBHandler.cpp}::create_custom_expr_from_thrift_obj(), custom_expressions_mutex_, get_session_ptr(), getConnectionInfo(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, THROW_DB_EXCEPTION, and toString().

                                                                                     {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
   stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
   check_read_only("create_custom_expression");
 
   auto session_ptr = stdlog.getConstSessionInfo();
   if (!session_ptr->get_currentUser().isSuper) {
     THROW_DB_EXCEPTION("Custom expressions can only be created by super users.")
   }
   auto& catalog = session_ptr->getCatalog();
   heavyai::unique_lock<heavyai::shared_mutex> write_lock(custom_expressions_mutex_);
   return catalog.createCustomExpression(
       create_custom_expr_from_thrift_obj(t_custom_expr, catalog));
 }

Here is the call graph for this function:

int32_t DBHandler::create_dashboard	(	const TSessionId &	session,
		const std::string &	dashboard_name,
		const std::string &	dashboard_state,
		const std::string &	image_hash,
		const std::string &	dashboard_metadata
	)

override

Definition at line 4615 of file DBHandler.cpp.

References cat(), CHECK, dbhandler::check_not_info_schema_db(), check_read_only(), AccessPrivileges::CREATE_DASHBOARD, anonymous_namespace{DBHandler.cpp}::dashboard_exists(), DashboardDBObjectType, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, g_allow_system_dashboard_update, get_session_ptr(), getConnectionInfo(), DashboardDescriptor::imageHash, heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, THROW_DB_EXCEPTION, toString(), DashboardDescriptor::user, and DashboardDescriptor::userId.

                                                                          {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
   stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
   auto session_ptr = stdlog.getConstSessionInfo();
   CHECK(session_ptr);
   check_read_only("create_dashboard");
   auto& cat = session_ptr->getCatalog();
   if (!g_allow_system_dashboard_update) {
     dbhandler::check_not_info_schema_db(cat.name(), true);
   }
 
   if (!session_ptr->checkDBAccessPrivileges(DBObjectType::DashboardDBObjectType,
                                             AccessPrivileges::CREATE_DASHBOARD)) {
     THROW_DB_EXCEPTION("Not enough privileges to create a dashboard.");
   }
 
   if (dashboard_exists(cat, session_ptr->get_currentUser().userId, dashboard_name)) {
     THROW_DB_EXCEPTION("Dashboard with name: " + dashboard_name + " already exists.");
   }
 
   DashboardDescriptor dd;
   dd.dashboardName = dashboard_name;
   dd.dashboardState = dashboard_state;
   dd.imageHash = image_hash;
   dd.dashboardMetadata = dashboard_metadata;
   dd.userId = session_ptr->get_currentUser().userId;
   dd.user = session_ptr->get_currentUser().userName;
 
   try {
     auto id = cat.createDashboard(dd);
     // TODO: transactionally unsafe
     SysCatalog::instance().createDBObject(
         session_ptr->get_currentUser(), dashboard_name, DashboardDBObjectType, cat, id);
     return id;
   } catch (const std::exception& e) {
     THROW_DB_EXCEPTION(e.what());
   }
 }

Here is the call graph for this function:

TColumnType DBHandler::create_geo_column	(	const TDatumType::type	type,
		const std::string &	name,
		const bool	is_array
	)

private

Definition at line 4951 of file DBHandler.cpp.

References setup::name, and run_benchmark_import::type.

                                                               {
   TColumnType ct;
   ct.col_name = name;
   ct.col_type.type = type;
   ct.col_type.is_array = is_array;
   return ct;
 }

void DBHandler::create_link	(	std::string &	_return,
		const TSessionId &	session,
		const std::string &	view_state,
		const std::string &	view_metadata
	)

override

Definition at line 4927 of file DBHandler.cpp.

References cat(), get_session_ptr(), getConnectionInfo(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, THROW_DB_EXCEPTION, toString(), LinkDescriptor::userId, LinkDescriptor::viewMetadata, and LinkDescriptor::viewState.

                                                             {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
   stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
   auto session_ptr = stdlog.getConstSessionInfo();
   // check_read_only("create_link");
   auto& cat = session_ptr->getCatalog();
 
   LinkDescriptor ld;
   ld.userId = session_ptr->get_currentUser().userId;
   ld.viewState = view_state;
   ld.viewMetadata = view_metadata;
 
   try {
     _return = cat.createLink(ld, 6);
   } catch (const std::exception& e) {
     THROW_DB_EXCEPTION(e.what());
   }
 }

Here is the call graph for this function:

std::shared_ptr<Catalog_Namespace::SessionInfo> DBHandler::create_new_session	(	TSessionId &	session,
		const std::string &	dbname,
		const Catalog_Namespace::UserMetadata &	user_meta,
		std::shared_ptr< Catalog_Namespace::Catalog >	cat
	)

private

template<typename... ARGS>

std::shared_ptr<query_state::QueryState> DBHandler::create_query_state ( ARGS &&... args )

inline

Definition at line 658 of file DBHandler.h.

References run_benchmark_import::args.

Referenced by convertResultSet(), get_table_details_impl(), get_tables_meta(), sql_execute(), sql_execute_df(), and sql_validate().

                                                                           {
     return query_states_.create(std::forward<ARGS>(args)...);
   }

Here is the caller graph for this function:

void DBHandler::create_table	(	const TSessionId &	session,
		const std::string &	table_name,
		const TRowDescriptor &	row_desc,
		const TCreateParams &	create_params
	)

override

Definition at line 4982 of file DBHandler.cpp.

References check_read_only(), getConnectionInfo(), ImportHelpers::is_reserved_name(), join(), heavyai::RequestInfo::json(), kENCODING_DATE_IN_DAYS, kENCODING_DICT, kENCODING_FIXED, kENCODING_GEOINT, kENCODING_NONE, logger::request_id(), heavyai::RequestInfo::requestId(), ImportHelpers::sanitize_name(), SET_REQUEST_ID, heavyai::RequestInfo::setRequestId(), sql_execute(), STDLOG, thrift_to_encoding(), thrift_to_encoding_name(), thrift_to_name(), THROW_DB_EXCEPTION, to_string(), and toString().

Referenced by importGeoTableSingle().

                                                                  {
   heavyai::RequestInfo request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto stdlog = STDLOG("table_name", table_name);
   stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
   check_read_only("create_table");
 
   if (ImportHelpers::is_reserved_name(table_name)) {
     THROW_DB_EXCEPTION("Invalid table name (reserved keyword): " + table_name);
   } else if (table_name != ImportHelpers::sanitize_name(table_name)) {
     THROW_DB_EXCEPTION("Invalid characters in table name: " + table_name);
   }
 
   auto rds = rd;
 
   std::string stmt{"CREATE TABLE " + table_name};
   std::vector<std::string> col_stmts;
 
   for (auto col : rds) {
     if (ImportHelpers::is_reserved_name(col.col_name)) {
       THROW_DB_EXCEPTION("Invalid column name (reserved keyword): " + col.col_name);
     } else if (col.col_name != ImportHelpers::sanitize_name(col.col_name)) {
       THROW_DB_EXCEPTION("Invalid characters in column name: " + col.col_name);
     }
     if (col.col_type.type == TDatumType::INTERVAL_DAY_TIME ||
         col.col_type.type == TDatumType::INTERVAL_YEAR_MONTH) {
       THROW_DB_EXCEPTION("Unsupported type: " + thrift_to_name(col.col_type) +
                          " for column: " + col.col_name);
     }
 
     if (col.col_type.type == TDatumType::DECIMAL) {
       // if no precision or scale passed in set to default 14,7
       if (col.col_type.precision == 0 && col.col_type.scale == 0) {
         col.col_type.precision = 14;
         col.col_type.scale = 7;
       }
     }
 
     std::string col_stmt;
     col_stmt.append(col.col_name + " " + thrift_to_name(col.col_type));
     if (col.__isset.default_value) {
       col_stmt.append(" DEFAULT " + col.default_value);
     }
 
     // As of 2016-06-27 the Immerse v1 frontend does not explicitly set the
     // `nullable` argument, leading this to default to false. Uncomment for v2.
     // if (!col.col_type.nullable) col_stmt.append(" NOT NULL");
 
     if (thrift_to_encoding(col.col_type.encoding) != kENCODING_NONE) {
       col_stmt.append(" ENCODING " + thrift_to_encoding_name(col.col_type));
       if (thrift_to_encoding(col.col_type.encoding) == kENCODING_DICT ||
           thrift_to_encoding(col.col_type.encoding) == kENCODING_FIXED ||
           thrift_to_encoding(col.col_type.encoding) == kENCODING_GEOINT ||
           thrift_to_encoding(col.col_type.encoding) == kENCODING_DATE_IN_DAYS) {
         col_stmt.append("(" + std::to_string(col.col_type.comp_param) + ")");
       }
     } else if (col.col_type.type == TDatumType::STR) {
       // non DICT encoded strings
       col_stmt.append(" ENCODING NONE");
     } else if (col.col_type.type == TDatumType::POINT ||
                col.col_type.type == TDatumType::MULTIPOINT ||
                col.col_type.type == TDatumType::LINESTRING ||
                col.col_type.type == TDatumType::MULTILINESTRING ||
                col.col_type.type == TDatumType::POLYGON ||
                col.col_type.type == TDatumType::MULTIPOLYGON) {
       // non encoded compressable geo
       if (col.col_type.scale == 4326) {
         col_stmt.append(" ENCODING NONE");
       }
     }
     col_stmts.push_back(col_stmt);
   }
 
   stmt.append(" (" + boost::algorithm::join(col_stmts, ", ") + ")");
 
   if (create_params.is_replicated) {
     stmt.append(" WITH (PARTITIONS = 'REPLICATED')");
   }
 
   stmt.append(";");
 
   TQueryResult ret;
   request_info.setRequestId(logger::request_id());
   sql_execute(ret, request_info.json(), stmt, true, "", -1, -1);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::string const DBHandler::createInMemoryCalciteSession ( const std::shared_ptr< Catalog_Namespace::Catalog > & catalog_ptr )

private

Definition at line 439 of file DBHandler.cpp.

References calcite_, Catalog_Namespace::CALCITE_SESSION_ID_LENGTH, calcite_sessions_, calcite_sessions_mtx_, CHECK, executor_device_type_, and generate_random_string().

Referenced by processCalciteRequest().

                                                                 {
   // We would create an in memory session for calcite with super user privileges which
   // would be used for getting all tables metadata when a user runs the query. The
   // session would be under the name of a proxy user/password which would only persist
   // till server's lifetime or execution of calcite query(in memory) whichever is the
   // earliest.
   heavyai::lock_guard<heavyai::shared_mutex> lg(calcite_sessions_mtx_);
   std::string session_id;
   do {
     session_id = generate_random_string(Catalog_Namespace::CALCITE_SESSION_ID_LENGTH);
   } while (calcite_sessions_.find(session_id) != calcite_sessions_.end());
   Catalog_Namespace::UserMetadata user_meta(-1,
                                             calcite_->getInternalSessionProxyUserName(),
                                             calcite_->getInternalSessionProxyPassword(),
                                             true,
                                             -1,
                                             true,
                                             false);
   const auto emplace_ret = calcite_sessions_.emplace(
       session_id,
       std::make_shared<Catalog_Namespace::SessionInfo>(
           catalog_ptr, user_meta, executor_device_type_, session_id));
   CHECK(emplace_ret.second);
   return session_id;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void DBHandler::createSimpleResult	(	TQueryResult &	_return,
		const ResultSet &	results,
		const bool	column_format,
		const std::string	label
	)

staticprivate

Definition at line 6175 of file DBHandler.cpp.

References CHECK, CHECK_EQ, and run_benchmark_import::label.

Referenced by convertExplain(), and convertResult().

                                                           {
   CHECK_EQ(size_t(1), results.rowCount());
   TColumnType proj_info;
   proj_info.col_name = label;
   proj_info.col_type.type = TDatumType::STR;
   proj_info.col_type.nullable = false;
   proj_info.col_type.is_array = false;
   _return.row_set.row_desc.push_back(proj_info);
   const auto crt_row = results.getNextRow(true, true);
   const auto tv = crt_row[0];
   CHECK(results.getNextRow(true, true).empty());
   const auto scalar_tv = boost::get<ScalarTargetValue>(&tv);
   CHECK(scalar_tv);
   const auto s_n = boost::get<NullableString>(scalar_tv);
   CHECK(s_n);
   const auto s = boost::get<std::string>(s_n);
   CHECK(s);
   if (column_format) {
     TColumn tcol;
     tcol.data.str_col.push_back(*s);
     tcol.nulls.push_back(false);
     _return.row_set.is_columnar = true;
     _return.row_set.columns.push_back(tcol);
   } else {
     TDatum explanation;
     explanation.val.str_val = *s;
     explanation.is_null = false;
     TRow trow;
     trow.cols.push_back(explanation);
     _return.row_set.is_columnar = false;
     _return.row_set.rows.push_back(trow);
   }
 }

Here is the caller graph for this function:

void DBHandler::deallocate_df	(	const TSessionId &	session,
		const TDataFrame &	df,
		const TDeviceType::type	device_type,
		const int32_t	device_id
	)

override

Definition at line 1415 of file DBHandler.cpp.

References CPU, data_mgr_, ArrowResultSet::deallocateArrowResultBuffer(), logger::ERROR, get_session_ptr(), GPU, handle_to_dev_ptr_mutex_, ipc_handle_to_dev_ptr_, LOG, heavyai::RequestInfo::requestId(), run_benchmark_import::result, heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, and STDLOG.

                                                        {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
   std::string serialized_cuda_handle = "";
   if (device_type == TDeviceType::GPU) {
     std::lock_guard<std::mutex> map_lock(handle_to_dev_ptr_mutex_);
     if (ipc_handle_to_dev_ptr_.count(df.df_handle) != size_t(1)) {
       TDBException ex;
       ex.error_msg = std::string(
           "Current data frame handle is not bookkept or been inserted "
           "twice");
       LOG(ERROR) << ex.error_msg;
       throw ex;
     }
     serialized_cuda_handle = ipc_handle_to_dev_ptr_[df.df_handle];
     ipc_handle_to_dev_ptr_.erase(df.df_handle);
   }
   std::vector<char> sm_handle(df.sm_handle.begin(), df.sm_handle.end());
   std::vector<char> df_handle(df.df_handle.begin(), df.df_handle.end());
   ArrowResult result{
       sm_handle, df.sm_size, df_handle, df.df_size, serialized_cuda_handle};
   ArrowResultSet::deallocateArrowResultBuffer(
       result,
       device_type == TDeviceType::CPU ? ExecutorDeviceType::CPU : ExecutorDeviceType::GPU,
       device_id,
       data_mgr_);
 }

Here is the call graph for this function:

void DBHandler::delete_custom_expressions	(	const TSessionId &	session,
		const std::vector< int32_t > &	custom_expression_ids,
		const bool	do_soft_delete
	)

override

Definition at line 4469 of file DBHandler.cpp.

References check_read_only(), custom_expressions_mutex_, get_session_ptr(), getConnectionInfo(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, THROW_DB_EXCEPTION, and toString().

                                {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
   stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
   check_read_only("delete_custom_expressions");
 
   auto session_ptr = stdlog.getConstSessionInfo();
   if (!session_ptr->get_currentUser().isSuper) {
     THROW_DB_EXCEPTION("Custom expressions can only be deleted by super users.")
   }
   auto& catalog = session_ptr->getCatalog();
   heavyai::unique_lock<heavyai::shared_mutex> write_lock(custom_expressions_mutex_);
   catalog.deleteCustomExpressions(custom_expression_ids, do_soft_delete);
 }

Here is the call graph for this function:

void DBHandler::delete_dashboard	(	const TSessionId &	session,
		const int32_t	dashboard_id
	)

override

Definition at line 4712 of file DBHandler.cpp.

References delete_dashboards().

                                                              {
   delete_dashboards(session_id_or_json, {dashboard_id});
 }

Here is the call graph for this function:

void DBHandler::delete_dashboards	(	const TSessionId &	session,
		const std::vector< int32_t > &	dashboard_ids
	)

override

Definition at line 4717 of file DBHandler.cpp.

References cat(), dbhandler::check_not_info_schema_db(), check_read_only(), g_allow_system_dashboard_update, get_session_ptr(), getConnectionInfo(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, THROW_DB_EXCEPTION, and toString().

Referenced by delete_dashboard().

                                                                            {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
   stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
   auto session_ptr = stdlog.getConstSessionInfo();
   check_read_only("delete_dashboards");
   auto& cat = session_ptr->getCatalog();
   if (!g_allow_system_dashboard_update) {
     dbhandler::check_not_info_schema_db(cat.name(), true);
   }
   // Checks will be performed in catalog
   try {
     cat.deleteMetadataForDashboards(dashboard_ids, session_ptr->get_currentUser());
   } catch (const std::exception& e) {
     THROW_DB_EXCEPTION(e.what());
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void DBHandler::detect_column_types	(	TDetectResult &	_return,
		const TSessionId &	session,
		const std::string &	file_name,
		const TCopyParams &	copy_params
	)

override

Definition at line 4129 of file DBHandler.cpp.

Referenced by importGeoTableSingle().

                                                            {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
   stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
   check_read_only("detect_column_types");
 
   bool is_raster = false;
   boost::filesystem::path file_path;
   import_export::CopyParams copy_params = thrift_to_copyparams(cp);
   if (copy_params.source_type != import_export::SourceType::kOdbc) {
     std::string file_name{file_name_in};
     if (path_is_relative(file_name)) {
       // assume relative paths are relative to data_path / import / <session>
       auto temp_file_path = import_path_ /
                             picosha2::hash256_hex_string(request_info.sessionId()) /
                             boost::filesystem::path(file_name).filename();
       file_name = temp_file_path.string();
     }
     validate_import_file_path_if_local(file_name);
 
     if ((copy_params.source_type == import_export::SourceType::kGeoFile ||
          copy_params.source_type == import_export::SourceType::kRasterFile) &&
         is_local_file(file_name)) {
       const shared::FilePathOptions options{copy_params.regex_path_filter,
                                             copy_params.file_sort_order_by,
                                             copy_params.file_sort_regex};
       auto file_paths = shared::local_glob_filter_sort_files(file_name, options, false);
       // For geo and raster detect, pick the first file, if multiple files are provided
       // (e.g. through file globbing).
       CHECK(!file_paths.empty());
       file_name = file_paths[0];
     }
 
     // if it's a geo or raster import, handle alternative paths (S3, HTTP, archive etc.)
     if (copy_params.source_type == import_export::SourceType::kGeoFile) {
       if (is_a_supported_archive_file(file_name)) {
         // find the archive file
         add_vsi_network_prefix(file_name);
         if (!import_export::Importer::gdalFileExists(file_name, copy_params)) {
           THROW_DB_EXCEPTION("Archive does not exist: " + file_name_in);
         }
         // find geo file in archive
         add_vsi_archive_prefix(file_name);
         std::string geo_file = find_first_geo_file_in_archive(file_name, copy_params);
         // prepare to detect that geo file
         if (geo_file.size()) {
           file_name = file_name + std::string("/") + geo_file;
         }
       } else {
         // prepare to detect geo file directly
         add_vsi_network_prefix(file_name);
         add_vsi_geo_prefix(file_name);
       }
     } else if (copy_params.source_type == import_export::SourceType::kRasterFile) {
       // prepare to detect raster file directly
       add_vsi_network_prefix(file_name);
       add_vsi_geo_prefix(file_name);
       is_raster = true;
     }
 
     file_path = boost::filesystem::path(file_name);
     // can be a s3 url
     if (!boost::istarts_with(file_name, "s3://")) {
       if (!boost::filesystem::path(file_name).is_absolute()) {
         file_path = import_path_ /
                     picosha2::hash256_hex_string(request_info.sessionId()) /
                     boost::filesystem::path(file_name).filename();
         file_name = file_path.string();
       }
 
       if (copy_params.source_type == import_export::SourceType::kGeoFile ||
           copy_params.source_type == import_export::SourceType::kRasterFile) {
         // check for geo or raster file
         if (!import_export::Importer::gdalFileOrDirectoryExists(file_name, copy_params)) {
           THROW_DB_EXCEPTION("File or directory \"" + file_path.string() +
                              "\" does not exist.")
         }
       } else {
         // check for regular file
         if (!shared::file_or_glob_path_exists(file_path.string())) {
           THROW_DB_EXCEPTION("File or directory \"" + file_path.string() +
                              "\" does not exist.");
         }
       }
     }
   }
 
   try {
     if (copy_params.source_type == import_export::SourceType::kDelimitedFile
 #ifdef ENABLE_IMPORT_PARQUET
         || (copy_params.source_type == import_export::SourceType::kParquetFile)
 #endif
     ) {
       import_export::Detector detector(file_path, copy_params);
       auto best_types = detector.getBestColumnTypes();
       std::vector<std::string> headers = detector.get_headers();
       copy_params = detector.get_copy_params();
 
       _return.copy_params = copyparams_to_thrift(copy_params);
       _return.row_set.row_desc.resize(best_types.size());
       for (size_t col_idx = 0; col_idx < best_types.size(); col_idx++) {
         TColumnType col;
         auto& ti = best_types[col_idx];
         col.col_type.precision = ti.get_precision();
         col.col_type.scale = ti.get_scale();
         col.col_type.comp_param = ti.get_comp_param();
         if (ti.is_geometry()) {
           // set this so encoding_to_thrift does the right thing
           ti.set_compression(copy_params.geo_coords_encoding);
           // fill in these directly
           col.col_type.precision = static_cast<int>(copy_params.geo_coords_type);
           col.col_type.scale = copy_params.geo_coords_srid;
           col.col_type.comp_param = copy_params.geo_coords_comp_param;
         }
         col.col_type.type = type_to_thrift(ti);
         col.col_type.encoding = encoding_to_thrift(ti);
         if (ti.is_array()) {
           col.col_type.is_array = true;
         }
         if (copy_params.sanitize_column_names) {
           col.col_name = ImportHelpers::sanitize_name(headers[col_idx]);
         } else {
           col.col_name = headers[col_idx];
         }
         col.is_reserved_keyword = ImportHelpers::is_reserved_name(col.col_name);
         _return.row_set.row_desc[col_idx] = col;
       }
       auto sample_data =
           detector.get_sample_rows(import_export::Detector::kDefaultSampleRowsCount);
 
       TRow sample_row;
       for (auto row : sample_data) {
         sample_row.cols.clear();
         for (const auto& s : row) {
           TDatum td;
           td.val.str_val = s;
           td.is_null = s.empty();
           sample_row.cols.push_back(td);
         }
         _return.row_set.rows.push_back(sample_row);
       }
     } else if (copy_params.source_type == import_export::SourceType::kGeoFile ||
                copy_params.source_type == import_export::SourceType::kRasterFile) {
       check_geospatial_files(file_path, copy_params);
       std::list<ColumnDescriptor> cds = import_export::Importer::gdalToColumnDescriptors(
           file_path.string(), is_raster, Geospatial::kGeoColumnName, copy_params);
       for (auto cd : cds) {
         if (copy_params.sanitize_column_names) {
           cd.columnName = ImportHelpers::sanitize_name(cd.columnName);
         }
         _return.row_set.row_desc.push_back(populateThriftColumnType(nullptr, &cd));
       }
       if (!is_raster) {
         // @TODO(se) support for raster?
         std::map<std::string, std::vector<std::string>> sample_data;
         import_export::Importer::readMetadataSampleGDAL(
             file_path.string(),
             Geospatial::kGeoColumnName,
             sample_data,
             import_export::Detector::kDefaultSampleRowsCount,
             copy_params);
         if (sample_data.size() > 0) {
           for (size_t i = 0; i < sample_data.begin()->second.size(); i++) {
             TRow sample_row;
             for (auto cd : cds) {
               TDatum td;
               td.val.str_val = sample_data[cd.sourceName].at(i);
               td.is_null = td.val.str_val.empty();
               sample_row.cols.push_back(td);
             }
             _return.row_set.rows.push_back(sample_row);
           }
         }
       }
       _return.copy_params = copyparams_to_thrift(copy_params);
     }
   } catch (const std::exception& e) {
     THROW_DB_EXCEPTION("detect_column_types error: " + std::string(e.what()));
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void DBHandler::disconnect ( const TSessionId & session )

override

Definition at line 572 of file DBHandler.cpp.

References get_session_ptr(), getConnectionInfo(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), sessions_store_, SET_REQUEST_ID, STDLOG, and toString().

                                                                {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto session_ptr = get_session_ptr(request_info.sessionId());
   auto stdlog = STDLOG(session_ptr, "client", getConnectionInfo().toString());
   sessions_store_->disconnect(request_info.sessionId());
 }

Here is the call graph for this function:

void DBHandler::disconnect_impl ( Catalog_Namespace::SessionInfoPtr & session_ptr )

private

Definition at line 580 of file DBHandler.cpp.

References LeafAggregator::disconnect(), leaf_aggregator_, LeafAggregator::leafCount(), render_group_assignment_map_, render_group_assignment_mutex_, and render_handler_.

Referenced by resetSessionsStore().

                                                                             {
   const auto session_id = session_ptr->get_session_id();
   std::exception_ptr leaf_exception = nullptr;
   try {
     if (leaf_aggregator_.leafCount() > 0) {
       leaf_aggregator_.disconnect(session_id);
     }
   } catch (...) {
     leaf_exception = std::current_exception();
   }
 
   {
     std::lock_guard<std::mutex> lock(render_group_assignment_mutex_);
     render_group_assignment_map_.erase(session_id);
   }
 
   if (render_handler_) {
     render_handler_->disconnect(session_id);
   }
 
   if (leaf_exception) {
     std::rethrow_exception(leaf_exception);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void DBHandler::dispatch_query_task	(	std::shared_ptr< QueryDispatchQueue::Task >	query_task,
		const bool	is_update_delete
	)

private

Definition at line 1679 of file DBHandler.cpp.

References CHECK, and dispatch_queue_.

Referenced by validate_rel_alg().

                                                                  {
   CHECK(dispatch_queue_);
   dispatch_queue_->submit(std::move(query_task), is_update_delete);
 }

Here is the caller graph for this function:

void DBHandler::emergency_shutdown ( )

Definition at line 7373 of file DBHandler.cpp.

References calcite_.

Referenced by shutdown().

                                    {
   if (calcite_) {
     calcite_->close_calcite_server(false);
   }
 }

Here is the caller graph for this function:

void DBHandler::execute_distributed_copy_statement	(	Parser::CopyTableStmt *	copy_stmt,
		const Catalog_Namespace::SessionInfo &	session_info
	)

private

Definition at line 6634 of file DBHandler.cpp.

Referenced by sql_execute_impl().

6636 {}

Here is the caller graph for this function:

void DBHandler::execute_next_render_step	(	TRenderStepResult &	_return,
		const TPendingRenderQuery &	pending_render,
		const TRenderAggDataMap &	merged_data
	)

override

Definition at line 7123 of file DBHandler.cpp.

References measure< TimeT >::execution(), logger::INFO, LOG, render_handler_, SET_REQUEST_ID, and THROW_DB_EXCEPTION.

                                                                                {
   // No SessionID is available
   SET_REQUEST_ID(0);
 
   if (!render_handler_) {
     THROW_DB_EXCEPTION("Backend rendering is disabled.");
   }
 
   LOG(INFO) << "execute_next_render_step: id:" << pending_render.id;
   auto time_ms = measure<>::execution([&]() {
     try {
       render_handler_->execute_next_render_step(_return, pending_render, merged_data);
     } catch (std::exception& e) {
       THROW_DB_EXCEPTION(e.what());
     }
   });
   LOG(INFO) << "execute_next_render_step-COMPLETED id: " << pending_render.id
             << ", time: " << time_ms << "ms ";
 }

Here is the call graph for this function:

void DBHandler::execute_query_step	(	TStepResult &	_return,
		const TPendingQuery &	pending_query,
		const TSubqueryId	subquery_id,
		const std::string &	start_time_str
	)

override

Definition at line 6879 of file DBHandler.cpp.

References measure< TimeT >::execution(), logger::INFO, leaf_handler_, LOG, SET_REQUEST_ID, and THROW_DB_EXCEPTION.

                                                                     {
   SET_REQUEST_ID(0);  // No SessionID is available
   if (!leaf_handler_) {
     THROW_DB_EXCEPTION("Distributed support is disabled.");
   }
   LOG(INFO) << "execute_query_step :  id:" << pending_query.id;
   auto time_ms = measure<>::execution([&]() {
     try {
       leaf_handler_->execute_query_step(
           _return, pending_query, subquery_id, start_time_str);
     } catch (std::exception& e) {
       THROW_DB_EXCEPTION(e.what());
     }
   });
   LOG(INFO) << "execute_query_step-COMPLETED " << time_ms << "ms";
 }

Here is the call graph for this function:

std::vector< PushedDownFilterInfo > DBHandler::execute_rel_alg	(	ExecutionResult &	_return,
		QueryStateProxy	query_state_proxy,
		const std::string &	query_ra,
		const bool	column_format,
		const ExecutorDeviceType	executor_device_type,
		const int32_t	first_n,
		const int32_t	at_most_n,
		const bool	just_validate,
		const bool	find_push_down_candidates,
		const ExplainInfo &	explain_info,
		const std::optional< size_t >	executor_index = `std::nullopt`
	)		const

private

Definition at line 5994 of file DBHandler.cpp.

References allow_loop_joins_, allow_multifrag_, cat(), query_state::QueryStateProxy::createTimer(), Default, measure< TimeT >::execution(), ExecutionResult::Explanation, g_dynamic_watchdog_time_limit, g_enable_columnar_output, g_enable_dynamic_watchdog, g_enable_runtime_query_interrupt, g_enable_watchdog, g_optimize_cuda_block_and_grid_sizes, g_pending_query_interrupt_freq, g_running_query_interrupt_freq, query_state::QueryState::getConstSessionInfo(), ExecutionResult::getExecutionTime(), Executor::getExecutor(), ExecutionResult::getPushedDownFilterInfo(), ExecutionResult::getRows(), SystemParameters::gpu_input_mem_limit, lockmgr::TableSchemaLockMgr::instance(), lockmgr::TableDataLockMgr::instance(), intel_jit_profile_, ExplainInfo::isCalciteExplain(), ExplainInfo::isJustExplain(), ExplainInfo::isOptimizedExplain(), ExplainInfo::isPlanExplain(), jit_debug_, Optimized, ExecutionResult::QueryResult, ExecutionResult::setExecutionTime(), ExecutionResult::setResultType(), system_parameters_, Executor::UNITARY_EXECUTOR_ID, and VLOG.

Referenced by execute_rel_alg_with_filter_push_down(), get_tables_meta_impl(), sql_execute_impl(), and validate_rel_alg().

                                                     {
   query_state::Timer timer = query_state_proxy.createTimer(__func__);
 
   VLOG(1) << "Table Schema Locks:\n" << lockmgr::TableSchemaLockMgr::instance();
   VLOG(1) << "Table Data Locks:\n" << lockmgr::TableDataLockMgr::instance();
 
   auto& cat = query_state_proxy->getConstSessionInfo()->getCatalog();
   auto executor = Executor::getExecutor(
       executor_index ? *executor_index : Executor::UNITARY_EXECUTOR_ID,
       jit_debug_ ? "/tmp" : "",
       jit_debug_ ? "mapdquery" : "",
       system_parameters_);
   RelAlgExecutor ra_executor(
       executor.get(), query_ra, query_state_proxy->shared_from_this());
   CompilationOptions co = {executor_device_type,
                            /*hoist_literals=*/true,
                            ExecutorOptLevel::Default,
                            g_enable_dynamic_watchdog,
                            /*allow_lazy_fetch=*/true,
                            /*filter_on_deleted_column=*/true,
                            explain_info.isOptimizedExplain()
                                ? ExecutorExplainType::Optimized
                                : ExecutorExplainType::Default,
                            intel_jit_profile_};
   auto validate_or_explain_query =
       explain_info.isJustExplain() || explain_info.isCalciteExplain() || just_validate;
   ExecutionOptions eo = {
       g_enable_columnar_output,
       false,
       allow_multifrag_,
       explain_info.isJustExplain(),
       allow_loop_joins_ || just_validate,
       g_enable_watchdog,
       jit_debug_,
       just_validate,
       g_enable_dynamic_watchdog,
       g_dynamic_watchdog_time_limit,
       find_push_down_candidates,
       explain_info.isCalciteExplain(),
       system_parameters_.gpu_input_mem_limit,
       g_enable_runtime_query_interrupt && !validate_or_explain_query &&
           !query_state_proxy->getConstSessionInfo()->get_session_id().empty(),
       g_running_query_interrupt_freq,
       g_pending_query_interrupt_freq,
       g_optimize_cuda_block_and_grid_sizes};
   auto execution_time_ms = _return.getExecutionTime() + measure<>::execution([&]() {
                              _return = ra_executor.executeRelAlgQuery(
                                  co, eo, explain_info.isPlanExplain(), nullptr);
                            });
   // reduce execution time by the time spent during queue waiting
   const auto rs = _return.getRows();
   if (rs) {
     execution_time_ms -= rs->getQueueTime();
   }
   _return.setExecutionTime(execution_time_ms);
   VLOG(1) << cat.getDataMgr().getSystemMemoryUsage();
   const auto& filter_push_down_info = _return.getPushedDownFilterInfo();
   if (!filter_push_down_info.empty()) {
     return filter_push_down_info;
   }
   if (explain_info.isJustExplain()) {
     _return.setResultType(ExecutionResult::Explanation);
   } else if (!explain_info.isCalciteExplain()) {
     _return.setResultType(ExecutionResult::QueryResult);
   }
   return {};
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void DBHandler::execute_rel_alg_with_filter_push_down	(	ExecutionResult &	_return,
		QueryStateProxy	query_state_proxy,
		std::string &	query_ra,
		const bool	column_format,
		const ExecutorDeviceType	executor_device_type,
		const int32_t	first_n,
		const int32_t	at_most_n,
		const bool	just_explain,
		const bool	just_calcite_explain,
		const std::vector< PushedDownFilterInfo > &	filter_push_down_requests
	)

private

Definition at line 6584 of file DBHandler.cpp.

References ExecutionResult::addExecutionTime(), execute_rel_alg(), ExecutionResult::Explanation, query_state::QueryState::getQueryStr(), ExplainInfo::IR, parse_to_ra(), system_parameters_, and ExecutionResult::updateResultSet().

Referenced by sql_execute_impl().

                                                                       {
   // collecting the selected filters' info to be sent to Calcite:
   std::vector<TFilterPushDownInfo> filter_push_down_info;
   for (const auto& req : filter_push_down_requests) {
     TFilterPushDownInfo filter_push_down_info_for_request;
     filter_push_down_info_for_request.input_prev = req.input_prev;
     filter_push_down_info_for_request.input_start = req.input_start;
     filter_push_down_info_for_request.input_next = req.input_next;
     filter_push_down_info.push_back(filter_push_down_info_for_request);
   }
   // deriving the new relational algebra plan with respect to the pushed down filters
   _return.addExecutionTime(measure<>::execution([&]() {
     query_ra = parse_to_ra(query_state_proxy,
                            query_state_proxy->getQueryStr(),
                            filter_push_down_info,
                            false,
                            system_parameters_)
                    .first.plan_result;
   }));
 
   if (is_calcite_explain) {
     // return the new ra as the result
     _return.updateResultSet(query_ra, ExecutionResult::Explanation);
     return;
   }
 
   // execute the new relational algebra plan:
   auto explain_info = ExplainInfo(ExplainInfo::ExplainType::IR);
   execute_rel_alg(_return,
                   query_state_proxy,
                   query_ra,
                   column_format,
                   executor_device_type,
                   first_n,
                   at_most_n,
                   /*just_validate=*/false,
                   /*find_push_down_candidates=*/false,
                   explain_info);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void DBHandler::executeDdl	(	TQueryResult &	_return,
		const std::string &	query_ra,
		std::shared_ptr< Catalog_Namespace::SessionInfo const >	session_ptr
	)

private

Definition at line 7878 of file DBHandler.cpp.

Referenced by sql_execute_impl().

                                                                    {
   DdlCommandExecutor executor = DdlCommandExecutor(query_ra, session_ptr);
   std::string commandStr = executor.commandStr();
 
   if (executor.isKillQuery()) {
     interruptQuery(*session_ptr, executor.getTargetQuerySessionToKill());
   } else {
     ExecutionResult result;
     int64_t execution_time_ms;
     if (executor.isShowQueries()) {
       // getQueries still requires Thrift cannot be nested into DdlCommandExecutor
       _return.execution_time_ms +=
           measure<>::execution([&]() { result = getQueries(session_ptr); });
     } else if (executor.isShowUserSessions()) {
       // getUserSessions still requires Thrift cannot be nested into DdlCommandExecutor
       _return.execution_time_ms +=
           measure<>::execution([&]() { result = getUserSessions(session_ptr); });
     } else if (executor.isAlterSystemClear()) {
       alterSystemClear(session_ptr->get_session_id(),
                        result,
                        executor.returnCacheType(),
                        execution_time_ms);
       _return.execution_time_ms += execution_time_ms;
 
     } else if (executor.isAlterSessionSet()) {
       alterSession(session_ptr->get_session_id(),
                    result,
                    executor.getSessionParameter(),
                    execution_time_ms);
       _return.execution_time_ms += execution_time_ms;
     } else {
       _return.execution_time_ms +=
           measure<>::execution([&]() { result = executor.execute(read_only_); });
     }
 
     if (!result.empty()) {
       // reduce execution time by the time spent during queue waiting
       _return.execution_time_ms -= result.getRows()->getQueueTime();
       convertResultSet(result, *session_ptr, commandStr, _return);
     }
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void DBHandler::executeDdl	(	ExecutionResult &	_return,
		const std::string &	query_ra,
		std::shared_ptr< Catalog_Namespace::SessionInfo const >	session_ptr
	)

private

Definition at line 7924 of file DBHandler.cpp.

                                                                    {
   DdlCommandExecutor executor = DdlCommandExecutor(query_ra, session_ptr);
   std::string commandStr = executor.commandStr();
 
   if (executor.isKillQuery()) {
     interruptQuery(*session_ptr, executor.getTargetQuerySessionToKill());
   } else {
     int64_t execution_time_ms;
     if (executor.isShowQueries()) {
       // getQueries still requires Thrift cannot be nested into DdlCommandExecutor
       execution_time_ms =
           measure<>::execution([&]() { _return = getQueries(session_ptr); });
     } else if (executor.isShowUserSessions()) {
       // getUserSessions still requires Thrift cannot be nested into DdlCommandExecutor
       execution_time_ms =
           measure<>::execution([&]() { _return = getUserSessions(session_ptr); });
     } else if (executor.isAlterSystemClear()) {
       alterSystemClear(session_ptr->get_session_id(),
                        _return,
                        executor.returnCacheType(),
                        execution_time_ms);
     } else if (executor.isAlterSessionSet()) {
       alterSession(session_ptr->get_session_id(),
                    _return,
                    executor.getSessionParameter(),
                    execution_time_ms);
     } else {
       execution_time_ms =
           measure<>::execution([&]() { _return = executor.execute(read_only_); });
     }
     _return.setExecutionTime(execution_time_ms);
   }
   if (_return.getResultType() == ExecutionResult::QueryResult) {
     // ResultType defaults to QueryResult => which can limit
     //   the number of lines output via ConvertRow... use CalciteDdl instead
     _return.setResultType(ExecutionResult::CalciteDdl);
   }
 }

Here is the call graph for this function:

std::unordered_map< std::string, std::unordered_set< std::string > > DBHandler::fill_column_names_by_table	(	std::vector< std::string > &	table_names,
		query_state::StdLog &	stdlog
	)

private

Definition at line 1629 of file DBHandler.cpp.

References get_table_details_impl().

Referenced by get_token_based_completions().

                                                                  {
   std::unordered_map<std::string, std::unordered_set<std::string>> column_names_by_table;
   for (auto it = table_names.begin(); it != table_names.end();) {
     TTableDetails table_details;
     try {
       get_table_details_impl(table_details, stdlog, *it, false, false);
     } catch (const TDBException& e) {
       // Remove the corrupted Table/View name from the list for further processing.
       it = table_names.erase(it);
       continue;
     }
     for (const auto& column_type : table_details.row_desc) {
       column_names_by_table[*it].emplace(column_type.col_name);
     }
     ++it;
   }
   return column_names_by_table;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void DBHandler::fillGeoColumns	(	const TSessionId &	session,
		const Catalog_Namespace::Catalog &	catalog,
		std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &	import_buffers,
		const ColumnDescriptor *	cd,
		size_t &	col_idx,
		size_t	num_rows,
		const std::string &	table_name,
		bool	assign_render_groups
	)

private

Definition at line 2957 of file DBHandler.cpp.

References CHECK, CHECK_EQ, ColumnDescriptor::columnName, ColumnDescriptor::columnType, g_enable_assign_render_groups, SQLTypeInfo::get_type(), Geospatial::GeoTypesFactory::getGeoColumns(), logger::INFO, IS_GEO_POLY, LOG, LOG_IF, render_group_assignment_map_, render_group_assignment_mutex_, import_export::Importer::set_geo_physical_import_buffer_columnar(), and THROW_DB_EXCEPTION.

Referenced by fillMissingBuffers(), load_table(), and loadTableBinaryColumnarInternal().

                                {
   auto geo_col_idx = col_idx - 1;
   const auto wkt_or_wkb_hex_column = import_buffers[geo_col_idx]->getGeoStringBuffer();
   std::vector<std::vector<double>> coords_column, bounds_column;
   std::vector<std::vector<int>> ring_sizes_column, poly_rings_column;
   std::vector<int> render_groups_column;
   SQLTypeInfo ti = cd->columnType;
   if (num_rows != wkt_or_wkb_hex_column->size() ||
       !Geospatial::GeoTypesFactory::getGeoColumns(wkt_or_wkb_hex_column,
                                                   ti,
                                                   coords_column,
                                                   bounds_column,
                                                   ring_sizes_column,
                                                   poly_rings_column,
                                                   true)) {
     std::ostringstream oss;
     oss << "Invalid geometry in column " << cd->columnName;
     THROW_DB_EXCEPTION(oss.str());
   }
 
   // start or continue assigning render groups for poly columns?
   if (IS_GEO_POLY(cd->columnType.get_type()) && assign_render_groups &&
       g_enable_assign_render_groups) {
     // get RGA to use
     import_export::RenderGroupAnalyzer* render_group_analyzer{};
     {
       // mutex the map access
       std::lock_guard<std::mutex> lock(render_group_assignment_mutex_);
 
       // emplace new RGA or fetch existing RGA from map
       auto [itr_table, emplaced_table] = render_group_assignment_map_.try_emplace(
           session_id, RenderGroupAssignmentTableMap());
       LOG_IF(INFO, emplaced_table)
           << "load_table_binary_columnar_polys: Creating Render Group Assignment "
              "Persistent Data for Session '"
           << session_id << "'";
       auto [itr_column, emplaced_column] =
           itr_table->second.try_emplace(table_name, RenderGroupAssignmentColumnMap());
       LOG_IF(INFO, emplaced_column)
           << "load_table_binary_columnar_polys: Creating Render Group Assignment "
              "Persistent Data for Table '"
           << table_name << "'";
       auto [itr_analyzer, emplaced_analyzer] = itr_column->second.try_emplace(
           cd->columnName, std::make_unique<import_export::RenderGroupAnalyzer>());
       LOG_IF(INFO, emplaced_analyzer)
           << "load_table_binary_columnar_polys: Creating Render Group Assignment "
              "Persistent Data for Column '"
           << cd->columnName << "'";
       render_group_analyzer = itr_analyzer->second.get();
       CHECK(render_group_analyzer);
 
       // seed new RGA from existing table/column, to handle appends
       if (emplaced_analyzer) {
         LOG(INFO) << "load_table_binary_columnar_polys: Seeding Render Groups from "
                      "existing table...";
         render_group_analyzer->seedFromExistingTableContents(
             catalog, table_name, cd->columnName);
         LOG(INFO) << "load_table_binary_columnar_polys: Done";
       }
     }
 
     // assign render groups for this set of bounds
     LOG(INFO) << "load_table_binary_columnar_polys: Assigning Render Groups...";
     render_groups_column.reserve(bounds_column.size());
     for (auto const& bounds : bounds_column) {
       CHECK_EQ(bounds.size(), 4u);
       int rg = render_group_analyzer->insertBoundsAndReturnRenderGroup(bounds);
       render_groups_column.push_back(rg);
     }
     LOG(INFO) << "load_table_binary_columnar_polys: Done";
   } else {
     // render groups all zero
     render_groups_column.resize(bounds_column.size(), 0);
   }
 
   // Populate physical columns, advance col_idx
   import_export::Importer::set_geo_physical_import_buffer_columnar(catalog,
                                                                    cd,
                                                                    import_buffers,
                                                                    col_idx,
                                                                    coords_column,
                                                                    bounds_column,
                                                                    ring_sizes_column,
                                                                    poly_rings_column,
                                                                    render_groups_column);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void DBHandler::fillMissingBuffers	(	const TSessionId &	session,
		const Catalog_Namespace::Catalog &	catalog,
		std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &	import_buffers,
		const std::list< const ColumnDescriptor * > &	cds,
		const std::vector< int > &	desc_id_to_column_id,
		size_t	num_rows,
		const std::string &	table_name,
		bool	assign_render_groups
	)

private

Definition at line 3052 of file DBHandler.cpp.

References CHECK, and fillGeoColumns().

Referenced by load_table(), load_table_binary(), load_table_binary_arrow(), and loadTableBinaryColumnarInternal().

                                {
   size_t skip_physical_cols = 0;
   size_t col_idx = 0, import_idx = 0;
   for (const auto& cd : cds) {
     if (skip_physical_cols > 0) {
       CHECK(cd->isGeoPhyCol);
       skip_physical_cols--;
       continue;
     } else if (cd->columnType.is_geometry()) {
       skip_physical_cols = cd->columnType.get_physical_cols();
     }
     if (desc_id_to_column_id[import_idx] == -1) {
       import_buffers[col_idx]->addDefaultValues(cd, num_rows);
       col_idx++;
       if (cd->columnType.is_geometry()) {
         fillGeoColumns(session_id,
                        catalog,
                        import_buffers,
                        cd,
                        col_idx,
                        num_rows,
                        table_name,
                        assign_render_groups);
       }
     } else {
       col_idx++;
       col_idx += skip_physical_cols;
     }
     import_idx++;
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

Catalog_Namespace::SessionInfoPtr DBHandler::findCalciteSession ( TSessionId const & session_id ) const

private

Definition at line 5900 of file DBHandler.cpp.

References calcite_sessions_, and calcite_sessions_mtx_.

Referenced by get_session_ptr().

                                         {
   heavyai::lock_guard<heavyai::shared_mutex> lg(calcite_sessions_mtx_);
   auto const itr = calcite_sessions_.find(session_id);
   return itr == calcite_sessions_.end() ? nullptr : itr->second;
 }

Here is the caller graph for this function:

TRowDescriptor DBHandler::fixup_row_descriptor	(	const TRowDescriptor &	row_desc,
		const Catalog_Namespace::Catalog &	cat
	)

private

Definition at line 6158 of file DBHandler.cpp.

References DictDescriptor::dictNBits, and Catalog_Namespace::Catalog::getMetadataForDict().

Referenced by get_table_details_impl(), and sql_validate().

                                                                    {
   TRowDescriptor fixedup_row_desc;
   for (const TColumnType& col_desc : row_desc) {
     auto fixedup_col_desc = col_desc;
     if (col_desc.col_type.encoding == TEncodingType::DICT &&
         col_desc.col_type.comp_param > 0) {
       const auto dd = cat.getMetadataForDict(col_desc.col_type.comp_param, false);
       fixedup_col_desc.col_type.comp_param = dd->dictNBits;
     }
     fixedup_row_desc.push_back(fixedup_col_desc);
   }
 
   return fixedup_row_desc;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void DBHandler::get_all_effective_roles_for_user	(	std::vector< std::string > &	_return,
		const TSessionId &	session,
		const std::string &	granteeName
	)

override

Definition at line 2130 of file DBHandler.cpp.

References get_session_ptr(), getAllRolesForUserImpl(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, and STDLOG.

                                                                                {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
   auto session_ptr = stdlog.getConstSessionInfo();
   getAllRolesForUserImpl(session_ptr, roles, granteeName, /*effective=*/true);
 }

Here is the call graph for this function:

void DBHandler::get_all_files_in_archive	(	std::vector< std::string > &	_return,
		const TSessionId &	session,
		const std::string &	archive_path,
		const TCopyParams &	copy_params
	)

override

Definition at line 5756 of file DBHandler.cpp.

References anonymous_namespace{DBHandler.cpp}::add_vsi_archive_prefix(), anonymous_namespace{DBHandler.cpp}::add_vsi_network_prefix(), import_export::Importer::gdalFileExists(), import_export::Importer::gdalGetAllFilesInArchive(), get_session_ptr(), getConnectionInfo(), import_path_, anonymous_namespace{DBHandler.cpp}::is_a_supported_archive_file(), anonymous_namespace{DBHandler.cpp}::path_is_relative(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, thrift_to_copyparams(), THROW_DB_EXCEPTION, toString(), and anonymous_namespace{DBHandler.cpp}::validate_import_file_path_if_local().

                                                                          {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()),
                        "get_all_files_in_archive",
                        archive_path_in);
   stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
 
   std::string archive_path(archive_path_in);
   if (path_is_relative(archive_path)) {
     // assume relative paths are relative to data_path / import / <session>
     auto file_path = import_path_ /
                      picosha2::hash256_hex_string(request_info.sessionId()) /
                      boost::filesystem::path(archive_path).filename();
     archive_path = file_path.string();
   }
   validate_import_file_path_if_local(archive_path);
 
   if (is_a_supported_archive_file(archive_path)) {
     // find the archive file
     add_vsi_network_prefix(archive_path);
     if (!import_export::Importer::gdalFileExists(archive_path,
                                                  thrift_to_copyparams(copy_params))) {
       THROW_DB_EXCEPTION("Archive does not exist: " + archive_path_in);
     }
     // find all files in archive
     add_vsi_archive_prefix(archive_path);
     _return = import_export::Importer::gdalGetAllFilesInArchive(
         archive_path, thrift_to_copyparams(copy_params));
     // prepend them all with original path
     for (auto& s : _return) {
       s = archive_path_in + '/' + s;
     }
   }
 }

Here is the call graph for this function:

void DBHandler::get_all_roles_for_user	(	std::vector< std::string > &	_return,
		const TSessionId &	session,
		const std::string &	granteeName
	)

override

Definition at line 2118 of file DBHandler.cpp.

References get_session_ptr(), getAllRolesForUserImpl(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, and STDLOG.

                                                                      {
   // WARNING: This function only returns directly granted roles.
   // See also: get_all_effective_roles_for_user() for all of a user's roles.
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
   auto session_ptr = stdlog.getConstSessionInfo();
   getAllRolesForUserImpl(session_ptr, roles, granteeName, /*effective=*/false);
 }

Here is the call graph for this function:

void DBHandler::get_completion_hints	(	std::vector< TCompletionHint > &	hints,
		const TSessionId &	session,
		const std::string &	sql,
		const int	cursor
	)

override

Definition at line 1520 of file DBHandler.cpp.

References anonymous_namespace{DBHandler.cpp}::extract_projection_tokens_for_completion(), get_completion_hints_unsorted(), get_session_ptr(), get_uc_compatible_table_names_by_column(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, gpu_enabled::sort(), STDLOG, and to_upper().

                                                        {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
   std::vector<std::string> visible_tables;  // Tables allowed for the given session.
   get_completion_hints_unsorted(hints, visible_tables, stdlog, sql, cursor);
   const auto proj_tokens = extract_projection_tokens_for_completion(sql);
   auto compatible_table_names = get_uc_compatible_table_names_by_column(
       proj_tokens.uc_column_names, visible_tables, stdlog);
   // Add the table qualifiers explicitly specified by the user.
   compatible_table_names.insert(proj_tokens.uc_column_table_qualifiers.begin(),
                                 proj_tokens.uc_column_table_qualifiers.end());
   // Sort the hints by category, from COLUMN (most specific) to KEYWORD.
   std::sort(
       hints.begin(),
       hints.end(),
       [&compatible_table_names](const TCompletionHint& lhs, const TCompletionHint& rhs) {
         if (lhs.type == TCompletionHintType::TABLE &&
             rhs.type == TCompletionHintType::TABLE) {
           // Between two tables, one which is compatible with the specified
           // projections and one which isn't, pick the one which is compatible.
           if (compatible_table_names.find(to_upper(lhs.hints.back())) !=
                   compatible_table_names.end() &&
               compatible_table_names.find(to_upper(rhs.hints.back())) ==
                   compatible_table_names.end()) {
             return true;
           }
         }
         return lhs.type < rhs.type;
       });
 }

Here is the call graph for this function:

void DBHandler::get_completion_hints_unsorted	(	std::vector< TCompletionHint > &	hints,
		std::vector< std::string > &	visible_tables,
		query_state::StdLog &	stdlog,
		const std::string &	sql,
		const int	cursor
	)

private

Definition at line 1555 of file DBHandler.cpp.

References calcite_, logger::ERROR, GET_PHYSICAL_TABLES_AND_VIEWS, get_tables_impl(), get_token_based_completions(), query_state::StdLog::getConstSessionInfo(), just_whitelisted_keyword_hints(), and LOG.

Referenced by get_completion_hints().

                                                                 {
   const auto& session_info = *stdlog.getConstSessionInfo();
   try {
     get_tables_impl(visible_tables, session_info, GET_PHYSICAL_TABLES_AND_VIEWS);
 
     // Filter out keywords suggested by Calcite which we don't support.
     hints = just_whitelisted_keyword_hints(
         calcite_->getCompletionHints(session_info, visible_tables, sql, cursor));
   } catch (const std::exception& e) {
     TDBException ex;
     ex.error_msg = std::string(e.what());
     LOG(ERROR) << ex.error_msg;
     throw ex;
   }
   boost::regex from_expr{R"(\s+from\s+)", boost::regex::extended | boost::regex::icase};
   const size_t length_to_cursor =
       cursor < 0 ? sql.size() : std::min(sql.size(), static_cast<size_t>(cursor));
   // Trust hints from Calcite after the FROM keyword.
   if (boost::regex_search(sql.cbegin(), sql.cbegin() + length_to_cursor, from_expr)) {
     return;
   }
   // Before FROM, the query is too incomplete for context-sensitive completions.
   get_token_based_completions(hints, stdlog, visible_tables, sql, cursor);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void DBHandler::get_custom_expressions	(	std::vector< TCustomExpression > &	_return,
		const TSessionId &	session
	)

override

Definition at line 4434 of file DBHandler.cpp.

References anonymous_namespace{DBHandler.cpp}::create_thrift_obj_from_custom_expr(), custom_expressions_mutex_, get_session_ptr(), getConnectionInfo(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, and toString().

                                                                              {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
   stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
 
   auto session_ptr = stdlog.getConstSessionInfo();
   auto& catalog = session_ptr->getCatalog();
   heavyai::shared_lock<heavyai::shared_mutex> read_lock(custom_expressions_mutex_);
   auto custom_expressions =
       catalog.getCustomExpressionsForUser(session_ptr->get_currentUser());
   for (const auto& custom_expression : custom_expressions) {
     _return.emplace_back(create_thrift_obj_from_custom_expr(*custom_expression, catalog));
   }
 }

Here is the call graph for this function:

void DBHandler::get_dashboard	(	TDashboard &	_return,
		const TSessionId &	session,
		const int32_t	dashboard_id
	)

override

Definition at line 4489 of file DBHandler.cpp.

References cat(), get_dashboard_impl(), get_session_ptr(), getConnectionInfo(), is_allowed_on_dashboard(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, THROW_DB_EXCEPTION, to_string(), toString(), Catalog_Namespace::UserMetadata::userName, and AccessPrivileges::VIEW_DASHBOARD.

                                                           {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
   stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
   auto session_ptr = stdlog.getConstSessionInfo();
   auto const& cat = session_ptr->getCatalog();
   Catalog_Namespace::UserMetadata user_meta;
   auto dash = cat.getMetadataForDashboard(dashboard_id);
   if (!dash) {
     THROW_DB_EXCEPTION("Dashboard with dashboard id " + std::to_string(dashboard_id) +
                        " doesn't exist");
   }
   if (!is_allowed_on_dashboard(
           *session_ptr, dash->dashboardId, AccessPrivileges::VIEW_DASHBOARD)) {
     THROW_DB_EXCEPTION("User has no view privileges for the dashboard with id " +
                        std::to_string(dashboard_id));
   }
   user_meta.userName = "";
   SysCatalog::instance().getMetadataForUserById(dash->userId, user_meta);
   dashboard = get_dashboard_impl(session_ptr, user_meta, dash);
 }

Here is the call graph for this function:

void DBHandler::get_dashboard_grantees	(	std::vector< TDashboardGrantees > &	_return,
		const TSessionId &	session,
		const int32_t	dashboard_id
	)

override

Definition at line 4881 of file DBHandler.cpp.

References cat(), DashboardPrivileges::CREATE_DASHBOARD, DashboardDBObjectType, DashboardPrivileges::DELETE_DASHBOARD, DashboardPrivileges::EDIT_DASHBOARD, get_session_ptr(), getConnectionInfo(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, THROW_DB_EXCEPTION, to_string(), toString(), Catalog_Namespace::UserMetadata::userId, Catalog_Namespace::UserMetadata::userName, and DashboardPrivileges::VIEW_DASHBOARD.

                                 {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
   stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
   auto session_ptr = stdlog.getConstSessionInfo();
   auto const& cat = session_ptr->getCatalog();
   Catalog_Namespace::UserMetadata user_meta;
   auto dash = cat.getMetadataForDashboard(dashboard_id);
   if (!dash) {
     THROW_DB_EXCEPTION("Dashboard id " + std::to_string(dashboard_id) +
                        " does not exist");
   } else if (session_ptr->get_currentUser().userId != dash->userId &&
              !session_ptr->get_currentUser().isSuper) {
     THROW_DB_EXCEPTION(
         "User should be either owner of dashboard or super user to access grantees");
   }
   std::vector<ObjectRoleDescriptor*> objectsList;
   objectsList = SysCatalog::instance().getMetadataForObject(
       cat.getCurrentDB().dbId,
       static_cast<int>(DBObjectType::DashboardDBObjectType),
       dashboard_id);  // By default objecttypecan be only dashabaords
   user_meta.userId = -1;
   user_meta.userName = "";
   SysCatalog::instance().getMetadataForUserById(dash->userId, user_meta);
   for (auto object : objectsList) {
     if (user_meta.userName == object->roleName) {
       // Mask owner
       continue;
     }
     TDashboardGrantees grantee;
     TDashboardPermissions perm;
     grantee.name = object->roleName;
     grantee.is_user = object->roleType;
     perm.create_ = object->privs.hasPermission(DashboardPrivileges::CREATE_DASHBOARD);
     perm.delete_ = object->privs.hasPermission(DashboardPrivileges::DELETE_DASHBOARD);
     perm.edit_ = object->privs.hasPermission(DashboardPrivileges::EDIT_DASHBOARD);
     perm.view_ = object->privs.hasPermission(DashboardPrivileges::VIEW_DASHBOARD);
     grantee.permissions = perm;
     dashboard_grantees.push_back(grantee);
   }
 }

Here is the call graph for this function:

TDashboard DBHandler::get_dashboard_impl	(	const std::shared_ptr< Catalog_Namespace::SessionInfo const > &	session_ptr,
		Catalog_Namespace::UserMetadata &	user_meta,
		const DashboardDescriptor *	dash,
		const bool	populate_state = `true`
	)

private

Definition at line 4536 of file DBHandler.cpp.

Referenced by get_dashboard(), and get_dashboards().

                                {
   auto const& cat = session_ptr->getCatalog();
   SysCatalog::instance().getMetadataForUserById(dash->userId, user_meta);
   auto objects_list = SysCatalog::instance().getMetadataForObject(
       cat.getCurrentDB().dbId,
       static_cast<int>(DBObjectType::DashboardDBObjectType),
       dash->dashboardId);
   TDashboard dashboard;
   dashboard.dashboard_name = dash->dashboardName;
   if (populate_state) {
     dashboard.dashboard_state = dash->dashboardState;
   }
   dashboard.image_hash = dash->imageHash;
   dashboard.update_time = dash->updateTime;
   dashboard.dashboard_metadata = dash->dashboardMetadata;
   dashboard.dashboard_id = dash->dashboardId;
   dashboard.dashboard_owner = dash->user;
   TDashboardPermissions perms;
   // Super user has all permissions.
   if (session_ptr->get_currentUser().isSuper) {
     perms.create_ = true;
     perms.delete_ = true;
     perms.edit_ = true;
     perms.view_ = true;
   } else {
     // Collect all grants on current user
     // add them to the permissions.
     auto obj_to_find =
         DBObject(dashboard.dashboard_id, DBObjectType::DashboardDBObjectType);
     obj_to_find.loadKey(cat);
     std::vector<std::string> grantees =
         SysCatalog::instance().getRoles(true,
                                         session_ptr->get_currentUser().isSuper,
                                         session_ptr->get_currentUser().userName);
     for (const auto& grantee : grantees) {
       DBObject* object_found;
       auto* gr = SysCatalog::instance().getGrantee(grantee);
       if (gr && (object_found = gr->findDbObject(obj_to_find.getObjectKey(), true))) {
         const auto obj_privs = object_found->getPrivileges();
         perms.create_ |= obj_privs.hasPermission(DashboardPrivileges::CREATE_DASHBOARD);
         perms.delete_ |= obj_privs.hasPermission(DashboardPrivileges::DELETE_DASHBOARD);
         perms.edit_ |= obj_privs.hasPermission(DashboardPrivileges::EDIT_DASHBOARD);
         perms.view_ |= obj_privs.hasPermission(DashboardPrivileges::VIEW_DASHBOARD);
       }
     }
   }
   dashboard.dashboard_permissions = perms;
   if (objects_list.empty() ||
       (objects_list.size() == 1 && objects_list[0]->roleName == user_meta.userName)) {
     dashboard.is_dash_shared = false;
   } else {
     dashboard.is_dash_shared = true;
   }
   return dashboard;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void DBHandler::get_dashboards	(	std::vector< TDashboard > &	_return,
		const TSessionId &	session
	)

override

Definition at line 4514 of file DBHandler.cpp.

References cat(), get_dashboard_impl(), get_session_ptr(), getConnectionInfo(), is_allowed_on_dashboard(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, toString(), Catalog_Namespace::UserMetadata::userName, and AccessPrivileges::VIEW_DASHBOARD.

                                                                      {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
   stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
   auto session_ptr = stdlog.getConstSessionInfo();
   auto const& cat = session_ptr->getCatalog();
   Catalog_Namespace::UserMetadata user_meta;
   const auto dashes = cat.getAllDashboardsMetadata();
   user_meta.userName = "";
   for (const auto dash : dashes) {
     if (is_allowed_on_dashboard(
             *session_ptr, dash->dashboardId, AccessPrivileges::VIEW_DASHBOARD)) {
       // dashboardState is intentionally not populated here
       // for payload reasons
       // use get_dashboard call to get state
       dashboards.push_back(get_dashboard_impl(session_ptr, user_meta, dash, false));
     }
   }
 }

Here is the call graph for this function:

void DBHandler::get_databases	(	std::vector< TDBInfo > &	_return,
		const TSessionId &	session
	)

override

Definition at line 2844 of file DBHandler.cpp.

References get_session_ptr(), getConnectionInfo(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, and toString().

                                                                     {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
   stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
   auto session_ptr = stdlog.getConstSessionInfo();
   const auto& user = session_ptr->get_currentUser();
   Catalog_Namespace::DBSummaryList dbs =
       SysCatalog::instance().getDatabaseListForUser(user);
   for (auto& db : dbs) {
     TDBInfo dbinfo;
     dbinfo.db_name = std::move(db.dbName);
     dbinfo.db_owner = std::move(db.dbOwnerName);
     dbinfos.push_back(std::move(dbinfo));
   }
 }

Here is the call graph for this function:

void DBHandler::get_db_object_privs	(	std::vector< TDBObject > &	_return,
		const TSessionId &	session,
		const std::string &	objectName,
		const TDBObjectType::type	type
	)

override

Definition at line 1999 of file DBHandler.cpp.

References AccessPrivileges::ALL_TABLE, cat(), DashboardDBObjectType, DatabaseDBObjectType, get_session_ptr(), DBObject::getObjectKey(), DBObject::loadKey(), heavyai::RequestInfo::requestId(), serialize_db_object(), ServerDBObjectType, heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, TableDBObjectType, THROW_DB_EXCEPTION, to_string(), and ViewDBObjectType.

                                                                   {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto session_ptr = get_session_ptr(request_info.sessionId());
   auto stdlog = STDLOG(session_ptr);
   const auto& cat = session_ptr->getCatalog();
   DBObjectType object_type;
   switch (type) {
     case TDBObjectType::DatabaseDBObjectType:
       object_type = DBObjectType::DatabaseDBObjectType;
       break;
     case TDBObjectType::TableDBObjectType:
       object_type = DBObjectType::TableDBObjectType;
       break;
     case TDBObjectType::DashboardDBObjectType:
       object_type = DBObjectType::DashboardDBObjectType;
       break;
     case TDBObjectType::ViewDBObjectType:
       object_type = DBObjectType::ViewDBObjectType;
       break;
     case TDBObjectType::ServerDBObjectType:
       object_type = DBObjectType::ServerDBObjectType;
       break;
     default:
       THROW_DB_EXCEPTION("Failed to get object privileges for " + objectName +
                          ": unknown object type (" + std::to_string(type) + ").");
   }
   DBObject object_to_find(objectName, object_type);
 
   // TODO(adb): Use DatabaseLock to protect method
   try {
     if (object_type == DashboardDBObjectType) {
       if (objectName == "") {
         object_to_find = DBObject(-1, object_type);
       } else {
         object_to_find = DBObject(std::stoi(objectName), object_type);
       }
     } else if ((object_type == TableDBObjectType || object_type == ViewDBObjectType) &&
                !objectName.empty()) {
       // special handling for view / table
       auto td = cat.getMetadataForTable(objectName, false);
       if (td) {
         object_type = td->isView ? ViewDBObjectType : TableDBObjectType;
         object_to_find = DBObject(objectName, object_type);
       }
     }
     object_to_find.loadKey(cat);
   } catch (const std::exception&) {
     THROW_DB_EXCEPTION("Object with name " + objectName + " does not exist.");
   }
 
   // object type on database level
   DBObject object_to_find_dblevel("", object_type);
   object_to_find_dblevel.loadKey(cat);
   // if user is superuser respond with a full priv
   if (session_ptr->get_currentUser().isSuper) {
     // using ALL_TABLE here to set max permissions
     DBObject dbObj{object_to_find.getObjectKey(),
                    AccessPrivileges::ALL_TABLE,
                    session_ptr->get_currentUser().userId};
     dbObj.setName("super");
     TDBObjects.push_back(
         serialize_db_object(session_ptr->get_currentUser().userName, dbObj));
   };
 
   std::vector<std::string> grantees =
       SysCatalog::instance().getRoles(true,
                                       session_ptr->get_currentUser().isSuper,
                                       session_ptr->get_currentUser().userName);
   for (const auto& grantee : grantees) {
     DBObject* object_found;
     auto* gr = SysCatalog::instance().getGrantee(grantee);
     if (gr && (object_found = gr->findDbObject(object_to_find.getObjectKey(), true))) {
       TDBObjects.push_back(serialize_db_object(grantee, *object_found));
     }
     // check object permissions on Database level
     if (gr &&
         (object_found = gr->findDbObject(object_to_find_dblevel.getObjectKey(), true))) {
       TDBObjects.push_back(serialize_db_object(grantee, *object_found));
     }
   }
 }

Here is the call graph for this function:

void DBHandler::get_db_objects_for_grantee	(	std::vector< TDBObject > &	_return,
		const TSessionId &	session,
		const std::string &	roleName
	)

override

Definition at line 1970 of file DBHandler.cpp.

References get_session_ptr(), heavyai::RequestInfo::requestId(), serialize_db_object(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, and THROW_DB_EXCEPTION.

                                                                       {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto session_ptr = get_session_ptr(request_info.sessionId());
   auto stdlog = STDLOG(session_ptr);
   auto const& user = session_ptr->get_currentUser();
   if (!user.isSuper &&
       !SysCatalog::instance().isRoleGrantedToGrantee(user.userName, roleName, false)) {
     return;
   }
   auto* rl = SysCatalog::instance().getGrantee(roleName);
   if (rl) {
     auto dbId = session_ptr->getCatalog().getCurrentDB().dbId;
     for (auto& dbObject : *rl->getDbObjects(true)) {
       if (dbObject.first.dbId != dbId) {
         // TODO (max): it doesn't scale well in case we have many DBs (not a typical
         // usecase for now, though)
         continue;
       }
       TDBObject tdbObject = serialize_db_object(roleName, *dbObject.second);
       TDBObjectsForRole.push_back(tdbObject);
     }
   } else {
     THROW_DB_EXCEPTION("User or role " + roleName + " does not exist.");
   }
 }

Here is the call graph for this function:

void DBHandler::get_device_parameters	(	std::map< std::string, std::string > &	_return,
		const TSessionId &	session
	)

override

Definition at line 7392 of file DBHandler.cpp.

References cpu_mode_only_, EXPOSE_THRIFT_MAP, get_session_ptr(), getConnectionInfo(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, and toString().

                                                                             {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
   stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
   auto params = ::get_device_parameters(cpu_mode_only_);
   for (auto item : params) {
     _return.insert(item);
   }
   EXPOSE_THRIFT_MAP(TDeviceType);
   EXPOSE_THRIFT_MAP(TDatumType);
   EXPOSE_THRIFT_MAP(TEncodingType);
   EXPOSE_THRIFT_MAP(TExtArgumentType);
   EXPOSE_THRIFT_MAP(TOutputBufferSizeType);
 }

Here is the call graph for this function:

void DBHandler::get_first_geo_file_in_archive	(	std::string &	_return,
		const TSessionId &	session,
		const std::string &	archive_path,
		const TCopyParams &	copy_params
	)

override

Definition at line 5709 of file DBHandler.cpp.

References anonymous_namespace{DBHandler.cpp}::add_vsi_archive_prefix(), anonymous_namespace{DBHandler.cpp}::add_vsi_network_prefix(), anonymous_namespace{DBHandler.cpp}::find_first_geo_file_in_archive(), import_export::Importer::gdalFileExists(), get_session_ptr(), getConnectionInfo(), import_path_, anonymous_namespace{DBHandler.cpp}::is_a_supported_archive_file(), anonymous_namespace{DBHandler.cpp}::path_is_relative(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, thrift_to_copyparams(), THROW_DB_EXCEPTION, toString(), and anonymous_namespace{DBHandler.cpp}::validate_import_file_path_if_local().

                                                                               {
   heavyai::RequestInfo const request_info(session_id_or_json);
   SET_REQUEST_ID(request_info.requestId());
   auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()),
                        "get_first_geo_file_in_archive",
                        archive_path_in);
   stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
 
   std::string archive_path(archive_path_in);
 
   if (path_is_relative(archive_path)) {
     // assume relative paths are relative to data_path / import / <session>
     auto file_path = import_path_ /
                      picosha2::hash256_hex_string(request_info.sessionId()) /
                      boost::filesystem::path(archive_path).filename();
     archive_path = file_path.string();
   }
   validate_import_file_path_if_local(archive_path);
 
   if (is_a_supported_archive_file(archive_path)) {
     // find the archive file
     add_vsi_network_prefix(archive_path);
     if (!import_export::Importer::gdalFileExists(archive_path,
                                                  thrift_to_copyparams(copy_params))) {
       THROW_DB_EXCEPTION("Archive does not exist: " + archive_path_in);
     }
     // find geo file in archive
     add_vsi_archive_prefix(archive_path);
     std::string geo_file =
         find_first_geo_file_in_archive(archive_path, thrift_to_copyparams(copy_params));
     // what did we get?
     if (geo_file.size()) {
       // prepend it with the original path
       _return = archive_path_in + std::string("/") + geo_file;
     } else {
       // just return the original path
       _return = archive_path_in;
     }
   } else {
     // just return the original path
     _return = archive_path_in;
   }
 }

Here is the call graph for this function:

void DBHandler::get_function_details	(	std::vector< TUserDefinedFunction > &	_return,
		const TSessionId &	session,
		const std::vector< std::string > &	udf_names
	)

override

Definition at line 7512 of file DBHandler.cpp.

References ExtensionFunctionsWhitelist::get_ext_funcs(), and ThriftSerializers::to_thrift().

                                                                             {
   for (const std::string& udf_name : udf_names) {
     for (auto udf : ExtensionFunctionsWhitelist::get_ext_funcs(udf_name)) {
       _return.emplace_back(ThriftSerializers::to_thrift(udf));
     }
   }
 }

Here is the call graph for this function:

void DBHandler::get_function_names ( std::vector< std::string > & _return,

Classes

Public Member Functions

Static Public Member Functions

Public Attributes

Protected Member Functions

Private Types

Private Member Functions

Static Private Member Functions

Private Attributes

Friends

Detailed Description

Member Typedef Documentation

Member Enumeration Documentation

Constructor & Destructor Documentation

Member Function Documentation