OmniSciDB  a5dc49c757
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
DBHandler Class Reference

#include <DBHandler.h>

+ Inheritance diagram for DBHandler:
+ Collaboration diagram for DBHandler:

Classes

struct  DeferredCopyFromSessions
 
struct  DeferredCopyFromState
 

Public Member Functions

 DBHandler (const std::vector< LeafHostInfo > &db_leaves, const std::vector< LeafHostInfo > &string_leaves, const std::string &base_data_path, const bool allow_multifrag, const bool jit_debug, const bool intel_jit_profile, const bool read_only, const bool allow_loop_joins, const bool enable_rendering, const bool renderer_prefer_igpu, const unsigned renderer_vulkan_timeout_ms, const bool renderer_use_parallel_executors, const bool enable_auto_clear_render_mem, const int render_oom_retry_threshold, const size_t render_mem_bytes, const size_t max_concurrent_render_sessions, const size_t reserved_gpu_mem, const bool render_compositor_use_last_gpu, const bool renderer_enable_slab_allocation, const size_t num_reader_threads, const AuthMetadata &authMetadata, SystemParameters &system_parameters, const bool legacy_syntax, const int idle_session_duration, const int max_session_duration, const std::string &udf_filename, const std::string &clang_path, const std::vector< std::string > &clang_options, const File_Namespace::DiskCacheConfig &disk_cache_config, const bool is_new_db)
 
void initialize (const bool is_new_db)
 
 ~DBHandler () override
 
void krb5_connect (TKrb5Session &session, const std::string &token, const std::string &dbname) override
 
void connect (TSessionId &session, const std::string &username, const std::string &passwd, const std::string &dbname) override
 
void disconnect (const TSessionId &session) override
 
void switch_database (const TSessionId &session, const std::string &dbname) override
 
void clone_session (TSessionId &session2, const TSessionId &session1) override
 
void get_server_status (TServerStatus &_return, const TSessionId &session) override
 
void get_status (std::vector< TServerStatus > &_return, const TSessionId &session) override
 
void get_hardware_info (TClusterHardwareInfo &_return, const TSessionId &session) override
 
bool hasTableAccessPrivileges (const TableDescriptor *td, const Catalog_Namespace::SessionInfo &session_info)
 
void get_tables (std::vector< std::string > &_return, const TSessionId &session) override
 
void get_tables_for_database (std::vector< std::string > &_return, const TSessionId &session, const std::string &database_name) override
 
void get_physical_tables (std::vector< std::string > &_return, const TSessionId &session) override
 
void get_views (std::vector< std::string > &_return, const TSessionId &session) override
 
void get_tables_meta (std::vector< TTableMeta > &_return, const TSessionId &session) override
 
void get_table_details (TTableDetails &_return, const TSessionId &session, const std::string &table_name) override
 
void get_table_details_for_database (TTableDetails &_return, const TSessionId &session, const std::string &table_name, const std::string &database_name) override
 
void get_internal_table_details (TTableDetails &_return, const TSessionId &session, const std::string &table_name, const bool include_system_columns) override
 
void get_internal_table_details_for_database (TTableDetails &_return, const TSessionId &session, const std::string &table_name, const std::string &database_name) override
 
void get_users (std::vector< std::string > &_return, const TSessionId &session) override
 
void get_databases (std::vector< TDBInfo > &_return, const TSessionId &session) override
 
void get_version (std::string &_return) override
 
void start_heap_profile (const TSessionId &session) override
 
void stop_heap_profile (const TSessionId &session) override
 
void get_heap_profile (std::string &_return, const TSessionId &session) override
 
void get_memory (std::vector< TNodeMemoryInfo > &_return, const TSessionId &session, const std::string &memory_level) override
 
void clear_cpu_memory (const TSessionId &session) override
 
void clear_gpu_memory (const TSessionId &session) override
 
void clearRenderMemory (const TSessionId &session)
 
void pause_executor_queue (const TSessionId &session)
 
void resume_executor_queue (const TSessionId &session)
 
void set_cur_session (const TSessionId &parent_session, const TSessionId &leaf_session, const std::string &start_time_str, const std::string &label, bool for_running_query_kernel) override
 
void invalidate_cur_session (const TSessionId &parent_session, const TSessionId &leaf_session, const std::string &start_time_str, const std::string &label, bool for_running_query_kernel) override
 
void set_table_epoch (const TSessionId &session, const int db_id, const int table_id, const int new_epoch) override
 
void set_table_epoch_by_name (const TSessionId &session, const std::string &table_name, const int new_epoch) override
 
int32_t get_table_epoch (const TSessionId &session, const int32_t db_id, const int32_t table_id) override
 
int32_t get_table_epoch_by_name (const TSessionId &session, const std::string &table_name) override
 
void get_table_epochs (std::vector< TTableEpochInfo > &_return, const TSessionId &session, const int32_t db_id, const int32_t table_id) override
 
void set_table_epochs (const TSessionId &session, const int32_t db_id, const std::vector< TTableEpochInfo > &table_epochs) override
 
void get_session_info (TSessionInfo &_return, const TSessionId &session) override
 
void set_leaf_info (const TSessionId &session, const TLeafInfo &info) override
 
void sql_execute (ExecutionResult &_return, const TSessionId &session, const std::string &query, const bool column_format, const int32_t first_n, const int32_t at_most_n, lockmgr::LockedTableDescriptors &locks)
 
void sql_execute (TQueryResult &_return, const TSessionId &session, const std::string &query, const bool column_format, const std::string &nonce, const int32_t first_n, const int32_t at_most_n) override
 
void get_completion_hints (std::vector< TCompletionHint > &hints, const TSessionId &session, const std::string &sql, const int cursor) override
 
void sql_execute_df (TDataFrame &_return, const TSessionId &session, const std::string &query, const TDeviceType::type device_type, const int32_t device_id, const int32_t first_n, const TArrowTransport::type transport_method) override
 
void sql_execute_gdf (TDataFrame &_return, const TSessionId &session, const std::string &query, const int32_t device_id, const int32_t first_n) override
 
void deallocate_df (const TSessionId &session, const TDataFrame &df, const TDeviceType::type device_type, const int32_t device_id) override
 
void interrupt (const TSessionId &query_session, const TSessionId &interrupt_session) override
 
void sql_validate (TRowDescriptor &_return, const TSessionId &session, const std::string &query) override
 
TExecuteMode::type getExecutionMode (const TSessionId &session)
 
void set_execution_mode (const TSessionId &session, const TExecuteMode::type mode) override
 
void render_vega (TRenderResult &_return, const TSessionId &session, const int64_t widget_id, const std::string &vega_json, const int32_t compression_level, const std::string &nonce) override
 
void get_result_row_for_pixel (TPixelTableRowResult &_return, const TSessionId &session, const int64_t widget_id, const TPixel &pixel, const std::map< std::string, std::vector< std::string >> &table_col_names, const bool column_format, const int32_t pixel_radius, const std::string &nonce) override
 
int32_t create_custom_expression (const TSessionId &session, const TCustomExpression &custom_expression) override
 
void get_custom_expressions (std::vector< TCustomExpression > &_return, const TSessionId &session) override
 
void update_custom_expression (const TSessionId &session, const int32_t id, const std::string &expression_json) override
 
void delete_custom_expressions (const TSessionId &session, const std::vector< int32_t > &custom_expression_ids, const bool do_soft_delete) override
 
void get_dashboard (TDashboard &_return, const TSessionId &session, const int32_t dashboard_id) override
 
void get_dashboards (std::vector< TDashboard > &_return, const TSessionId &session) override
 
int32_t create_dashboard (const TSessionId &session, const std::string &dashboard_name, const std::string &dashboard_state, const std::string &image_hash, const std::string &dashboard_metadata) override
 
void replace_dashboard (const TSessionId &session, const int32_t dashboard_id, const std::string &dashboard_name, const std::string &dashboard_owner, const std::string &dashboard_state, const std::string &image_hash, const std::string &dashboard_metadata) override
 
void delete_dashboard (const TSessionId &session, const int32_t dashboard_id) override
 
void share_dashboards (const TSessionId &session, const std::vector< int32_t > &dashboard_ids, const std::vector< std::string > &groups, const TDashboardPermissions &permissions) override
 
void delete_dashboards (const TSessionId &session, const std::vector< int32_t > &dashboard_ids) override
 
void share_dashboard (const TSessionId &session, const int32_t dashboard_id, const std::vector< std::string > &groups, const std::vector< std::string > &objects, const TDashboardPermissions &permissions, const bool grant_role) override
 
void unshare_dashboards (const TSessionId &session, const std::vector< int32_t > &dashboard_ids, const std::vector< std::string > &groups, const TDashboardPermissions &permissions) override
 
void unshare_dashboard (const TSessionId &session, const int32_t dashboard_id, const std::vector< std::string > &groups, const std::vector< std::string > &objects, const TDashboardPermissions &permissions) override
 
void get_dashboard_grantees (std::vector< TDashboardGrantees > &_return, const TSessionId &session, const int32_t dashboard_id) override
 
void get_link_view (TFrontendView &_return, const TSessionId &session, const std::string &link) override
 
void create_link (std::string &_return, const TSessionId &session, const std::string &view_state, const std::string &view_metadata) override
 
void load_table_binary (const TSessionId &session, const std::string &table_name, const std::vector< TRow > &rows, const std::vector< std::string > &column_names) override
 
void load_table_binary_columnar (const TSessionId &session, const std::string &table_name, const std::vector< TColumn > &cols, const std::vector< std::string > &column_names) override
 
void load_table_binary_arrow (const TSessionId &session, const std::string &table_name, const std::string &arrow_stream, const bool use_column_names) override
 
void load_table (const TSessionId &session, const std::string &table_name, const std::vector< TStringRow > &rows, const std::vector< std::string > &column_names) override
 
void detect_column_types (TDetectResult &_return, const TSessionId &session, const std::string &file_name, const TCopyParams &copy_params) override
 
void create_table (const TSessionId &session, const std::string &table_name, const TRowDescriptor &row_desc, const TCreateParams &create_params) override
 
void import_table (const TSessionId &session, const std::string &table_name, const std::string &file_name, const TCopyParams &copy_params) override
 
void import_geo_table (const TSessionId &session, const std::string &table_name, const std::string &file_name, const TCopyParams &copy_params, const TRowDescriptor &row_desc, const TCreateParams &create_params) override
 
void import_table_status (TImportStatus &_return, const TSessionId &session, const std::string &import_id) override
 
void get_first_geo_file_in_archive (std::string &_return, const TSessionId &session, const std::string &archive_path, const TCopyParams &copy_params) override
 
void get_all_files_in_archive (std::vector< std::string > &_return, const TSessionId &session, const std::string &archive_path, const TCopyParams &copy_params) override
 
void get_layers_in_geo_file (std::vector< TGeoFileLayerInfo > &_return, const TSessionId &session, const std::string &file_name, const TCopyParams &copy_params) override
 
int64_t query_get_outer_fragment_count (const TSessionId &session, const std::string &select_query) override
 
void check_table_consistency (TTableMeta &_return, const TSessionId &session, const int32_t table_id) override
 
void start_query (TPendingQuery &_return, const TSessionId &leaf_session, const TSessionId &parent_session, const std::string &serialized_rel_alg_dag, const std::string &start_time_str, const bool just_explain, const std::vector< int64_t > &outer_fragment_indices) override
 
void execute_query_step (TStepResult &_return, const TPendingQuery &pending_query, const TSubqueryId subquery_id, const std::string &start_time_str) override
 
void broadcast_serialized_rows (const TSerializedRows &serialized_rows, const TRowDescriptor &row_desc, const TQueryId query_id, const TSubqueryId subquery_id, const bool is_final_subquery_result) override
 
void start_render_query (TPendingRenderQuery &_return, const TSessionId &session, const int64_t widget_id, const int16_t node_idx, const std::string &vega_json) override
 
void execute_next_render_step (TRenderStepResult &_return, const TPendingRenderQuery &pending_render, const TRenderAggDataMap &merged_data) override
 
void insert_data (const TSessionId &session, const TInsertData &insert_data) override
 
void insert_chunks (const TSessionId &session, const TInsertChunks &insert_chunks) override
 
void checkpoint (const TSessionId &session, const int32_t table_id) override
 
void get_roles (std::vector< std::string > &_return, const TSessionId &session) override
 
bool has_role (const TSessionId &sessionId, const std::string &granteeName, const std::string &roleName) override
 
bool has_object_privilege (const TSessionId &sessionId, const std::string &granteeName, const std::string &objectName, const TDBObjectType::type object_type, const TDBObjectPermissions &permissions) override
 
void get_db_objects_for_grantee (std::vector< TDBObject > &_return, const TSessionId &session, const std::string &roleName) override
 
void get_db_object_privs (std::vector< TDBObject > &_return, const TSessionId &session, const std::string &objectName, const TDBObjectType::type type) override
 
void get_all_roles_for_user (std::vector< std::string > &_return, const TSessionId &session, const std::string &granteeName) override
 
void get_all_effective_roles_for_user (std::vector< std::string > &_return, const TSessionId &session, const std::string &granteeName) override
 
std::vector< std::string > get_valid_groups (const TSessionId &session, int32_t dashboard_id, std::vector< std::string > groups)
 
void set_license_key (TLicenseInfo &_return, const TSessionId &session, const std::string &key, const std::string &nonce) override
 
void get_license_claims (TLicenseInfo &_return, const TSessionId &session, const std::string &nonce) override
 
void get_device_parameters (std::map< std::string, std::string > &_return, const TSessionId &session) override
 
void register_runtime_extension_functions (const TSessionId &session, const std::vector< TUserDefinedFunction > &udfs, const std::vector< TUserDefinedTableFunction > &udtfs, const std::map< std::string, std::string > &device_ir_map) override
 
void get_function_names (std::vector< std::string > &_return, const TSessionId &session) override
 
void get_runtime_function_names (std::vector< std::string > &_return, const TSessionId &session) override
 
void get_function_details (std::vector< TUserDefinedFunction > &_return, const TSessionId &session, const std::vector< std::string > &udf_names) override
 
void get_table_function_names (std::vector< std::string > &_return, const TSessionId &session) override
 
void get_runtime_table_function_names (std::vector< std::string > &_return, const TSessionId &session) override
 
void get_table_function_details (std::vector< TUserDefinedTableFunction > &_return, const TSessionId &session, const std::vector< std::string > &udtf_names) override
 
void shutdown ()
 
void emergency_shutdown ()
 
TSessionId getInvalidSessionId () const
 
void internal_connect (TSessionId &session, const std::string &username, const std::string &dbname)
 
bool isAggregator () const
 
bool checkInMemorySystemTableQuery (const std::unordered_set< shared::TableKey > &tables_selected_from) const
 
template<typename... ARGS>
std::shared_ptr
< query_state::QueryState
create_query_state (ARGS &&...args)
 
Catalog_Namespace::SessionInfo get_session_copy (const TSessionId &session_id)
 
void get_tables_meta_impl (std::vector< TTableMeta > &_return, QueryStateProxy query_state_proxy, const Catalog_Namespace::SessionInfo &session_info, const bool with_table_locks=true)
 
void resizeDispatchQueue (size_t queue_size)
 

Static Public Member Functions

static size_t max_bytes_for_thrift ()
 

Public Attributes

std::shared_ptr
< Data_Namespace::DataMgr
data_mgr_
 
LeafAggregator leaf_aggregator_
 
std::vector< LeafHostInfodb_leaves_
 
std::vector< LeafHostInfostring_leaves_
 
const std::string base_data_path_
 
boost::filesystem::path import_path_
 
ExecutorDeviceType executor_device_type_
 
std::default_random_engine random_gen_
 
std::uniform_int_distribution
< int64_t > 
session_id_dist_
 
const bool jit_debug_
 
const bool intel_jit_profile_
 
bool allow_multifrag_
 
const bool read_only_
 
const bool allow_loop_joins_
 
bool cpu_mode_only_
 
heavyai::shared_mutex sessions_mutex_
 
std::mutex render_mutex_
 
int64_t start_time_
 
const AuthMetadataauthMetadata_
 
SystemParameterssystem_parameters_
 
std::shared_ptr< QueryEnginequery_engine_
 
std::unique_ptr< RenderHandlerrender_handler_
 
std::unique_ptr
< HeavyDBAggHandler
agg_handler_
 
std::unique_ptr
< HeavyDBLeafHandler
leaf_handler_
 
std::shared_ptr< Calcitecalcite_
 
const bool legacy_syntax_
 
std::unique_ptr
< QueryDispatchQueue
dispatch_queue_
 

Protected Member Functions

std::shared_ptr
< Catalog_Namespace::SessionInfo
get_session_ptr (const TSessionId &session_id)
 
ConnectionInfo getConnectionInfo () const
 

Private Member Functions

void init_executor_resource_mgr ()
 
void validate_configurations ()
 
std::shared_ptr
< Catalog_Namespace::SessionInfo
create_new_session (TSessionId &session, const std::string &dbname, const Catalog_Namespace::UserMetadata &user_meta, std::shared_ptr< Catalog_Namespace::Catalog > cat)
 
void connect_impl (TSessionId &session, const std::string &passwd, const std::string &dbname, const Catalog_Namespace::UserMetadata &user_meta, std::shared_ptr< Catalog_Namespace::Catalog > cat, query_state::StdLog &stdlog)
 
void disconnect_impl (Catalog_Namespace::SessionInfoPtr &session_ptr)
 
void check_table_load_privileges (const Catalog_Namespace::SessionInfo &session_info, const std::string &table_name)
 
void get_tables_impl (std::vector< std::string > &table_names, const Catalog_Namespace::SessionInfo &, const GetTablesType get_tables_type, const std::string &database_name={})
 
void get_table_details_impl (TTableDetails &_return, query_state::StdLog &stdlog, const std::string &table_name, const bool get_system, const bool get_physical, const std::string &database_name={})
 
void getAllRolesForUserImpl (std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr, std::vector< std::string > &roles, const std::string &granteeName, bool effective)
 
void check_read_only (const std::string &str)
 
void validateGroups (const std::vector< std::string > &groups)
 
void validateDashboardIdsForSharing (const Catalog_Namespace::SessionInfo &session_info, const std::vector< int32_t > &dashboard_ids)
 
void shareOrUnshareDashboards (const TSessionId &session, const std::vector< int32_t > &dashboard_ids, const std::vector< std::string > &groups, const TDashboardPermissions &permissions, const bool do_share)
 
std::pair< TPlanResult,
lockmgr::LockedTableDescriptors
parse_to_ra (QueryStateProxy, const std::string &query_str, const std::vector< TFilterPushDownInfo > &filter_push_down_info, const bool acquire_locks, const SystemParameters &system_parameters, bool check_privileges=true)
 
void sql_execute_local (TQueryResult &_return, const QueryStateProxy &query_state_proxy, const std::shared_ptr< Catalog_Namespace::SessionInfo > session_ptr, const std::string &query_str, const bool column_format, const std::string &nonce, const int32_t first_n, const int32_t at_most_n, const bool use_calcite)
 
int64_t process_deferred_copy_from (const TSessionId &session_id)
 
void sql_execute_impl (ExecutionResult &_return, QueryStateProxy, const bool column_format, const ExecutorDeviceType executor_device_type, const int32_t first_n, const int32_t at_most_n, const bool use_calcite, lockmgr::LockedTableDescriptors &locks)
 
bool user_can_access_table (const Catalog_Namespace::SessionInfo &, const TableDescriptor *td, const AccessPrivileges acess_priv)
 
void execute_distributed_copy_statement (Parser::CopyTableStmt *, const Catalog_Namespace::SessionInfo &session_info)
 
TPlanResult processCalciteRequest (QueryStateProxy, const std::shared_ptr< Catalog_Namespace::Catalog > &cat, const std::string &query_str, const std::vector< TFilterPushDownInfo > &filter_push_down_info, const SystemParameters &system_parameters, const bool check_privileges)
 
TRowDescriptor validateRelAlg (const std::string &query_ra, QueryStateProxy query_state_proxy)
 
void dispatch_query_task (std::shared_ptr< QueryDispatchQueue::Task > query_task, const bool is_update_delete)
 
std::vector< PushedDownFilterInfoexecute_rel_alg (ExecutionResult &_return, QueryStateProxy, const std::string &query_ra, const bool column_format, const ExecutorDeviceType executor_device_type, const int32_t first_n, const int32_t at_most_n, const bool just_validate, const bool find_push_down_candidates, const ExplainInfo &explain_info, const std::optional< size_t > executor_index=std::nullopt) const
 
void execute_rel_alg_with_filter_push_down (ExecutionResult &_return, QueryStateProxy, std::string &query_ra, const bool column_format, const ExecutorDeviceType executor_device_type, const int32_t first_n, const int32_t at_most_n, const bool just_explain, const bool just_calcite_explain, const std::vector< PushedDownFilterInfo > &filter_push_down_requests)
 
void executeDdl (TQueryResult &_return, const std::string &query_ra, std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
 
void executeDdl (ExecutionResult &_return, const std::string &query_ra, std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
 
TColumnType populateThriftColumnType (const Catalog_Namespace::Catalog *cat, const ColumnDescriptor *cd)
 
void set_execution_mode_nolock (Catalog_Namespace::SessionInfo *session_ptr, const TExecuteMode::type mode)
 
char unescape_char (std::string str)
 
import_export::CopyParams thrift_to_copyparams (const TCopyParams &cp)
 
TCopyParams copyparams_to_thrift (const import_export::CopyParams &cp)
 
void check_geospatial_files (const boost::filesystem::path file_path, const import_export::CopyParams &copy_params)
 
void render_rel_alg (TRenderResult &_return, const std::string &query_ra, const std::string &query_str, const Catalog_Namespace::SessionInfo &session_info, const std::string &render_type, const bool is_projection_query)
 
TColumnType create_geo_column (const TDatumType::type type, const std::string &name, const bool is_array)
 
void convertResultSet (ExecutionResult &result, const Catalog_Namespace::SessionInfo &session_info, const std::string &query_state_str, TQueryResult &_return)
 
std::vector< TargetMetaInfogetTargetMetaInfo (const std::vector< std::shared_ptr< Analyzer::TargetEntry >> &targets) const
 
std::vector< std::string > getTargetNames (const std::vector< TargetMetaInfo > &targets) const
 
std::vector< std::string > getTargetNames (const std::vector< std::shared_ptr< Analyzer::TargetEntry >> &targets) const
 
void get_completion_hints_unsorted (std::vector< TCompletionHint > &hints, std::vector< std::string > &visible_tables, query_state::StdLog &stdlog, const std::string &sql, const int cursor)
 
void get_token_based_completions (std::vector< TCompletionHint > &hints, query_state::StdLog &stdlog, std::vector< std::string > &visible_tables, const std::string &sql, const int cursor)
 
std::unordered_map
< std::string,
std::unordered_set
< std::string > > 
fill_column_names_by_table (std::vector< std::string > &table_names, query_state::StdLog &stdlog)
 
TDashboard get_dashboard_impl (const std::shared_ptr< Catalog_Namespace::SessionInfo const > &session_ptr, Catalog_Namespace::UserMetadata &user_meta, const DashboardDescriptor *dash, const bool populate_state=true)
 
std::unordered_set< std::string > get_uc_compatible_table_names_by_column (const std::unordered_set< std::string > &uc_column_names, std::vector< std::string > &table_names, query_state::StdLog &stdlog)
 
std::unique_ptr
< lockmgr::AbstractLockContainer
< const TableDescriptor * > > 
prepare_loader_generic (const Catalog_Namespace::SessionInfo &session_info, const std::string &table_name, size_t num_cols, std::unique_ptr< import_export::Loader > *loader, std::vector< std::unique_ptr< import_export::TypedImportBuffer >> *import_buffers, const std::vector< std::string > &column_names, std::string load_type)
 
void fillGeoColumns (const TSessionId &session, const Catalog_Namespace::Catalog &catalog, std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &import_buffers, const ColumnDescriptor *cd, size_t &col_idx, size_t num_rows, const std::string &table_name)
 
void fillMissingBuffers (const TSessionId &session, const Catalog_Namespace::Catalog &catalog, std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &import_buffers, const std::list< const ColumnDescriptor * > &cds, const std::vector< int > &desc_id_to_column_id, size_t num_rows, const std::string &table_name)
 
Catalog_Namespace::SessionInfoPtr findCalciteSession (TSessionId const &) const
 
void check_and_invalidate_sessions (Parser::DDLStmt *ddl)
 
std::string const createInMemoryCalciteSession (const std::shared_ptr< Catalog_Namespace::Catalog > &catalog_ptr)
 
void removeInMemoryCalciteSession (const std::string &session_id)
 
ExecutionResult getUserSessions (std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
 
ExecutionResult getQueries (std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
 
void get_queries_info (std::vector< TQueryInfo > &_return, const TSessionId &session) override
 
void interruptQuery (const Catalog_Namespace::SessionInfo &session_info, const std::string &target_session)
 
void alterSystemClear (const std::string &sesson_id, ExecutionResult &result, const std::string &cache_type, int64_t &execution_time_ms)
 
void alterSession (const std::string &sesson_id, ExecutionResult &result, const std::pair< std::string, std::string > &session_parameter, int64_t &execution_time_ms)
 
TRole::type getServerRole () const
 
void importGeoTableGlobFilterSort (const TSessionId &session, const std::string &table_name, const std::string &file_name, const import_export::CopyParams &copy_params, const TRowDescriptor &row_desc, const TCreateParams &create_params)
 
void importGeoTableSingle (const TSessionId &session, const std::string &table_name, const std::string &file_name, const import_export::CopyParams &copy_params, const TRowDescriptor &row_desc, const TCreateParams &create_params)
 
void resetSessionsStore ()
 

Static Private Member Functions

static void value_to_thrift_column (const TargetValue &tv, const SQLTypeInfo &ti, TColumn &column)
 
static TDatum value_to_thrift (const TargetValue &tv, const SQLTypeInfo &ti)
 
static void convertData (TQueryResult &_return, ExecutionResult &result, const QueryStateProxy &query_state_proxy, const bool column_format, const int32_t first_n, const int32_t at_most_n)
 
static void convertExplain (TQueryResult &_return, const ResultSet &results, const bool column_format)
 
static void convertResult (TQueryResult &_return, const ResultSet &results, const bool column_format)
 
static void convertRows (TQueryResult &_return, QueryStateProxy query_state_proxy, const std::vector< TargetMetaInfo > &targets, const ResultSet &results, const bool column_format, const int32_t first_n, const int32_t at_most_n)
 
static void createSimpleResult (TQueryResult &_return, const ResultSet &results, const bool column_format, const std::string label)
 
static bool has_database_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permissions)
 
static bool has_table_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permission)
 
static bool has_dashboard_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permissions)
 
static bool has_view_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permissions)
 
static bool has_server_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permissions)
 

Private Attributes

std::atomic< bool > initialized_ {false}
 
query_state::QueryStates query_states_
 
std::unordered_map
< std::string,
Catalog_Namespace::SessionInfoPtr
calcite_sessions_
 
heavyai::shared_mutex calcite_sessions_mtx_
 
bool super_user_rights_
 
const int idle_session_duration_
 
const int max_session_duration_
 
const bool enable_rendering_
 
const bool renderer_prefer_igpu_
 
const unsigned renderer_vulkan_timeout_
 
const bool renderer_use_parallel_executors_
 
const bool enable_auto_clear_render_mem_
 
const int render_oom_retry_threshold_
 
const size_t render_mem_bytes_
 
const size_t max_concurrent_render_sessions_
 
const size_t reserved_gpu_mem_
 
const bool render_compositor_use_last_gpu_
 
const bool renderer_enable_slab_allocation_
 
const size_t num_reader_threads_
 
const
File_Namespace::DiskCacheConfig
disk_cache_config_
 
const std::string & udf_filename_
 
const std::string & clang_path_
 
const std::vector< std::string > & clang_options_
 
int32_t max_num_sessions_ {-1}
 
std::unique_ptr
< Catalog_Namespace::SessionsStore
sessions_store_
 
DeferredCopyFromSessions deferred_copy_from_sessions
 
std::mutex handle_to_dev_ptr_mutex_
 
std::unordered_map
< std::string, std::string > 
ipc_handle_to_dev_ptr_
 
std::map< const std::string,
const PermissionFuncPtr
permissionFuncMap_
 
heavyai::shared_mutex custom_expressions_mutex_
 

Friends

class RenderHandler::Impl
 
class HeavyDBAggHandler
 
class HeavyDBLeafHandler
 
void run_warmup_queries (std::shared_ptr< DBHandler > handler, std::string base_path, std::string query_file_path)
 

Detailed Description

Definition at line 165 of file DBHandler.h.

Constructor & Destructor Documentation

DBHandler::DBHandler ( const std::vector< LeafHostInfo > &  db_leaves,
const std::vector< LeafHostInfo > &  string_leaves,
const std::string &  base_data_path,
const bool  allow_multifrag,
const bool  jit_debug,
const bool  intel_jit_profile,
const bool  read_only,
const bool  allow_loop_joins,
const bool  enable_rendering,
const bool  renderer_prefer_igpu,
const unsigned  renderer_vulkan_timeout_ms,
const bool  renderer_use_parallel_executors,
const bool  enable_auto_clear_render_mem,
const int  render_oom_retry_threshold,
const size_t  render_mem_bytes,
const size_t  max_concurrent_render_sessions,
const size_t  reserved_gpu_mem,
const bool  render_compositor_use_last_gpu,
const bool  renderer_enable_slab_allocation,
const size_t  num_reader_threads,
const AuthMetadata authMetadata,
SystemParameters system_parameters,
const bool  legacy_syntax,
const int  idle_session_duration,
const int  max_session_duration,
const std::string &  udf_filename,
const std::string &  clang_path,
const std::vector< std::string > &  clang_options,
const File_Namespace::DiskCacheConfig disk_cache_config,
const bool  is_new_db 
)

Definition at line 169 of file DBHandler.cpp.

205  : leaf_aggregator_(db_leaves)
206  , db_leaves_(db_leaves)
207  , string_leaves_(string_leaves)
208  , base_data_path_(base_data_path)
209  , random_gen_(std::random_device{}())
210  , session_id_dist_(0, INT32_MAX)
211  , jit_debug_(jit_debug)
212  , intel_jit_profile_(intel_jit_profile)
213  , allow_multifrag_(allow_multifrag)
214  , read_only_(read_only)
215  , allow_loop_joins_(allow_loop_joins)
216  , authMetadata_(authMetadata)
217  , system_parameters_(system_parameters)
218  , legacy_syntax_(legacy_syntax)
219  , dispatch_queue_(
220  std::make_unique<QueryDispatchQueue>(system_parameters.num_executors))
221  , super_user_rights_(false)
222  , idle_session_duration_(idle_session_duration * 60)
223  , max_session_duration_(max_session_duration * 60)
224  , enable_rendering_(enable_rendering)
225  , renderer_prefer_igpu_(renderer_prefer_igpu)
226  , renderer_vulkan_timeout_(renderer_vulkan_timeout_ms)
227  , renderer_use_parallel_executors_(renderer_use_parallel_executors)
228  , enable_auto_clear_render_mem_(enable_auto_clear_render_mem)
229  , render_oom_retry_threshold_(render_oom_retry_threshold)
230  , render_mem_bytes_(render_mem_bytes)
231  , max_concurrent_render_sessions_(max_concurrent_render_sessions)
232  , reserved_gpu_mem_(reserved_gpu_mem)
233  , render_compositor_use_last_gpu_(render_compositor_use_last_gpu)
234  , renderer_enable_slab_allocation_{renderer_enable_slab_allocation}
235  , num_reader_threads_(num_reader_threads)
236 #ifdef ENABLE_GEOS
237  , libgeos_so_filename_(libgeos_so_filename)
238 #endif
239 #ifdef HAVE_TORCH_TFS
240  , torch_lib_path_(torch_lib_path)
241 #endif
242  , disk_cache_config_(disk_cache_config)
243  , udf_filename_(udf_filename)
244  , clang_path_(clang_path)
245  , clang_options_(clang_options)
246  , max_num_sessions_(-1) {
247  LOG(INFO) << "HeavyDB Server " << MAPD_RELEASE;
248  initialize(is_new_db);
250 }
std::vector< LeafHostInfo > string_leaves_
Definition: DBHandler.h:636
const std::vector< std::string > & clang_options_
Definition: DBHandler.h:983
std::unique_ptr< QueryDispatchQueue > dispatch_queue_
Definition: DBHandler.h:660
void resetSessionsStore()
Definition: DBHandler.cpp:365
const bool renderer_use_parallel_executors_
Definition: DBHandler.h:965
const std::string & udf_filename_
Definition: DBHandler.h:981
const int render_oom_retry_threshold_
Definition: DBHandler.h:967
#define LOG(tag)
Definition: Logger.h:285
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:634
const unsigned renderer_vulkan_timeout_
Definition: DBHandler.h:964
const std::string base_data_path_
Definition: DBHandler.h:637
void initialize(const bool is_new_db)
Definition: DBHandler.cpp:382
const bool jit_debug_
Definition: DBHandler.h:642
const size_t render_mem_bytes_
Definition: DBHandler.h:968
const size_t reserved_gpu_mem_
Definition: DBHandler.h:970
const bool render_compositor_use_last_gpu_
Definition: DBHandler.h:971
const int max_session_duration_
Definition: DBHandler.h:960
std::vector< LeafHostInfo > db_leaves_
Definition: DBHandler.h:635
const File_Namespace::DiskCacheConfig & disk_cache_config_
Definition: DBHandler.h:980
const std::string & clang_path_
Definition: DBHandler.h:982
SystemParameters & system_parameters_
Definition: DBHandler.h:652
const size_t num_reader_threads_
Definition: DBHandler.h:973
int32_t max_num_sessions_
Definition: DBHandler.h:984
const bool enable_auto_clear_render_mem_
Definition: DBHandler.h:966
const bool renderer_prefer_igpu_
Definition: DBHandler.h:963
const bool allow_loop_joins_
Definition: DBHandler.h:646
const bool enable_rendering_
Definition: DBHandler.h:962
const bool intel_jit_profile_
Definition: DBHandler.h:643
bool super_user_rights_
Definition: DBHandler.h:956
const bool renderer_enable_slab_allocation_
Definition: DBHandler.h:972
const bool read_only_
Definition: DBHandler.h:645
const bool legacy_syntax_
Definition: DBHandler.h:658
const int idle_session_duration_
Definition: DBHandler.h:959
const size_t max_concurrent_render_sessions_
Definition: DBHandler.h:969
bool allow_multifrag_
Definition: DBHandler.h:644
static const std::string MAPD_RELEASE
Definition: release.h:42
const AuthMetadata & authMetadata_
Definition: DBHandler.h:651
std::default_random_engine random_gen_
Definition: DBHandler.h:640
std::uniform_int_distribution< int64_t > session_id_dist_
Definition: DBHandler.h:641
DBHandler::~DBHandler ( )
override

Definition at line 576 of file DBHandler.cpp.

References shutdown().

576  {
577  shutdown();
578 }
void shutdown()
Definition: DBHandler.cpp:7551

+ Here is the call graph for this function:

Member Function Documentation

void DBHandler::alterSession ( const std::string &  sesson_id,
ExecutionResult result,
const std::pair< std::string, std::string > &  session_parameter,
int64_t &  execution_time_ms 
)
private

Definition at line 8045 of file DBHandler.cpp.

References measure< TimeT >::execution(), set_execution_mode(), switch_database(), to_upper(), and run_benchmark_import::type.

Referenced by executeDdl().

8048  {
8049  result = ExecutionResult();
8050  if (session_parameter.first == "EXECUTOR_DEVICE") {
8051  std::string parameter_value = to_upper(session_parameter.second);
8052  TExecuteMode::type executorType;
8053  if (parameter_value == "GPU") {
8054  executorType = TExecuteMode::type::GPU;
8055  } else if (parameter_value == "CPU") {
8056  executorType = TExecuteMode::type::CPU;
8057  } else {
8058  throw std::runtime_error("Cannot set the " + session_parameter.first + " to " +
8059  session_parameter.second +
8060  ". Valid options are CPU and GPU");
8061  }
8062  execution_time_ms =
8063  measure<>::execution([&]() { set_execution_mode(session_id, executorType); });
8064  } else if (session_parameter.first == "CURRENT_DATABASE") {
8065  execution_time_ms = measure<>::execution(
8066  [&]() { switch_database(session_id, session_parameter.second); });
8067  }
8068 }
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
void set_execution_mode(const TSessionId &session, const TExecuteMode::type mode) override
Definition: DBHandler.cpp:3087
std::string to_upper(const std::string &str)
void switch_database(const TSessionId &session, const std::string &dbname) override
Definition: DBHandler.cpp:747

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::alterSystemClear ( const std::string &  sesson_id,
ExecutionResult result,
const std::string &  cache_type,
int64_t &  execution_time_ms 
)
private

Definition at line 8029 of file DBHandler.cpp.

References clear_cpu_memory(), clear_gpu_memory(), clearRenderMemory(), measure< TimeT >::execution(), and to_upper().

Referenced by executeDdl().

8032  {
8033  result = ExecutionResult();
8034  if (to_upper(cache_type) == "CPU") {
8035  execution_time_ms = measure<>::execution([&]() { clear_cpu_memory(session_id); });
8036  } else if (to_upper(cache_type) == "GPU") {
8037  execution_time_ms = measure<>::execution([&]() { clear_gpu_memory(session_id); });
8038  } else if (to_upper(cache_type) == "RENDER") {
8039  execution_time_ms = measure<>::execution([&]() { clearRenderMemory(session_id); });
8040  } else {
8041  throw std::runtime_error("Invalid cache type. Valid values are CPU,GPU or RENDER");
8042  }
8043 }
void clearRenderMemory(const TSessionId &session)
Definition: DBHandler.cpp:2934
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
std::string to_upper(const std::string &str)
void clear_gpu_memory(const TSessionId &session) override
Definition: DBHandler.cpp:2887
void clear_cpu_memory(const TSessionId &session) override
Definition: DBHandler.cpp:2911

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::broadcast_serialized_rows ( const TSerializedRows &  serialized_rows,
const TRowDescriptor &  row_desc,
const TQueryId  query_id,
const TSubqueryId  subquery_id,
const bool  is_final_subquery_result 
)
override

Definition at line 7106 of file DBHandler.cpp.

References measure< TimeT >::execution(), logger::INFO, leaf_handler_, LOG, and THROW_DB_EXCEPTION.

7110  {
7111  if (!leaf_handler_) {
7112  THROW_DB_EXCEPTION("Distributed support is disabled.");
7113  }
7114  LOG(INFO) << "BROADCAST-SERIALIZED-ROWS id:" << query_id;
7115  auto time_ms = measure<>::execution([&]() {
7116  try {
7117  leaf_handler_->broadcast_serialized_rows(
7118  serialized_rows, row_desc, query_id, subquery_id, is_final_subquery_result);
7119  } catch (std::exception& e) {
7120  THROW_DB_EXCEPTION(e.what());
7121  }
7122  });
7123  LOG(INFO) << "BROADCAST-SERIALIZED-ROWS COMPLETED " << time_ms << "ms";
7124 }
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:285
std::unique_ptr< HeavyDBLeafHandler > leaf_handler_
Definition: DBHandler.h:656
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the call graph for this function:

void DBHandler::check_and_invalidate_sessions ( Parser::DDLStmt ddl)
private

Definition at line 6433 of file DBHandler.cpp.

References sessions_store_.

6433  {
6434  if (const auto drop_db_stmt = dynamic_cast<Parser::DropDBStmt*>(ddl)) {
6435  sessions_store_->eraseByDB(*drop_db_stmt->getDatabaseName());
6436  } else if (const auto rename_db_stmt = dynamic_cast<Parser::RenameDBStmt*>(ddl)) {
6437  sessions_store_->eraseByDB(*rename_db_stmt->getPreviousDatabaseName());
6438  } else if (const auto drop_user_stmt = dynamic_cast<Parser::DropUserStmt*>(ddl)) {
6439  sessions_store_->eraseByUser(*drop_user_stmt->getUserName());
6440  } else if (const auto rename_user_stmt = dynamic_cast<Parser::RenameUserStmt*>(ddl)) {
6441  sessions_store_->eraseByUser(*rename_user_stmt->getOldUserName());
6442  }
6443 }
std::unique_ptr< Catalog_Namespace::SessionsStore > sessions_store_
Definition: DBHandler.h:985
void DBHandler::check_geospatial_files ( const boost::filesystem::path  file_path,
const import_export::CopyParams copy_params 
)
private

Definition at line 5170 of file DBHandler.cpp.

References import_export::Importer::gdalFileExists().

Referenced by detect_column_types(), and importGeoTableSingle().

5171  {
5172  const std::list<std::string> shp_ext{".shp", ".shx", ".dbf"};
5173  if (std::find(shp_ext.begin(),
5174  shp_ext.end(),
5175  boost::algorithm::to_lower_copy(file_path.extension().string())) !=
5176  shp_ext.end()) {
5177  for (auto ext : shp_ext) {
5178  auto aux_file = file_path;
5180  aux_file.replace_extension(boost::algorithm::to_upper_copy(ext)).string(),
5181  copy_params) &&
5183  aux_file.replace_extension(ext).string(), copy_params)) {
5184  throw std::runtime_error("required file for shapefile does not exist: " +
5185  aux_file.filename().string());
5186  }
5187  }
5188  }
5189 }
static bool gdalFileExists(const std::string &path, const CopyParams &copy_params)
Definition: Importer.cpp:5053

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::check_read_only ( const std::string &  str)
private

Definition at line 580 of file DBHandler.cpp.

References read_only_, and THROW_DB_EXCEPTION.

Referenced by create_custom_expression(), create_dashboard(), create_table(), delete_custom_expressions(), delete_dashboards(), detect_column_types(), import_table(), importGeoTableSingle(), prepare_loader_generic(), replace_dashboard(), set_license_key(), shareOrUnshareDashboards(), sql_execute_impl(), and update_custom_expression().

580  {
581  if (DBHandler::read_only_) {
582  THROW_DB_EXCEPTION(str + " disabled: server running in read-only mode.");
583  }
584 }
const bool read_only_
Definition: DBHandler.h:645
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the caller graph for this function:

void DBHandler::check_table_consistency ( TTableMeta &  _return,
const TSessionId &  session,
const int32_t  table_id 
)
override

Definition at line 7037 of file DBHandler.cpp.

References get_session_ptr(), leaf_handler_, heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, and THROW_DB_EXCEPTION.

7039  {
7040  heavyai::RequestInfo const request_info(session_id_or_json);
7041  SET_REQUEST_ID(request_info.requestId());
7042  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
7043  if (!leaf_handler_) {
7044  THROW_DB_EXCEPTION("Distributed support is disabled.");
7045  }
7046  try {
7047  leaf_handler_->check_table_consistency(_return, request_info.sessionId(), table_id);
7048  } catch (std::exception& e) {
7049  THROW_DB_EXCEPTION(e.what());
7050  }
7051 }
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
std::unique_ptr< HeavyDBLeafHandler > leaf_handler_
Definition: DBHandler.h:656
#define STDLOG(...)
Definition: QueryState.h:234
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the call graph for this function:

void DBHandler::check_table_load_privileges ( const Catalog_Namespace::SessionInfo session_info,
const std::string &  table_name 
)
private

Definition at line 6169 of file DBHandler.cpp.

References cat(), Catalog_Namespace::SessionInfo::get_currentUser(), Catalog_Namespace::SessionInfo::getCatalog(), AccessPrivileges::INSERT_INTO_TABLE, lockmgr::instance(), DBObject::loadKey(), DBObject::setPrivileges(), TableDBObjectType, and THROW_DB_EXCEPTION.

Referenced by import_table(), importGeoTableSingle(), and prepare_loader_generic().

6171  {
6172  auto user_metadata = session_info.get_currentUser();
6173  auto& cat = session_info.getCatalog();
6174  DBObject dbObject(table_name, TableDBObjectType);
6175  dbObject.loadKey(cat);
6176  dbObject.setPrivileges(AccessPrivileges::INSERT_INTO_TABLE);
6177  std::vector<DBObject> privObjects;
6178  privObjects.push_back(dbObject);
6179  if (!SysCatalog::instance().checkPrivileges(user_metadata, privObjects)) {
6180  THROW_DB_EXCEPTION("Violation of access privileges: user " +
6181  user_metadata.userLoggable() +
6182  " has no insert privileges for table " + table_name + ".");
6183  }
6184 }
std::string cat(Ts &&...args)
static const AccessPrivileges INSERT_INTO_TABLE
Definition: DBObject.h:161
Catalog & getCatalog() const
Definition: SessionInfo.h:75
T & instance()
Definition: LockMgr.cpp:101
const UserMetadata & get_currentUser() const
Definition: SessionInfo.h:88
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool DBHandler::checkInMemorySystemTableQuery ( const std::unordered_set< shared::TableKey > &  tables_selected_from) const

Definition at line 8184 of file DBHandler.cpp.

References CHECK, anonymous_namespace{DBHandler.cpp}::check_and_reset_in_memory_system_table(), Catalog_Namespace::SysCatalog::getCatalog(), Catalog_Namespace::SysCatalog::instance(), and shared::kInfoSchemaDbName.

8185  {
8186  bool is_in_memory_system_table_query{false};
8187  const auto info_schema_catalog =
8189  if (info_schema_catalog) {
8190  for (const auto& table_key : selected_table_keys) {
8191  if (table_key.db_id == info_schema_catalog->getDatabaseId()) {
8192  auto td = info_schema_catalog->getMetadataForTable(table_key.table_id, false);
8193  CHECK(td);
8194  if (check_and_reset_in_memory_system_table(*info_schema_catalog, *td)) {
8195  is_in_memory_system_table_query = true;
8196  }
8197  }
8198  }
8199  }
8200  return is_in_memory_system_table_query;
8201 }
const std::string kInfoSchemaDbName
static SysCatalog & instance()
Definition: SysCatalog.h:343
std::shared_ptr< Catalog > getCatalog(const std::string &dbName)
#define CHECK(condition)
Definition: Logger.h:291
bool check_and_reset_in_memory_system_table(const Catalog &catalog, const TableDescriptor &td)
Definition: DBHandler.cpp:6843

+ Here is the call graph for this function:

void DBHandler::checkpoint ( const TSessionId &  session,
const int32_t  table_id 
)
override

Definition at line 7352 of file DBHandler.cpp.

References cat(), get_session_ptr(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, and STDLOG.

7352  {
7353  heavyai::RequestInfo const request_info(session_id_or_json);
7354  SET_REQUEST_ID(request_info.requestId());
7355  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
7356  auto session_ptr = stdlog.getConstSessionInfo();
7357  auto& cat = session_ptr->getCatalog();
7358  cat.checkpoint(table_id);
7359 }
std::string cat(Ts &&...args)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

void DBHandler::clear_cpu_memory ( const TSessionId &  session)
override

Definition at line 2911 of file DBHandler.cpp.

References Executor::clearMemory(), Data_Namespace::CPU_LEVEL, get_session_ptr(), anonymous_namespace{DBHandler.cpp}::pause_and_resume_executor_queue(), render_handler_, heavyai::RequestInfo::requestId(), resume_executor_queue(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, and THROW_DB_EXCEPTION.

Referenced by alterSystemClear().

2911  {
2912  heavyai::RequestInfo const request_info(session_id_or_json);
2913  SET_REQUEST_ID(request_info.requestId());
2914  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
2915  auto session_ptr = stdlog.getConstSessionInfo();
2916  if (!session_ptr->get_currentUser().isSuper) {
2917  THROW_DB_EXCEPTION("Superuser privilege is required to run clear_cpu_memory");
2918  }
2920  // clear renderer memory first
2921  // this will block until any running render finishes
2922  if (render_handler_) {
2923  render_handler_->clear_cpu_memory();
2924  }
2925  // then clear the QE memory
2926  // the renderer will have disconnected from any QE memory
2927  try {
2929  } catch (const std::exception& e) {
2930  THROW_DB_EXCEPTION(e.what());
2931  }
2932 }
void resume_executor_queue(const TSessionId &session)
Definition: DBHandler.cpp:2964
static void clearMemory(const Data_Namespace::MemoryLevel memory_level)
Definition: Execute.cpp:535
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:654
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
#define STDLOG(...)
Definition: QueryState.h:234
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::clear_gpu_memory ( const TSessionId &  session)
override

Definition at line 2887 of file DBHandler.cpp.

References Executor::clearMemory(), get_session_ptr(), getConnectionInfo(), Data_Namespace::GPU_LEVEL, anonymous_namespace{DBHandler.cpp}::pause_and_resume_executor_queue(), render_handler_, heavyai::RequestInfo::requestId(), resume_executor_queue(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, THROW_DB_EXCEPTION, and toString().

Referenced by alterSystemClear().

2887  {
2888  heavyai::RequestInfo const request_info(session_id_or_json);
2889  SET_REQUEST_ID(request_info.requestId());
2890  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
2891  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
2892  auto session_ptr = stdlog.getConstSessionInfo();
2893  if (!session_ptr->get_currentUser().isSuper) {
2894  THROW_DB_EXCEPTION("Superuser privilege is required to run clear_gpu_memory");
2895  }
2897  // clear renderer memory first
2898  // this will block until any running render finishes
2899  if (render_handler_) {
2900  render_handler_->clear_gpu_memory();
2901  }
2902  // then clear the QE memory
2903  // the renderer will have disconnected from any QE memory
2904  try {
2906  } catch (const std::exception& e) {
2907  THROW_DB_EXCEPTION(e.what());
2908  }
2909 }
void resume_executor_queue(const TSessionId &session)
Definition: DBHandler.cpp:2964
static void clearMemory(const Data_Namespace::MemoryLevel memory_level)
Definition: Execute.cpp:535
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:654
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1703
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1790
#define STDLOG(...)
Definition: QueryState.h:234
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::clearRenderMemory ( const TSessionId &  session)

Definition at line 2934 of file DBHandler.cpp.

References get_session_ptr(), getConnectionInfo(), anonymous_namespace{DBHandler.cpp}::pause_and_resume_executor_queue(), render_handler_, heavyai::RequestInfo::requestId(), resume_executor_queue(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, THROW_DB_EXCEPTION, and toString().

Referenced by alterSystemClear().

2934  {
2935  heavyai::RequestInfo const request_info(session_id_or_json);
2936  SET_REQUEST_ID(request_info.requestId());
2937  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
2938  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
2939  auto session_ptr = stdlog.getConstSessionInfo();
2940  if (!session_ptr->get_currentUser().isSuper) {
2941  THROW_DB_EXCEPTION("Superuser privilege is required to run clear_render_memory");
2942  }
2943  if (render_handler_) {
2945  render_handler_->clear_cpu_memory();
2946  render_handler_->clear_gpu_memory();
2947  }
2948 }
void resume_executor_queue(const TSessionId &session)
Definition: DBHandler.cpp:2964
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:654
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1703
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1790
#define STDLOG(...)
Definition: QueryState.h:234
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::clone_session ( TSessionId &  session2,
const TSessionId &  session1 
)
override

Definition at line 768 of file DBHandler.cpp.

References cat(), LeafAggregator::clone_session(), executor_device_type_, get_session_ptr(), getConnectionInfo(), logger::INFO, leaf_aggregator_, LeafAggregator::leafCount(), LOG, heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), sessions_store_, SET_REQUEST_ID, STDLOG, THROW_DB_EXCEPTION, toString(), and Catalog_Namespace::UserMetadata::userLoggable().

769  {
770  heavyai::RequestInfo const request_info(session1_id_or_json);
771  SET_REQUEST_ID(request_info.requestId());
772  auto session1_ptr = get_session_ptr(request_info.sessionId());
773  auto stdlog = STDLOG(session1_ptr);
774  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
775 
776  try {
777  const Catalog_Namespace::UserMetadata& user_meta = session1_ptr->get_currentUser();
778  std::shared_ptr<Catalog> cat = session1_ptr->get_catalog_ptr();
779  auto session2_ptr = sessions_store_->add(user_meta, cat, executor_device_type_);
780  session2_id = session2_ptr->get_session_id();
781  LOG(INFO) << "User " << user_meta.userLoggable() << " connected to database "
782  << cat->name();
783  if (leaf_aggregator_.leafCount() > 0) {
784  leaf_aggregator_.clone_session(request_info.sessionId(), session2_id);
785  return;
786  }
787  } catch (std::exception& e) {
788  THROW_DB_EXCEPTION(e.what());
789  }
790 }
std::string cat(Ts &&...args)
void clone_session(const TSessionId session1, const TSessionId session2)
#define LOG(tag)
Definition: Logger.h:285
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:634
ExecutorDeviceType executor_device_type_
Definition: DBHandler.h:639
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
size_t leafCount() const
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1703
std::unique_ptr< Catalog_Namespace::SessionsStore > sessions_store_
Definition: DBHandler.h:985
std::string userLoggable() const
Definition: SysCatalog.cpp:158
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1790
#define STDLOG(...)
Definition: QueryState.h:234
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the call graph for this function:

void DBHandler::connect ( TSessionId &  session,
const std::string &  username,
const std::string &  passwd,
const std::string &  dbname 
)
override

Definition at line 658 of file DBHandler.cpp.

References AccessPrivileges::ACCESS, cat(), connect_impl(), DatabaseDBObjectType, getConnectionInfo(), lockmgr::instance(), DBObject::loadKey(), logger::set_new_request_id(), DBObject::setPrivileges(), STDLOG, super_user_rights_, THROW_DB_EXCEPTION, toString(), and Catalog_Namespace::UserMetadata::userLoggable().

661  {
663  auto stdlog = STDLOG(); // session_info set by connect_impl()
664  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
665  std::string username2 = username; // login() may reset username given as argument
666  std::string dbname2 = dbname; // login() may reset dbname given as argument
668  std::shared_ptr<Catalog> cat = nullptr;
669  try {
670  cat = SysCatalog::instance().login(
671  dbname2, username2, passwd, user_meta, !super_user_rights_);
672  } catch (std::exception& e) {
673  stdlog.appendNameValuePairs("user", username, "db", dbname, "exception", e.what());
674  THROW_DB_EXCEPTION(e.what());
675  }
676 
677  DBObject dbObject(dbname2, DatabaseDBObjectType);
678  dbObject.loadKey(*cat);
679  dbObject.setPrivileges(AccessPrivileges::ACCESS);
680  std::vector<DBObject> dbObjects;
681  dbObjects.push_back(dbObject);
682  if (!SysCatalog::instance().checkPrivileges(user_meta, dbObjects)) {
683  stdlog.appendNameValuePairs(
684  "user", username, "db", dbname, "exception", "Missing Privileges");
685  THROW_DB_EXCEPTION("Unauthorized Access: user " + user_meta.userLoggable() +
686  " is not allowed to access database " + dbname2 + ".");
687  }
688  connect_impl(session_id, passwd, dbname2, user_meta, cat, stdlog);
689 
690  // if pki auth session_id will come back encrypted with user pubkey
691  SysCatalog::instance().check_for_session_encryption(passwd, session_id);
692 }
std::string cat(Ts &&...args)
void connect_impl(TSessionId &session, const std::string &passwd, const std::string &dbname, const Catalog_Namespace::UserMetadata &user_meta, std::shared_ptr< Catalog_Namespace::Catalog > cat, query_state::StdLog &stdlog)
Definition: DBHandler.cpp:694
RequestId set_new_request_id()
Definition: Logger.cpp:891
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1703
bool super_user_rights_
Definition: DBHandler.h:956
T & instance()
Definition: LockMgr.cpp:101
static const AccessPrivileges ACCESS
Definition: DBObject.h:153
std::string userLoggable() const
Definition: SysCatalog.cpp:158
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1790
#define STDLOG(...)
Definition: QueryState.h:234
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the call graph for this function:

void DBHandler::connect_impl ( TSessionId &  session,
const std::string &  passwd,
const std::string &  dbname,
const Catalog_Namespace::UserMetadata user_meta,
std::shared_ptr< Catalog_Namespace::Catalog cat,
query_state::StdLog stdlog 
)
private

Definition at line 694 of file DBHandler.cpp.

References query_state::StdLog::appendNameValuePairs(), executor_device_type_, getConnectionInfo(), query_state::StdLog::getConstSessionInfo(), logger::INFO, lockmgr::instance(), join(), LOG, sessions_store_, query_state::StdLog::setSessionInfo(), super_user_rights_, toString(), and Catalog_Namespace::UserMetadata::userLoggable().

Referenced by connect(), and internal_connect().

699  {
700  // TODO(sy): Is there any reason to have dbname as a parameter
701  // here when the cat parameter already provides cat->name()?
702  // Should dbname and cat->name() ever differ?
703  auto session_ptr = sessions_store_->add(user_meta, cat, executor_device_type_);
704  session_id = session_ptr->get_session_id();
705  LOG(INFO) << "User " << user_meta.userLoggable() << " connected to database " << dbname;
706  stdlog.setSessionInfo(session_ptr);
707  session_ptr->set_connection_info(getConnectionInfo().toString());
708  if (!super_user_rights_) { // no need to connect to leaf_aggregator_ at this time
709  // while doing warmup
710  }
711  auto const roles =
712  stdlog.getConstSessionInfo()->get_currentUser().isSuper
713  ? std::vector<std::string>{{"super"}}
714  : SysCatalog::instance().getRoles(
715  false, false, stdlog.getConstSessionInfo()->get_currentUser().userName);
716  stdlog.appendNameValuePairs("roles", boost::algorithm::join(roles, ","));
717 }
std::string cat(Ts &&...args)
#define LOG(tag)
Definition: Logger.h:285
std::string join(T const &container, std::string const &delim)
void setSessionInfo(std::shared_ptr< Catalog_Namespace::SessionInfo >)
Definition: QueryState.cpp:282
ExecutorDeviceType executor_device_type_
Definition: DBHandler.h:639
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1703
void appendNameValuePairs(Pairs &&...pairs)
Definition: QueryState.h:312
bool super_user_rights_
Definition: DBHandler.h:956
std::unique_ptr< Catalog_Namespace::SessionsStore > sessions_store_
Definition: DBHandler.h:985
T & instance()
Definition: LockMgr.cpp:101
std::shared_ptr< Catalog_Namespace::SessionInfo const > getConstSessionInfo() const
Definition: QueryState.cpp:150
std::string userLoggable() const
Definition: SysCatalog.cpp:158
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1790

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::convertData ( TQueryResult &  _return,
ExecutionResult result,
const QueryStateProxy query_state_proxy,
const bool  column_format,
const int32_t  first_n,
const int32_t  at_most_n 
)
staticprivate

Definition at line 1250 of file DBHandler.cpp.

References ExecutionResult::CalciteDdl, convertExplain(), convertResult(), convertRows(), ExecutionResult::empty(), ExecutionResult::Explanation, ExecutionResult::getExecutionTime(), ExecutionResult::getResultType(), ExecutionResult::getRows(), ExecutionResult::getTargetsMeta(), ExecutionResult::QueryResult, and ExecutionResult::SimpleResult.

Referenced by convertResultSet(), get_tables_meta_impl(), sql_execute_local(), and validateRelAlg().

1255  {
1256  _return.execution_time_ms += result.getExecutionTime();
1257  if (result.empty()) {
1258  return;
1259  }
1260 
1261  switch (result.getResultType()) {
1263  convertRows(_return,
1264  query_state_proxy,
1265  result.getTargetsMeta(),
1266  *result.getRows(),
1267  column_format,
1268  first_n,
1269  at_most_n);
1270  break;
1272  convertResult(_return, *result.getRows(), true);
1273  break;
1275  convertExplain(_return, *result.getRows(), true);
1276  break;
1278  convertRows(_return,
1279  query_state_proxy,
1280  result.getTargetsMeta(),
1281  *result.getRows(),
1282  column_format,
1283  -1,
1284  -1);
1285  break;
1286  }
1287 }
RType getResultType() const
static void convertExplain(TQueryResult &_return, const ResultSet &results, const bool column_format)
Definition: DBHandler.cpp:6405
static void convertResult(TQueryResult &_return, const ResultSet &results, const bool column_format)
Definition: DBHandler.cpp:6411
const std::vector< TargetMetaInfo > & getTargetsMeta() const
const std::shared_ptr< ResultSet > & getRows() const
static void convertRows(TQueryResult &_return, QueryStateProxy query_state_proxy, const std::vector< TargetMetaInfo > &targets, const ResultSet &results, const bool column_format, const int32_t first_n, const int32_t at_most_n)
Definition: DBHandler.cpp:6313
int64_t getExecutionTime() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::convertExplain ( TQueryResult &  _return,
const ResultSet results,
const bool  column_format 
)
staticprivate

Definition at line 6405 of file DBHandler.cpp.

References createSimpleResult().

Referenced by convertData().

6407  {
6408  createSimpleResult(_return, results, column_format, "Explanation");
6409 }
static void createSimpleResult(TQueryResult &_return, const ResultSet &results, const bool column_format, const std::string label)
Definition: DBHandler.cpp:6368

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::convertResult ( TQueryResult &  _return,
const ResultSet results,
const bool  column_format 
)
staticprivate

Definition at line 6411 of file DBHandler.cpp.

References createSimpleResult().

Referenced by convertData().

6413  {
6414  createSimpleResult(_return, results, column_format, "Result");
6415 }
static void createSimpleResult(TQueryResult &_return, const ResultSet &results, const bool column_format, const std::string label)
Definition: DBHandler.cpp:6368

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::convertResultSet ( ExecutionResult result,
const Catalog_Namespace::SessionInfo session_info,
const std::string &  query_state_str,
TQueryResult &  _return 
)
private

Definition at line 7746 of file DBHandler.cpp.

References CHECK, convertData(), create_query_state(), query_state::QueryState::createQueryStateProxy(), Catalog_Namespace::SessionInfo::get_session_id(), get_session_ptr(), and ExecutionResult::getDataPtr().

Referenced by executeDdl().

7749  {
7750  // Stuff ResultSet into _return (which is a TQueryResult)
7751  // calls convertRows, but after some setup using session_info
7752 
7753  auto session_ptr = get_session_ptr(session_info.get_session_id());
7754  CHECK(session_ptr);
7755  auto qs = create_query_state(session_ptr, query_state_str);
7757 
7758  // heavysql only accepts column format as being 'VALID",
7759  // assume that heavydb should only return column format
7760  int32_t nRows = result.getDataPtr()->rowCount();
7761 
7762  convertData(_return,
7763  result,
7764  qsp,
7765  /*column_format=*/true,
7766  /*first_n=*/nRows,
7767  /*at_most_n=*/nRows);
7768 }
static void convertData(TQueryResult &_return, ExecutionResult &result, const QueryStateProxy &query_state_proxy, const bool column_format, const int32_t first_n, const int32_t at_most_n)
Definition: DBHandler.cpp:1250
QueryStateProxy createQueryStateProxy()
Definition: QueryState.cpp:71
std::shared_ptr< query_state::QueryState > create_query_state(ARGS &&...args)
Definition: DBHandler.h:663
const ResultSetPtr & getDataPtr() const
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
std::string get_session_id() const
Definition: SessionInfo.h:93
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::convertRows ( TQueryResult &  _return,
QueryStateProxy  query_state_proxy,
const std::vector< TargetMetaInfo > &  targets,
const ResultSet results,
const bool  column_format,
const int32_t  first_n,
const int32_t  at_most_n 
)
staticprivate

Definition at line 6313 of file DBHandler.cpp.

References query_state::QueryStateProxy::createTimer(), ThriftSerializers::target_meta_infos_to_thrift(), THROW_DB_EXCEPTION, to_string(), value_to_thrift(), and value_to_thrift_column().

Referenced by convertData().

6319  {
6320  query_state::Timer timer = query_state_proxy.createTimer(__func__);
6321  _return.row_set.row_desc = ThriftSerializers::target_meta_infos_to_thrift(targets);
6322  int32_t fetched{0};
6323  if (column_format) {
6324  _return.row_set.is_columnar = true;
6325  std::vector<TColumn> tcolumns(results.colCount());
6326  while (first_n == -1 || fetched < first_n) {
6327  const auto crt_row = results.getNextRow(true, true);
6328  if (crt_row.empty()) {
6329  break;
6330  }
6331  ++fetched;
6332  if (at_most_n >= 0 && fetched > at_most_n) {
6333  THROW_DB_EXCEPTION("The result contains more rows than the specified cap of " +
6334  std::to_string(at_most_n));
6335  }
6336  for (size_t i = 0; i < results.colCount(); ++i) {
6337  const auto agg_result = crt_row[i];
6338  value_to_thrift_column(agg_result, targets[i].get_type_info(), tcolumns[i]);
6339  }
6340  }
6341  for (size_t i = 0; i < results.colCount(); ++i) {
6342  _return.row_set.columns.push_back(tcolumns[i]);
6343  }
6344  } else {
6345  _return.row_set.is_columnar = false;
6346  while (first_n == -1 || fetched < first_n) {
6347  const auto crt_row = results.getNextRow(true, true);
6348  if (crt_row.empty()) {
6349  break;
6350  }
6351  ++fetched;
6352  if (at_most_n >= 0 && fetched > at_most_n) {
6353  THROW_DB_EXCEPTION("The result contains more rows than the specified cap of " +
6354  std::to_string(at_most_n));
6355  }
6356  TRow trow;
6357  trow.cols.reserve(results.colCount());
6358  for (size_t i = 0; i < results.colCount(); ++i) {
6359  const auto agg_result = crt_row[i];
6360  trow.cols.push_back(value_to_thrift(agg_result, targets[i].get_type_info()));
6361  }
6362  _return.row_set.rows.push_back(trow);
6363  }
6364  }
6365 }
TRowDescriptor target_meta_infos_to_thrift(const std::vector< TargetMetaInfo > &targets)
static void value_to_thrift_column(const TargetValue &tv, const SQLTypeInfo &ti, TColumn &column)
Definition: DBHandler.cpp:978
Timer createTimer(char const *event_name)
Definition: QueryState.cpp:129
std::string to_string(char const *&&v)
static TDatum value_to_thrift(const TargetValue &tv, const SQLTypeInfo &ti)
Definition: DBHandler.cpp:1103
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

TCopyParams DBHandler::copyparams_to_thrift ( const import_export::CopyParams cp)
private

Definition at line 4049 of file DBHandler.cpp.

References import_export::CopyParams::add_metadata_columns, import_export::CopyParams::array_begin, import_export::CopyParams::array_delim, import_export::CopyParams::array_end, CHECK, import_export::CopyParams::connection_string, import_export::CopyParams::credential_string, import_export::CopyParams::delimiter, import_export::CopyParams::dsn, import_export::CopyParams::escape, import_export::CopyParams::geo_coords_comp_param, import_export::CopyParams::geo_coords_encoding, import_export::CopyParams::geo_coords_srid, import_export::CopyParams::geo_coords_type, import_export::CopyParams::geo_explode_collections, import_export::CopyParams::geo_layer_name, import_export::CopyParams::geo_validate_geometry, import_export::CopyParams::has_header, import_export::kAuto, import_export::kAutoDetect, import_export::kDelimitedFile, import_export::kDouble, kENCODING_GEOINT, import_export::kFile, import_export::kFloat, import_export::kGeoFile, kGEOGRAPHY, kGEOMETRY, import_export::kHasHeader, import_export::kInt, import_export::kNoHeader, import_export::kNone, import_export::kOdbc, import_export::kParquetFile, import_export::kPoint, import_export::kRasterFile, import_export::kSmallInt, import_export::kWorld, import_export::CopyParams::line_delim, import_export::CopyParams::null_str, import_export::CopyParams::password, import_export::CopyParams::quote, import_export::CopyParams::quoted, import_export::CopyParams::raster_drop_if_all_null, import_export::CopyParams::raster_import_bands, import_export::CopyParams::raster_import_dimensions, import_export::CopyParams::raster_point_compute_angle, import_export::CopyParams::raster_point_transform, import_export::CopyParams::raster_point_type, import_export::CopyParams::raster_scanlines_per_thread, import_export::CopyParams::s3_access_key, import_export::CopyParams::s3_endpoint, import_export::CopyParams::s3_region, import_export::CopyParams::s3_secret_key, import_export::CopyParams::s3_session_token, import_export::CopyParams::sanitize_column_names, import_export::CopyParams::source_srid, import_export::CopyParams::source_type, import_export::CopyParams::sql_order_by, import_export::CopyParams::sql_select, import_export::CopyParams::threads, import_export::CopyParams::trim_spaces, and import_export::CopyParams::username.

Referenced by detect_column_types(), and importGeoTableSingle().

4049  {
4050  TCopyParams copy_params;
4051  copy_params.delimiter = cp.delimiter;
4052  copy_params.null_str = cp.null_str;
4053  switch (cp.has_header) {
4055  copy_params.has_header = TImportHeaderRow::AUTODETECT;
4056  break;
4058  copy_params.has_header = TImportHeaderRow::NO_HEADER;
4059  break;
4061  copy_params.has_header = TImportHeaderRow::HAS_HEADER;
4062  break;
4063  default:
4064  CHECK(false);
4065  }
4066  copy_params.quoted = cp.quoted;
4067  copy_params.quote = cp.quote;
4068  copy_params.escape = cp.escape;
4069  copy_params.line_delim = cp.line_delim;
4070  copy_params.array_delim = cp.array_delim;
4071  copy_params.array_begin = cp.array_begin;
4072  copy_params.array_end = cp.array_end;
4073  copy_params.threads = cp.threads;
4074  copy_params.s3_access_key = cp.s3_access_key;
4075  copy_params.s3_secret_key = cp.s3_secret_key;
4076  copy_params.s3_session_token = cp.s3_session_token;
4077  copy_params.s3_region = cp.s3_region;
4078  copy_params.s3_endpoint = cp.s3_endpoint;
4079  switch (cp.source_type) {
4081  copy_params.source_type = TSourceType::DELIMITED_FILE;
4082  break;
4084  copy_params.source_type = TSourceType::GEO_FILE;
4085  break;
4087  copy_params.source_type = TSourceType::PARQUET_FILE;
4088  break;
4090  copy_params.source_type = TSourceType::RASTER_FILE;
4091  break;
4093  copy_params.source_type = TSourceType::ODBC;
4094  break;
4095  default:
4096  CHECK(false);
4097  }
4098  switch (cp.geo_coords_encoding) {
4099  case kENCODING_GEOINT:
4100  copy_params.geo_coords_encoding = TEncodingType::GEOINT;
4101  break;
4102  default:
4103  copy_params.geo_coords_encoding = TEncodingType::NONE;
4104  break;
4105  }
4106  copy_params.geo_coords_comp_param = cp.geo_coords_comp_param;
4107  switch (cp.geo_coords_type) {
4108  case kGEOGRAPHY:
4109  copy_params.geo_coords_type = TDatumType::GEOGRAPHY;
4110  break;
4111  case kGEOMETRY:
4112  copy_params.geo_coords_type = TDatumType::GEOMETRY;
4113  break;
4114  default:
4115  CHECK(false);
4116  }
4117  copy_params.geo_coords_srid = cp.geo_coords_srid;
4118  copy_params.sanitize_column_names = cp.sanitize_column_names;
4119  copy_params.geo_layer_name = cp.geo_layer_name;
4120  copy_params.geo_assign_render_groups = false;
4121  copy_params.geo_explode_collections = cp.geo_explode_collections;
4122  copy_params.source_srid = cp.source_srid;
4123  switch (cp.raster_point_type) {
4125  copy_params.raster_point_type = TRasterPointType::NONE;
4126  break;
4128  copy_params.raster_point_type = TRasterPointType::AUTO;
4129  break;
4131  copy_params.raster_point_type = TRasterPointType::SMALLINT;
4132  break;
4134  copy_params.raster_point_type = TRasterPointType::INT;
4135  break;
4137  copy_params.raster_point_type = TRasterPointType::FLOAT;
4138  break;
4140  copy_params.raster_point_type = TRasterPointType::DOUBLE;
4141  break;
4143  copy_params.raster_point_type = TRasterPointType::POINT;
4144  break;
4145  default:
4146  CHECK(false);
4147  }
4148  copy_params.raster_import_bands = cp.raster_import_bands;
4149  copy_params.raster_scanlines_per_thread = cp.raster_scanlines_per_thread;
4150  switch (cp.raster_point_transform) {
4152  copy_params.raster_point_transform = TRasterPointTransform::NONE;
4153  break;
4155  copy_params.raster_point_transform = TRasterPointTransform::AUTO;
4156  break;
4158  copy_params.raster_point_transform = TRasterPointTransform::FILE;
4159  break;
4161  copy_params.raster_point_transform = TRasterPointTransform::WORLD;
4162  break;
4163  default:
4164  CHECK(false);
4165  }
4166  copy_params.raster_point_compute_angle = cp.raster_point_compute_angle;
4167  copy_params.raster_import_dimensions = cp.raster_import_dimensions;
4168  copy_params.odbc_dsn = cp.dsn;
4169  copy_params.odbc_connection_string = cp.connection_string;
4170  copy_params.odbc_sql_select = cp.sql_select;
4171  copy_params.odbc_sql_order_by = cp.sql_order_by;
4172  copy_params.odbc_username = cp.username;
4173  copy_params.odbc_password = cp.password;
4174  copy_params.odbc_credential_string = cp.credential_string;
4175  copy_params.add_metadata_columns = cp.add_metadata_columns;
4176  copy_params.trim_spaces = cp.trim_spaces;
4177  copy_params.geo_validate_geometry = cp.geo_validate_geometry;
4178  copy_params.raster_drop_if_all_null = cp.raster_drop_if_all_null;
4179  return copy_params;
4180 }
std::string s3_secret_key
Definition: CopyParams.h:62
int32_t raster_scanlines_per_thread
Definition: CopyParams.h:90
std::string connection_string
Definition: CopyParams.h:105
std::string raster_import_dimensions
Definition: CopyParams.h:93
std::string add_metadata_columns
Definition: CopyParams.h:94
ImportHeaderRow has_header
Definition: CopyParams.h:46
RasterPointType raster_point_type
Definition: CopyParams.h:88
std::string sql_order_by
Definition: CopyParams.h:98
import_export::SourceType source_type
Definition: CopyParams.h:57
std::string geo_layer_name
Definition: CopyParams.h:81
std::string s3_session_token
Definition: CopyParams.h:63
std::string raster_import_bands
Definition: CopyParams.h:89
#define CHECK(condition)
Definition: Logger.h:291
std::string s3_access_key
Definition: CopyParams.h:61
RasterPointTransform raster_point_transform
Definition: CopyParams.h:91
std::string credential_string
Definition: CopyParams.h:102
EncodingType geo_coords_encoding
Definition: CopyParams.h:76

+ Here is the caller graph for this function:

int32_t DBHandler::create_custom_expression ( const TSessionId &  session,
const TCustomExpression &  custom_expression 
)
override

Definition at line 4625 of file DBHandler.cpp.

References check_read_only(), anonymous_namespace{DBHandler.cpp}::create_custom_expr_from_thrift_obj(), custom_expressions_mutex_, get_session_ptr(), getConnectionInfo(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, THROW_DB_EXCEPTION, and toString().

4626  {
4627  heavyai::RequestInfo const request_info(session_id_or_json);
4628  SET_REQUEST_ID(request_info.requestId());
4629  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
4630  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4631  check_read_only("create_custom_expression");
4632 
4633  auto session_ptr = stdlog.getConstSessionInfo();
4634  if (!session_ptr->get_currentUser().isSuper) {
4635  THROW_DB_EXCEPTION("Custom expressions can only be created by super users.")
4636  }
4637  auto& catalog = session_ptr->getCatalog();
4639  return catalog.createCustomExpression(
4640  create_custom_expr_from_thrift_obj(t_custom_expr, catalog));
4641 }
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
std::unique_lock< T > unique_lock
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1703
heavyai::shared_mutex custom_expressions_mutex_
Definition: DBHandler.h:1073
void check_read_only(const std::string &str)
Definition: DBHandler.cpp:580
std::unique_ptr< Catalog_Namespace::CustomExpression > create_custom_expr_from_thrift_obj(const TCustomExpression &t_custom_expr, const Catalog &catalog)
Definition: DBHandler.cpp:4582
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1790
#define STDLOG(...)
Definition: QueryState.h:234
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the call graph for this function:

int32_t DBHandler::create_dashboard ( const TSessionId &  session,
const std::string &  dashboard_name,
const std::string &  dashboard_state,
const std::string &  image_hash,
const std::string &  dashboard_metadata 
)
override

Definition at line 4824 of file DBHandler.cpp.

References cat(), CHECK, dbhandler::check_not_info_schema_db(), check_read_only(), AccessPrivileges::CREATE_DASHBOARD, anonymous_namespace{DBHandler.cpp}::dashboard_exists(), DashboardDBObjectType, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, g_allow_system_dashboard_update, get_session_ptr(), getConnectionInfo(), DashboardDescriptor::imageHash, lockmgr::instance(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, THROW_DB_EXCEPTION, toString(), DashboardDescriptor::user, and DashboardDescriptor::userId.

4828  {
4829  heavyai::RequestInfo const request_info(session_id_or_json);
4830  SET_REQUEST_ID(request_info.requestId());
4831  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
4832  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4833  auto session_ptr = stdlog.getConstSessionInfo();
4834  CHECK(session_ptr);
4835  check_read_only("create_dashboard");
4836  auto& cat = session_ptr->getCatalog();
4839  }
4840 
4841  if (!session_ptr->checkDBAccessPrivileges(DBObjectType::DashboardDBObjectType,
4843  THROW_DB_EXCEPTION("Not enough privileges to create a dashboard.");
4844  }
4845 
4846  if (dashboard_exists(cat, session_ptr->get_currentUser().userId, dashboard_name)) {
4847  THROW_DB_EXCEPTION("Dashboard with name: " + dashboard_name + " already exists.");
4848  }
4849 
4851  dd.dashboardName = dashboard_name;
4852  dd.dashboardState = dashboard_state;
4853  dd.imageHash = image_hash;
4854  dd.dashboardMetadata = dashboard_metadata;
4855  dd.userId = session_ptr->get_currentUser().userId;
4856  dd.user = session_ptr->get_currentUser().userName;
4857 
4858  try {
4859  auto id = cat.createDashboard(dd);
4860  // TODO: transactionally unsafe
4861  SysCatalog::instance().createDBObject(
4862  session_ptr->get_currentUser(), dashboard_name, DashboardDBObjectType, cat, id);
4863  return id;
4864  } catch (const std::exception& e) {
4865  THROW_DB_EXCEPTION(e.what());
4866  }
4867 }
std::string cat(Ts &&...args)
static const AccessPrivileges CREATE_DASHBOARD
Definition: DBObject.h:170
void check_not_info_schema_db(const std::string &db_name, bool throw_db_exception)
Definition: DBHandler.cpp:4811
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1703
T & instance()
Definition: LockMgr.cpp:101
void check_read_only(const std::string &str)
Definition: DBHandler.cpp:580
bool g_allow_system_dashboard_update
Definition: DBHandler.cpp:124
bool dashboard_exists(const Catalog_Namespace::Catalog &cat, const int32_t user_id, const std::string &dashboard_name)
Definition: DBHandler.cpp:152
#define CHECK(condition)
Definition: Logger.h:291
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1790
#define STDLOG(...)
Definition: QueryState.h:234
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the call graph for this function:

TColumnType DBHandler::create_geo_column ( const TDatumType::type  type,
const std::string &  name,
const bool  is_array 
)
private

Definition at line 5160 of file DBHandler.cpp.

References setup::name, and run_benchmark_import::type.

5162  {
5163  TColumnType ct;
5164  ct.col_name = name;
5165  ct.col_type.type = type;
5166  ct.col_type.is_array = is_array;
5167  return ct;
5168 }
string name
Definition: setup.in.py:72
void DBHandler::create_link ( std::string &  _return,
const TSessionId &  session,
const std::string &  view_state,
const std::string &  view_metadata 
)
override

Definition at line 5136 of file DBHandler.cpp.

References cat(), get_session_ptr(), getConnectionInfo(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, THROW_DB_EXCEPTION, toString(), LinkDescriptor::userId, LinkDescriptor::viewMetadata, and LinkDescriptor::viewState.

5139  {
5140  heavyai::RequestInfo const request_info(session_id_or_json);
5141  SET_REQUEST_ID(request_info.requestId());
5142  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
5143  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
5144  auto session_ptr = stdlog.getConstSessionInfo();
5145  // check_read_only("create_link");
5146  auto& cat = session_ptr->getCatalog();
5147 
5148  LinkDescriptor ld;
5149  ld.userId = session_ptr->get_currentUser().userId;
5150  ld.viewState = view_state;
5151  ld.viewMetadata = view_metadata;
5152 
5153  try {
5154  _return = cat.createLink(ld, 6);
5155  } catch (const std::exception& e) {
5156  THROW_DB_EXCEPTION(e.what());
5157  }
5158 }
std::string cat(Ts &&...args)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1703
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1790
#define STDLOG(...)
Definition: QueryState.h:234
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the call graph for this function:

std::shared_ptr<Catalog_Namespace::SessionInfo> DBHandler::create_new_session ( TSessionId &  session,
const std::string &  dbname,
const Catalog_Namespace::UserMetadata user_meta,
std::shared_ptr< Catalog_Namespace::Catalog cat 
)
private
template<typename... ARGS>
std::shared_ptr<query_state::QueryState> DBHandler::create_query_state ( ARGS &&...  args)
inline

Definition at line 663 of file DBHandler.h.

References run_benchmark_import::args.

Referenced by convertResultSet(), get_table_details_impl(), get_tables_meta(), sql_execute(), sql_execute_df(), and sql_validate().

663  {
664  return query_states_.create(std::forward<ARGS>(args)...);
665  }
CircleBuffer::value_type create(ARGS &&...args)
Definition: QueryState.h:194
query_state::QueryStates query_states_
Definition: DBHandler.h:950

+ Here is the caller graph for this function:

void DBHandler::create_table ( const TSessionId &  session,
const std::string &  table_name,
const TRowDescriptor &  row_desc,
const TCreateParams &  create_params 
)
override

Definition at line 5191 of file DBHandler.cpp.

References check_read_only(), getConnectionInfo(), ImportHelpers::is_reserved_name(), join(), heavyai::RequestInfo::json(), kENCODING_DATE_IN_DAYS, kENCODING_DICT, kENCODING_FIXED, kENCODING_GEOINT, kENCODING_NONE, logger::request_id(), heavyai::RequestInfo::requestId(), ImportHelpers::sanitize_name(), SET_REQUEST_ID, heavyai::RequestInfo::setRequestId(), sql_execute(), STDLOG, thrift_to_encoding(), thrift_to_encoding_name(), thrift_to_name(), THROW_DB_EXCEPTION, to_string(), and toString().

Referenced by importGeoTableSingle().

5194  {
5195  heavyai::RequestInfo request_info(session_id_or_json);
5196  SET_REQUEST_ID(request_info.requestId());
5197  auto stdlog = STDLOG("table_name", table_name);
5198  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
5199  check_read_only("create_table");
5200 
5201  if (ImportHelpers::is_reserved_name(table_name)) {
5202  THROW_DB_EXCEPTION("Invalid table name (reserved keyword): " + table_name);
5203  } else if (table_name != ImportHelpers::sanitize_name(table_name)) {
5204  THROW_DB_EXCEPTION("Invalid characters in table name: " + table_name);
5205  }
5206 
5207  auto rds = rd;
5208 
5209  std::string stmt{"CREATE TABLE " + table_name};
5210  std::vector<std::string> col_stmts;
5211 
5212  for (auto col : rds) {
5213  if (ImportHelpers::is_reserved_name(col.col_name)) {
5214  THROW_DB_EXCEPTION("Invalid column name (reserved keyword): " + col.col_name);
5215  } else if (col.col_name != ImportHelpers::sanitize_name(col.col_name)) {
5216  THROW_DB_EXCEPTION("Invalid characters in column name: " + col.col_name);
5217  }
5218  if (col.col_type.type == TDatumType::INTERVAL_DAY_TIME ||
5219  col.col_type.type == TDatumType::INTERVAL_YEAR_MONTH) {
5220  THROW_DB_EXCEPTION("Unsupported type: " + thrift_to_name(col.col_type) +
5221  " for column: " + col.col_name);
5222  }
5223 
5224  if (col.col_type.type == TDatumType::DECIMAL) {
5225  // if no precision or scale passed in set to default 14,7
5226  if (col.col_type.precision == 0 && col.col_type.scale == 0) {
5227  col.col_type.precision = 14;
5228  col.col_type.scale = 7;
5229  }
5230  }
5231 
5232  std::string col_stmt;
5233  col_stmt.append(col.col_name + " " + thrift_to_name(col.col_type));
5234  if (col.__isset.default_value) {
5235  col_stmt.append(" DEFAULT " + col.default_value);
5236  }
5237 
5238  // As of 2016-06-27 the Immerse v1 frontend does not explicitly set the
5239  // `nullable` argument, leading this to default to false. Uncomment for v2.
5240  // if (!col.col_type.nullable) col_stmt.append(" NOT NULL");
5241 
5242  if (thrift_to_encoding(col.col_type.encoding) != kENCODING_NONE) {
5243  col_stmt.append(" ENCODING " + thrift_to_encoding_name(col.col_type));
5244  if (thrift_to_encoding(col.col_type.encoding) == kENCODING_DICT ||
5245  thrift_to_encoding(col.col_type.encoding) == kENCODING_FIXED ||
5246  thrift_to_encoding(col.col_type.encoding) == kENCODING_GEOINT ||
5247  thrift_to_encoding(col.col_type.encoding) == kENCODING_DATE_IN_DAYS) {
5248  col_stmt.append("(" + std::to_string(col.col_type.comp_param) + ")");
5249  }
5250  } else if (col.col_type.type == TDatumType::STR) {
5251  // non DICT encoded strings
5252  col_stmt.append(" ENCODING NONE");
5253  } else if (col.col_type.type == TDatumType::POINT ||
5254  col.col_type.type == TDatumType::MULTIPOINT ||
5255  col.col_type.type == TDatumType::LINESTRING ||
5256  col.col_type.type == TDatumType::MULTILINESTRING ||
5257  col.col_type.type == TDatumType::POLYGON ||
5258  col.col_type.type == TDatumType::MULTIPOLYGON) {
5259  // non encoded compressable geo
5260  if (col.col_type.scale == 4326) {
5261  col_stmt.append(" ENCODING NONE");
5262  }
5263  }
5264  col_stmts.push_back(col_stmt);
5265  }
5266 
5267  stmt.append(" (" + boost::algorithm::join(col_stmts, ", ") + ")");
5268 
5269  if (create_params.is_replicated) {
5270  stmt.append(" WITH (PARTITIONS = 'REPLICATED')");
5271  }
5272 
5273  stmt.append(";");
5274 
5275  TQueryResult ret;
5276  request_info.setRequestId(logger::request_id());
5277  sql_execute(ret, request_info.json(), stmt, true, "", -1, -1);
5278 }
EncodingType thrift_to_encoding(const TEncodingType::type tEncodingType)
void sql_execute(ExecutionResult &_return, const TSessionId &session, const std::string &query, const bool column_format, const int32_t first_n, const int32_t at_most_n, lockmgr::LockedTableDescriptors &locks)
Definition: DBHandler.cpp:1368
std::string join(T const &container, std::string const &delim)
std::string to_string(char const *&&v)
bool is_reserved_name(const std::string &name)
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1703
std::string thrift_to_encoding_name(const TTypeInfo &ti)
std::string thrift_to_name(const TTypeInfo &ti)
std::string sanitize_name(const std::string &name, const bool underscore=false)
void check_read_only(const std::string &str)
Definition: DBHandler.cpp:580
RequestId request_id()
Definition: Logger.cpp:876
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1790
#define STDLOG(...)
Definition: QueryState.h:234
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string const DBHandler::createInMemoryCalciteSession ( const std::shared_ptr< Catalog_Namespace::Catalog > &  catalog_ptr)
private

Definition at line 586 of file DBHandler.cpp.

References calcite_, Catalog_Namespace::CALCITE_SESSION_ID_LENGTH, calcite_sessions_, calcite_sessions_mtx_, CHECK, executor_device_type_, and generate_random_string().

Referenced by processCalciteRequest().

587  {
588  // We would create an in memory session for calcite with super user privileges which
589  // would be used for getting all tables metadata when a user runs the query. The
590  // session would be under the name of a proxy user/password which would only persist
591  // till server's lifetime or execution of calcite query(in memory) whichever is the
592  // earliest.
594  std::string session_id;
595  do {
597  } while (calcite_sessions_.find(session_id) != calcite_sessions_.end());
598  Catalog_Namespace::UserMetadata user_meta(-1,
599  calcite_->getInternalSessionProxyUserName(),
600  calcite_->getInternalSessionProxyPassword(),
601  true,
602  -1,
603  true,
604  false);
605  const auto emplace_ret = calcite_sessions_.emplace(
606  session_id,
607  std::make_shared<Catalog_Namespace::SessionInfo>(
608  catalog_ptr, user_meta, executor_device_type_, session_id));
609  CHECK(emplace_ret.second);
610  return session_id;
611 }
std::lock_guard< T > lock_guard
std::unordered_map< std::string, Catalog_Namespace::SessionInfoPtr > calcite_sessions_
Definition: DBHandler.h:951
const size_t CALCITE_SESSION_ID_LENGTH
Definition: SessionInfo.h:126
std::string generate_random_string(const size_t len)
ExecutorDeviceType executor_device_type_
Definition: DBHandler.h:639
heavyai::shared_mutex calcite_sessions_mtx_
Definition: DBHandler.h:952
std::shared_ptr< Calcite > calcite_
Definition: DBHandler.h:657
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::createSimpleResult ( TQueryResult &  _return,
const ResultSet results,
const bool  column_format,
const std::string  label 
)
staticprivate

Definition at line 6368 of file DBHandler.cpp.

References CHECK, CHECK_EQ, and run_benchmark_import::label.

Referenced by convertExplain(), and convertResult().

6371  {
6372  CHECK_EQ(size_t(1), results.rowCount());
6373  TColumnType proj_info;
6374  proj_info.col_name = label;
6375  proj_info.col_type.type = TDatumType::STR;
6376  proj_info.col_type.nullable = false;
6377  proj_info.col_type.is_array = false;
6378  _return.row_set.row_desc.push_back(proj_info);
6379  const auto crt_row = results.getNextRow(true, true);
6380  const auto tv = crt_row[0];
6381  CHECK(results.getNextRow(true, true).empty());
6382  const auto scalar_tv = boost::get<ScalarTargetValue>(&tv);
6383  CHECK(scalar_tv);
6384  const auto s_n = boost::get<NullableString>(scalar_tv);
6385  CHECK(s_n);
6386  const auto s = boost::get<std::string>(s_n);
6387  CHECK(s);
6388  if (column_format) {
6389  TColumn tcol;
6390  tcol.data.str_col.push_back(*s);
6391  tcol.nulls.push_back(false);
6392  _return.row_set.is_columnar = true;
6393  _return.row_set.columns.push_back(tcol);
6394  } else {
6395  TDatum explanation;
6396  explanation.val.str_val = *s;
6397  explanation.is_null = false;
6398  TRow trow;
6399  trow.cols.push_back(explanation);
6400  _return.row_set.is_columnar = false;
6401  _return.row_set.rows.push_back(trow);
6402  }
6403 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the caller graph for this function:

void DBHandler::deallocate_df ( const TSessionId &  session,
const TDataFrame &  df,
const TDeviceType::type  device_type,
const int32_t  device_id 
)
override

Definition at line 1562 of file DBHandler.cpp.

References CPU, data_mgr_, ArrowResultSet::deallocateArrowResultBuffer(), logger::ERROR, get_session_ptr(), GPU, handle_to_dev_ptr_mutex_, ipc_handle_to_dev_ptr_, LOG, heavyai::RequestInfo::requestId(), run_benchmark_import::result, heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, and STDLOG.

1565  {
1566  heavyai::RequestInfo const request_info(session_id_or_json);
1567  SET_REQUEST_ID(request_info.requestId());
1568  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
1569  std::string serialized_cuda_handle = "";
1570  if (device_type == TDeviceType::GPU) {
1571  std::lock_guard<std::mutex> map_lock(handle_to_dev_ptr_mutex_);
1572  if (ipc_handle_to_dev_ptr_.count(df.df_handle) != size_t(1)) {
1573  TDBException ex;
1574  ex.error_msg = std::string(
1575  "Current data frame handle is not bookkept or been inserted "
1576  "twice");
1577  LOG(ERROR) << ex.error_msg;
1578  throw ex;
1579  }
1580  serialized_cuda_handle = ipc_handle_to_dev_ptr_[df.df_handle];
1581  ipc_handle_to_dev_ptr_.erase(df.df_handle);
1582  }
1583  std::vector<char> sm_handle(df.sm_handle.begin(), df.sm_handle.end());
1584  std::vector<char> df_handle(df.df_handle.begin(), df.df_handle.end());
1586  sm_handle, df.sm_size, df_handle, df.df_size, serialized_cuda_handle};
1588  result,
1589  device_type == TDeviceType::CPU ? ExecutorDeviceType::CPU : ExecutorDeviceType::GPU,
1590  device_id,
1591  data_mgr_);
1592 }
std::mutex handle_to_dev_ptr_mutex_
Definition: DBHandler.h:1021
#define LOG(tag)
Definition: Logger.h:285
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
static void deallocateArrowResultBuffer(const ArrowResult &result, const ExecutorDeviceType device_type, const size_t device_id, std::shared_ptr< Data_Namespace::DataMgr > &data_mgr)
std::shared_ptr< Data_Namespace::DataMgr > data_mgr_
Definition: DBHandler.h:632
std::unordered_map< std::string, std::string > ipc_handle_to_dev_ptr_
Definition: DBHandler.h:1022
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

void DBHandler::delete_custom_expressions ( const TSessionId &  session,
const std::vector< int32_t > &  custom_expression_ids,
const bool  do_soft_delete 
)
override

Definition at line 4678 of file DBHandler.cpp.

References check_read_only(), custom_expressions_mutex_, get_session_ptr(), getConnectionInfo(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, THROW_DB_EXCEPTION, and toString().

4681  {
4682  heavyai::RequestInfo const request_info(session_id_or_json);
4683  SET_REQUEST_ID(request_info.requestId());
4684  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
4685  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4686  check_read_only("delete_custom_expressions");
4687 
4688  auto session_ptr = stdlog.getConstSessionInfo();
4689  if (!session_ptr->get_currentUser().isSuper) {
4690  THROW_DB_EXCEPTION("Custom expressions can only be deleted by super users.")
4691  }
4692  auto& catalog = session_ptr->getCatalog();
4694  catalog.deleteCustomExpressions(custom_expression_ids, do_soft_delete);
4695 }
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
std::unique_lock< T > unique_lock
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1703
heavyai::shared_mutex custom_expressions_mutex_
Definition: DBHandler.h:1073
void check_read_only(const std::string &str)
Definition: DBHandler.cpp:580
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1790
#define STDLOG(...)
Definition: QueryState.h:234
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the call graph for this function:

void DBHandler::delete_dashboard ( const TSessionId &  session,
const int32_t  dashboard_id 
)
override

Definition at line 4921 of file DBHandler.cpp.

References delete_dashboards().

4922  {
4923  delete_dashboards(session_id_or_json, {dashboard_id});
4924 }
void delete_dashboards(const TSessionId &session, const std::vector< int32_t > &dashboard_ids) override
Definition: DBHandler.cpp:4926

+ Here is the call graph for this function:

void DBHandler::delete_dashboards ( const TSessionId &  session,
const std::vector< int32_t > &  dashboard_ids 
)
override

Definition at line 4926 of file DBHandler.cpp.

References cat(), dbhandler::check_not_info_schema_db(), check_read_only(), g_allow_system_dashboard_update, get_session_ptr(), getConnectionInfo(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, THROW_DB_EXCEPTION, and toString().

Referenced by delete_dashboard().

4927  {
4928  heavyai::RequestInfo const request_info(session_id_or_json);
4929  SET_REQUEST_ID(request_info.requestId());
4930  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
4931  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4932  auto session_ptr = stdlog.getConstSessionInfo();
4933  check_read_only("delete_dashboards");
4934  auto& cat = session_ptr->getCatalog();
4937  }
4938  // Checks will be performed in catalog
4939  try {
4940  cat.deleteMetadataForDashboards(dashboard_ids, session_ptr->get_currentUser());
4941  } catch (const std::exception& e) {
4942  THROW_DB_EXCEPTION(e.what());
4943  }
4944 }
std::string cat(Ts &&...args)
void check_not_info_schema_db(const std::string &db_name, bool throw_db_exception)
Definition: DBHandler.cpp:4811
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1703
void check_read_only(const std::string &str)
Definition: DBHandler.cpp:580
bool g_allow_system_dashboard_update
Definition: DBHandler.cpp:124
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1790
#define STDLOG(...)
Definition: QueryState.h:234
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::detect_column_types ( TDetectResult &  _return,
const TSessionId &  session,
const std::string &  file_name,
const TCopyParams &  copy_params 
)
override

Definition at line 4340 of file DBHandler.cpp.

References anonymous_namespace{DBHandler.cpp}::add_vsi_archive_prefix(), anonymous_namespace{DBHandler.cpp}::add_vsi_geo_prefix(), anonymous_namespace{DBHandler.cpp}::add_vsi_network_prefix(), CHECK, check_geospatial_files(), check_read_only(), copyparams_to_thrift(), encoding_to_thrift(), shared::file_or_glob_path_exists(), import_export::CopyParams::file_sort_order_by, import_export::CopyParams::file_sort_regex, anonymous_namespace{DBHandler.cpp}::find_first_geo_file_in_archive(), import_export::Importer::gdalFileExists(), import_export::Importer::gdalFileOrDirectoryExists(), import_export::Importer::gdalToColumnDescriptors(), import_export::CopyParams::geo_coords_comp_param, import_export::CopyParams::geo_coords_encoding, import_export::CopyParams::geo_coords_srid, import_export::CopyParams::geo_coords_type, import_export::DataStreamSink::get_copy_params(), import_export::Detector::get_headers(), import_export::Detector::get_sample_rows(), get_session_ptr(), import_export::Detector::getBestColumnTypes(), getConnectionInfo(), import_path_, anonymous_namespace{DBHandler.cpp}::is_a_supported_archive_file(), anonymous_namespace{DBHandler.cpp}::is_local_file(), ImportHelpers::is_reserved_name(), shared::kDefaultSampleRowsCount, import_export::kDelimitedFile, Geospatial::kGeoColumnName, import_export::kGeoFile, import_export::kOdbc, import_export::kParquetFile, import_export::kRasterFile, shared::local_glob_filter_sort_files(), anonymous_namespace{DBHandler.cpp}::path_is_relative(), populateThriftColumnType(), import_export::Importer::readMetadataSampleGDAL(), import_export::CopyParams::regex_path_filter, heavyai::RequestInfo::requestId(), import_export::CopyParams::sanitize_column_names, ImportHelpers::sanitize_name(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, import_export::CopyParams::source_type, STDLOG, thrift_to_copyparams(), THROW_DB_EXCEPTION, toString(), type_to_thrift(), and anonymous_namespace{DBHandler.cpp}::validate_import_file_path_if_local().

Referenced by importGeoTableSingle().

4343  {
4344  heavyai::RequestInfo const request_info(session_id_or_json);
4345  SET_REQUEST_ID(request_info.requestId());
4346  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
4347  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4348  check_read_only("detect_column_types");
4349 
4350  bool is_raster = false;
4351  boost::filesystem::path file_path;
4353  if (copy_params.source_type != import_export::SourceType::kOdbc) {
4354  std::string file_name{file_name_in};
4355  if (path_is_relative(file_name)) {
4356  // assume relative paths are relative to data_path / import / <session>
4357  auto temp_file_path = import_path_ /
4358  picosha2::hash256_hex_string(request_info.sessionId()) /
4359  boost::filesystem::path(file_name).filename();
4360  file_name = temp_file_path.string();
4361  }
4363 
4364  if ((copy_params.source_type == import_export::SourceType::kGeoFile ||
4366  is_local_file(file_name)) {
4367  const shared::FilePathOptions options{copy_params.regex_path_filter,
4368  copy_params.file_sort_order_by,
4369  copy_params.file_sort_regex};
4370  auto file_paths = shared::local_glob_filter_sort_files(file_name, options, false);
4371  // For geo and raster detect, pick the first file, if multiple files are provided
4372  // (e.g. through file globbing).
4373  CHECK(!file_paths.empty());
4374  file_name = file_paths[0];
4375  }
4376 
4377  // if it's a geo or raster import, handle alternative paths (S3, HTTP, archive etc.)
4378  if (copy_params.source_type == import_export::SourceType::kGeoFile) {
4379  if (is_a_supported_archive_file(file_name)) {
4380  // find the archive file
4381  add_vsi_network_prefix(file_name);
4382  if (!import_export::Importer::gdalFileExists(file_name, copy_params)) {
4383  THROW_DB_EXCEPTION("Archive does not exist: " + file_name_in);
4384  }
4385  // find geo file in archive
4386  add_vsi_archive_prefix(file_name);
4387  std::string geo_file = find_first_geo_file_in_archive(file_name, copy_params);
4388  // prepare to detect that geo file
4389  if (geo_file.size()) {
4390  file_name = file_name + std::string("/") + geo_file;
4391  }
4392  } else {
4393  // prepare to detect geo file directly
4394  add_vsi_network_prefix(file_name);
4395  add_vsi_geo_prefix(file_name);
4396  }
4397  } else if (copy_params.source_type == import_export::SourceType::kRasterFile) {
4398  // prepare to detect raster file directly
4399  add_vsi_network_prefix(file_name);
4400  add_vsi_geo_prefix(file_name);
4401  is_raster = true;
4402  }
4403 
4404  file_path = boost::filesystem::path(file_name);
4405  // can be a s3 url
4406  if (!boost::istarts_with(file_name, "s3://")) {
4407  if (!boost::filesystem::path(file_name).is_absolute()) {
4408  file_path = import_path_ /
4409  picosha2::hash256_hex_string(request_info.sessionId()) /
4410  boost::filesystem::path(file_name).filename();
4411  file_name = file_path.string();
4412  }
4413 
4414  if (copy_params.source_type == import_export::SourceType::kGeoFile ||
4416  // check for geo or raster file
4417  if (!import_export::Importer::gdalFileOrDirectoryExists(file_name, copy_params)) {
4418  THROW_DB_EXCEPTION("File or directory \"" + file_path.string() +
4419  "\" does not exist.")
4420  }
4421  } else {
4422  // check for regular file
4423  if (!shared::file_or_glob_path_exists(file_path.string())) {
4424  THROW_DB_EXCEPTION("File or directory \"" + file_path.string() +
4425  "\" does not exist.");
4426  }
4427  }
4428  }
4429  }
4430 
4431  try {
4433 #ifdef ENABLE_IMPORT_PARQUET
4435 #endif
4436  ) {
4437  import_export::Detector detector(file_path, copy_params);
4438  auto best_types = detector.getBestColumnTypes();
4439  std::vector<std::string> headers = detector.get_headers();
4440  copy_params = detector.get_copy_params();
4441 
4442  _return.copy_params = copyparams_to_thrift(copy_params);
4443  _return.row_set.row_desc.resize(best_types.size());
4444  for (size_t col_idx = 0; col_idx < best_types.size(); col_idx++) {
4445  TColumnType col;
4446  auto& ti = best_types[col_idx];
4447  col.col_type.precision = ti.get_precision();
4448  col.col_type.scale = ti.get_scale();
4449  col.col_type.comp_param = ti.get_comp_param();
4450  if (ti.is_geometry()) {
4451  // set this so encoding_to_thrift does the right thing
4452  ti.set_compression(copy_params.geo_coords_encoding);
4453  // fill in these directly
4454  col.col_type.precision = static_cast<int>(copy_params.geo_coords_type);
4455  col.col_type.scale = copy_params.geo_coords_srid;
4456  col.col_type.comp_param = copy_params.geo_coords_comp_param;
4457  }
4458  col.col_type.type = type_to_thrift(ti);
4459  col.col_type.encoding = encoding_to_thrift(ti);
4460  if (ti.is_array()) {
4461  col.col_type.is_array = true;
4462  }
4463  if (copy_params.sanitize_column_names) {
4464  col.col_name = ImportHelpers::sanitize_name(headers[col_idx]);
4465  } else {
4466  col.col_name = headers[col_idx];
4467  }
4468  col.is_reserved_keyword = ImportHelpers::is_reserved_name(col.col_name);
4469  _return.row_set.row_desc[col_idx] = col;
4470  }
4471  auto sample_data = detector.get_sample_rows(shared::kDefaultSampleRowsCount);
4472 
4473  TRow sample_row;
4474  for (auto row : sample_data) {
4475  sample_row.cols.clear();
4476  for (const auto& s : row) {
4477  TDatum td;
4478  td.val.str_val = s;
4479  td.is_null = s.empty();
4480  sample_row.cols.push_back(td);
4481  }
4482  _return.row_set.rows.push_back(sample_row);
4483  }
4484  } else if (copy_params.source_type == import_export::SourceType::kGeoFile ||
4486  check_geospatial_files(file_path, copy_params);
4487  std::list<ColumnDescriptor> cds = import_export::Importer::gdalToColumnDescriptors(
4488  file_path.string(), is_raster, Geospatial::kGeoColumnName, copy_params);
4489  for (auto cd : cds) {
4490  if (copy_params.sanitize_column_names) {
4491  cd.columnName = ImportHelpers::sanitize_name(cd.columnName);
4492  }
4493  _return.row_set.row_desc.push_back(populateThriftColumnType(nullptr, &cd));
4494  }
4495  if (!is_raster) {
4496  // @TODO(se) support for raster?
4497  std::map<std::string, std::vector<std::string>> sample_data;
4500  sample_data,
4502  copy_params);
4503  if (sample_data.size() > 0) {
4504  for (size_t i = 0; i < sample_data.begin()->second.size(); i++) {
4505  TRow sample_row;
4506  for (auto cd : cds) {
4507  TDatum td;
4508  td.val.str_val = sample_data[cd.sourceName].at(i);
4509  td.is_null = td.val.str_val.empty();
4510  sample_row.cols.push_back(td);
4511  }
4512  _return.row_set.rows.push_back(sample_row);
4513  }
4514  }
4515  }
4516  _return.copy_params = copyparams_to_thrift(copy_params);
4517  }
4518  } catch (const std::exception& e) {
4519  THROW_DB_EXCEPTION("detect_column_types error: " + std::string(e.what()));
4520  }
4521 }
boost::filesystem::path import_path_
Definition: DBHandler.h:638
void add_vsi_archive_prefix(std::string &path)
Definition: DBHandler.cpp:4212
bool is_a_supported_archive_file(const std::string &path)
Definition: DBHandler.cpp:4278
TDatumType::type type_to_thrift(const SQLTypeInfo &type_info)
bool path_is_relative(const std::string &path)
Definition: DBHandler.cpp:4246
TCopyParams copyparams_to_thrift(const import_export::CopyParams &cp)
Definition: DBHandler.cpp:4049
static bool gdalFileExists(const std::string &path, const CopyParams &copy_params)
Definition: Importer.cpp:5053
const std::string kGeoColumnName
Definition: ColumnNames.h:23
bool is_reserved_name(const std::string &name)
std::optional< std::string > regex_path_filter
Definition: CopyParams.h:85
TColumnType populateThriftColumnType(const Catalog_Namespace::Catalog *cat, const ColumnDescriptor *cd)
Definition: DBHandler.cpp:2363
void check_geospatial_files(const boost::filesystem::path file_path, const import_export::CopyParams &copy_params)
Definition: DBHandler.cpp:5170
static void readMetadataSampleGDAL(const std::string &fileName, const std::string &geoColumnName, std::map< std::string, std::vector< std::string >> &metadata, int rowLimit, const CopyParams &copy_params)
Definition: Importer.cpp:4625
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
void validate_import_file_path_if_local(const std::string &file_path)
Definition: DBHandler.cpp:4332
import_export::SourceType source_type
Definition: CopyParams.h:57
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1703
static bool gdalFileOrDirectoryExists(const std::string &path, const CopyParams &copy_params)
Definition: Importer.cpp:5058
bool file_or_glob_path_exists(const std::string &path)
std::string sanitize_name(const std::string &name, const bool underscore=false)
void check_read_only(const std::string &str)
Definition: DBHandler.cpp:580
std::string find_first_geo_file_in_archive(const std::string &archive_path, const import_export::CopyParams &copy_params)
Definition: DBHandler.cpp:4291
void add_vsi_network_prefix(std::string &path)
Definition: DBHandler.cpp:4183
#define CHECK(condition)
Definition: Logger.h:291
static const std::list< ColumnDescriptor > gdalToColumnDescriptors(const std::string &fileName, const bool is_raster, const std::string &geoColumnName, const CopyParams &copy_params)
Definition: Importer.cpp:4820
std::vector< std::string > local_glob_filter_sort_files(const std::string &file_path, const FilePathOptions &options, const bool recurse)
import_export::CopyParams thrift_to_copyparams(const TCopyParams &cp)
Definition: DBHandler.cpp:3854
static constexpr size_t kDefaultSampleRowsCount
std::optional< std::string > file_sort_order_by
Definition: CopyParams.h:86
void add_vsi_geo_prefix(std::string &path)
Definition: DBHandler.cpp:4205
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1790
#define STDLOG(...)
Definition: QueryState.h:234
std::optional< std::string > file_sort_regex
Definition: CopyParams.h:87
TEncodingType::type encoding_to_thrift(const SQLTypeInfo &type_info)
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139
EncodingType geo_coords_encoding
Definition: CopyParams.h:76
bool is_local_file(const std::string &file_path)
Definition: DBHandler.cpp:4326

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::disconnect ( const TSessionId &  session)
override

Definition at line 719 of file DBHandler.cpp.

References get_session_ptr(), getConnectionInfo(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), sessions_store_, SET_REQUEST_ID, STDLOG, and toString().

719  {
720  heavyai::RequestInfo const request_info(session_id_or_json);
721  SET_REQUEST_ID(request_info.requestId());
722  auto session_ptr = get_session_ptr(request_info.sessionId());
723  auto stdlog = STDLOG(session_ptr, "client", getConnectionInfo().toString());
724  sessions_store_->disconnect(request_info.sessionId());
725 }
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1703
std::unique_ptr< Catalog_Namespace::SessionsStore > sessions_store_
Definition: DBHandler.h:985
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1790
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

void DBHandler::disconnect_impl ( Catalog_Namespace::SessionInfoPtr session_ptr)
private

Definition at line 727 of file DBHandler.cpp.

References LeafAggregator::disconnect(), leaf_aggregator_, LeafAggregator::leafCount(), and render_handler_.

Referenced by resetSessionsStore().

727  {
728  const auto session_id = session_ptr->get_session_id();
729  std::exception_ptr leaf_exception = nullptr;
730  try {
731  if (leaf_aggregator_.leafCount() > 0) {
732  leaf_aggregator_.disconnect(session_id);
733  }
734  } catch (...) {
735  leaf_exception = std::current_exception();
736  }
737 
738  if (render_handler_) {
739  render_handler_->disconnect(session_id);
740  }
741 
742  if (leaf_exception) {
743  std::rethrow_exception(leaf_exception);
744  }
745 }
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:634
void disconnect(const TSessionId session)
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:654
size_t leafCount() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::dispatch_query_task ( std::shared_ptr< QueryDispatchQueue::Task query_task,
const bool  is_update_delete 
)
private

Definition at line 1820 of file DBHandler.cpp.

References CHECK, and dispatch_queue_.

Referenced by validateRelAlg().

1821  {
1823  dispatch_queue_->submit(std::move(query_task), is_update_delete);
1824 }
std::unique_ptr< QueryDispatchQueue > dispatch_queue_
Definition: DBHandler.h:660
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the caller graph for this function:

void DBHandler::emergency_shutdown ( )

Definition at line 7565 of file DBHandler.cpp.

References calcite_.

Referenced by shutdown().

7565  {
7566  if (calcite_) {
7567  calcite_->close_calcite_server(false);
7568  }
7569 }
std::shared_ptr< Calcite > calcite_
Definition: DBHandler.h:657

+ Here is the caller graph for this function:

void DBHandler::execute_distributed_copy_statement ( Parser::CopyTableStmt copy_stmt,
const Catalog_Namespace::SessionInfo session_info 
)
private

Definition at line 6838 of file DBHandler.cpp.

Referenced by sql_execute_impl().

6840  {}

+ Here is the caller graph for this function:

void DBHandler::execute_next_render_step ( TRenderStepResult &  _return,
const TPendingRenderQuery &  pending_render,
const TRenderAggDataMap &  merged_data 
)
override

Definition at line 7330 of file DBHandler.cpp.

References measure< TimeT >::execution(), logger::INFO, LOG, render_handler_, SET_REQUEST_ID, and THROW_DB_EXCEPTION.

7332  {
7333  // No SessionID is available
7334  SET_REQUEST_ID(0);
7335 
7336  if (!render_handler_) {
7337  THROW_DB_EXCEPTION("Backend rendering is disabled.");
7338  }
7339 
7340  LOG(INFO) << "execute_next_render_step: id:" << pending_render.id;
7341  auto time_ms = measure<>::execution([&]() {
7342  try {
7343  render_handler_->execute_next_render_step(_return, pending_render, merged_data);
7344  } catch (std::exception& e) {
7345  THROW_DB_EXCEPTION(e.what());
7346  }
7347  });
7348  LOG(INFO) << "execute_next_render_step-COMPLETED id: " << pending_render.id
7349  << ", time: " << time_ms << "ms ";
7350 }
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:285
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:654
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the call graph for this function:

void DBHandler::execute_query_step ( TStepResult &  _return,
const TPendingQuery &  pending_query,
const TSubqueryId  subquery_id,
const std::string &  start_time_str 
)
override

Definition at line 7086 of file DBHandler.cpp.

References measure< TimeT >::execution(), logger::INFO, leaf_handler_, LOG, SET_REQUEST_ID, and THROW_DB_EXCEPTION.

7089  {
7090  SET_REQUEST_ID(0); // No SessionID is available
7091  if (!leaf_handler_) {
7092  THROW_DB_EXCEPTION("Distributed support is disabled.");
7093  }
7094  LOG(INFO) << "execute_query_step : id:" << pending_query.id;
7095  auto time_ms = measure<>::execution([&]() {
7096  try {
7097  leaf_handler_->execute_query_step(
7098  _return, pending_query, subquery_id, start_time_str);
7099  } catch (std::exception& e) {
7100  THROW_DB_EXCEPTION(e.what());
7101  }
7102  });
7103  LOG(INFO) << "execute_query_step-COMPLETED " << time_ms << "ms";
7104 }
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:285
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
std::unique_ptr< HeavyDBLeafHandler > leaf_handler_
Definition: DBHandler.h:656
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the call graph for this function:

std::vector< PushedDownFilterInfo > DBHandler::execute_rel_alg ( ExecutionResult _return,
QueryStateProxy  query_state_proxy,
const std::string &  query_ra,
const bool  column_format,
const ExecutorDeviceType  executor_device_type,
const int32_t  first_n,
const int32_t  at_most_n,
const bool  just_validate,
const bool  find_push_down_candidates,
const ExplainInfo explain_info,
const std::optional< size_t >  executor_index = std::nullopt 
) const
private

Definition at line 6206 of file DBHandler.cpp.

References allow_loop_joins_, allow_multifrag_, query_state::QueryStateProxy::createTimer(), Default, measure< TimeT >::execution(), ExecutionResult::Explanation, g_dynamic_watchdog_time_limit, g_enable_columnar_output, g_enable_dynamic_watchdog, g_enable_runtime_query_interrupt, g_enable_watchdog, g_optimize_cuda_block_and_grid_sizes, g_pending_query_interrupt_freq, g_running_query_interrupt_freq, query_state::QueryState::getConstSessionInfo(), ExecutionResult::getExecutionTime(), Executor::getExecutor(), ExecutionResult::getPushedDownFilterInfo(), ExecutionResult::getRows(), SystemParameters::gpu_input_mem_limit, lockmgr::TableSchemaLockMgr::instance(), lockmgr::TableDataLockMgr::instance(), intel_jit_profile_, ExplainInfo::isCalciteExplain(), ExplainInfo::isJustExplain(), ExplainInfo::isOptimizedExplain(), ExplainInfo::isPlanExplain(), ExplainInfo::isVerbose(), jit_debug_, Optimized, ExecutionResult::QueryResult, ExecutionResult::setExecutionTime(), ExecutionResult::setResultType(), system_parameters_, Executor::UNITARY_EXECUTOR_ID, and VLOG.

Referenced by execute_rel_alg_with_filter_push_down(), get_tables_meta_impl(), sql_execute_impl(), and validateRelAlg().

6217  {
6218  query_state::Timer timer = query_state_proxy.createTimer(__func__);
6219  VLOG(1) << "Table Schema Locks:\n" << lockmgr::TableSchemaLockMgr::instance();
6220  VLOG(1) << "Table Data Locks:\n" << lockmgr::TableDataLockMgr::instance();
6221  auto executor = Executor::getExecutor(
6222  executor_index ? *executor_index : Executor::UNITARY_EXECUTOR_ID,
6223  jit_debug_ ? "/tmp" : "",
6224  jit_debug_ ? "mapdquery" : "",
6226  RelAlgExecutor ra_executor(
6227  executor.get(), query_ra, query_state_proxy->shared_from_this());
6228  CompilationOptions co = {executor_device_type,
6229  /*hoist_literals=*/true,
6232  /*allow_lazy_fetch=*/true,
6233  /*filter_on_deleted_column=*/true,
6234  explain_info.isOptimizedExplain()
6238  auto validate_or_explain_query =
6239  explain_info.isJustExplain() || explain_info.isCalciteExplain() || just_validate;
6240  ExecutionOptions eo = {
6242  false,
6244  explain_info.isJustExplain(),
6245  allow_loop_joins_ || just_validate,
6247  jit_debug_,
6248  just_validate,
6251  find_push_down_candidates,
6252  explain_info.isCalciteExplain(),
6254  g_enable_runtime_query_interrupt && !validate_or_explain_query &&
6255  !query_state_proxy->getConstSessionInfo()->get_session_id().empty(),
6259  auto execution_time_ms =
6260  _return.getExecutionTime() + measure<>::execution([&]() {
6261  _return = ra_executor.executeRelAlgQuery(
6262  co, eo, explain_info.isPlanExplain(), explain_info.isVerbose(), nullptr);
6263  });
6264  // reduce execution time by the time spent during queue waiting
6265  const auto rs = _return.getRows();
6266  if (rs) {
6267  execution_time_ms -= rs->getQueueTime();
6268  }
6269  _return.setExecutionTime(execution_time_ms);
6270  const auto& filter_push_down_info = _return.getPushedDownFilterInfo();
6271  if (!filter_push_down_info.empty()) {
6272  return filter_push_down_info;
6273  }
6274  if (explain_info.isJustExplain()) {
6276  } else if (!explain_info.isCalciteExplain()) {
6278  }
6279  return {};
6280 }
double g_running_query_interrupt_freq
Definition: Execute.cpp:141
static TableSchemaLockMgr & instance()
Definition: LockMgr.h:40
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
bool isVerbose() const
Definition: ParserWrapper.h:83
unsigned g_pending_query_interrupt_freq
Definition: Execute.cpp:140
const bool jit_debug_
Definition: DBHandler.h:642
bool g_enable_dynamic_watchdog
Definition: Execute.cpp:81
Timer createTimer(char const *event_name)
Definition: QueryState.cpp:129
static TableDataLockMgr & instance()
Definition: LockMgr.h:78
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:513
bool g_enable_columnar_output
Definition: Execute.cpp:106
const std::shared_ptr< ResultSet > & getRows() const
bool g_enable_watchdog
SystemParameters & system_parameters_
Definition: DBHandler.h:652
bool isOptimizedExplain() const
Definition: ParserWrapper.h:70
bool g_optimize_cuda_block_and_grid_sizes
Definition: Execute.cpp:169
void setResultType(RType type)
bool isJustExplain() const
Definition: ParserWrapper.h:52
const bool allow_loop_joins_
Definition: DBHandler.h:646
const bool intel_jit_profile_
Definition: DBHandler.h:643
const std::vector< PushedDownFilterInfo > & getPushedDownFilterInfo() const
void setExecutionTime(int64_t execution_time_ms)
bool isCalciteExplain() const
Definition: ParserWrapper.h:71
bool allow_multifrag_
Definition: DBHandler.h:644
static constexpr ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:423
unsigned g_dynamic_watchdog_time_limit
Definition: Execute.cpp:92
std::shared_ptr< Catalog_Namespace::SessionInfo const > getConstSessionInfo() const
Definition: QueryState.cpp:84
int64_t getExecutionTime() const
bool g_enable_runtime_query_interrupt
Definition: Execute.cpp:137
#define VLOG(n)
Definition: Logger.h:388
bool isPlanExplain() const
Definition: ParserWrapper.h:78

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::execute_rel_alg_with_filter_push_down ( ExecutionResult _return,
QueryStateProxy  query_state_proxy,
std::string &  query_ra,
const bool  column_format,
const ExecutorDeviceType  executor_device_type,
const int32_t  first_n,
const int32_t  at_most_n,
const bool  just_explain,
const bool  just_calcite_explain,
const std::vector< PushedDownFilterInfo > &  filter_push_down_requests 
)
private

Definition at line 6794 of file DBHandler.cpp.

References ExecutionResult::addExecutionTime(), execute_rel_alg(), query_state::QueryState::getQueryStr(), ExplainInfo::None, parse_to_ra(), and system_parameters_.

Referenced by sql_execute_impl().

6804  {
6805  // collecting the selected filters' info to be sent to Calcite:
6806  std::vector<TFilterPushDownInfo> filter_push_down_info;
6807  for (const auto& req : filter_push_down_requests) {
6808  TFilterPushDownInfo filter_push_down_info_for_request;
6809  filter_push_down_info_for_request.input_prev = req.input_prev;
6810  filter_push_down_info_for_request.input_start = req.input_start;
6811  filter_push_down_info_for_request.input_next = req.input_next;
6812  filter_push_down_info.push_back(filter_push_down_info_for_request);
6813  }
6814  // deriving the new relational algebra plan with respect to the pushed down filters
6815  _return.addExecutionTime(measure<>::execution([&]() {
6816  query_ra = parse_to_ra(query_state_proxy,
6817  query_state_proxy->getQueryStr(),
6818  filter_push_down_info,
6819  false,
6821  .first.plan_result;
6822  }));
6823 
6824  // execute the new relational algebra plan:
6825  auto explain_info = ExplainInfo(ExplainInfo::ExplainType::None);
6826  execute_rel_alg(_return,
6827  query_state_proxy,
6828  query_ra,
6829  column_format,
6830  executor_device_type,
6831  first_n,
6832  at_most_n,
6833  /*just_validate=*/false,
6834  /*find_push_down_candidates=*/false,
6835  explain_info);
6836 }
std::vector< PushedDownFilterInfo > execute_rel_alg(ExecutionResult &_return, QueryStateProxy, const std::string &query_ra, const bool column_format, const ExecutorDeviceType executor_device_type, const int32_t first_n, const int32_t at_most_n, const bool just_validate, const bool find_push_down_candidates, const ExplainInfo &explain_info, const std::optional< size_t > executor_index=std::nullopt) const
Definition: DBHandler.cpp:6206
std::string const & getQueryStr() const
Definition: QueryState.h:159
void addExecutionTime(int64_t execution_time_ms)
SystemParameters & system_parameters_
Definition: DBHandler.h:652
std::pair< TPlanResult, lockmgr::LockedTableDescriptors > parse_to_ra(QueryStateProxy, const std::string &query_str, const std::vector< TFilterPushDownInfo > &filter_push_down_info, const bool acquire_locks, const SystemParameters &system_parameters, bool check_privileges=true)
Definition: DBHandler.cpp:6913

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::executeDdl ( TQueryResult &  _return,
const std::string &  query_ra,
std::shared_ptr< Catalog_Namespace::SessionInfo const >  session_ptr 
)
private

Definition at line 8070 of file DBHandler.cpp.

References alterSession(), alterSystemClear(), DdlCommandExecutor::commandStr(), convertResultSet(), ExecutionResult::empty(), DdlCommandExecutor::execute(), measure< TimeT >::execution(), getQueries(), ExecutionResult::getRows(), DdlCommandExecutor::getSessionParameter(), DdlCommandExecutor::getTargetQuerySessionToKill(), getUserSessions(), interruptQuery(), DdlCommandExecutor::isAlterSessionSet(), DdlCommandExecutor::isAlterSystemClear(), DdlCommandExecutor::isAlterSystemControlExecutorQueue(), DdlCommandExecutor::isKillQuery(), DdlCommandExecutor::isShowQueries(), DdlCommandExecutor::isShowUserSessions(), pause_executor_queue(), read_only_, run_benchmark_import::result, resume_executor_queue(), DdlCommandExecutor::returnCacheType(), and DdlCommandExecutor::returnQueueAction().

Referenced by sql_execute_impl().

8073  {
8074  DdlCommandExecutor executor = DdlCommandExecutor(query_ra, session_ptr);
8075  std::string commandStr = executor.commandStr();
8076 
8077  if (executor.isKillQuery()) {
8078  interruptQuery(*session_ptr, executor.getTargetQuerySessionToKill());
8079  } else {
8081  int64_t execution_time_ms;
8082  if (executor.isShowQueries()) {
8083  // getQueries still requires Thrift cannot be nested into DdlCommandExecutor
8084  _return.execution_time_ms +=
8085  measure<>::execution([&]() { result = getQueries(session_ptr); });
8086  } else if (executor.isShowUserSessions()) {
8087  // getUserSessions still requires Thrift cannot be nested into DdlCommandExecutor
8088  _return.execution_time_ms +=
8089  measure<>::execution([&]() { result = getUserSessions(session_ptr); });
8090  } else if (executor.isAlterSystemClear()) {
8091  alterSystemClear(session_ptr->get_session_id(),
8092  result,
8093  executor.returnCacheType(),
8094  execution_time_ms);
8095  _return.execution_time_ms += execution_time_ms;
8096 
8097  } else if (executor.isAlterSessionSet()) {
8098  alterSession(session_ptr->get_session_id(),
8099  result,
8100  executor.getSessionParameter(),
8101  execution_time_ms);
8102  _return.execution_time_ms += execution_time_ms;
8103  } else if (executor.isAlterSystemControlExecutorQueue()) {
8104  result = ExecutionResult();
8105  if (executor.returnQueueAction() == "PAUSE") {
8106  _return.execution_time_ms += measure<>::execution(
8107  [&]() { pause_executor_queue(session_ptr->get_session_id()); });
8108  } else if (executor.returnQueueAction() == "RESUME") {
8109  _return.execution_time_ms += measure<>::execution(
8110  [&]() { resume_executor_queue(session_ptr->get_session_id()); });
8111  } else {
8112  throw std::runtime_error("Unknown queue command.");
8113  }
8114  } else {
8115  _return.execution_time_ms +=
8116  measure<>::execution([&]() { result = executor.execute(read_only_); });
8117  }
8118 
8119  if (!result.empty()) {
8120  // reduce execution time by the time spent during queue waiting
8121  _return.execution_time_ms -= result.getRows()->getQueueTime();
8122  convertResultSet(result, *session_ptr, commandStr, _return);
8123  }
8124  }
8125 }
void resume_executor_queue(const TSessionId &session)
Definition: DBHandler.cpp:2964
const std::string getTargetQuerySessionToKill() const
const std::string commandStr() const
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
std::pair< std::string, std::string > getSessionParameter() const
void convertResultSet(ExecutionResult &result, const Catalog_Namespace::SessionInfo &session_info, const std::string &query_state_str, TQueryResult &_return)
Definition: DBHandler.cpp:7746
bool isAlterSystemClear() const
const std::shared_ptr< ResultSet > & getRows() const
bool isShowUserSessions() const
void alterSession(const std::string &sesson_id, ExecutionResult &result, const std::pair< std::string, std::string > &session_parameter, int64_t &execution_time_ms)
Definition: DBHandler.cpp:8045
bool isAlterSessionSet() const
ExecutionResult execute(bool read_only_mode)
std::string returnQueueAction() const
void pause_executor_queue(const TSessionId &session)
Definition: DBHandler.cpp:2950
const bool read_only_
Definition: DBHandler.h:645
std::string returnCacheType() const
void interruptQuery(const Catalog_Namespace::SessionInfo &session_info, const std::string &target_session)
Definition: DBHandler.cpp:7940
bool isAlterSystemControlExecutorQueue() const
ExecutionResult getQueries(std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
Definition: DBHandler.cpp:7813
ExecutionResult getUserSessions(std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
Definition: DBHandler.cpp:7775
void alterSystemClear(const std::string &sesson_id, ExecutionResult &result, const std::string &cache_type, int64_t &execution_time_ms)
Definition: DBHandler.cpp:8029

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::executeDdl ( ExecutionResult _return,
const std::string &  query_ra,
std::shared_ptr< Catalog_Namespace::SessionInfo const >  session_ptr 
)
private

Definition at line 8127 of file DBHandler.cpp.

References alterSession(), alterSystemClear(), ExecutionResult::CalciteDdl, DdlCommandExecutor::commandStr(), DdlCommandExecutor::execute(), measure< TimeT >::execution(), getQueries(), ExecutionResult::getResultType(), DdlCommandExecutor::getSessionParameter(), DdlCommandExecutor::getTargetQuerySessionToKill(), getUserSessions(), interruptQuery(), DdlCommandExecutor::isAlterSessionSet(), DdlCommandExecutor::isAlterSystemClear(), DdlCommandExecutor::isAlterSystemControlExecutorQueue(), DdlCommandExecutor::isKillQuery(), DdlCommandExecutor::isShowQueries(), DdlCommandExecutor::isShowUserSessions(), pause_executor_queue(), ExecutionResult::QueryResult, read_only_, resume_executor_queue(), DdlCommandExecutor::returnCacheType(), DdlCommandExecutor::returnQueueAction(), ExecutionResult::setExecutionTime(), and ExecutionResult::setResultType().

8130  {
8131  DdlCommandExecutor executor = DdlCommandExecutor(query_ra, session_ptr);
8132  std::string commandStr = executor.commandStr();
8133 
8134  if (executor.isKillQuery()) {
8135  interruptQuery(*session_ptr, executor.getTargetQuerySessionToKill());
8136  } else {
8137  int64_t execution_time_ms;
8138  if (executor.isShowQueries()) {
8139  // getQueries still requires Thrift cannot be nested into DdlCommandExecutor
8140  execution_time_ms =
8141  measure<>::execution([&]() { _return = getQueries(session_ptr); });
8142  } else if (executor.isShowUserSessions()) {
8143  // getUserSessions still requires Thrift cannot be nested into DdlCommandExecutor
8144  execution_time_ms =
8145  measure<>::execution([&]() { _return = getUserSessions(session_ptr); });
8146  } else if (executor.isAlterSystemClear()) {
8147  alterSystemClear(session_ptr->get_session_id(),
8148  _return,
8149  executor.returnCacheType(),
8150  execution_time_ms);
8151  } else if (executor.isAlterSessionSet()) {
8152  alterSession(session_ptr->get_session_id(),
8153  _return,
8154  executor.getSessionParameter(),
8155  execution_time_ms);
8156  } else if (executor.isAlterSystemControlExecutorQueue()) {
8157  _return = ExecutionResult();
8158  if (executor.returnQueueAction() == "PAUSE") {
8159  execution_time_ms = measure<>::execution(
8160  [&]() { pause_executor_queue(session_ptr->get_session_id()); });
8161  } else if (executor.returnQueueAction() == "RESUME") {
8162  execution_time_ms = measure<>::execution(
8163  [&]() { resume_executor_queue(session_ptr->get_session_id()); });
8164  } else {
8165  throw std::runtime_error("Unknwon queue command.");
8166  }
8167  } else {
8168  execution_time_ms =
8169  measure<>::execution([&]() { _return = executor.execute(read_only_); });
8170  }
8171  _return.setExecutionTime(execution_time_ms);
8172  }
8173  if (_return.getResultType() == ExecutionResult::QueryResult) {
8174  // ResultType defaults to QueryResult => which can limit
8175  // the number of lines output via ConvertRow... use CalciteDdl instead
8177  }
8178 }
RType getResultType() const
void resume_executor_queue(const TSessionId &session)
Definition: DBHandler.cpp:2964
const std::string getTargetQuerySessionToKill() const
const std::string commandStr() const
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
std::pair< std::string, std::string > getSessionParameter() const
bool isAlterSystemClear() const
bool isShowUserSessions() const
void alterSession(const std::string &sesson_id, ExecutionResult &result, const std::pair< std::string, std::string > &session_parameter, int64_t &execution_time_ms)
Definition: DBHandler.cpp:8045
void setResultType(RType type)
bool isAlterSessionSet() const
ExecutionResult execute(bool read_only_mode)
std::string returnQueueAction() const
void pause_executor_queue(const TSessionId &session)
Definition: DBHandler.cpp:2950
const bool read_only_
Definition: DBHandler.h:645
std::string returnCacheType() const
void setExecutionTime(int64_t execution_time_ms)
void interruptQuery(const Catalog_Namespace::SessionInfo &session_info, const std::string &target_session)
Definition: DBHandler.cpp:7940
bool isAlterSystemControlExecutorQueue() const
ExecutionResult getQueries(std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
Definition: DBHandler.cpp:7813
ExecutionResult getUserSessions(std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
Definition: DBHandler.cpp:7775
void alterSystemClear(const std::string &sesson_id, ExecutionResult &result, const std::string &cache_type, int64_t &execution_time_ms)
Definition: DBHandler.cpp:8029

+ Here is the call graph for this function:

std::unordered_map< std::string, std::unordered_set< std::string > > DBHandler::fill_column_names_by_table ( std::vector< std::string > &  table_names,
query_state::StdLog stdlog 
)
private

Definition at line 1770 of file DBHandler.cpp.

References get_table_details_impl().

Referenced by get_token_based_completions().

1771  {
1772  std::unordered_map<std::string, std::unordered_set<std::string>> column_names_by_table;
1773  for (auto it = table_names.begin(); it != table_names.end();) {
1774  TTableDetails table_details;
1775  try {
1776  get_table_details_impl(table_details, stdlog, *it, false, false);
1777  } catch (const TDBException& e) {
1778  // Remove the corrupted Table/View name from the list for further processing.
1779  it = table_names.erase(it);
1780  continue;
1781  }
1782  for (const auto& column_type : table_details.row_desc) {
1783  column_names_by_table[*it].emplace(column_type.col_name);
1784  }
1785  ++it;
1786  }
1787  return column_names_by_table;
1788 }
void get_table_details_impl(TTableDetails &_return, query_state::StdLog &stdlog, const std::string &table_name, const bool get_system, const bool get_physical, const std::string &database_name={})
Definition: DBHandler.cpp:2529

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::fillGeoColumns ( const TSessionId &  session,
const Catalog_Namespace::Catalog catalog,
std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &  import_buffers,
const ColumnDescriptor cd,
size_t &  col_idx,
size_t  num_rows,
const std::string &  table_name 
)
private

Definition at line 3237 of file DBHandler.cpp.

References ColumnDescriptor::columnName, ColumnDescriptor::columnType, Geospatial::GeoTypesFactory::getGeoColumns(), import_export::Importer::set_geo_physical_import_buffer_columnar(), and THROW_DB_EXCEPTION.

Referenced by fillMissingBuffers(), load_table(), load_table_binary_arrow(), and load_table_binary_columnar().

3244  {
3245  auto geo_col_idx = col_idx - 1;
3246  const auto wkt_or_wkb_hex_column = import_buffers[geo_col_idx]->getGeoStringBuffer();
3247  std::vector<std::vector<double>> coords_column, bounds_column;
3248  std::vector<std::vector<int>> ring_sizes_column, poly_rings_column;
3249  SQLTypeInfo ti = cd->columnType;
3250  const bool validate_with_geos_if_available = false;
3251  if (num_rows != wkt_or_wkb_hex_column->size() ||
3252  !Geospatial::GeoTypesFactory::getGeoColumns(wkt_or_wkb_hex_column,
3253  ti,
3254  coords_column,
3255  bounds_column,
3256  ring_sizes_column,
3257  poly_rings_column,
3258  validate_with_geos_if_available)) {
3259  std::ostringstream oss;
3260  oss << "Invalid geometry in column " << cd->columnName;
3261  THROW_DB_EXCEPTION(oss.str());
3262  }
3263 
3264  // Populate physical columns, advance col_idx
3266  cd,
3267  import_buffers,
3268  col_idx,
3269  coords_column,
3270  bounds_column,
3271  ring_sizes_column,
3272  poly_rings_column);
3273 }
static void set_geo_physical_import_buffer_columnar(const Catalog_Namespace::Catalog &catalog, const ColumnDescriptor *cd, std::vector< std::unique_ptr< TypedImportBuffer >> &import_buffers, size_t &col_idx, std::vector< std::vector< double >> &coords_column, std::vector< std::vector< double >> &bounds_column, std::vector< std::vector< int >> &ring_sizes_column, std::vector< std::vector< int >> &poly_rings_column)
Definition: Importer.cpp:1731
static bool getGeoColumns(const std::string &wkt_or_wkb_hex, SQLTypeInfo &ti, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, const bool validate_with_geos_if_available)
Definition: Types.cpp:1121
SQLTypeInfo columnType
std::string columnName
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::fillMissingBuffers ( const TSessionId &  session,
const Catalog_Namespace::Catalog catalog,
std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &  import_buffers,
const std::list< const ColumnDescriptor * > &  cds,
const std::vector< int > &  desc_id_to_column_id,
size_t  num_rows,
const std::string &  table_name 
)
private

Definition at line 3275 of file DBHandler.cpp.

References CHECK, and fillGeoColumns().

Referenced by load_table(), load_table_binary(), load_table_binary_arrow(), and load_table_binary_columnar().

3282  {
3283  size_t skip_physical_cols = 0;
3284  size_t col_idx = 0, import_idx = 0;
3285  for (const auto& cd : cds) {
3286  if (skip_physical_cols > 0) {
3287  CHECK(cd->isGeoPhyCol);
3288  skip_physical_cols--;
3289  continue;
3290  } else if (cd->columnType.is_geometry()) {
3291  skip_physical_cols = cd->columnType.get_physical_cols();
3292  }
3293  if (desc_id_to_column_id[import_idx] == -1) {
3294  import_buffers[col_idx]->addDefaultValues(cd, num_rows);
3295  col_idx++;
3296  if (cd->columnType.is_geometry()) {
3298  session_id, catalog, import_buffers, cd, col_idx, num_rows, table_name);
3299  }
3300  } else {
3301  col_idx++;
3302  col_idx += skip_physical_cols;
3303  }
3304  import_idx++;
3305  }
3306 }
void fillGeoColumns(const TSessionId &session, const Catalog_Namespace::Catalog &catalog, std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &import_buffers, const ColumnDescriptor *cd, size_t &col_idx, size_t num_rows, const std::string &table_name)
Definition: DBHandler.cpp:3237
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

Catalog_Namespace::SessionInfoPtr DBHandler::findCalciteSession ( TSessionId const &  session_id) const
private

Definition at line 6112 of file DBHandler.cpp.

References calcite_sessions_, and calcite_sessions_mtx_.

Referenced by get_session_ptr().

6113  {
6115  auto const itr = calcite_sessions_.find(session_id);
6116  return itr == calcite_sessions_.end() ? nullptr : itr->second;
6117 }
std::lock_guard< T > lock_guard
std::unordered_map< std::string, Catalog_Namespace::SessionInfoPtr > calcite_sessions_
Definition: DBHandler.h:951
heavyai::shared_mutex calcite_sessions_mtx_
Definition: DBHandler.h:952

+ Here is the caller graph for this function:

void DBHandler::get_all_effective_roles_for_user ( std::vector< std::string > &  _return,
const TSessionId &  session,
const std::string &  granteeName 
)
override

Definition at line 2293 of file DBHandler.cpp.

References get_session_ptr(), getAllRolesForUserImpl(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, and STDLOG.

2295  {
2296  heavyai::RequestInfo const request_info(session_id_or_json);
2297  SET_REQUEST_ID(request_info.requestId());
2298  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
2299  auto session_ptr = stdlog.getConstSessionInfo();
2300  getAllRolesForUserImpl(session_ptr, roles, granteeName, /*effective=*/true);
2301 }
void getAllRolesForUserImpl(std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr, std::vector< std::string > &roles, const std::string &granteeName, bool effective)
Definition: DBHandler.cpp:2248
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

void DBHandler::get_all_files_in_archive ( std::vector< std::string > &  _return,
const TSessionId &  session,
const std::string &  archive_path,
const TCopyParams &  copy_params 
)
override

Definition at line 5968 of file DBHandler.cpp.

References anonymous_namespace{DBHandler.cpp}::add_vsi_archive_prefix(), anonymous_namespace{DBHandler.cpp}::add_vsi_network_prefix(), import_export::Importer::gdalFileExists(), import_export::Importer::gdalGetAllFilesInArchive(), get_session_ptr(), getConnectionInfo(), import_path_, anonymous_namespace{DBHandler.cpp}::is_a_supported_archive_file(), anonymous_namespace{DBHandler.cpp}::path_is_relative(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, thrift_to_copyparams(), THROW_DB_EXCEPTION, toString(), and anonymous_namespace{DBHandler.cpp}::validate_import_file_path_if_local().

5971  {
5972  heavyai::RequestInfo const request_info(session_id_or_json);
5973  SET_REQUEST_ID(request_info.requestId());
5974  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()),
5975  "get_all_files_in_archive",
5976  archive_path_in);
5977  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
5978 
5979  std::string archive_path(archive_path_in);
5980  if (path_is_relative(archive_path)) {
5981  // assume relative paths are relative to data_path / import / <session>
5982  auto file_path = import_path_ /
5983  picosha2::hash256_hex_string(request_info.sessionId()) /
5984  boost::filesystem::path(archive_path).filename();
5985  archive_path = file_path.string();
5986  }
5987  validate_import_file_path_if_local(archive_path);
5988 
5989  if (is_a_supported_archive_file(archive_path)) {
5990  // find the archive file
5991  add_vsi_network_prefix(archive_path);
5992  if (!import_export::Importer::gdalFileExists(archive_path,
5993  thrift_to_copyparams(copy_params))) {
5994  THROW_DB_EXCEPTION("Archive does not exist: " + archive_path_in);
5995  }
5996  // find all files in archive
5997  add_vsi_archive_prefix(archive_path);
5999  archive_path, thrift_to_copyparams(copy_params));
6000  // prepend them all with original path
6001  for (auto& s : _return) {
6002  s = archive_path_in + '/' + s;
6003  }
6004  }
6005 }
boost::filesystem::path import_path_
Definition: DBHandler.h:638
void add_vsi_archive_prefix(std::string &path)
Definition: DBHandler.cpp:4212
bool is_a_supported_archive_file(const std::string &path)
Definition: DBHandler.cpp:4278
bool path_is_relative(const std::string &path)
Definition: DBHandler.cpp:4246
static bool gdalFileExists(const std::string &path, const CopyParams &copy_params)
Definition: Importer.cpp:5053
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
void validate_import_file_path_if_local(const std::string &file_path)
Definition: DBHandler.cpp:4332
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1703
void add_vsi_network_prefix(std::string &path)
Definition: DBHandler.cpp:4183
static std::vector< std::string > gdalGetAllFilesInArchive(const std::string &archive_path, const CopyParams &copy_params)
Definition: Importer.cpp:5130
import_export::CopyParams thrift_to_copyparams(const TCopyParams &cp)
Definition: DBHandler.cpp:3854
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1790
#define STDLOG(...)
Definition: QueryState.h:234
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the call graph for this function:

void DBHandler::get_all_roles_for_user ( std::vector< std::string > &  _return,
const TSessionId &  session,
const std::string &  granteeName 
)
override

Definition at line 2281 of file DBHandler.cpp.

References get_session_ptr(), getAllRolesForUserImpl(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, and STDLOG.

2283  {
2284  // WARNING: This function only returns directly granted roles.
2285  // See also: get_all_effective_roles_for_user() for all of a user's roles.
2286  heavyai::RequestInfo const request_info(session_id_or_json);
2287  SET_REQUEST_ID(request_info.requestId());
2288  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
2289  auto session_ptr = stdlog.getConstSessionInfo();
2290  getAllRolesForUserImpl(session_ptr, roles, granteeName, /*effective=*/false);
2291 }
void getAllRolesForUserImpl(std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr, std::vector< std::string > &roles, const std::string &granteeName, bool effective)
Definition: DBHandler.cpp:2248
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

void DBHandler::get_completion_hints ( std::vector< TCompletionHint > &  hints,
const TSessionId &  session,
const std::string &  sql,
const int  cursor 
)
override

Definition at line 1661 of file DBHandler.cpp.

References anonymous_namespace{DBHandler.cpp}::extract_projection_tokens_for_completion(), get_completion_hints_unsorted(), get_session_ptr(), get_uc_compatible_table_names_by_column(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, gpu_enabled::sort(), STDLOG, and to_upper().

1664  {
1665  heavyai::RequestInfo const request_info(session_id_or_json);
1666  SET_REQUEST_ID(request_info.requestId());
1667  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
1668  std::vector<std::string> visible_tables; // Tables allowed for the given session.
1669  get_completion_hints_unsorted(hints, visible_tables, stdlog, sql, cursor);
1670  const auto proj_tokens = extract_projection_tokens_for_completion(sql);
1671  auto compatible_table_names = get_uc_compatible_table_names_by_column(
1672  proj_tokens.uc_column_names, visible_tables, stdlog);
1673  // Add the table qualifiers explicitly specified by the user.
1674  compatible_table_names.insert(proj_tokens.uc_column_table_qualifiers.begin(),
1675  proj_tokens.uc_column_table_qualifiers.end());
1676  // Sort the hints by category, from COLUMN (most specific) to KEYWORD.
1677  std::sort(
1678  hints.begin(),
1679  hints.end(),
1680  [&compatible_table_names](const TCompletionHint& lhs, const TCompletionHint& rhs) {
1681  if (lhs.type == TCompletionHintType::TABLE &&
1682  rhs.type == TCompletionHintType::TABLE) {
1683  // Between two tables, one which is compatible with the specified
1684  // projections and one which isn't, pick the one which is compatible.
1685  if (compatible_table_names.find(to_upper(lhs.hints.back())) !=
1686  compatible_table_names.end() &&
1687  compatible_table_names.find(to_upper(rhs.hints.back())) ==
1688  compatible_table_names.end()) {
1689  return true;
1690  }
1691  }
1692  return lhs.type < rhs.type;
1693  });
1694 }
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
ProjectionTokensForCompletion extract_projection_tokens_for_completion(const std::string &sql)
Definition: DBHandler.cpp:1637
void get_completion_hints_unsorted(std::vector< TCompletionHint > &hints, std::vector< std::string > &visible_tables, query_state::StdLog &stdlog, const std::string &sql, const int cursor)
Definition: DBHandler.cpp:1696
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
std::string to_upper(const std::string &str)
std::unordered_set< std::string > get_uc_compatible_table_names_by_column(const std::unordered_set< std::string > &uc_column_names, std::vector< std::string > &table_names, query_state::StdLog &stdlog)
Definition: DBHandler.cpp:1795
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

void DBHandler::get_completion_hints_unsorted ( std::vector< TCompletionHint > &  hints,
std::vector< std::string > &  visible_tables,
query_state::StdLog stdlog,
const std::string &  sql,
const int  cursor 
)
private

Definition at line 1696 of file DBHandler.cpp.

References calcite_, logger::ERROR, GET_PHYSICAL_TABLES_AND_VIEWS, get_tables_impl(), get_token_based_completions(), query_state::StdLog::getConstSessionInfo(), just_whitelisted_keyword_hints(), and LOG.

Referenced by get_completion_hints().

1700  {
1701  const auto& session_info = *stdlog.getConstSessionInfo();
1702  try {
1703  get_tables_impl(visible_tables, session_info, GET_PHYSICAL_TABLES_AND_VIEWS);
1704 
1705  // Filter out keywords suggested by Calcite which we don't support.
1707  calcite_->getCompletionHints(session_info, visible_tables, sql, cursor));
1708  } catch (const std::exception& e) {
1709  TDBException ex;
1710  ex.error_msg = std::string(e.what());
1711  LOG(ERROR) << ex.error_msg;
1712  throw ex;
1713  }
1714  boost::regex from_expr{R"(\s+from\s+)", boost::regex::extended | boost::regex::icase};
1715  const size_t length_to_cursor =
1716  cursor < 0 ? sql.size() : std::min(sql.size(), static_cast<size_t>(cursor));
1717  // Trust hints from Calcite after the FROM keyword.
1718  if (boost::regex_search(sql.cbegin(), sql.cbegin() + length_to_cursor, from_expr)) {
1719  return;
1720  }
1721  // Before FROM, the query is too incomplete for context-sensitive completions.
1722  get_token_based_completions(hints, stdlog, visible_tables, sql, cursor);
1723 }
#define LOG(tag)
Definition: Logger.h:285
std::vector< TCompletionHint > just_whitelisted_keyword_hints(const std::vector< TCompletionHint > &hints)
void get_tables_impl(std::vector< std::string > &table_names, const Catalog_Namespace::SessionInfo &, const GetTablesType get_tables_type, const std::string &database_name={})
Definition: DBHandler.cpp:2675
void get_token_based_completions(std::vector< TCompletionHint > &hints, query_state::StdLog &stdlog, std::vector< std::string > &visible_tables, const std::string &sql, const int cursor)
Definition: DBHandler.cpp:1725
std::shared_ptr< Calcite > calcite_
Definition: DBHandler.h:657
std::shared_ptr< Catalog_Namespace::SessionInfo const > getConstSessionInfo() const
Definition: QueryState.cpp:150

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::get_custom_expressions ( std::vector< TCustomExpression > &  _return,
const TSessionId &  session 
)
override

Definition at line 4643 of file DBHandler.cpp.

References anonymous_namespace{DBHandler.cpp}::create_thrift_obj_from_custom_expr(), custom_expressions_mutex_, get_session_ptr(), getConnectionInfo(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, and toString().

4644  {
4645  heavyai::RequestInfo const request_info(session_id_or_json);
4646  SET_REQUEST_ID(request_info.requestId());
4647  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
4648  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4649 
4650  auto session_ptr = stdlog.getConstSessionInfo();
4651  auto& catalog = session_ptr->getCatalog();
4653  auto custom_expressions =
4654  catalog.getCustomExpressionsForUser(session_ptr->get_currentUser());
4655  for (const auto& custom_expression : custom_expressions) {
4656  _return.emplace_back(create_thrift_obj_from_custom_expr(*custom_expression, catalog));
4657  }
4658 }
std::shared_lock< T > shared_lock
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1703
heavyai::shared_mutex custom_expressions_mutex_
Definition: DBHandler.h:1073
TCustomExpression create_thrift_obj_from_custom_expr(const CustomExpression &custom_expr, const Catalog &catalog)
Definition: DBHandler.cpp:4601
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1790
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

void DBHandler::get_dashboard ( TDashboard &  _return,
const TSessionId &  session,
const int32_t  dashboard_id 
)
override

Definition at line 4698 of file DBHandler.cpp.

References cat(), get_dashboard_impl(), get_session_ptr(), getConnectionInfo(), lockmgr::instance(), is_allowed_on_dashboard(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, THROW_DB_EXCEPTION, to_string(), toString(), Catalog_Namespace::UserMetadata::userName, and AccessPrivileges::VIEW_DASHBOARD.

4700  {
4701  heavyai::RequestInfo const request_info(session_id_or_json);
4702  SET_REQUEST_ID(request_info.requestId());
4703  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
4704  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4705  auto session_ptr = stdlog.getConstSessionInfo();
4706  auto const& cat = session_ptr->getCatalog();
4708  auto dash = cat.getMetadataForDashboard(dashboard_id);
4709  if (!dash) {
4710  THROW_DB_EXCEPTION("Dashboard with dashboard id " + std::to_string(dashboard_id) +
4711  " doesn't exist");
4712  }
4714  *session_ptr, dash->dashboardId, AccessPrivileges::VIEW_DASHBOARD)) {
4715  THROW_DB_EXCEPTION("User has no view privileges for the dashboard with id " +
4716  std::to_string(dashboard_id));
4717  }
4718  user_meta.userName = "";
4719  SysCatalog::instance().getMetadataForUserById(dash->userId, user_meta);
4720  dashboard = get_dashboard_impl(session_ptr, user_meta, dash);
4721 }
static const AccessPrivileges VIEW_DASHBOARD
Definition: DBObject.h:171
std::string cat(Ts &&...args)
std::string to_string(char const *&&v)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
TDashboard get_dashboard_impl(const std::shared_ptr< Catalog_Namespace::SessionInfo const > &session_ptr, Catalog_Namespace::UserMetadata &user_meta, const DashboardDescriptor *dash, const bool populate_state=true)
Definition: DBHandler.cpp:4745
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1703
T & instance()
Definition: LockMgr.cpp:101
static bool is_allowed_on_dashboard(const Catalog_Namespace::SessionInfo &session_info, int32_t dashboard_id, AccessPrivileges requestedPermissions)
Definition: DBHandler.cpp:4565
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1790
#define STDLOG(...)
Definition: QueryState.h:234
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the call graph for this function:

void DBHandler::get_dashboard_grantees ( std::vector< TDashboardGrantees > &  _return,
const TSessionId &  session,
const int32_t  dashboard_id 
)
override

Definition at line 5090 of file DBHandler.cpp.

References cat(), DashboardPrivileges::CREATE_DASHBOARD, DashboardDBObjectType, DashboardPrivileges::DELETE_DASHBOARD, DashboardPrivileges::EDIT_DASHBOARD, get_session_ptr(), getConnectionInfo(), lockmgr::instance(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, THROW_DB_EXCEPTION, to_string(), toString(), Catalog_Namespace::UserMetadata::userId, Catalog_Namespace::UserMetadata::userName, and DashboardPrivileges::VIEW_DASHBOARD.

5093  {
5094  heavyai::RequestInfo const request_info(session_id_or_json);
5095  SET_REQUEST_ID(request_info.requestId());
5096  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
5097  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
5098  auto session_ptr = stdlog.getConstSessionInfo();
5099  auto const& cat = session_ptr->getCatalog();
5101  auto dash = cat.getMetadataForDashboard(dashboard_id);
5102  if (!dash) {
5103  THROW_DB_EXCEPTION("Dashboard id " + std::to_string(dashboard_id) +
5104  " does not exist");
5105  } else if (session_ptr->get_currentUser().userId != dash->userId &&
5106  !session_ptr->get_currentUser().isSuper) {
5108  "User should be either owner of dashboard or super user to access grantees");
5109  }
5110  std::vector<ObjectRoleDescriptor*> objectsList;
5111  objectsList = SysCatalog::instance().getMetadataForObject(
5112  cat.getCurrentDB().dbId,
5113  static_cast<int>(DBObjectType::DashboardDBObjectType),
5114  dashboard_id); // By default objecttypecan be only dashabaords
5115  user_meta.userId = -1;
5116  user_meta.userName = "";
5117  SysCatalog::instance().getMetadataForUserById(dash->userId, user_meta);
5118  for (auto object : objectsList) {
5119  if (user_meta.userName == object->roleName) {
5120  // Mask owner
5121  continue;
5122  }
5123  TDashboardGrantees grantee;
5124  TDashboardPermissions perm;
5125  grantee.name = object->roleName;
5126  grantee.is_user = object->roleType;
5127  perm.create_ = object->privs.hasPermission(DashboardPrivileges::CREATE_DASHBOARD);
5128  perm.delete_ = object->privs.hasPermission(DashboardPrivileges::DELETE_DASHBOARD);
5129  perm.edit_ = object->privs.hasPermission(DashboardPrivileges::EDIT_DASHBOARD);
5130  perm.view_ = object->privs.hasPermission(DashboardPrivileges::VIEW_DASHBOARD);
5131  grantee.permissions = perm;
5132  dashboard_grantees.push_back(grantee);
5133  }
5134 }
std::string cat(Ts &&...args)
std::string to_string(char const *&&v)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1703
static const int32_t EDIT_DASHBOARD
Definition: DBObject.h:104
static const int32_t DELETE_DASHBOARD
Definition: DBObject.h:102
T & instance()
Definition: LockMgr.cpp:101
static const int32_t VIEW_DASHBOARD
Definition: DBObject.h:103
static const int32_t CREATE_DASHBOARD
Definition: DBObject.h:101
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1790
#define STDLOG(...)
Definition: QueryState.h:234
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the call graph for this function:

TDashboard DBHandler::get_dashboard_impl ( const std::shared_ptr< Catalog_Namespace::SessionInfo const > &  session_ptr,
Catalog_Namespace::UserMetadata user_meta,
const DashboardDescriptor dash,
const bool  populate_state = true 
)
private

Definition at line 4745 of file DBHandler.cpp.

References cat(), DashboardPrivileges::CREATE_DASHBOARD, DashboardDBObjectType, DashboardDescriptor::dashboardId, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, DashboardPrivileges::DELETE_DASHBOARD, DashboardPrivileges::EDIT_DASHBOARD, DBObject::getPrivileges(), AccessPrivileges::hasPermission(), DashboardDescriptor::imageHash, lockmgr::instance(), DashboardDescriptor::updateTime, DashboardDescriptor::user, DashboardDescriptor::userId, Catalog_Namespace::UserMetadata::userName, and DashboardPrivileges::VIEW_DASHBOARD.

Referenced by get_dashboard(), and get_dashboards().

4749  {
4750  auto const& cat = session_ptr->getCatalog();
4751  SysCatalog::instance().getMetadataForUserById(dash->userId, user_meta);
4752  auto objects_list = SysCatalog::instance().getMetadataForObject(
4753  cat.getCurrentDB().dbId,
4754  static_cast<int>(DBObjectType::DashboardDBObjectType),
4755  dash->dashboardId);
4756  TDashboard dashboard;
4757  dashboard.dashboard_name = dash->dashboardName;
4758  if (populate_state) {
4759  dashboard.dashboard_state = dash->dashboardState;
4760  }
4761  dashboard.image_hash = dash->imageHash;
4762  dashboard.update_time = dash->updateTime;
4763  dashboard.dashboard_metadata = dash->dashboardMetadata;
4764  dashboard.dashboard_id = dash->dashboardId;
4765  dashboard.dashboard_owner = dash->user;
4766  TDashboardPermissions perms;
4767  // Super user has all permissions.
4768  if (session_ptr->get_currentUser().isSuper) {
4769  perms.create_ = true;
4770  perms.delete_ = true;
4771  perms.edit_ = true;
4772  perms.view_ = true;
4773  } else {
4774  // Collect all grants on current user
4775  // add them to the permissions.
4776  auto obj_to_find =
4777  DBObject(dashboard.dashboard_id, DBObjectType::DashboardDBObjectType);
4778  obj_to_find.loadKey(cat);
4779  std::vector<std::string> grantees =
4780  SysCatalog::instance().getRoles(true,
4781  session_ptr->get_currentUser().isSuper,
4782  session_ptr->get_currentUser().userName);
4783  for (const auto& grantee : grantees) {
4784  DBObject* object_found;
4785  auto* gr = SysCatalog::instance().getGrantee(grantee);
4786  if (gr && (object_found = gr->findDbObject(obj_to_find.getObjectKey(), true))) {
4787  const auto obj_privs = object_found->getPrivileges();
4788  perms.create_ |= obj_privs.hasPermission(DashboardPrivileges::CREATE_DASHBOARD);
4789  perms.delete_ |= obj_privs.hasPermission(DashboardPrivileges::DELETE_DASHBOARD);
4790  perms.edit_ |= obj_privs.hasPermission(DashboardPrivileges::EDIT_DASHBOARD);
4791  perms.view_ |= obj_privs.hasPermission(DashboardPrivileges::VIEW_DASHBOARD);
4792  }
4793  }
4794  }
4795  dashboard.dashboard_permissions = perms;
4796  if (objects_list.empty() ||
4797  (objects_list.size() == 1 && objects_list[0]->roleName == user_meta.userName)) {
4798  dashboard.is_dash_shared = false;
4799  } else {
4800  dashboard.is_dash_shared = true;
4801  }
4802  return dashboard;
4803 }
std::string cat(Ts &&...args)
static const int32_t EDIT_DASHBOARD
Definition: DBObject.h:104
static const int32_t DELETE_DASHBOARD
Definition: DBObject.h:102
const AccessPrivileges & getPrivileges() const
Definition: DBObject.h:226
T & instance()
Definition: LockMgr.cpp:101
static const int32_t VIEW_DASHBOARD
Definition: DBObject.h:103
static const int32_t CREATE_DASHBOARD
Definition: DBObject.h:101
bool hasPermission(int permission) const
Definition: DBObject.h:141

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::get_dashboards ( std::vector< TDashboard > &  _return,
const TSessionId &  session 
)
override

Definition at line 4723 of file DBHandler.cpp.

References cat(), get_dashboard_impl(), get_session_ptr(), getConnectionInfo(), is_allowed_on_dashboard(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, toString(), Catalog_Namespace::UserMetadata::userName, and AccessPrivileges::VIEW_DASHBOARD.

4724  {
4725  heavyai::RequestInfo const request_info(session_id_or_json);
4726  SET_REQUEST_ID(request_info.requestId());
4727  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
4728  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4729  auto session_ptr = stdlog.getConstSessionInfo();
4730  auto const& cat = session_ptr->getCatalog();
4732  const auto dashes = cat.getAllDashboardsMetadata();
4733  user_meta.userName = "";
4734  for (const auto dash : dashes) {
4736  *session_ptr, dash->dashboardId, AccessPrivileges::VIEW_DASHBOARD)) {
4737  // dashboardState is intentionally not populated here
4738  // for payload reasons
4739  // use get_dashboard call to get state
4740  dashboards.push_back(get_dashboard_impl(session_ptr, user_meta, dash, false));
4741  }
4742  }
4743 }
static const AccessPrivileges VIEW_DASHBOARD
Definition: DBObject.h:171
std::string cat(Ts &&...args)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
TDashboard get_dashboard_impl(const std::shared_ptr< Catalog_Namespace::SessionInfo const > &session_ptr, Catalog_Namespace::UserMetadata &user_meta, const DashboardDescriptor *dash, const bool populate_state=true)
Definition: DBHandler.cpp:4745
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1703
static bool is_allowed_on_dashboard(const Catalog_Namespace::SessionInfo &session_info, int32_t dashboard_id, AccessPrivileges requestedPermissions)
Definition: DBHandler.cpp:4565
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1790
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

void DBHandler::get_databases ( std::vector< TDBInfo > &  _return,
const TSessionId &  session 
)
override

Definition at line 3056 of file DBHandler.cpp.

References get_session_ptr(), getConnectionInfo(), lockmgr::instance(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, and toString().

3057  {
3058  heavyai::RequestInfo const request_info(session_id_or_json);
3059  SET_REQUEST_ID(request_info.requestId());
3060  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
3061  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
3062  auto session_ptr = stdlog.getConstSessionInfo();
3063  const auto& user = session_ptr->get_currentUser();
3065  SysCatalog::instance().getDatabaseListForUser(user);
3066  for (auto& db : dbs) {
3067  TDBInfo dbinfo;
3068  dbinfo.db_name = std::move(db.dbName);
3069  dbinfo.db_owner = std::move(db.dbOwnerName);
3070  dbinfos.push_back(std::move(dbinfo));
3071  }
3072 }
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1703
T & instance()
Definition: LockMgr.cpp:101
std::list< DBSummary > DBSummaryList
Definition: SysCatalog.h:145
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1790
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

void DBHandler::get_db_object_privs ( std::vector< TDBObject > &  _return,
const TSessionId &  session,
const std::string &  objectName,
const TDBObjectType::type  type 
)
override

Definition at line 2162 of file DBHandler.cpp.

References AccessPrivileges::ALL_TABLE, cat(), DashboardDBObjectType, DatabaseDBObjectType, get_session_ptr(), DBObject::getObjectKey(), lockmgr::instance(), DBObject::loadKey(), heavyai::RequestInfo::requestId(), serialize_db_object(), ServerDBObjectType, heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, TableDBObjectType, THROW_DB_EXCEPTION, to_string(), and ViewDBObjectType.

2165  {
2166  heavyai::RequestInfo const request_info(session_id_or_json);
2167  SET_REQUEST_ID(request_info.requestId());
2168  auto session_ptr = get_session_ptr(request_info.sessionId());
2169  auto stdlog = STDLOG(session_ptr);
2170  const auto& cat = session_ptr->getCatalog();
2171  DBObjectType object_type;
2172  switch (type) {
2174  object_type = DBObjectType::DatabaseDBObjectType;
2175  break;
2177  object_type = DBObjectType::TableDBObjectType;
2178  break;
2181  break;
2183  object_type = DBObjectType::ViewDBObjectType;
2184  break;
2186  object_type = DBObjectType::ServerDBObjectType;
2187  break;
2188  default:
2189  THROW_DB_EXCEPTION("Failed to get object privileges for " + objectName +
2190  ": unknown object type (" + std::to_string(type) + ").");
2191  }
2192  DBObject object_to_find(objectName, object_type);
2193 
2194  // TODO(adb): Use DatabaseLock to protect method
2195  try {
2196  if (object_type == DashboardDBObjectType) {
2197  if (objectName == "") {
2198  object_to_find = DBObject(-1, object_type);
2199  } else {
2200  object_to_find = DBObject(std::stoi(objectName), object_type);
2201  }
2202  } else if ((object_type == TableDBObjectType || object_type == ViewDBObjectType) &&
2203  !objectName.empty()) {
2204  // special handling for view / table
2205  auto td = cat.getMetadataForTable(objectName, false);
2206  if (td) {
2207  object_type = td->isView ? ViewDBObjectType : TableDBObjectType;
2208  object_to_find = DBObject(objectName, object_type);
2209  }
2210  }
2211  object_to_find.loadKey(cat);
2212  } catch (const std::exception&) {
2213  THROW_DB_EXCEPTION("Object with name " + objectName + " does not exist.");
2214  }
2215 
2216  // object type on database level
2217  DBObject object_to_find_dblevel("", object_type);
2218  object_to_find_dblevel.loadKey(cat);
2219  // if user is superuser respond with a full priv
2220  if (session_ptr->get_currentUser().isSuper) {
2221  // using ALL_TABLE here to set max permissions
2222  DBObject dbObj{object_to_find.getObjectKey(),
2224  session_ptr->get_currentUser().userId};
2225  dbObj.setName("super");
2226  TDBObjects.push_back(
2227  serialize_db_object(session_ptr->get_currentUser().userName, dbObj));
2228  };
2229 
2230  std::vector<std::string> grantees =
2231  SysCatalog::instance().getRoles(true,
2232  session_ptr->get_currentUser().isSuper,
2233  session_ptr->get_currentUser().userName);
2234  for (const auto& grantee : grantees) {
2235  DBObject* object_found;
2236  auto* gr = SysCatalog::instance().getGrantee(grantee);
2237  if (gr && (object_found = gr->findDbObject(object_to_find.getObjectKey(), true))) {
2238  TDBObjects.push_back(serialize_db_object(grantee, *object_found));
2239  }
2240  // check object permissions on Database level
2241  if (gr &&
2242  (object_found = gr->findDbObject(object_to_find_dblevel.getObjectKey(), true))) {
2243  TDBObjects.push_back(serialize_db_object(grantee, *object_found));
2244  }
2245  }
2246 }
std::string cat(Ts &&...args)
DBObjectKey getObjectKey() const
Definition: DBObject.h:221
DBObjectType
Definition: DBObject.h:40
std::string to_string(char const *&&v)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
static TDBObject serialize_db_object(const std::string &roleName, const DBObject &inObject)
Definition: DBHandler.cpp:1919
T & instance()
Definition: LockMgr.cpp:101
static const AccessPrivileges ALL_TABLE
Definition: DBObject.h:157
#define STDLOG(...)
Definition: QueryState.h:234
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the call graph for this function:

void DBHandler::get_db_objects_for_grantee ( std::vector< TDBObject > &  _return,
const TSessionId &  session,
const std::string &  roleName 
)
override

Definition at line 2133 of file DBHandler.cpp.

References get_session_ptr(), lockmgr::instance(), heavyai::RequestInfo::requestId(), serialize_db_object(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, and THROW_DB_EXCEPTION.

2135  {
2136  heavyai::RequestInfo const request_info(session_id_or_json);
2137  SET_REQUEST_ID(request_info.requestId());
2138  auto session_ptr = get_session_ptr(request_info.sessionId());
2139  auto stdlog = STDLOG(session_ptr);
2140  auto const& user = session_ptr->get_currentUser();
2141  if (!user.isSuper &&
2142  !SysCatalog::instance().isRoleGrantedToGrantee(user.userName, roleName, false)) {
2143  return;
2144  }
2145  auto* rl = SysCatalog::instance().getGrantee(roleName);
2146  if (rl) {
2147  auto dbId = session_ptr->getCatalog().getCurrentDB().dbId;
2148  for (auto& dbObject : *rl->getDbObjects(true)) {
2149  if (dbObject.first.dbId != dbId) {
2150  // TODO (max): it doesn't scale well in case we have many DBs (not a typical
2151  // usecase for now, though)
2152  continue;
2153  }
2154  TDBObject tdbObject = serialize_db_object(roleName, *dbObject.second);
2155  TDBObjectsForRole.push_back(tdbObject);
2156  }
2157  } else {
2158  THROW_DB_EXCEPTION("User or role " + roleName + " does not exist.");
2159  }
2160 }
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
static TDBObject serialize_db_object(const std::string &roleName, const DBObject &inObject)
Definition: DBHandler.cpp:1919
T & instance()
Definition: LockMgr.cpp:101
#define STDLOG(...)
Definition: QueryState.h:234
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the call graph for this function:

void DBHandler::get_device_parameters ( std::map< std::string, std::string > &  _return,
const TSessionId &  session 
)
override

Definition at line 7584 of file DBHandler.cpp.

References cpu_mode_only_, EXPOSE_THRIFT_MAP, get_session_ptr(), getConnectionInfo(), report::params, heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, and toString().

7585  {
7586  heavyai::RequestInfo const request_info(session_id_or_json);
7587  SET_REQUEST_ID(request_info.requestId());
7588  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
7589  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
7591  for (auto item : params) {
7592  _return.insert(item);
7593  }
7594  EXPOSE_THRIFT_MAP(TDeviceType);
7595  EXPOSE_THRIFT_MAP(TDatumType);
7596  EXPOSE_THRIFT_MAP(TEncodingType);
7597  EXPOSE_THRIFT_MAP(TExtArgumentType);
7598  EXPOSE_THRIFT_MAP(TOutputBufferSizeType);
7599 }
#define EXPOSE_THRIFT_MAP(TYPENAME)
Definition: DBHandler.cpp:7573
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1703
dictionary params
Definition: report.py:27
void get_device_parameters(std::map< std::string, std::string > &_return, const TSessionId &session) override
Definition: DBHandler.cpp:7584
bool cpu_mode_only_
Definition: DBHandler.h:647
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1790
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

void DBHandler::get_first_geo_file_in_archive ( std::string &  _return,
const TSessionId &  session,
const std::string &  archive_path,
const TCopyParams &  copy_params 
)
override

Definition at line 5921 of file DBHandler.cpp.

References anonymous_namespace{DBHandler.cpp}::add_vsi_archive_prefix(), anonymous_namespace{DBHandler.cpp}::add_vsi_network_prefix(), anonymous_namespace{DBHandler.cpp}::find_first_geo_file_in_archive(), import_export::Importer::gdalFileExists(), get_session_ptr(), getConnectionInfo(), import_path_, anonymous_namespace{DBHandler.cpp}::is_a_supported_archive_file(), anonymous_namespace{DBHandler.cpp}::path_is_relative(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, thrift_to_copyparams(), THROW_DB_EXCEPTION, toString(), and anonymous_namespace{DBHandler.cpp}::validate_import_file_path_if_local().

5924  {
5925  heavyai::RequestInfo const request_info(session_id_or_json);
5926  SET_REQUEST_ID(request_info.requestId());
5927  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()),
5928  "get_first_geo_file_in_archive",
5929  archive_path_in);
5930  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
5931 
5932  std::string archive_path(archive_path_in);
5933 
5934  if (path_is_relative(archive_path)) {
5935  // assume relative paths are relative to data_path / import / <session>
5936  auto file_path = import_path_ /
5937  picosha2::hash256_hex_string(request_info.sessionId()) /
5938  boost::filesystem::path(archive_path).filename();
5939  archive_path = file_path.string();
5940  }
5941  validate_import_file_path_if_local(archive_path);
5942 
5943  if (is_a_supported_archive_file(archive_path)) {
5944  // find the archive file
5945  add_vsi_network_prefix(archive_path);
5946  if (!import_export::Importer::gdalFileExists(archive_path,
5947  thrift_to_copyparams(copy_params))) {
5948  THROW_DB_EXCEPTION("Archive does not exist: " + archive_path_in);
5949  }
5950  // find geo file in archive
5951  add_vsi_archive_prefix(archive_path);
5952  std::string geo_file =
5953  find_first_geo_file_in_archive(archive_path, thrift_to_copyparams(copy_params));
5954  // what did we get?
5955  if (geo_file.size()) {
5956  // prepend it with the original path
5957  _return = archive_path_in + std::string("/") + geo_file;
5958  } else {
5959  // just return the original path
5960  _return = archive_path_in;
5961  }
5962  } else {
5963  // just return the original path
5964  _return = archive_path_in;
5965  }
5966 }
boost::filesystem::path import_path_
Definition: DBHandler.h:638
void add_vsi_archive_prefix(std::string &path)
Definition: DBHandler.cpp:4212
bool is_a_supported_archive_file(const std::string &path)
Definition: DBHandler.cpp:4278
bool path_is_relative(const std::string &path)
Definition: DBHandler.cpp:4246
static bool gdalFileExists(const std::string &path, const CopyParams &copy_params)
Definition: Importer.cpp:5053
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
void validate_import_file_path_if_local(const std::string &file_path)
Definition: DBHandler.cpp:4332
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1703
std::string find_first_geo_file_in_archive(const std::string &archive_path, const import_export::CopyParams &copy_params)
Definition: DBHandler.cpp:4291
void add_vsi_network_prefix(std::string &path)
Definition: DBHandler.cpp:4183
import_export::CopyParams thrift_to_copyparams(const TCopyParams &cp)
Definition: DBHandler.cpp:3854
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1790
#define STDLOG(...)
Definition: QueryState.h:234
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:139

+ Here is the call graph for this function:

void DBHandler::get_function_details ( std::vector< TUserDefinedFunction > &  _return,
const TSessionId &  session,
const std::vector< std::string > &  udf_names 
)
override

Definition at line 7704 of file DBHandler.cpp.

References ExtensionFunctionsWhitelist::get_ext_funcs(), and ThriftSerializers::to_thrift().

7706  {
7707  for (const std::string& udf_name : udf_names) {
7708  for (auto udf : ExtensionFunctionsWhitelist::get_ext_funcs(udf_name)) {
7709  _return.emplace_back(ThriftSerializers::to_thrift(udf));
7710  }
7711  }
7712 }
static std::vector< ExtensionFunction > get_ext_funcs(const std::string &name)
TExtArgumentType::type to_thrift(const ExtArgumentType &t)

+ Here is the call graph for this function:

void DBHandler::get_function_names ( std::vector< std::string > &  _return,
const TSessionId &  session 
)
override

Definition at line 7684 of file DBHandler.cpp.

References ExtensionFunctionsWhitelist::get_udfs_name().

7685  {
7686  for (auto udf_name :
7687  ExtensionFunctionsWhitelist::get_udfs_name(/* is_runtime */ false)) {
7688  if (std::find(_return.begin(), _return.end(), udf_name) == _return.end()) {
7689  _return.emplace_back(udf_name);
7690  }
7691  }
7692 }
static std::unordered_set< std::string > get_udfs_name(const bool is_runtime)

+ Here is the call graph for this function:

void DBHandler::get_hardware_info ( TClusterHardwareInfo &  _return,
const TSessionId &  session 
)
override

Definition at line 923 of file DBHandler.cpp.

References data_mgr_, get_session_ptr(), getConnectionInfo(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, and toString().

924  {
925  heavyai::RequestInfo const request_info(session_id_or_json);
926  SET_REQUEST_ID(request_info.requestId());
927  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
928  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
929  THardwareInfo ret;
930  const auto cuda_mgr = data_mgr_->getCudaMgr();
931  if (cuda_mgr) {
932  ret.num_gpu_hw = cuda_mgr->getDeviceCount();
933  ret.start_gpu = cuda_mgr->getStartGpu();
934  if (ret.start_gpu >= 0) {
935  ret.num_gpu_allocated = cuda_mgr->getDeviceCount() - cuda_mgr->getStartGpu();
936  // ^ This will break as soon as we allow non contiguous GPU allocations to MapD
937  }
938  for (int16_t device_id = 0; device_id < ret.num_gpu_hw; device_id++) {
939  TGpuSpecification gpu_spec;
940  auto deviceProperties = cuda_mgr->getDeviceProperties(device_id);
941  gpu_spec.num_sm = deviceProperties->numMPs;
942  gpu_spec.clock_frequency_kHz = deviceProperties->clockKhz;
943  gpu_spec.memory = deviceProperties->globalMem;
944  gpu_spec.compute_capability_major = deviceProperties->computeMajor;
945  gpu_spec.compute_capability_minor = deviceProperties->computeMinor;
946  ret.gpu_info.push_back(gpu_spec);
947  }
948  }
949 
950  // start hardware/OS dependent code
951  ret.num_cpu_hw = std::thread::hardware_concurrency();
952  // ^ This might return diffrent results in case of hyper threading
953  // end hardware/OS dependent code
954 
955  _return.hardware_info.push_back(ret);
956 }
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:6147
#define SET_REQUEST_ID(parent_request_id)
Definition: DBHandler.cpp:133
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1703
std::shared_ptr< Data_Namespace::DataMgr > data_mgr_
Definition: DBHandler.h:632
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1790
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

void DBHandler::get_heap_profile ( std::string &  _return,
const TSessionId &  session 
)
override

Definition at line 6119 of file DBHandler.cpp.

References get_session_ptr(), heavyai::RequestInfo::requestId(), heavyai::RequestInfo::sessionId(), SET_REQUEST_ID, STDLOG, and THROW_DB_EXCEPTION.

6120  {
6121  heavyai::RequestInfo const request_info(session_id_or_json);
6122  SET_REQUEST_ID(request_info.requestId());
6123  auto stdlog = STDLOG(get_session_ptr(request_info.sessionId()));
6124 #ifdef HAVE_PROFILER
6125  if (!IsHeapProfilerRunning()) {
6126  THROW_DB_EXCEPTION("Profiler not running");
6127  }
6128  auto profile_buff = GetHeapProfile();