OmniSciDB  cde582ebc3
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
DBHandler Class Reference

#include <DBHandler.h>

+ Inheritance diagram for DBHandler:
+ Collaboration diagram for DBHandler:

Classes

struct  DeferredCopyFromSessions
 
struct  DeferredCopyFromState
 

Public Member Functions

 DBHandler (const std::vector< LeafHostInfo > &db_leaves, const std::vector< LeafHostInfo > &string_leaves, const std::string &base_data_path, const bool allow_multifrag, const bool jit_debug, const bool intel_jit_profile, const bool read_only, const bool allow_loop_joins, const bool enable_rendering, const bool renderer_use_ppll_polys, const bool renderer_prefer_igpu, const unsigned renderer_vulkan_timeout_ms, const bool enable_auto_clear_render_mem, const int render_oom_retry_threshold, const size_t render_mem_bytes, const size_t max_concurrent_render_sessions, const size_t reserved_gpu_mem, const bool render_compositor_use_last_gpu, const size_t num_reader_threads, const AuthMetadata &authMetadata, SystemParameters &system_parameters, const bool legacy_syntax, const int idle_session_duration, const int max_session_duration, const std::string &udf_filename, const std::string &clang_path, const std::vector< std::string > &clang_options, const File_Namespace::DiskCacheConfig &disk_cache_config, const bool is_new_db)
 
void initialize (const bool is_new_db)
 
 ~DBHandler () override
 
void krb5_connect (TKrb5Session &session, const std::string &token, const std::string &dbname) override
 
void connect (TSessionId &session, const std::string &username, const std::string &passwd, const std::string &dbname) override
 
void disconnect (const TSessionId &session) override
 
void switch_database (const TSessionId &session, const std::string &dbname) override
 
void clone_session (TSessionId &session2, const TSessionId &session1) override
 
void get_server_status (TServerStatus &_return, const TSessionId &session) override
 
void get_status (std::vector< TServerStatus > &_return, const TSessionId &session) override
 
void get_hardware_info (TClusterHardwareInfo &_return, const TSessionId &session) override
 
bool hasTableAccessPrivileges (const TableDescriptor *td, const Catalog_Namespace::SessionInfo &session_info)
 
void get_tables (std::vector< std::string > &_return, const TSessionId &session) override
 
void get_tables_for_database (std::vector< std::string > &_return, const TSessionId &session, const std::string &database_name) override
 
void get_physical_tables (std::vector< std::string > &_return, const TSessionId &session) override
 
void get_views (std::vector< std::string > &_return, const TSessionId &session) override
 
void get_tables_meta (std::vector< TTableMeta > &_return, const TSessionId &session) override
 
void get_table_details (TTableDetails &_return, const TSessionId &session, const std::string &table_name) override
 
void get_table_details_for_database (TTableDetails &_return, const TSessionId &session, const std::string &table_name, const std::string &database_name) override
 
void get_internal_table_details (TTableDetails &_return, const TSessionId &session, const std::string &table_name, const bool include_system_columns) override
 
void get_internal_table_details_for_database (TTableDetails &_return, const TSessionId &session, const std::string &table_name, const std::string &database_name) override
 
void get_users (std::vector< std::string > &_return, const TSessionId &session) override
 
void get_databases (std::vector< TDBInfo > &_return, const TSessionId &session) override
 
void get_version (std::string &_return) override
 
void start_heap_profile (const TSessionId &session) override
 
void stop_heap_profile (const TSessionId &session) override
 
void get_heap_profile (std::string &_return, const TSessionId &session) override
 
void get_memory (std::vector< TNodeMemoryInfo > &_return, const TSessionId &session, const std::string &memory_level) override
 
void clear_cpu_memory (const TSessionId &session) override
 
void clear_gpu_memory (const TSessionId &session) override
 
void clearRenderMemory (const TSessionId &session)
 
void set_cur_session (const TSessionId &parent_session, const TSessionId &leaf_session, const std::string &start_time_str, const std::string &label, bool for_running_query_kernel) override
 
void invalidate_cur_session (const TSessionId &parent_session, const TSessionId &leaf_session, const std::string &start_time_str, const std::string &label, bool for_running_query_kernel) override
 
void set_table_epoch (const TSessionId &session, const int db_id, const int table_id, const int new_epoch) override
 
void set_table_epoch_by_name (const TSessionId &session, const std::string &table_name, const int new_epoch) override
 
int32_t get_table_epoch (const TSessionId &session, const int32_t db_id, const int32_t table_id) override
 
int32_t get_table_epoch_by_name (const TSessionId &session, const std::string &table_name) override
 
void get_table_epochs (std::vector< TTableEpochInfo > &_return, const TSessionId &session, const int32_t db_id, const int32_t table_id) override
 
void set_table_epochs (const TSessionId &session, const int32_t db_id, const std::vector< TTableEpochInfo > &table_epochs) override
 
void get_session_info (TSessionInfo &_return, const TSessionId &session) override
 
void set_leaf_info (const TSessionId &session, const TLeafInfo &info) override
 
void sql_execute (ExecutionResult &_return, const TSessionId &session, const std::string &query, const bool column_format, const int32_t first_n, const int32_t at_most_n, lockmgr::LockedTableDescriptors &locks)
 
void sql_execute (TQueryResult &_return, const TSessionId &session, const std::string &query, const bool column_format, const std::string &nonce, const int32_t first_n, const int32_t at_most_n) override
 
void get_completion_hints (std::vector< TCompletionHint > &hints, const TSessionId &session, const std::string &sql, const int cursor) override
 
void sql_execute_df (TDataFrame &_return, const TSessionId &session, const std::string &query, const TDeviceType::type device_type, const int32_t device_id, const int32_t first_n, const TArrowTransport::type transport_method) override
 
void sql_execute_gdf (TDataFrame &_return, const TSessionId &session, const std::string &query, const int32_t device_id, const int32_t first_n) override
 
void deallocate_df (const TSessionId &session, const TDataFrame &df, const TDeviceType::type device_type, const int32_t device_id) override
 
void interrupt (const TSessionId &query_session, const TSessionId &interrupt_session) override
 
void sql_validate (TRowDescriptor &_return, const TSessionId &session, const std::string &query) override
 
TExecuteMode::type getExecutionMode (const TSessionId &session)
 
void set_execution_mode (const TSessionId &session, const TExecuteMode::type mode) override
 
void render_vega (TRenderResult &_return, const TSessionId &session, const int64_t widget_id, const std::string &vega_json, const int32_t compression_level, const std::string &nonce) override
 
void get_result_row_for_pixel (TPixelTableRowResult &_return, const TSessionId &session, const int64_t widget_id, const TPixel &pixel, const std::map< std::string, std::vector< std::string >> &table_col_names, const bool column_format, const int32_t pixel_radius, const std::string &nonce) override
 
int32_t create_custom_expression (const TSessionId &session, const TCustomExpression &custom_expression) override
 
void get_custom_expressions (std::vector< TCustomExpression > &_return, const TSessionId &session) override
 
void update_custom_expression (const TSessionId &session, const int32_t id, const std::string &expression_json) override
 
void delete_custom_expressions (const TSessionId &session, const std::vector< int32_t > &custom_expression_ids, const bool do_soft_delete) override
 
void get_dashboard (TDashboard &_return, const TSessionId &session, const int32_t dashboard_id) override
 
void get_dashboards (std::vector< TDashboard > &_return, const TSessionId &session) override
 
int32_t create_dashboard (const TSessionId &session, const std::string &dashboard_name, const std::string &dashboard_state, const std::string &image_hash, const std::string &dashboard_metadata) override
 
void replace_dashboard (const TSessionId &session, const int32_t dashboard_id, const std::string &dashboard_name, const std::string &dashboard_owner, const std::string &dashboard_state, const std::string &image_hash, const std::string &dashboard_metadata) override
 
void delete_dashboard (const TSessionId &session, const int32_t dashboard_id) override
 
void share_dashboards (const TSessionId &session, const std::vector< int32_t > &dashboard_ids, const std::vector< std::string > &groups, const TDashboardPermissions &permissions) override
 
void delete_dashboards (const TSessionId &session, const std::vector< int32_t > &dashboard_ids) override
 
void share_dashboard (const TSessionId &session, const int32_t dashboard_id, const std::vector< std::string > &groups, const std::vector< std::string > &objects, const TDashboardPermissions &permissions, const bool grant_role) override
 
void unshare_dashboards (const TSessionId &session, const std::vector< int32_t > &dashboard_ids, const std::vector< std::string > &groups, const TDashboardPermissions &permissions) override
 
void unshare_dashboard (const TSessionId &session, const int32_t dashboard_id, const std::vector< std::string > &groups, const std::vector< std::string > &objects, const TDashboardPermissions &permissions) override
 
void get_dashboard_grantees (std::vector< TDashboardGrantees > &_return, const TSessionId &session, const int32_t dashboard_id) override
 
void get_link_view (TFrontendView &_return, const TSessionId &session, const std::string &link) override
 
void create_link (std::string &_return, const TSessionId &session, const std::string &view_state, const std::string &view_metadata) override
 
void load_table_binary (const TSessionId &session, const std::string &table_name, const std::vector< TRow > &rows, const std::vector< std::string > &column_names) override
 
void load_table_binary_columnar (const TSessionId &session, const std::string &table_name, const std::vector< TColumn > &cols, const std::vector< std::string > &column_names) override
 
void load_table_binary_columnar_polys (const TSessionId &session, const std::string &table_name, const std::vector< TColumn > &cols, const std::vector< std::string > &column_names, const bool assign_render_groups) override
 
void load_table_binary_arrow (const TSessionId &session, const std::string &table_name, const std::string &arrow_stream, const bool use_column_names) override
 
void load_table (const TSessionId &session, const std::string &table_name, const std::vector< TStringRow > &rows, const std::vector< std::string > &column_names) override
 
void detect_column_types (TDetectResult &_return, const TSessionId &session, const std::string &file_name, const TCopyParams &copy_params) override
 
void create_table (const TSessionId &session, const std::string &table_name, const TRowDescriptor &row_desc, const TCreateParams &create_params) override
 
void import_table (const TSessionId &session, const std::string &table_name, const std::string &file_name, const TCopyParams &copy_params) override
 
void import_geo_table (const TSessionId &session, const std::string &table_name, const std::string &file_name, const TCopyParams &copy_params, const TRowDescriptor &row_desc, const TCreateParams &create_params) override
 
void import_table_status (TImportStatus &_return, const TSessionId &session, const std::string &import_id) override
 
void get_first_geo_file_in_archive (std::string &_return, const TSessionId &session, const std::string &archive_path, const TCopyParams &copy_params) override
 
void get_all_files_in_archive (std::vector< std::string > &_return, const TSessionId &session, const std::string &archive_path, const TCopyParams &copy_params) override
 
void get_layers_in_geo_file (std::vector< TGeoFileLayerInfo > &_return, const TSessionId &session, const std::string &file_name, const TCopyParams &copy_params) override
 
int64_t query_get_outer_fragment_count (const TSessionId &session, const std::string &select_query) override
 
void check_table_consistency (TTableMeta &_return, const TSessionId &session, const int32_t table_id) override
 
void start_query (TPendingQuery &_return, const TSessionId &leaf_session, const TSessionId &parent_session, const std::string &query_ra, const std::string &start_time_str, const bool just_explain, const std::vector< int64_t > &outer_fragment_indices) override
 
void execute_query_step (TStepResult &_return, const TPendingQuery &pending_query, const TSubqueryId subquery_id, const std::string &start_time_str) override
 
void broadcast_serialized_rows (const TSerializedRows &serialized_rows, const TRowDescriptor &row_desc, const TQueryId query_id, const TSubqueryId subquery_id, const bool is_final_subquery_result) override
 
void start_render_query (TPendingRenderQuery &_return, const TSessionId &session, const int64_t widget_id, const int16_t node_idx, const std::string &vega_json) override
 
void execute_next_render_step (TRenderStepResult &_return, const TPendingRenderQuery &pending_render, const TRenderAggDataMap &merged_data) override
 
void insert_data (const TSessionId &session, const TInsertData &insert_data) override
 
void insert_chunks (const TSessionId &session, const TInsertChunks &insert_chunks) override
 
void checkpoint (const TSessionId &session, const int32_t table_id) override
 
void get_roles (std::vector< std::string > &_return, const TSessionId &session) override
 
bool has_role (const TSessionId &sessionId, const std::string &granteeName, const std::string &roleName) override
 
bool has_object_privilege (const TSessionId &sessionId, const std::string &granteeName, const std::string &objectName, const TDBObjectType::type object_type, const TDBObjectPermissions &permissions) override
 
void get_db_objects_for_grantee (std::vector< TDBObject > &_return, const TSessionId &session, const std::string &roleName) override
 
void get_db_object_privs (std::vector< TDBObject > &_return, const TSessionId &session, const std::string &objectName, const TDBObjectType::type type) override
 
void get_all_roles_for_user (std::vector< std::string > &_return, const TSessionId &session, const std::string &granteeName) override
 
void get_all_effective_roles_for_user (std::vector< std::string > &_return, const TSessionId &session, const std::string &granteeName) override
 
std::vector< std::string > get_valid_groups (const TSessionId &session, int32_t dashboard_id, std::vector< std::string > groups)
 
void set_license_key (TLicenseInfo &_return, const TSessionId &session, const std::string &key, const std::string &nonce) override
 
void get_license_claims (TLicenseInfo &_return, const TSessionId &session, const std::string &nonce) override
 
void get_device_parameters (std::map< std::string, std::string > &_return, const TSessionId &session) override
 
void register_runtime_extension_functions (const TSessionId &session, const std::vector< TUserDefinedFunction > &udfs, const std::vector< TUserDefinedTableFunction > &udtfs, const std::map< std::string, std::string > &device_ir_map) override
 
void get_function_names (std::vector< std::string > &_return, const TSessionId &session) override
 
void get_runtime_function_names (std::vector< std::string > &_return, const TSessionId &session) override
 
void get_function_details (std::vector< TUserDefinedFunction > &_return, const TSessionId &session, const std::vector< std::string > &udf_names) override
 
void get_table_function_names (std::vector< std::string > &_return, const TSessionId &session) override
 
void get_runtime_table_function_names (std::vector< std::string > &_return, const TSessionId &session) override
 
void get_table_function_details (std::vector< TUserDefinedTableFunction > &_return, const TSessionId &session, const std::vector< std::string > &udtf_names) override
 
void shutdown ()
 
void emergency_shutdown ()
 
TSessionId getInvalidSessionId () const
 
void internal_connect (TSessionId &session, const std::string &username, const std::string &dbname)
 
bool isAggregator () const
 
template<typename... ARGS>
std::shared_ptr
< query_state::QueryState
create_query_state (ARGS &&...args)
 
Catalog_Namespace::SessionInfo get_session_copy (const TSessionId &session)
 
std::shared_ptr
< Catalog_Namespace::SessionInfo
get_session_copy_ptr (const TSessionId &session)
 
void get_tables_meta_impl (std::vector< TTableMeta > &_return, QueryStateProxy query_state_proxy, const Catalog_Namespace::SessionInfo &session_info, const bool with_table_locks=true)
 
void resizeDispatchQueue (size_t queue_size)
 
template<>
SessionMap::iterator get_session_it_unsafe (const TSessionId &session, heavyai::shared_lock< heavyai::shared_mutex > &read_lock)
 
template<>
SessionMap::iterator get_session_it_unsafe (const TSessionId &session, heavyai::unique_lock< heavyai::shared_mutex > &write_lock)
 
template<>
void expire_idle_sessions_unsafe (heavyai::unique_lock< heavyai::shared_mutex > &write_lock)
 

Static Public Member Functions

static size_t max_bytes_for_thrift ()
 

Public Attributes

std::shared_ptr
< Data_Namespace::DataMgr
data_mgr_
 
LeafAggregator leaf_aggregator_
 
std::vector< LeafHostInfodb_leaves_
 
std::vector< LeafHostInfostring_leaves_
 
const std::string base_data_path_
 
boost::filesystem::path import_path_
 
ExecutorDeviceType executor_device_type_
 
std::default_random_engine random_gen_
 
std::uniform_int_distribution
< int64_t > 
session_id_dist_
 
const bool jit_debug_
 
const bool intel_jit_profile_
 
bool allow_multifrag_
 
const bool read_only_
 
const bool allow_loop_joins_
 
bool cpu_mode_only_
 
heavyai::shared_mutex sessions_mutex_
 
std::mutex render_mutex_
 
int64_t start_time_
 
const AuthMetadataauthMetadata_
 
SystemParameterssystem_parameters_
 
std::shared_ptr< QueryEnginequery_engine_
 
std::unique_ptr< RenderHandlerrender_handler_
 
std::unique_ptr
< HeavyDBAggHandler
agg_handler_
 
std::unique_ptr
< HeavyDBLeafHandler
leaf_handler_
 
std::shared_ptr< Calcitecalcite_
 
const bool legacy_syntax_
 
std::unique_ptr
< QueryDispatchQueue
dispatch_queue_
 

Protected Member Functions

std::shared_ptr
< Catalog_Namespace::SessionInfo
get_session_ptr (const TSessionId &session_id)
 
ConnectionInfo getConnectionInfo () const
 

Private Types

enum  AssignRenderGroupsMode { AssignRenderGroupsMode::kNone, AssignRenderGroupsMode::kAssign, AssignRenderGroupsMode::kCleanUp }
 
using RenderGroupAssignmentColumnMap = std::unordered_map< std::string, std::unique_ptr< import_export::RenderGroupAnalyzer >>
 
using RenderGroupAssignmentTableMap = std::unordered_map< std::string, RenderGroupAssignmentColumnMap >
 
using RenderGroupAnalyzerSessionMap = std::unordered_map< TSessionId, RenderGroupAssignmentTableMap >
 

Private Member Functions

std::shared_ptr
< Catalog_Namespace::SessionInfo
create_new_session (TSessionId &session, const std::string &dbname, const Catalog_Namespace::UserMetadata &user_meta, std::shared_ptr< Catalog_Namespace::Catalog > cat)
 
void connect_impl (TSessionId &session, const std::string &passwd, const std::string &dbname, const Catalog_Namespace::UserMetadata &user_meta, std::shared_ptr< Catalog_Namespace::Catalog > cat, query_state::StdLog &stdlog)
 
void disconnect_impl (const SessionMap::iterator &session_it, heavyai::unique_lock< heavyai::shared_mutex > &write_lock)
 
void check_table_load_privileges (const TSessionId &session, const std::string &table_name)
 
void check_table_load_privileges (const Catalog_Namespace::SessionInfo &session_info, const std::string &table_name)
 
void get_tables_impl (std::vector< std::string > &table_names, const Catalog_Namespace::SessionInfo &, const GetTablesType get_tables_type, const std::string &database_name={})
 
void get_table_details_impl (TTableDetails &_return, query_state::StdLog &stdlog, const std::string &table_name, const bool get_system, const bool get_physical, const std::string &database_name={})
 
void getAllRolesForUserImpl (std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr, std::vector< std::string > &roles, const TSessionId &sessionId, const std::string &granteeName, bool effective)
 
void check_read_only (const std::string &str)
 
void check_session_exp_unsafe (const SessionMap::iterator &session_it)
 
void validateGroups (const std::vector< std::string > &groups)
 
void validateDashboardIdsForSharing (const Catalog_Namespace::SessionInfo &session_info, const std::vector< int32_t > &dashboard_ids)
 
void shareOrUnshareDashboards (const TSessionId &session, const std::vector< int32_t > &dashboard_ids, const std::vector< std::string > &groups, const TDashboardPermissions &permissions, const bool do_share)
 
std::shared_ptr< const
Catalog_Namespace::SessionInfo
get_const_session_ptr (const TSessionId &session)
 
template<typename SESSION_MAP_LOCK >
SessionMap::iterator get_session_it_unsafe (const TSessionId &session, SESSION_MAP_LOCK &lock)
 
template<typename SESSION_MAP_LOCK >
void expire_idle_sessions_unsafe (SESSION_MAP_LOCK &lock)
 
std::pair< TPlanResult,
lockmgr::LockedTableDescriptors
parse_to_ra (QueryStateProxy, const std::string &query_str, const std::vector< TFilterPushDownInfo > &filter_push_down_info, const bool acquire_locks, const SystemParameters &system_parameters, bool check_privileges=true)
 
void sql_execute_local (TQueryResult &_return, const QueryStateProxy &query_state_proxy, const std::shared_ptr< Catalog_Namespace::SessionInfo > session_ptr, const std::string &query_str, const bool column_format, const std::string &nonce, const int32_t first_n, const int32_t at_most_n, const bool use_calcite)
 
int64_t process_deferred_copy_from (const TSessionId &session_id)
 
void sql_execute_impl (ExecutionResult &_return, QueryStateProxy, const bool column_format, const ExecutorDeviceType executor_device_type, const int32_t first_n, const int32_t at_most_n, const bool use_calcite, lockmgr::LockedTableDescriptors &locks)
 
bool user_can_access_table (const Catalog_Namespace::SessionInfo &, const TableDescriptor *td, const AccessPrivileges acess_priv)
 
void execute_distributed_copy_statement (Parser::CopyTableStmt *, const Catalog_Namespace::SessionInfo &session_info)
 
TQueryResult validate_rel_alg (const std::string &query_ra, QueryStateProxy)
 
std::vector< PushedDownFilterInfoexecute_rel_alg (ExecutionResult &_return, QueryStateProxy, const std::string &query_ra, const bool column_format, const ExecutorDeviceType executor_device_type, const int32_t first_n, const int32_t at_most_n, const bool just_validate, const bool find_push_down_candidates, const ExplainInfo &explain_info, const std::optional< size_t > executor_index=std::nullopt) const
 
void execute_rel_alg_with_filter_push_down (ExecutionResult &_return, QueryStateProxy, std::string &query_ra, const bool column_format, const ExecutorDeviceType executor_device_type, const int32_t first_n, const int32_t at_most_n, const bool just_explain, const bool just_calcite_explain, const std::vector< PushedDownFilterInfo > &filter_push_down_requests)
 
void executeDdl (TQueryResult &_return, const std::string &query_ra, std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
 
void executeDdl (ExecutionResult &_return, const std::string &query_ra, std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
 
TColumnType populateThriftColumnType (const Catalog_Namespace::Catalog *cat, const ColumnDescriptor *cd)
 
TRowDescriptor fixup_row_descriptor (const TRowDescriptor &row_desc, const Catalog_Namespace::Catalog &cat)
 
void set_execution_mode_nolock (Catalog_Namespace::SessionInfo *session_ptr, const TExecuteMode::type mode)
 
char unescape_char (std::string str)
 
import_export::CopyParams thrift_to_copyparams (const TCopyParams &cp)
 
TCopyParams copyparams_to_thrift (const import_export::CopyParams &cp)
 
void check_geospatial_files (const boost::filesystem::path file_path, const import_export::CopyParams &copy_params)
 
void render_rel_alg (TRenderResult &_return, const std::string &query_ra, const std::string &query_str, const Catalog_Namespace::SessionInfo &session_info, const std::string &render_type, const bool is_projection_query)
 
TColumnType create_geo_column (const TDatumType::type type, const std::string &name, const bool is_array)
 
void convertResultSet (ExecutionResult &result, const Catalog_Namespace::SessionInfo &session_info, const std::string &query_state_str, TQueryResult &_return)
 
std::vector< TargetMetaInfogetTargetMetaInfo (const std::vector< std::shared_ptr< Analyzer::TargetEntry >> &targets) const
 
std::vector< std::string > getTargetNames (const std::vector< TargetMetaInfo > &targets) const
 
std::vector< std::string > getTargetNames (const std::vector< std::shared_ptr< Analyzer::TargetEntry >> &targets) const
 
void get_completion_hints_unsorted (std::vector< TCompletionHint > &hints, std::vector< std::string > &visible_tables, query_state::StdLog &stdlog, const std::string &sql, const int cursor)
 
void get_token_based_completions (std::vector< TCompletionHint > &hints, query_state::StdLog &stdlog, std::vector< std::string > &visible_tables, const std::string &sql, const int cursor)
 
std::unordered_map
< std::string,
std::unordered_set
< std::string > > 
fill_column_names_by_table (std::vector< std::string > &table_names, query_state::StdLog &stdlog)
 
TDashboard get_dashboard_impl (const std::shared_ptr< Catalog_Namespace::SessionInfo const > &session_ptr, Catalog_Namespace::UserMetadata &user_meta, const DashboardDescriptor *dash, const bool populate_state=true)
 
std::unordered_set< std::string > get_uc_compatible_table_names_by_column (const std::unordered_set< std::string > &uc_column_names, std::vector< std::string > &table_names, query_state::StdLog &stdlog)
 
std::unique_ptr
< lockmgr::AbstractLockContainer
< const TableDescriptor * > > 
prepare_loader_generic (const Catalog_Namespace::SessionInfo &session_info, const std::string &table_name, size_t num_cols, std::unique_ptr< import_export::Loader > *loader, std::vector< std::unique_ptr< import_export::TypedImportBuffer >> *import_buffers, const std::vector< std::string > &column_names, std::string load_type)
 
void fillGeoColumns (const TSessionId &session, const Catalog_Namespace::Catalog &catalog, std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &import_buffers, const ColumnDescriptor *cd, size_t &col_idx, size_t num_rows, const std::string &table_name, bool assign_render_groups)
 
void fillMissingBuffers (const TSessionId &session, const Catalog_Namespace::Catalog &catalog, std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &import_buffers, const std::list< const ColumnDescriptor * > &cds, const std::vector< int > &desc_id_to_column_id, size_t num_rows, const std::string &table_name, bool assign_render_groups)
 
void check_and_invalidate_sessions (Parser::DDLStmt *ddl)
 
template<typename STMT_TYPE >
void invalidate_sessions (std::string &name, STMT_TYPE *stmt)
 
std::string const createInMemoryCalciteSession (const std::shared_ptr< Catalog_Namespace::Catalog > &catalog_ptr)
 
bool isInMemoryCalciteSession (const Catalog_Namespace::UserMetadata user_meta)
 
void removeInMemoryCalciteSession (const std::string &session_id)
 
ExecutionResult getUserSessions (std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
 
ExecutionResult getQueries (std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
 
void get_queries_info (std::vector< TQueryInfo > &_return, const TSessionId &session) override
 
void interruptQuery (const Catalog_Namespace::SessionInfo &session_info, const std::string &target_session)
 
void alterSystemClear (const std::string &sesson_id, ExecutionResult &result, const std::string &cache_type, int64_t &execution_time_ms)
 
void alterSession (const std::string &sesson_id, ExecutionResult &result, const std::pair< std::string, std::string > &session_parameter, int64_t &execution_time_ms)
 
void loadTableBinaryColumnarInternal (const TSessionId &session, const std::string &table_name, const std::vector< TColumn > &cols, const std::vector< std::string > &column_names, const AssignRenderGroupsMode assign_render_groups_mode)
 
TRole::type getServerRole () const
 
void importGeoTableGlobFilterSort (const TSessionId &session, const std::string &table_name, const std::string &file_name, const import_export::CopyParams &copy_params, const TRowDescriptor &row_desc, const TCreateParams &create_params)
 
void importGeoTableSingle (const TSessionId &session, const std::string &table_name, const std::string &file_name, const import_export::CopyParams &copy_params, const TRowDescriptor &row_desc, const TCreateParams &create_params)
 

Static Private Member Functions

static void value_to_thrift_column (const TargetValue &tv, const SQLTypeInfo &ti, TColumn &column)
 
static TDatum value_to_thrift (const TargetValue &tv, const SQLTypeInfo &ti)
 
static void convertData (TQueryResult &_return, ExecutionResult &result, const QueryStateProxy &query_state_proxy, const bool column_format, const int32_t first_n, const int32_t at_most_n)
 
static void convertExplain (TQueryResult &_return, const ResultSet &results, const bool column_format)
 
static void convertResult (TQueryResult &_return, const ResultSet &results, const bool column_format)
 
static void convertRows (TQueryResult &_return, QueryStateProxy query_state_proxy, const std::vector< TargetMetaInfo > &targets, const ResultSet &results, const bool column_format, const int32_t first_n, const int32_t at_most_n)
 
static void createSimpleResult (TQueryResult &_return, const ResultSet &results, const bool column_format, const std::string label)
 
static bool has_database_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permissions)
 
static bool has_table_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permission)
 
static bool has_dashboard_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permissions)
 
static bool has_view_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permissions)
 
static bool has_server_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permissions)
 

Private Attributes

std::atomic< bool > initialized_ {false}
 
query_state::QueryStates query_states_
 
SessionMap sessions_
 
bool super_user_rights_
 
const int idle_session_duration_
 
const int max_session_duration_
 
const bool enable_rendering_
 
const bool renderer_use_ppll_polys_
 
const bool renderer_prefer_igpu_
 
const unsigned renderer_vulkan_timeout_
 
const bool enable_auto_clear_render_mem_
 
const int render_oom_retry_threshold_
 
const size_t max_concurrent_render_sessions_
 
const size_t reserved_gpu_mem_
 
const bool render_compositor_use_last_gpu_
 
const size_t render_mem_bytes_
 
const size_t num_reader_threads_
 
const
File_Namespace::DiskCacheConfig
disk_cache_config_
 
const std::string & udf_filename_
 
const std::string & clang_path_
 
const std::vector< std::string > & clang_options_
 
DeferredCopyFromSessions deferred_copy_from_sessions
 
std::mutex handle_to_dev_ptr_mutex_
 
std::unordered_map
< std::string, std::string > 
ipc_handle_to_dev_ptr_
 
std::map< const std::string,
const PermissionFuncPtr
permissionFuncMap_
 
RenderGroupAnalyzerSessionMap render_group_assignment_map_
 
std::mutex render_group_assignment_mutex_
 
heavyai::shared_mutex custom_expressions_mutex_
 

Friends

class RenderHandler::Impl
 
class HeavyDBAggHandler
 
class HeavyDBLeafHandler
 
void run_warmup_queries (std::shared_ptr< DBHandler > handler, std::string base_path, std::string query_file_path)
 

Detailed Description

Definition at line 166 of file DBHandler.h.

Member Typedef Documentation

using DBHandler::RenderGroupAnalyzerSessionMap = std::unordered_map<TSessionId, RenderGroupAssignmentTableMap>
private

Definition at line 1106 of file DBHandler.h.

using DBHandler::RenderGroupAssignmentColumnMap = std::unordered_map<std::string, std::unique_ptr<import_export::RenderGroupAnalyzer>>
private

Definition at line 1102 of file DBHandler.h.

using DBHandler::RenderGroupAssignmentTableMap = std::unordered_map<std::string, RenderGroupAssignmentColumnMap>
private

Definition at line 1104 of file DBHandler.h.

Member Enumeration Documentation

Enumerator
kNone 
kAssign 
kCleanUp 

Definition at line 1089 of file DBHandler.h.

1089 { kNone, kAssign, kCleanUp };

Constructor & Destructor Documentation

DBHandler::DBHandler ( const std::vector< LeafHostInfo > &  db_leaves,
const std::vector< LeafHostInfo > &  string_leaves,
const std::string &  base_data_path,
const bool  allow_multifrag,
const bool  jit_debug,
const bool  intel_jit_profile,
const bool  read_only,
const bool  allow_loop_joins,
const bool  enable_rendering,
const bool  renderer_use_ppll_polys,
const bool  renderer_prefer_igpu,
const unsigned  renderer_vulkan_timeout_ms,
const bool  enable_auto_clear_render_mem,
const int  render_oom_retry_threshold,
const size_t  render_mem_bytes,
const size_t  max_concurrent_render_sessions,
const size_t  reserved_gpu_mem,
const bool  render_compositor_use_last_gpu,
const size_t  num_reader_threads,
const AuthMetadata authMetadata,
SystemParameters system_parameters,
const bool  legacy_syntax,
const int  idle_session_duration,
const int  max_session_duration,
const std::string &  udf_filename,
const std::string &  clang_path,
const std::vector< std::string > &  clang_options,
const File_Namespace::DiskCacheConfig disk_cache_config,
const bool  is_new_db 
)

Definition at line 238 of file DBHandler.cpp.

270  : leaf_aggregator_(db_leaves)
271  , db_leaves_(db_leaves)
272  , string_leaves_(string_leaves)
273  , base_data_path_(base_data_path)
274  , random_gen_(std::random_device{}())
275  , session_id_dist_(0, INT32_MAX)
276  , jit_debug_(jit_debug)
277  , intel_jit_profile_(intel_jit_profile)
278  , allow_multifrag_(allow_multifrag)
279  , read_only_(read_only)
280  , allow_loop_joins_(allow_loop_joins)
281  , authMetadata_(authMetadata)
282  , system_parameters_(system_parameters)
283  , legacy_syntax_(legacy_syntax)
284  , dispatch_queue_(
285  std::make_unique<QueryDispatchQueue>(system_parameters.num_executors))
286  , super_user_rights_(false)
287  , idle_session_duration_(idle_session_duration * 60)
288  , max_session_duration_(max_session_duration * 60)
289  , enable_rendering_(enable_rendering)
290  , renderer_use_ppll_polys_(renderer_use_ppll_polys)
291  , renderer_prefer_igpu_(renderer_prefer_igpu)
292  , renderer_vulkan_timeout_(renderer_vulkan_timeout_ms)
293  , enable_auto_clear_render_mem_(enable_auto_clear_render_mem)
294  , render_oom_retry_threshold_(render_oom_retry_threshold)
295  , max_concurrent_render_sessions_(max_concurrent_render_sessions)
296  , reserved_gpu_mem_(reserved_gpu_mem)
297  , render_compositor_use_last_gpu_(render_compositor_use_last_gpu)
298  , render_mem_bytes_(render_mem_bytes)
299  , num_reader_threads_(num_reader_threads)
300 #ifdef ENABLE_GEOS
301  , libgeos_so_filename_(libgeos_so_filename)
302 #endif
303  , disk_cache_config_(disk_cache_config)
304  , udf_filename_(udf_filename)
305  , clang_path_(clang_path)
306  , clang_options_(clang_options)
307 
308 {
309  LOG(INFO) << "HeavyDB Server " << MAPD_RELEASE;
310  initialize(is_new_db);
311 }
std::vector< LeafHostInfo > string_leaves_
Definition: DBHandler.h:627
const std::vector< std::string > & clang_options_
Definition: DBHandler.h:976
std::unique_ptr< QueryDispatchQueue > dispatch_queue_
Definition: DBHandler.h:651
const std::string & udf_filename_
Definition: DBHandler.h:974
const int render_oom_retry_threshold_
Definition: DBHandler.h:964
#define LOG(tag)
Definition: Logger.h:216
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:625
const unsigned renderer_vulkan_timeout_
Definition: DBHandler.h:962
const std::string base_data_path_
Definition: DBHandler.h:628
void initialize(const bool is_new_db)
Definition: DBHandler.cpp:313
const bool jit_debug_
Definition: DBHandler.h:633
const size_t render_mem_bytes_
Definition: DBHandler.h:968
const bool renderer_use_ppll_polys_
Definition: DBHandler.h:960
const size_t reserved_gpu_mem_
Definition: DBHandler.h:966
const bool render_compositor_use_last_gpu_
Definition: DBHandler.h:967
const int max_session_duration_
Definition: DBHandler.h:957
std::vector< LeafHostInfo > db_leaves_
Definition: DBHandler.h:626
const File_Namespace::DiskCacheConfig & disk_cache_config_
Definition: DBHandler.h:973
const std::string & clang_path_
Definition: DBHandler.h:975
SystemParameters & system_parameters_
Definition: DBHandler.h:643
const size_t num_reader_threads_
Definition: DBHandler.h:969
const bool enable_auto_clear_render_mem_
Definition: DBHandler.h:963
const bool renderer_prefer_igpu_
Definition: DBHandler.h:961
const bool allow_loop_joins_
Definition: DBHandler.h:637
const bool enable_rendering_
Definition: DBHandler.h:959
const bool intel_jit_profile_
Definition: DBHandler.h:634
bool super_user_rights_
Definition: DBHandler.h:953
const bool read_only_
Definition: DBHandler.h:636
const bool legacy_syntax_
Definition: DBHandler.h:649
const int idle_session_duration_
Definition: DBHandler.h:956
const size_t max_concurrent_render_sessions_
Definition: DBHandler.h:965
bool allow_multifrag_
Definition: DBHandler.h:635
static const std::string MAPD_RELEASE
Definition: release.h:42
const AuthMetadata & authMetadata_
Definition: DBHandler.h:642
std::default_random_engine random_gen_
Definition: DBHandler.h:631
std::uniform_int_distribution< int64_t > session_id_dist_
Definition: DBHandler.h:632
DBHandler::~DBHandler ( )
override

Definition at line 496 of file DBHandler.cpp.

References shutdown().

496  {
497  shutdown();
498 }
void shutdown()
Definition: DBHandler.cpp:7308

+ Here is the call graph for this function:

Member Function Documentation

void DBHandler::alterSession ( const std::string &  sesson_id,
ExecutionResult result,
const std::pair< std::string, std::string > &  session_parameter,
int64_t &  execution_time_ms 
)
private

Definition at line 7824 of file DBHandler.cpp.

References measure< TimeT >::execution(), set_execution_mode(), switch_database(), to_upper(), and run_benchmark_import::type.

Referenced by executeDdl().

7827  {
7828  result = ExecutionResult();
7829  if (session_parameter.first == "EXECUTOR_DEVICE") {
7830  std::string parameter_value = to_upper(session_parameter.second);
7831  TExecuteMode::type executorType;
7832  if (parameter_value == "GPU") {
7833  executorType = TExecuteMode::type::GPU;
7834  } else if (parameter_value == "CPU") {
7835  executorType = TExecuteMode::type::CPU;
7836  } else {
7837  throw std::runtime_error("Cannot set the " + session_parameter.first + " to " +
7838  session_parameter.second +
7839  ". Valid options are CPU and GPU");
7840  }
7841  execution_time_ms =
7842  measure<>::execution([&]() { set_execution_mode(session_id, executorType); });
7843  } else if (session_parameter.first == "CURRENT_DATABASE") {
7844  execution_time_ms = measure<>::execution(
7845  [&]() { switch_database(session_id, session_parameter.second); });
7846  }
7847 }
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
void set_execution_mode(const TSessionId &session, const TExecuteMode::type mode) override
Definition: DBHandler.cpp:2914
std::string to_upper(const std::string &str)
void switch_database(const TSessionId &session, const std::string &dbname) override
Definition: DBHandler.cpp:721

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::alterSystemClear ( const std::string &  sesson_id,
ExecutionResult result,
const std::string &  cache_type,
int64_t &  execution_time_ms 
)
private

Definition at line 7808 of file DBHandler.cpp.

References clear_cpu_memory(), clear_gpu_memory(), clearRenderMemory(), measure< TimeT >::execution(), and to_upper().

Referenced by executeDdl().

7811  {
7812  result = ExecutionResult();
7813  if (to_upper(cache_type) == "CPU") {
7814  execution_time_ms = measure<>::execution([&]() { clear_cpu_memory(session_id); });
7815  } else if (to_upper(cache_type) == "GPU") {
7816  execution_time_ms = measure<>::execution([&]() { clear_gpu_memory(session_id); });
7817  } else if (to_upper(cache_type) == "RENDER") {
7818  execution_time_ms = measure<>::execution([&]() { clearRenderMemory(session_id); });
7819  } else {
7820  throw std::runtime_error("Invalid cache type. Valid values are CPU,GPU or RENDER");
7821  }
7822 }
void clearRenderMemory(const TSessionId &session)
Definition: DBHandler.cpp:2791
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
std::string to_upper(const std::string &str)
void clear_gpu_memory(const TSessionId &session) override
Definition: DBHandler.cpp:2749
void clear_cpu_memory(const TSessionId &session) override
Definition: DBHandler.cpp:2770

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::broadcast_serialized_rows ( const TSerializedRows &  serialized_rows,
const TRowDescriptor &  row_desc,
const TQueryId  query_id,
const TSubqueryId  subquery_id,
const bool  is_final_subquery_result 
)
override

Definition at line 6854 of file DBHandler.cpp.

References measure< TimeT >::execution(), logger::INFO, leaf_handler_, LOG, logger::query_id(), and THROW_DB_EXCEPTION.

6858  {
6859  if (!leaf_handler_) {
6860  THROW_DB_EXCEPTION("Distributed support is disabled.");
6861  }
6862  LOG(INFO) << "BROADCAST-SERIALIZED-ROWS id:" << query_id;
6863  auto time_ms = measure<>::execution([&]() {
6864  try {
6865  leaf_handler_->broadcast_serialized_rows(
6866  serialized_rows, row_desc, query_id, subquery_id, is_final_subquery_result);
6867  } catch (std::exception& e) {
6868  THROW_DB_EXCEPTION(e.what());
6869  }
6870  });
6871  LOG(INFO) << "BROADCAST-SERIALIZED-ROWS COMPLETED " << time_ms << "ms";
6872 }
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:216
std::unique_ptr< HeavyDBLeafHandler > leaf_handler_
Definition: DBHandler.h:647
QueryId query_id()
Definition: Logger.cpp:473
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

void DBHandler::check_and_invalidate_sessions ( Parser::DDLStmt ddl)
private

Definition at line 6227 of file DBHandler.cpp.

References invalidate_sessions().

6227  {
6228  const auto drop_db_stmt = dynamic_cast<Parser::DropDBStmt*>(ddl);
6229  if (drop_db_stmt) {
6230  invalidate_sessions(*drop_db_stmt->getDatabaseName(), drop_db_stmt);
6231  return;
6232  }
6233  const auto rename_db_stmt = dynamic_cast<Parser::RenameDBStmt*>(ddl);
6234  if (rename_db_stmt) {
6235  invalidate_sessions(*rename_db_stmt->getPreviousDatabaseName(), rename_db_stmt);
6236  return;
6237  }
6238  const auto drop_user_stmt = dynamic_cast<Parser::DropUserStmt*>(ddl);
6239  if (drop_user_stmt) {
6240  invalidate_sessions(*drop_user_stmt->getUserName(), drop_user_stmt);
6241  return;
6242  }
6243  const auto rename_user_stmt = dynamic_cast<Parser::RenameUserStmt*>(ddl);
6244  if (rename_user_stmt) {
6245  invalidate_sessions(*rename_user_stmt->getOldUserName(), rename_user_stmt);
6246  return;
6247  }
6248 }
void invalidate_sessions(std::string &name, STMT_TYPE *stmt)
Definition: DBHandler.h:1033

+ Here is the call graph for this function:

void DBHandler::check_geospatial_files ( const boost::filesystem::path  file_path,
const import_export::CopyParams copy_params 
)
private

Definition at line 4960 of file DBHandler.cpp.

References import_export::Importer::gdalFileExists().

Referenced by detect_column_types(), and importGeoTableSingle().

4961  {
4962  const std::list<std::string> shp_ext{".shp", ".shx", ".dbf"};
4963  if (std::find(shp_ext.begin(),
4964  shp_ext.end(),
4965  boost::algorithm::to_lower_copy(file_path.extension().string())) !=
4966  shp_ext.end()) {
4967  for (auto ext : shp_ext) {
4968  auto aux_file = file_path;
4970  aux_file.replace_extension(boost::algorithm::to_upper_copy(ext)).string(),
4971  copy_params) &&
4973  aux_file.replace_extension(ext).string(), copy_params)) {
4974  throw std::runtime_error("required file for shapefile does not exist: " +
4975  aux_file.filename().string());
4976  }
4977  }
4978  }
4979 }
static bool gdalFileExists(const std::string &path, const CopyParams &copy_params)
Definition: Importer.cpp:5136

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::check_read_only ( const std::string &  str)
private

Definition at line 500 of file DBHandler.cpp.

References read_only_, and THROW_DB_EXCEPTION.

Referenced by create_custom_expression(), create_dashboard(), create_table(), delete_custom_expressions(), delete_dashboards(), detect_column_types(), import_table(), importGeoTableSingle(), prepare_loader_generic(), replace_dashboard(), set_license_key(), shareOrUnshareDashboards(), sql_execute_impl(), and update_custom_expression().

500  {
501  if (DBHandler::read_only_) {
502  THROW_DB_EXCEPTION(str + " disabled: server running in read-only mode.");
503  }
504 }
const bool read_only_
Definition: DBHandler.h:636
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the caller graph for this function:

void DBHandler::check_session_exp_unsafe ( const SessionMap::iterator &  session_it)
private

Definition at line 5869 of file DBHandler.cpp.

References idle_session_duration_, logger::INFO, isInMemoryCalciteSession(), LOG, max_session_duration_, and run_benchmark_import::start_time.

Referenced by expire_idle_sessions_unsafe(), and get_session_it_unsafe().

5869  {
5870  if (session_it->second.use_count() > 2 ||
5871  isInMemoryCalciteSession(session_it->second->get_currentUser())) {
5872  // SessionInfo is being used in more than one active operation. Original copy + one
5873  // stored in StdLog. Skip the checks.
5874  return;
5875  }
5876  time_t last_used_time = session_it->second->get_last_used_time();
5877  time_t start_time = session_it->second->get_start_time();
5878  const auto current_session_duration = time(0) - last_used_time;
5879  if (current_session_duration > idle_session_duration_) {
5880  LOG(INFO) << "Session " << session_it->second->get_public_session_id()
5881  << " idle duration " << current_session_duration
5882  << " seconds exceeds maximum idle duration " << idle_session_duration_
5883  << " seconds. Invalidating session.";
5884  throw ForceDisconnect("Idle Session Timeout. User should re-authenticate.");
5885  }
5886  const auto total_session_duration = time(0) - start_time;
5887  if (total_session_duration > max_session_duration_) {
5888  LOG(INFO) << "Session " << session_it->second->get_public_session_id()
5889  << " total duration " << total_session_duration
5890  << " seconds exceeds maximum total session duration "
5891  << max_session_duration_ << " seconds. Invalidating session.";
5892  throw ForceDisconnect("Maximum active Session Timeout. User should re-authenticate.");
5893  }
5894 }
#define LOG(tag)
Definition: Logger.h:216
const int max_session_duration_
Definition: DBHandler.h:957
const int idle_session_duration_
Definition: DBHandler.h:956
bool isInMemoryCalciteSession(const Catalog_Namespace::UserMetadata user_meta)
Definition: DBHandler.cpp:533

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::check_table_consistency ( TTableMeta &  _return,
const TSessionId &  session,
const int32_t  table_id 
)
override

Definition at line 6791 of file DBHandler.cpp.

References get_session_ptr(), leaf_handler_, STDLOG, and THROW_DB_EXCEPTION.

6793  {
6794  auto stdlog = STDLOG(get_session_ptr(session));
6795  if (!leaf_handler_) {
6796  THROW_DB_EXCEPTION("Distributed support is disabled.");
6797  }
6798  try {
6799  leaf_handler_->check_table_consistency(_return, session, table_id);
6800  } catch (std::exception& e) {
6801  THROW_DB_EXCEPTION(e.what());
6802  }
6803 }
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
std::unique_ptr< HeavyDBLeafHandler > leaf_handler_
Definition: DBHandler.h:647
#define STDLOG(...)
Definition: QueryState.h:235
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

void DBHandler::check_table_load_privileges ( const TSessionId &  session,
const std::string &  table_name 
)
private

Definition at line 5953 of file DBHandler.cpp.

References get_session_copy().

Referenced by import_table(), importGeoTableSingle(), and prepare_loader_generic().

5954  {
5955  const auto session_info = get_session_copy(session);
5956  check_table_load_privileges(session_info, table_name);
5957 }
void check_table_load_privileges(const TSessionId &session, const std::string &table_name)
Definition: DBHandler.cpp:5953
Catalog_Namespace::SessionInfo get_session_copy(const TSessionId &session)
Definition: DBHandler.cpp:5901

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::check_table_load_privileges ( const Catalog_Namespace::SessionInfo session_info,
const std::string &  table_name 
)
private

Definition at line 5936 of file DBHandler.cpp.

References cat(), Catalog_Namespace::SessionInfo::get_currentUser(), Catalog_Namespace::SessionInfo::getCatalog(), AccessPrivileges::INSERT_INTO_TABLE, DBObject::loadKey(), DBObject::setPrivileges(), TableDBObjectType, and THROW_DB_EXCEPTION.

5938  {
5939  auto user_metadata = session_info.get_currentUser();
5940  auto& cat = session_info.getCatalog();
5941  DBObject dbObject(table_name, TableDBObjectType);
5942  dbObject.loadKey(cat);
5943  dbObject.setPrivileges(AccessPrivileges::INSERT_INTO_TABLE);
5944  std::vector<DBObject> privObjects;
5945  privObjects.push_back(dbObject);
5946  if (!SysCatalog::instance().checkPrivileges(user_metadata, privObjects)) {
5947  THROW_DB_EXCEPTION("Violation of access privileges: user " +
5948  user_metadata.userLoggable() +
5949  " has no insert privileges for table " + table_name + ".");
5950  }
5951 }
std::string cat(Ts &&...args)
static const AccessPrivileges INSERT_INTO_TABLE
Definition: DBObject.h:161
Catalog & getCatalog() const
Definition: SessionInfo.h:65
const UserMetadata & get_currentUser() const
Definition: SessionInfo.h:71
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

void DBHandler::checkpoint ( const TSessionId &  session,
const int32_t  table_id 
)
override

Definition at line 7088 of file DBHandler.cpp.

References cat(), get_session_ptr(), and STDLOG.

7088  {
7089  auto stdlog = STDLOG(get_session_ptr(session));
7090  auto session_ptr = stdlog.getConstSessionInfo();
7091  auto& cat = session_ptr->getCatalog();
7092  cat.checkpoint(table_id);
7093 }
std::string cat(Ts &&...args)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

void DBHandler::clear_cpu_memory ( const TSessionId &  session)
override

Definition at line 2770 of file DBHandler.cpp.

References LeafAggregator::clear_leaf_cpu_memory(), Executor::clearMemory(), Data_Namespace::CPU_LEVEL, get_session_ptr(), getConnectionInfo(), leaf_aggregator_, LeafAggregator::leafCount(), render_handler_, STDLOG, THROW_DB_EXCEPTION, and toString().

Referenced by alterSystemClear().

2770  {
2771  auto stdlog = STDLOG(get_session_ptr(session));
2772  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
2773  auto session_ptr = stdlog.getConstSessionInfo();
2774  if (!session_ptr->get_currentUser().isSuper) {
2775  THROW_DB_EXCEPTION("Superuser privilege is required to run clear_cpu_memory");
2776  }
2777  try {
2779  } catch (const std::exception& e) {
2780  THROW_DB_EXCEPTION(e.what());
2781  }
2782  if (render_handler_) {
2783  render_handler_->clear_cpu_memory();
2784  }
2785 
2786  if (leaf_aggregator_.leafCount() > 0) {
2788  }
2789 }
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:625
static void clearMemory(const Data_Namespace::MemoryLevel memory_level)
Definition: Execute.cpp:492
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
void clear_leaf_cpu_memory(const TSessionId session)
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:645
size_t leafCount() const
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1448
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1737
#define STDLOG(...)
Definition: QueryState.h:235
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::clear_gpu_memory ( const TSessionId &  session)
override

Definition at line 2749 of file DBHandler.cpp.

References LeafAggregator::clear_leaf_gpu_memory(), Executor::clearMemory(), get_session_ptr(), getConnectionInfo(), Data_Namespace::GPU_LEVEL, leaf_aggregator_, LeafAggregator::leafCount(), render_handler_, STDLOG, THROW_DB_EXCEPTION, and toString().

Referenced by alterSystemClear().

2749  {
2750  auto stdlog = STDLOG(get_session_ptr(session));
2751  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
2752  auto session_ptr = stdlog.getConstSessionInfo();
2753  if (!session_ptr->get_currentUser().isSuper) {
2754  THROW_DB_EXCEPTION("Superuser privilege is required to run clear_gpu_memory");
2755  }
2756  try {
2758  } catch (const std::exception& e) {
2759  THROW_DB_EXCEPTION(e.what());
2760  }
2761  if (render_handler_) {
2762  render_handler_->clear_gpu_memory();
2763  }
2764 
2765  if (leaf_aggregator_.leafCount() > 0) {
2767  }
2768 }
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:625
void clear_leaf_gpu_memory(const TSessionId session)
static void clearMemory(const Data_Namespace::MemoryLevel memory_level)
Definition: Execute.cpp:492
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:645
size_t leafCount() const
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1448
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1737
#define STDLOG(...)
Definition: QueryState.h:235
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::clearRenderMemory ( const TSessionId &  session)

Definition at line 2791 of file DBHandler.cpp.

References get_session_ptr(), getConnectionInfo(), render_handler_, STDLOG, THROW_DB_EXCEPTION, and toString().

Referenced by alterSystemClear().

2791  {
2792  auto stdlog = STDLOG(get_session_ptr(session));
2793  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
2794  auto session_ptr = stdlog.getConstSessionInfo();
2795  if (!session_ptr->get_currentUser().isSuper) {
2796  THROW_DB_EXCEPTION("Superuser privilege is required to run clear_render_memory");
2797  }
2798  if (render_handler_) {
2799  render_handler_->clear_cpu_memory();
2800  render_handler_->clear_gpu_memory();
2801  }
2802 }
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:645
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1448
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1737
#define STDLOG(...)
Definition: QueryState.h:235
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::clone_session ( TSessionId &  session2,
const TSessionId &  session1 
)
override

Definition at line 742 of file DBHandler.cpp.

References cat(), LeafAggregator::clone_session(), create_new_session(), get_session_it_unsafe(), get_session_ptr(), getConnectionInfo(), leaf_aggregator_, LeafAggregator::leafCount(), sessions_mutex_, STDLOG, THROW_DB_EXCEPTION, and toString().

742  {
743  auto stdlog = STDLOG(get_session_ptr(session1));
744  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
746  auto session_it = get_session_it_unsafe(session1, write_lock);
747 
748  try {
749  const Catalog_Namespace::UserMetadata& user_meta =
750  session_it->second->get_currentUser();
751  std::shared_ptr<Catalog> cat = session_it->second->get_catalog_ptr();
752  auto session2_ptr = create_new_session(session2, cat->name(), user_meta, cat);
753  if (leaf_aggregator_.leafCount() > 0) {
754  leaf_aggregator_.clone_session(session1, session2);
755  return;
756  }
757  } catch (std::exception& e) {
758  THROW_DB_EXCEPTION(e.what());
759  }
760 }
std::string cat(Ts &&...args)
void clone_session(const TSessionId session1, const TSessionId session2)
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:625
std::shared_ptr< Catalog_Namespace::SessionInfo > create_new_session(TSessionId &session, const std::string &dbname, const Catalog_Namespace::UserMetadata &user_meta, std::shared_ptr< Catalog_Namespace::Catalog > cat)
Definition: DBHandler.cpp:621
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
std::unique_lock< T > unique_lock
size_t leafCount() const
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1448
heavyai::shared_mutex sessions_mutex_
Definition: DBHandler.h:639
SessionMap::iterator get_session_it_unsafe(const TSessionId &session, SESSION_MAP_LOCK &lock)
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1737
#define STDLOG(...)
Definition: QueryState.h:235
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

void DBHandler::connect ( TSessionId &  session,
const std::string &  username,
const std::string &  passwd,
const std::string &  dbname 
)
override

Definition at line 586 of file DBHandler.cpp.

References AccessPrivileges::ACCESS, cat(), connect_impl(), DatabaseDBObjectType, getConnectionInfo(), DBObject::loadKey(), DBObject::setPrivileges(), STDLOG, super_user_rights_, THROW_DB_EXCEPTION, toString(), and Catalog_Namespace::UserMetadata::userLoggable().

589  {
590  auto stdlog = STDLOG(); // session_info set by connect_impl()
591  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
592  std::string username2 = username; // login() may reset username given as argument
593  std::string dbname2 = dbname; // login() may reset dbname given as argument
595  std::shared_ptr<Catalog> cat = nullptr;
596  try {
597  cat = SysCatalog::instance().login(
598  dbname2, username2, passwd, user_meta, !super_user_rights_);
599  } catch (std::exception& e) {
600  stdlog.appendNameValuePairs("user", username, "db", dbname, "exception", e.what());
601  THROW_DB_EXCEPTION(e.what());
602  }
603 
604  DBObject dbObject(dbname2, DatabaseDBObjectType);
605  dbObject.loadKey(*cat);
606  dbObject.setPrivileges(AccessPrivileges::ACCESS);
607  std::vector<DBObject> dbObjects;
608  dbObjects.push_back(dbObject);
609  if (!SysCatalog::instance().checkPrivileges(user_meta, dbObjects)) {
610  stdlog.appendNameValuePairs(
611  "user", username, "db", dbname, "exception", "Missing Privileges");
612  THROW_DB_EXCEPTION("Unauthorized Access: user " + user_meta.userLoggable() +
613  " is not allowed to access database " + dbname2 + ".");
614  }
615  connect_impl(session, passwd, dbname2, user_meta, cat, stdlog);
616 
617  // if pki auth session will come back encrypted with user pubkey
618  SysCatalog::instance().check_for_session_encryption(passwd, session);
619 }
std::string cat(Ts &&...args)
void connect_impl(TSessionId &session, const std::string &passwd, const std::string &dbname, const Catalog_Namespace::UserMetadata &user_meta, std::shared_ptr< Catalog_Namespace::Catalog > cat, query_state::StdLog &stdlog)
Definition: DBHandler.cpp:639
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1448
bool super_user_rights_
Definition: DBHandler.h:953
static const AccessPrivileges ACCESS
Definition: DBObject.h:153
std::string userLoggable() const
Definition: SysCatalog.cpp:127
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1737
#define STDLOG(...)
Definition: QueryState.h:235
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

void DBHandler::connect_impl ( TSessionId &  session,
const std::string &  passwd,
const std::string &  dbname,
const Catalog_Namespace::UserMetadata user_meta,
std::shared_ptr< Catalog_Namespace::Catalog cat,
query_state::StdLog stdlog 
)
private

Definition at line 639 of file DBHandler.cpp.

References query_state::StdLog::appendNameValuePairs(), LeafAggregator::connect(), create_new_session(), expire_idle_sessions_unsafe(), getConnectionInfo(), query_state::StdLog::getConstSessionInfo(), join(), leaf_aggregator_, LeafAggregator::leafCount(), SystemParameters::num_sessions, sessions_, sessions_mutex_, query_state::StdLog::setSessionInfo(), super_user_rights_, system_parameters_, THROW_DB_EXCEPTION, toString(), and Catalog_Namespace::UserMetadata::userName.

Referenced by connect(), and internal_connect().

644  {
645  // TODO(sy): Is there any reason to have dbname as a parameter
646  // here when the cat parameter already provides cat->name()?
647  // Should dbname and cat->name() ever differ?
648  {
652  sessions_.size() + 1 > static_cast<size_t>(system_parameters_.num_sessions)) {
653  THROW_DB_EXCEPTION("Too many active sessions");
654  }
655  }
656  {
658  auto session_ptr = create_new_session(session, dbname, user_meta, cat);
659  stdlog.setSessionInfo(session_ptr);
660  session_ptr->set_connection_info(getConnectionInfo().toString());
661  if (!super_user_rights_) { // no need to connect to leaf_aggregator_ at this time
662  // while doing warmup
663  if (leaf_aggregator_.leafCount() > 0) {
664  leaf_aggregator_.connect(*session_ptr, user_meta.userName, passwd, dbname);
665  return;
666  }
667  }
668  }
669  auto const roles =
670  stdlog.getConstSessionInfo()->get_currentUser().isSuper
671  ? std::vector<std::string>{{"super"}}
672  : SysCatalog::instance().getRoles(
673  false, false, stdlog.getConstSessionInfo()->get_currentUser().userName);
674  stdlog.appendNameValuePairs("roles", boost::algorithm::join(roles, ","));
675 }
std::lock_guard< T > lock_guard
std::string cat(Ts &&...args)
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:625
std::shared_ptr< Catalog_Namespace::SessionInfo > create_new_session(TSessionId &session, const std::string &dbname, const Catalog_Namespace::UserMetadata &user_meta, std::shared_ptr< Catalog_Namespace::Catalog > cat)
Definition: DBHandler.cpp:621
std::string join(T const &container, std::string const &delim)
void connect(const Catalog_Namespace::SessionInfo &parent_session_info, const std::string &user, const std::string &passwd, const std::string &dbname)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void setSessionInfo(std::shared_ptr< Catalog_Namespace::SessionInfo >)
Definition: QueryState.cpp:286
void expire_idle_sessions_unsafe(SESSION_MAP_LOCK &lock)
std::unique_lock< T > unique_lock
SystemParameters & system_parameters_
Definition: DBHandler.h:643
size_t leafCount() const
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1448
heavyai::shared_mutex sessions_mutex_
Definition: DBHandler.h:639
void appendNameValuePairs(Pairs &&...pairs)
Definition: QueryState.h:313
bool super_user_rights_
Definition: DBHandler.h:953
std::shared_ptr< Catalog_Namespace::SessionInfo const > getConstSessionInfo() const
Definition: QueryState.cpp:159
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1737
SessionMap sessions_
Definition: DBHandler.h:951
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::convertData ( TQueryResult &  _return,
ExecutionResult result,
const QueryStateProxy query_state_proxy,
const bool  column_format,
const int32_t  first_n,
const int32_t  at_most_n 
)
staticprivate

Definition at line 1207 of file DBHandler.cpp.

References ExecutionResult::CalciteDdl, convertExplain(), convertResult(), convertRows(), ExecutionResult::empty(), ExecutionResult::Explanation, ExecutionResult::getExecutionTime(), ExecutionResult::getResultType(), ExecutionResult::getRows(), ExecutionResult::getTargetsMeta(), ExecutionResult::QueryResult, and ExecutionResult::SimpleResult.

Referenced by convertResultSet(), get_tables_meta_impl(), sql_execute_local(), and validate_rel_alg().

1212  {
1213  _return.execution_time_ms += result.getExecutionTime();
1214  if (result.empty()) {
1215  return;
1216  }
1217 
1218  switch (result.getResultType()) {
1220  convertRows(_return,
1221  query_state_proxy,
1222  result.getTargetsMeta(),
1223  *result.getRows(),
1224  column_format,
1225  first_n,
1226  at_most_n);
1227  break;
1229  convertResult(_return, *result.getRows(), true);
1230  break;
1232  convertExplain(_return, *result.getRows(), true);
1233  break;
1235  convertRows(_return,
1236  query_state_proxy,
1237  result.getTargetsMeta(),
1238  *result.getRows(),
1239  column_format,
1240  -1,
1241  -1);
1242  break;
1243  }
1244 }
RType getResultType() const
static void convertExplain(TQueryResult &_return, const ResultSet &results, const bool column_format)
Definition: DBHandler.cpp:6200
static void convertResult(TQueryResult &_return, const ResultSet &results, const bool column_format)
Definition: DBHandler.cpp:6206
const std::vector< TargetMetaInfo > & getTargetsMeta() const
const std::shared_ptr< ResultSet > & getRows() const
static void convertRows(TQueryResult &_return, QueryStateProxy query_state_proxy, const std::vector< TargetMetaInfo > &targets, const ResultSet &results, const bool column_format, const int32_t first_n, const int32_t at_most_n)
Definition: DBHandler.cpp:6092
int64_t getExecutionTime() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::convertExplain ( TQueryResult &  _return,
const ResultSet results,
const bool  column_format 
)
staticprivate

Definition at line 6200 of file DBHandler.cpp.

References createSimpleResult().

Referenced by convertData().

6202  {
6203  createSimpleResult(_return, results, column_format, "Explanation");
6204 }
static void createSimpleResult(TQueryResult &_return, const ResultSet &results, const bool column_format, const std::string label)
Definition: DBHandler.cpp:6163

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::convertResult ( TQueryResult &  _return,
const ResultSet results,
const bool  column_format 
)
staticprivate

Definition at line 6206 of file DBHandler.cpp.

References createSimpleResult().

Referenced by convertData().

6208  {
6209  createSimpleResult(_return, results, column_format, "Result");
6210 }
static void createSimpleResult(TQueryResult &_return, const ResultSet &results, const bool column_format, const std::string label)
Definition: DBHandler.cpp:6163

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::convertResultSet ( ExecutionResult result,
const Catalog_Namespace::SessionInfo session_info,
const std::string &  query_state_str,
TQueryResult &  _return 
)
private

Definition at line 7498 of file DBHandler.cpp.

References CHECK, convertData(), create_query_state(), Catalog_Namespace::SessionInfo::get_session_id(), get_session_ptr(), and ExecutionResult::getDataPtr().

Referenced by executeDdl().

7501  {
7502  // Stuff ResultSet into _return (which is a TQueryResult)
7503  // calls convertRows, but after some setup using session_info
7504 
7505  auto session_ptr = get_session_ptr(session_info.get_session_id());
7506  CHECK(session_ptr);
7507  auto qs = create_query_state(session_ptr, query_state_str);
7508  QueryStateProxy qsp = qs->createQueryStateProxy();
7509 
7510  // heavysql only accepts column format as being 'VALID",
7511  // assume that heavydb should only return column format
7512  int32_t nRows = result.getDataPtr()->rowCount();
7513 
7514  convertData(_return,
7515  result,
7516  qsp,
7517  /*column_format=*/true,
7518  /*first_n=*/nRows,
7519  /*at_most_n=*/nRows);
7520 }
static void convertData(TQueryResult &_return, ExecutionResult &result, const QueryStateProxy &query_state_proxy, const bool column_format, const int32_t first_n, const int32_t at_most_n)
Definition: DBHandler.cpp:1207
std::shared_ptr< query_state::QueryState > create_query_state(ARGS &&...args)
Definition: DBHandler.h:654
const ResultSetPtr & getDataPtr() const
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
std::string get_session_id() const
Definition: SessionInfo.h:76
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::convertRows ( TQueryResult &  _return,
QueryStateProxy  query_state_proxy,
const std::vector< TargetMetaInfo > &  targets,
const ResultSet results,
const bool  column_format,
const int32_t  first_n,
const int32_t  at_most_n 
)
staticprivate

Definition at line 6092 of file DBHandler.cpp.

References query_state::QueryStateProxy::createTimer(), ThriftSerializers::target_meta_infos_to_thrift(), THROW_DB_EXCEPTION, to_string(), value_to_thrift(), and value_to_thrift_column().

Referenced by convertData().

6098  {
6099  query_state::Timer timer = query_state_proxy.createTimer(__func__);
6100  _return.row_set.row_desc = ThriftSerializers::target_meta_infos_to_thrift(targets);
6101  int32_t fetched{0};
6102  if (column_format) {
6103  _return.row_set.is_columnar = true;
6104  std::vector<TColumn> tcolumns(results.colCount());
6105  while (first_n == -1 || fetched < first_n) {
6106  const auto crt_row = results.getNextRow(true, true);
6107  if (crt_row.empty()) {
6108  break;
6109  }
6110  ++fetched;
6111  if (at_most_n >= 0 && fetched > at_most_n) {
6112  THROW_DB_EXCEPTION("The result contains more rows than the specified cap of " +
6113  std::to_string(at_most_n));
6114  }
6115  for (size_t i = 0; i < results.colCount(); ++i) {
6116  const auto agg_result = crt_row[i];
6117  value_to_thrift_column(agg_result, targets[i].get_type_info(), tcolumns[i]);
6118  }
6119  }
6120  for (size_t i = 0; i < results.colCount(); ++i) {
6121  _return.row_set.columns.push_back(tcolumns[i]);
6122  }
6123  } else {
6124  _return.row_set.is_columnar = false;
6125  while (first_n == -1 || fetched < first_n) {
6126  const auto crt_row = results.getNextRow(true, true);
6127  if (crt_row.empty()) {
6128  break;
6129  }
6130  ++fetched;
6131  if (at_most_n >= 0 && fetched > at_most_n) {
6132  THROW_DB_EXCEPTION("The result contains more rows than the specified cap of " +
6133  std::to_string(at_most_n));
6134  }
6135  TRow trow;
6136  trow.cols.reserve(results.colCount());
6137  for (size_t i = 0; i < results.colCount(); ++i) {
6138  const auto agg_result = crt_row[i];
6139  trow.cols.push_back(value_to_thrift(agg_result, targets[i].get_type_info()));
6140  }
6141  _return.row_set.rows.push_back(trow);
6142  }
6143  }
6144 }
TRowDescriptor target_meta_infos_to_thrift(const std::vector< TargetMetaInfo > &targets)
static void value_to_thrift_column(const TargetValue &tv, const SQLTypeInfo &ti, TColumn &column)
Definition: DBHandler.cpp:939
Timer createTimer(char const *event_name)
Definition: QueryState.cpp:138
std::string to_string(char const *&&v)
static TDatum value_to_thrift(const TargetValue &tv, const SQLTypeInfo &ti)
Definition: DBHandler.cpp:1062
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

TCopyParams DBHandler::copyparams_to_thrift ( const import_export::CopyParams cp)
private

Definition at line 3875 of file DBHandler.cpp.

References import_export::CopyParams::add_metadata_columns, import_export::CopyParams::array_begin, import_export::CopyParams::array_delim, import_export::CopyParams::array_end, CHECK, import_export::CopyParams::connection_string, import_export::CopyParams::credential_string, import_export::CopyParams::delimiter, import_export::CopyParams::dsn, import_export::CopyParams::escape, import_export::CopyParams::geo_assign_render_groups, import_export::CopyParams::geo_coords_comp_param, import_export::CopyParams::geo_coords_encoding, import_export::CopyParams::geo_coords_srid, import_export::CopyParams::geo_coords_type, import_export::CopyParams::geo_explode_collections, import_export::CopyParams::geo_layer_name, import_export::CopyParams::has_header, import_export::kAuto, import_export::kAutoDetect, import_export::kDelimitedFile, import_export::kDouble, kENCODING_GEOINT, import_export::kFile, import_export::kFloat, import_export::kGeoFile, kGEOGRAPHY, kGEOMETRY, import_export::kHasHeader, import_export::kInt, import_export::kNoHeader, import_export::kNone, import_export::kOdbc, import_export::kParquetFile, import_export::kPoint, import_export::kRasterFile, import_export::kSmallInt, import_export::kWorld, import_export::CopyParams::line_delim, import_export::CopyParams::null_str, import_export::CopyParams::password, import_export::CopyParams::quote, import_export::CopyParams::quoted, import_export::CopyParams::raster_import_bands, import_export::CopyParams::raster_import_dimensions, import_export::CopyParams::raster_point_compute_angle, import_export::CopyParams::raster_point_transform, import_export::CopyParams::raster_point_type, import_export::CopyParams::raster_scanlines_per_thread, import_export::CopyParams::s3_access_key, import_export::CopyParams::s3_endpoint, import_export::CopyParams::s3_region, import_export::CopyParams::s3_secret_key, import_export::CopyParams::s3_session_token, import_export::CopyParams::sanitize_column_names, import_export::CopyParams::source_srid, import_export::CopyParams::source_type, import_export::CopyParams::sql_order_by, import_export::CopyParams::sql_select, import_export::CopyParams::threads, import_export::CopyParams::trim_spaces, and import_export::CopyParams::username.

Referenced by detect_column_types(), and importGeoTableSingle().

3875  {
3876  TCopyParams copy_params;
3877  copy_params.delimiter = cp.delimiter;
3878  copy_params.null_str = cp.null_str;
3879  switch (cp.has_header) {
3881  copy_params.has_header = TImportHeaderRow::AUTODETECT;
3882  break;
3884  copy_params.has_header = TImportHeaderRow::NO_HEADER;
3885  break;
3887  copy_params.has_header = TImportHeaderRow::HAS_HEADER;
3888  break;
3889  default:
3890  CHECK(false);
3891  }
3892  copy_params.quoted = cp.quoted;
3893  copy_params.quote = cp.quote;
3894  copy_params.escape = cp.escape;
3895  copy_params.line_delim = cp.line_delim;
3896  copy_params.array_delim = cp.array_delim;
3897  copy_params.array_begin = cp.array_begin;
3898  copy_params.array_end = cp.array_end;
3899  copy_params.threads = cp.threads;
3900  copy_params.s3_access_key = cp.s3_access_key;
3901  copy_params.s3_secret_key = cp.s3_secret_key;
3902  copy_params.s3_session_token = cp.s3_session_token;
3903  copy_params.s3_region = cp.s3_region;
3904  copy_params.s3_endpoint = cp.s3_endpoint;
3905  switch (cp.source_type) {
3907  copy_params.source_type = TSourceType::DELIMITED_FILE;
3908  break;
3910  copy_params.source_type = TSourceType::GEO_FILE;
3911  break;
3913  copy_params.source_type = TSourceType::PARQUET_FILE;
3914  break;
3916  copy_params.source_type = TSourceType::RASTER_FILE;
3917  break;
3919  copy_params.source_type = TSourceType::ODBC;
3920  break;
3921  default:
3922  CHECK(false);
3923  }
3924  switch (cp.geo_coords_encoding) {
3925  case kENCODING_GEOINT:
3926  copy_params.geo_coords_encoding = TEncodingType::GEOINT;
3927  break;
3928  default:
3929  copy_params.geo_coords_encoding = TEncodingType::NONE;
3930  break;
3931  }
3932  copy_params.geo_coords_comp_param = cp.geo_coords_comp_param;
3933  switch (cp.geo_coords_type) {
3934  case kGEOGRAPHY:
3935  copy_params.geo_coords_type = TDatumType::GEOGRAPHY;
3936  break;
3937  case kGEOMETRY:
3938  copy_params.geo_coords_type = TDatumType::GEOMETRY;
3939  break;
3940  default:
3941  CHECK(false);
3942  }
3943  copy_params.geo_coords_srid = cp.geo_coords_srid;
3944  copy_params.sanitize_column_names = cp.sanitize_column_names;
3945  copy_params.geo_layer_name = cp.geo_layer_name;
3946  copy_params.geo_assign_render_groups = cp.geo_assign_render_groups;
3947  copy_params.geo_explode_collections = cp.geo_explode_collections;
3948  copy_params.source_srid = cp.source_srid;
3949  switch (cp.raster_point_type) {
3951  copy_params.raster_point_type = TRasterPointType::NONE;
3952  break;
3954  copy_params.raster_point_type = TRasterPointType::AUTO;
3955  break;
3957  copy_params.raster_point_type = TRasterPointType::SMALLINT;
3958  break;
3960  copy_params.raster_point_type = TRasterPointType::INT;
3961  break;
3963  copy_params.raster_point_type = TRasterPointType::FLOAT;
3964  break;
3966  copy_params.raster_point_type = TRasterPointType::DOUBLE;
3967  break;
3969  copy_params.raster_point_type = TRasterPointType::POINT;
3970  break;
3971  default:
3972  CHECK(false);
3973  }
3974  copy_params.raster_import_bands = cp.raster_import_bands;
3975  copy_params.raster_scanlines_per_thread = cp.raster_scanlines_per_thread;
3976  switch (cp.raster_point_transform) {
3978  copy_params.raster_point_transform = TRasterPointTransform::NONE;
3979  break;
3981  copy_params.raster_point_transform = TRasterPointTransform::AUTO;
3982  break;
3984  copy_params.raster_point_transform = TRasterPointTransform::FILE;
3985  break;
3987  copy_params.raster_point_transform = TRasterPointTransform::WORLD;
3988  break;
3989  default:
3990  CHECK(false);
3991  }
3992  copy_params.raster_point_compute_angle = cp.raster_point_compute_angle;
3993  copy_params.raster_import_dimensions = cp.raster_import_dimensions;
3994  copy_params.odbc_dsn = cp.dsn;
3995  copy_params.odbc_connection_string = cp.connection_string;
3996  copy_params.odbc_sql_select = cp.sql_select;
3997  copy_params.odbc_sql_order_by = cp.sql_order_by;
3998  copy_params.odbc_username = cp.username;
3999  copy_params.odbc_password = cp.password;
4000  copy_params.odbc_credential_string = cp.credential_string;
4001  copy_params.add_metadata_columns = cp.add_metadata_columns;
4002  copy_params.trim_spaces = cp.trim_spaces;
4003  return copy_params;
4004 }
std::string s3_secret_key
Definition: CopyParams.h:62
int32_t raster_scanlines_per_thread
Definition: CopyParams.h:89
std::string connection_string
Definition: CopyParams.h:103
std::string raster_import_dimensions
Definition: CopyParams.h:92
std::string add_metadata_columns
Definition: CopyParams.h:93
ImportHeaderRow has_header
Definition: CopyParams.h:46
RasterPointType raster_point_type
Definition: CopyParams.h:87
std::string sql_order_by
Definition: CopyParams.h:96
import_export::SourceType source_type
Definition: CopyParams.h:57
std::string geo_layer_name
Definition: CopyParams.h:80
std::string s3_session_token
Definition: CopyParams.h:63
std::string raster_import_bands
Definition: CopyParams.h:88
#define CHECK(condition)
Definition: Logger.h:222
std::string s3_access_key
Definition: CopyParams.h:61
RasterPointTransform raster_point_transform
Definition: CopyParams.h:90
std::string credential_string
Definition: CopyParams.h:100
EncodingType geo_coords_encoding
Definition: CopyParams.h:75

+ Here is the caller graph for this function:

int32_t DBHandler::create_custom_expression ( const TSessionId &  session,
const TCustomExpression &  custom_expression 
)
override

Definition at line 4446 of file DBHandler.cpp.

References check_read_only(), anonymous_namespace{DBHandler.cpp}::create_custom_expr_from_thrift_obj(), custom_expressions_mutex_, get_session_ptr(), getConnectionInfo(), STDLOG, THROW_DB_EXCEPTION, and toString().

4447  {
4448  auto stdlog = STDLOG(get_session_ptr(session));
4449  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4450  check_read_only("create_custom_expression");
4451 
4452  auto session_ptr = stdlog.getConstSessionInfo();
4453  if (!session_ptr->get_currentUser().isSuper) {
4454  THROW_DB_EXCEPTION("Custom expressions can only be created by super users.")
4455  }
4456  auto& catalog = session_ptr->getCatalog();
4458  return catalog.createCustomExpression(
4459  create_custom_expr_from_thrift_obj(t_custom_expr, catalog));
4460 }
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
std::unique_lock< T > unique_lock
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1448
heavyai::shared_mutex custom_expressions_mutex_
Definition: DBHandler.h:1109
void check_read_only(const std::string &str)
Definition: DBHandler.cpp:500
std::unique_ptr< Catalog_Namespace::CustomExpression > create_custom_expr_from_thrift_obj(const TCustomExpression &t_custom_expr, const Catalog &catalog)
Definition: DBHandler.cpp:4403
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1737
#define STDLOG(...)
Definition: QueryState.h:235
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

int32_t DBHandler::create_dashboard ( const TSessionId &  session,
const std::string &  dashboard_name,
const std::string &  dashboard_state,
const std::string &  image_hash,
const std::string &  dashboard_metadata 
)
override

Definition at line 4633 of file DBHandler.cpp.

References cat(), CHECK, dbhandler::check_not_info_schema_db(), check_read_only(), AccessPrivileges::CREATE_DASHBOARD, anonymous_namespace{DBHandler.cpp}::dashboard_exists(), DashboardDBObjectType, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, g_allow_system_dashboard_update, get_session_ptr(), getConnectionInfo(), DashboardDescriptor::imageHash, STDLOG, THROW_DB_EXCEPTION, toString(), DashboardDescriptor::user, and DashboardDescriptor::userId.

4637  {
4638  auto stdlog = STDLOG(get_session_ptr(session));
4639  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4640  auto session_ptr = stdlog.getConstSessionInfo();
4641  CHECK(session_ptr);
4642  check_read_only("create_dashboard");
4643  auto& cat = session_ptr->getCatalog();
4646  }
4647 
4648  if (!session_ptr->checkDBAccessPrivileges(DBObjectType::DashboardDBObjectType,
4650  THROW_DB_EXCEPTION("Not enough privileges to create a dashboard.");
4651  }
4652 
4653  if (dashboard_exists(cat, session_ptr->get_currentUser().userId, dashboard_name)) {
4654  THROW_DB_EXCEPTION("Dashboard with name: " + dashboard_name + " already exists.");
4655  }
4656 
4658  dd.dashboardName = dashboard_name;
4659  dd.dashboardState = dashboard_state;
4660  dd.imageHash = image_hash;
4661  dd.dashboardMetadata = dashboard_metadata;
4662  dd.userId = session_ptr->get_currentUser().userId;
4663  dd.user = session_ptr->get_currentUser().userName;
4664 
4665  try {
4666  auto id = cat.createDashboard(dd);
4667  // TODO: transactionally unsafe
4668  SysCatalog::instance().createDBObject(
4669  session_ptr->get_currentUser(), dashboard_name, DashboardDBObjectType, cat, id);
4670  return id;
4671  } catch (const std::exception& e) {
4672  THROW_DB_EXCEPTION(e.what());
4673  }
4674 }
std::string cat(Ts &&...args)
static const AccessPrivileges CREATE_DASHBOARD
Definition: DBObject.h:170
void check_not_info_schema_db(const std::string &db_name, bool throw_db_exception)
Definition: DBHandler.cpp:4620
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1448
void check_read_only(const std::string &str)
Definition: DBHandler.cpp:500
bool g_allow_system_dashboard_update
Definition: DBHandler.cpp:121
bool dashboard_exists(const Catalog_Namespace::Catalog &cat, const int32_t user_id, const std::string &dashboard_name)
Definition: DBHandler.cpp:141
#define CHECK(condition)
Definition: Logger.h:222
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1737
#define STDLOG(...)
Definition: QueryState.h:235
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

TColumnType DBHandler::create_geo_column ( const TDatumType::type  type,
const std::string &  name,
const bool  is_array 
)
private

Definition at line 4950 of file DBHandler.cpp.

References setup::name, and run_benchmark_import::type.

4952  {
4953  TColumnType ct;
4954  ct.col_name = name;
4955  ct.col_type.type = type;
4956  ct.col_type.is_array = is_array;
4957  return ct;
4958 }
string name
Definition: setup.in.py:72
void DBHandler::create_link ( std::string &  _return,
const TSessionId &  session,
const std::string &  view_state,
const std::string &  view_metadata 
)
override

Definition at line 4928 of file DBHandler.cpp.

References cat(), get_session_ptr(), getConnectionInfo(), STDLOG, THROW_DB_EXCEPTION, toString(), LinkDescriptor::userId, LinkDescriptor::viewMetadata, and LinkDescriptor::viewState.

4931  {
4932  auto stdlog = STDLOG(get_session_ptr(session));
4933  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4934  auto session_ptr = stdlog.getConstSessionInfo();
4935  // check_read_only("create_link");
4936  auto& cat = session_ptr->getCatalog();
4937 
4938  LinkDescriptor ld;
4939  ld.userId = session_ptr->get_currentUser().userId;
4940  ld.viewState = view_state;
4941  ld.viewMetadata = view_metadata;
4942 
4943  try {
4944  _return = cat.createLink(ld, 6);
4945  } catch (const std::exception& e) {
4946  THROW_DB_EXCEPTION(e.what());
4947  }
4948 }
std::string cat(Ts &&...args)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1448
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1737
#define STDLOG(...)
Definition: QueryState.h:235
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

std::shared_ptr< Catalog_Namespace::SessionInfo > DBHandler::create_new_session ( TSessionId &  session,
const std::string &  dbname,
const Catalog_Namespace::UserMetadata user_meta,
std::shared_ptr< Catalog_Namespace::Catalog cat 
)
private

Definition at line 621 of file DBHandler.cpp.

References CHECK, executor_device_type_, generate_random_string(), logger::INFO, LOG, sessions_, and Catalog_Namespace::UserMetadata::userLoggable().

Referenced by clone_session(), and connect_impl().

625  {
626  do {
627  session = generate_random_string(32);
628  } while (sessions_.find(session) != sessions_.end());
629  std::pair<SessionMap::iterator, bool> emplace_retval =
630  sessions_.emplace(session,
631  std::make_shared<Catalog_Namespace::SessionInfo>(
632  cat, user_meta, executor_device_type_, session));
633  CHECK(emplace_retval.second);
634  auto& session_ptr = emplace_retval.first->second;
635  LOG(INFO) << "User " << user_meta.userLoggable() << " connected to database " << dbname;
636  return session_ptr;
637 }
std::string cat(Ts &&...args)
#define LOG(tag)
Definition: Logger.h:216
std::string generate_random_string(const size_t len)
ExecutorDeviceType executor_device_type_
Definition: DBHandler.h:630
#define CHECK(condition)
Definition: Logger.h:222
std::string userLoggable() const
Definition: SysCatalog.cpp:127
SessionMap sessions_
Definition: DBHandler.h:951

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename... ARGS>
std::shared_ptr<query_state::QueryState> DBHandler::create_query_state ( ARGS &&...  args)
inline

Definition at line 654 of file DBHandler.h.

References run_benchmark_import::args.

Referenced by convertResultSet(), get_table_details_impl(), get_tables_meta(), sql_execute(), sql_execute_df(), and sql_validate().

654  {
655  return query_states_.create(std::forward<ARGS>(args)...);
656  }
CircleBuffer::value_type create(ARGS &&...args)
Definition: QueryState.h:195
query_state::QueryStates query_states_
Definition: DBHandler.h:950

+ Here is the caller graph for this function:

void DBHandler::create_table ( const TSessionId &  session,
const std::string &  table_name,
const TRowDescriptor &  row_desc,
const TCreateParams &  create_params 
)
override

Definition at line 4981 of file DBHandler.cpp.

References check_read_only(), getConnectionInfo(), ImportHelpers::is_reserved_name(), join(), kENCODING_DATE_IN_DAYS, kENCODING_DICT, kENCODING_FIXED, kENCODING_GEOINT, kENCODING_NONE, ImportHelpers::sanitize_name(), sql_execute(), STDLOG, thrift_to_encoding(), thrift_to_encoding_name(), thrift_to_name(), THROW_DB_EXCEPTION, to_string(), and toString().

Referenced by importGeoTableSingle().

4984  {
4985  // @TODO(se) remove file_type which is unused
4986  auto stdlog = STDLOG("table_name", table_name);
4987  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4988  check_read_only("create_table");
4989 
4990  if (ImportHelpers::is_reserved_name(table_name)) {
4991  THROW_DB_EXCEPTION("Invalid table name (reserved keyword): " + table_name);
4992  } else if (table_name != ImportHelpers::sanitize_name(table_name)) {
4993  THROW_DB_EXCEPTION("Invalid characters in table name: " + table_name);
4994  }
4995 
4996  auto rds = rd;
4997 
4998  std::string stmt{"CREATE TABLE " + table_name};
4999  std::vector<std::string> col_stmts;
5000 
5001  for (auto col : rds) {
5002  if (ImportHelpers::is_reserved_name(col.col_name)) {
5003  THROW_DB_EXCEPTION("Invalid column name (reserved keyword): " + col.col_name);
5004  } else if (col.col_name != ImportHelpers::sanitize_name(col.col_name)) {
5005  THROW_DB_EXCEPTION("Invalid characters in column name: " + col.col_name);
5006  }
5007  if (col.col_type.type == TDatumType::INTERVAL_DAY_TIME ||
5008  col.col_type.type == TDatumType::INTERVAL_YEAR_MONTH) {
5009  THROW_DB_EXCEPTION("Unsupported type: " + thrift_to_name(col.col_type) +
5010  " for column: " + col.col_name);
5011  }
5012 
5013  if (col.col_type.type == TDatumType::DECIMAL) {
5014  // if no precision or scale passed in set to default 14,7
5015  if (col.col_type.precision == 0 && col.col_type.scale == 0) {
5016  col.col_type.precision = 14;
5017  col.col_type.scale = 7;
5018  }
5019  }
5020 
5021  std::string col_stmt;
5022  col_stmt.append(col.col_name + " " + thrift_to_name(col.col_type));
5023  if (col.__isset.default_value) {
5024  col_stmt.append(" DEFAULT " + col.default_value);
5025  }
5026 
5027  // As of 2016-06-27 the Immerse v1 frontend does not explicitly set the
5028  // `nullable` argument, leading this to default to false. Uncomment for v2.
5029  // if (!col.col_type.nullable) col_stmt.append(" NOT NULL");
5030 
5031  if (thrift_to_encoding(col.col_type.encoding) != kENCODING_NONE) {
5032  col_stmt.append(" ENCODING " + thrift_to_encoding_name(col.col_type));
5033  if (thrift_to_encoding(col.col_type.encoding) == kENCODING_DICT ||
5034  thrift_to_encoding(col.col_type.encoding) == kENCODING_FIXED ||
5035  thrift_to_encoding(col.col_type.encoding) == kENCODING_GEOINT ||
5036  thrift_to_encoding(col.col_type.encoding) == kENCODING_DATE_IN_DAYS) {
5037  col_stmt.append("(" + std::to_string(col.col_type.comp_param) + ")");
5038  }
5039  } else if (col.col_type.type == TDatumType::STR) {
5040  // non DICT encoded strings
5041  col_stmt.append(" ENCODING NONE");
5042  } else if (col.col_type.type == TDatumType::POINT ||
5043  col.col_type.type == TDatumType::LINESTRING ||
5044  col.col_type.type == TDatumType::POLYGON ||
5045  col.col_type.type == TDatumType::MULTIPOLYGON) {
5046  // non encoded compressable geo
5047  if (col.col_type.scale == 4326) {
5048  col_stmt.append(" ENCODING NONE");
5049  }
5050  }
5051  col_stmts.push_back(col_stmt);
5052  }
5053 
5054  stmt.append(" (" + boost::algorithm::join(col_stmts, ", ") + ")");
5055 
5056  if (create_params.is_replicated) {
5057  stmt.append(" WITH (PARTITIONS = 'REPLICATED')");
5058  }
5059 
5060  stmt.append(";");
5061 
5062  TQueryResult ret;
5063  sql_execute(ret, session, stmt, true, "", -1, -1);
5064 }
EncodingType thrift_to_encoding(const TEncodingType::type tEncodingType)
void sql_execute(ExecutionResult &_return, const TSessionId &session, const std::string &query, const bool column_format, const int32_t first_n, const int32_t at_most_n, lockmgr::LockedTableDescriptors &locks)
Definition: DBHandler.cpp:1323
std::string join(T const &container, std::string const &delim)
std::string to_string(char const *&&v)
bool is_reserved_name(const std::string &name)
std::string sanitize_name(const std::string &name)
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1448
std::string thrift_to_encoding_name(const TTypeInfo &ti)
std::string thrift_to_name(const TTypeInfo &ti)
void check_read_only(const std::string &str)
Definition: DBHandler.cpp:500
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1737
#define STDLOG(...)
Definition: QueryState.h:235
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string const DBHandler::createInMemoryCalciteSession ( const std::shared_ptr< Catalog_Namespace::Catalog > &  catalog_ptr)
private

Definition at line 506 of file DBHandler.cpp.

References calcite_, CHECK, executor_device_type_, generate_random_string(), sessions_, and sessions_mutex_.

Referenced by parse_to_ra().

507  {
508  // We would create an in memory session for calcite with super user privileges which
509  // would be used for getting all tables metadata when a user runs the query. The
510  // session would be under the name of a proxy user/password which would only persist
511  // till server's lifetime or execution of calcite query(in memory) whichever is the
512  // earliest.
514  std::string session_id;
515  do {
516  session_id = generate_random_string(64);
517  } while (sessions_.find(session_id) != sessions_.end());
518  Catalog_Namespace::UserMetadata user_meta(-1,
519  calcite_->getInternalSessionProxyUserName(),
520  calcite_->getInternalSessionProxyPassword(),
521  true,
522  -1,
523  true,
524  false);
525  const auto emplace_ret =
526  sessions_.emplace(session_id,
527  std::make_shared<Catalog_Namespace::SessionInfo>(
528  catalog_ptr, user_meta, executor_device_type_, session_id));
529  CHECK(emplace_ret.second);
530  return session_id;
531 }
std::lock_guard< T > lock_guard
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::string generate_random_string(const size_t len)
ExecutorDeviceType executor_device_type_
Definition: DBHandler.h:630
heavyai::shared_mutex sessions_mutex_
Definition: DBHandler.h:639
std::shared_ptr< Calcite > calcite_
Definition: DBHandler.h:648
#define CHECK(condition)
Definition: Logger.h:222
SessionMap sessions_
Definition: DBHandler.h:951

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::createSimpleResult ( TQueryResult &  _return,
const ResultSet results,
const bool  column_format,
const std::string  label 
)
staticprivate

Definition at line 6163 of file DBHandler.cpp.

References CHECK, CHECK_EQ, and run_benchmark_import::label.

Referenced by convertExplain(), and convertResult().

6166  {
6167  CHECK_EQ(size_t(1), results.rowCount());
6168  TColumnType proj_info;
6169  proj_info.col_name = label;
6170  proj_info.col_type.type = TDatumType::STR;
6171  proj_info.col_type.nullable = false;
6172  proj_info.col_type.is_array = false;
6173  _return.row_set.row_desc.push_back(proj_info);
6174  const auto crt_row = results.getNextRow(true, true);
6175  const auto tv = crt_row[0];
6176  CHECK(results.getNextRow(true, true).empty());
6177  const auto scalar_tv = boost::get<ScalarTargetValue>(&tv);
6178  CHECK(scalar_tv);
6179  const auto s_n = boost::get<NullableString>(scalar_tv);
6180  CHECK(s_n);
6181  const auto s = boost::get<std::string>(s_n);
6182  CHECK(s);
6183  if (column_format) {
6184  TColumn tcol;
6185  tcol.data.str_col.push_back(*s);
6186  tcol.nulls.push_back(false);
6187  _return.row_set.is_columnar = true;
6188  _return.row_set.columns.push_back(tcol);
6189  } else {
6190  TDatum explanation;
6191  explanation.val.str_val = *s;
6192  explanation.is_null = false;
6193  TRow trow;
6194  trow.cols.push_back(explanation);
6195  _return.row_set.is_columnar = false;
6196  _return.row_set.rows.push_back(trow);
6197  }
6198 }
#define CHECK_EQ(x, y)
Definition: Logger.h:230
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the caller graph for this function:

void DBHandler::deallocate_df ( const TSessionId &  session,
const TDataFrame &  df,
const TDeviceType::type  device_type,
const int32_t  device_id 
)
override

Definition at line 1509 of file DBHandler.cpp.

References CPU, data_mgr_, ArrowResultSet::deallocateArrowResultBuffer(), logger::ERROR, get_session_ptr(), GPU, handle_to_dev_ptr_mutex_, ipc_handle_to_dev_ptr_, LOG, run_benchmark_import::result, and STDLOG.

1512  {
1513  auto stdlog = STDLOG(get_session_ptr(session));
1514  std::string serialized_cuda_handle = "";
1515  if (device_type == TDeviceType::GPU) {
1516  std::lock_guard<std::mutex> map_lock(handle_to_dev_ptr_mutex_);
1517  if (ipc_handle_to_dev_ptr_.count(df.df_handle) != size_t(1)) {
1518  TDBException ex;
1519  ex.error_msg = std::string(
1520  "Current data frame handle is not bookkept or been inserted "
1521  "twice");
1522  LOG(ERROR) << ex.error_msg;
1523  throw ex;
1524  }
1525  serialized_cuda_handle = ipc_handle_to_dev_ptr_[df.df_handle];
1526  ipc_handle_to_dev_ptr_.erase(df.df_handle);
1527  }
1528  std::vector<char> sm_handle(df.sm_handle.begin(), df.sm_handle.end());
1529  std::vector<char> df_handle(df.df_handle.begin(), df.df_handle.end());
1531  sm_handle, df.sm_size, df_handle, df.df_size, serialized_cuda_handle};
1533  result,
1534  device_type == TDeviceType::CPU ? ExecutorDeviceType::CPU : ExecutorDeviceType::GPU,
1535  device_id,
1536  data_mgr_);
1537 }
std::mutex handle_to_dev_ptr_mutex_
Definition: DBHandler.h:1012
#define LOG(tag)
Definition: Logger.h:216
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
static void deallocateArrowResultBuffer(const ArrowResult &result, const ExecutorDeviceType device_type, const size_t device_id, std::shared_ptr< Data_Namespace::DataMgr > &data_mgr)
std::shared_ptr< Data_Namespace::DataMgr > data_mgr_
Definition: DBHandler.h:623
std::unordered_map< std::string, std::string > ipc_handle_to_dev_ptr_
Definition: DBHandler.h:1013
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

void DBHandler::delete_custom_expressions ( const TSessionId &  session,
const std::vector< int32_t > &  custom_expression_ids,
const bool  do_soft_delete 
)
override

Definition at line 4493 of file DBHandler.cpp.

References check_read_only(), custom_expressions_mutex_, get_session_ptr(), getConnectionInfo(), STDLOG, THROW_DB_EXCEPTION, and toString().

4496  {
4497  auto stdlog = STDLOG(get_session_ptr(session));
4498  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4499  check_read_only("delete_custom_expressions");
4500 
4501  auto session_ptr = stdlog.getConstSessionInfo();
4502  if (!session_ptr->get_currentUser().isSuper) {
4503  THROW_DB_EXCEPTION("Custom expressions can only be deleted by super users.")
4504  }
4505  auto& catalog = session_ptr->getCatalog();
4507  catalog.deleteCustomExpressions(custom_expression_ids, do_soft_delete);
4508 }
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
std::unique_lock< T > unique_lock
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1448
heavyai::shared_mutex custom_expressions_mutex_
Definition: DBHandler.h:1109
void check_read_only(const std::string &str)
Definition: DBHandler.cpp:500
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1737
#define STDLOG(...)
Definition: QueryState.h:235
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

void DBHandler::delete_dashboard ( const TSessionId &  session,
const int32_t  dashboard_id 
)
override

Definition at line 4726 of file DBHandler.cpp.

References delete_dashboards().

4726  {
4727  delete_dashboards(session, {dashboard_id});
4728 }
void delete_dashboards(const TSessionId &session, const std::vector< int32_t > &dashboard_ids) override
Definition: DBHandler.cpp:4730

+ Here is the call graph for this function:

void DBHandler::delete_dashboards ( const TSessionId &  session,
const std::vector< int32_t > &  dashboard_ids 
)
override

Definition at line 4730 of file DBHandler.cpp.

References cat(), dbhandler::check_not_info_schema_db(), check_read_only(), g_allow_system_dashboard_update, get_session_ptr(), getConnectionInfo(), STDLOG, THROW_DB_EXCEPTION, and toString().

Referenced by delete_dashboard().

4731  {
4732  auto stdlog = STDLOG(get_session_ptr(session));
4733  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4734  auto session_ptr = stdlog.getConstSessionInfo();
4735  check_read_only("delete_dashboards");
4736  auto& cat = session_ptr->getCatalog();
4739  }
4740  // Checks will be performed in catalog
4741  try {
4742  cat.deleteMetadataForDashboards(dashboard_ids, session_ptr->get_currentUser());
4743  } catch (const std::exception& e) {
4744  THROW_DB_EXCEPTION(e.what());
4745  }
4746 }
std::string cat(Ts &&...args)
void check_not_info_schema_db(const std::string &db_name, bool throw_db_exception)
Definition: DBHandler.cpp:4620
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1448
void check_read_only(const std::string &str)
Definition: DBHandler.cpp:500
bool g_allow_system_dashboard_update
Definition: DBHandler.cpp:121
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1737
#define STDLOG(...)
Definition: QueryState.h:235
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::detect_column_types ( TDetectResult &  _return,
const TSessionId &  session,
const std::string &  file_name,
const TCopyParams &  copy_params 
)
override

Definition at line 4164 of file DBHandler.cpp.

References anonymous_namespace{DBHandler.cpp}::add_vsi_archive_prefix(), anonymous_namespace{DBHandler.cpp}::add_vsi_geo_prefix(), anonymous_namespace{DBHandler.cpp}::add_vsi_network_prefix(), CHECK, check_geospatial_files(), check_read_only(), copyparams_to_thrift(), encoding_to_thrift(), shared::file_or_glob_path_exists(), import_export::CopyParams::file_sort_order_by, import_export::CopyParams::file_sort_regex, anonymous_namespace{DBHandler.cpp}::find_first_geo_file_in_archive(), import_export::Importer::gdalFileExists(), import_export::Importer::gdalFileOrDirectoryExists(), import_export::Importer::gdalToColumnDescriptors(), import_export::CopyParams::geo_coords_comp_param, import_export::CopyParams::geo_coords_encoding, import_export::CopyParams::geo_coords_srid, import_export::CopyParams::geo_coords_type, import_export::DataStreamSink::get_copy_params(), import_export::Detector::get_headers(), import_export::Detector::get_sample_rows(), get_session_ptr(), import_export::Detector::getBestColumnTypes(), getConnectionInfo(), import_path_, anonymous_namespace{DBHandler.cpp}::is_a_supported_archive_file(), anonymous_namespace{DBHandler.cpp}::is_local_file(), ImportHelpers::is_reserved_name(), import_export::Detector::kDefaultSampleRowsCount, import_export::kDelimitedFile, Geospatial::kGeoColumnName, import_export::kGeoFile, import_export::kOdbc, import_export::kParquetFile, import_export::kRasterFile, shared::local_glob_filter_sort_files(), anonymous_namespace{DBHandler.cpp}::path_is_relative(), populateThriftColumnType(), import_export::Importer::readMetadataSampleGDAL(), import_export::CopyParams::regex_path_filter, import_export::CopyParams::sanitize_column_names, ImportHelpers::sanitize_name(), import_export::CopyParams::source_type, STDLOG, thrift_to_copyparams(), THROW_DB_EXCEPTION, toString(), type_to_thrift(), and anonymous_namespace{DBHandler.cpp}::validate_import_file_path_if_local().

Referenced by importGeoTableSingle().

4167  {
4168  auto stdlog = STDLOG(get_session_ptr(session));
4169  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4170  check_read_only("detect_column_types");
4171 
4172  bool is_raster = false;
4173  boost::filesystem::path file_path;
4175  if (copy_params.source_type != import_export::SourceType::kOdbc) {
4176  std::string file_name{file_name_in};
4177  if (path_is_relative(file_name)) {
4178  // assume relative paths are relative to data_path / import / <session>
4179  auto temp_file_path = import_path_ / picosha2::hash256_hex_string(session) /
4180  boost::filesystem::path(file_name).filename();
4181  file_name = temp_file_path.string();
4182  }
4184 
4185  if ((copy_params.source_type == import_export::SourceType::kGeoFile ||
4187  is_local_file(file_name)) {
4188  const shared::FilePathOptions options{copy_params.regex_path_filter,
4189  copy_params.file_sort_order_by,
4190  copy_params.file_sort_regex};
4191  auto file_paths = shared::local_glob_filter_sort_files(file_name, options, false);
4192  // For geo and raster detect, pick the first file, if multiple files are provided
4193  // (e.g. through file globbing).
4194  CHECK(!file_paths.empty());
4195  file_name = file_paths[0];
4196  }
4197 
4198  // if it's a geo or raster import, handle alternative paths (S3, HTTP, archive etc.)
4199  if (copy_params.source_type == import_export::SourceType::kGeoFile) {
4200  if (is_a_supported_archive_file(file_name)) {
4201  // find the archive file
4202  add_vsi_network_prefix(file_name);
4203  if (!import_export::Importer::gdalFileExists(file_name, copy_params)) {
4204  THROW_DB_EXCEPTION("Archive does not exist: " + file_name_in);
4205  }
4206  // find geo file in archive
4207  add_vsi_archive_prefix(file_name);
4208  std::string geo_file = find_first_geo_file_in_archive(file_name, copy_params);
4209  // prepare to detect that geo file
4210  if (geo_file.size()) {
4211  file_name = file_name + std::string("/") + geo_file;
4212  }
4213  } else {
4214  // prepare to detect geo file directly
4215  add_vsi_network_prefix(file_name);
4216  add_vsi_geo_prefix(file_name);
4217  }
4218  } else if (copy_params.source_type == import_export::SourceType::kRasterFile) {
4219  // prepare to detect raster file directly
4220  add_vsi_network_prefix(file_name);
4221  add_vsi_geo_prefix(file_name);
4222  is_raster = true;
4223  }
4224 
4225  file_path = boost::filesystem::path(file_name);
4226  // can be a s3 url
4227  if (!boost::istarts_with(file_name, "s3://")) {
4228  if (!boost::filesystem::path(file_name).is_absolute()) {
4229  file_path = import_path_ / picosha2::hash256_hex_string(session) /
4230  boost::filesystem::path(file_name).filename();
4231  file_name = file_path.string();
4232  }
4233 
4234  if (copy_params.source_type == import_export::SourceType::kGeoFile ||
4236  // check for geo or raster file
4237  if (!import_export::Importer::gdalFileOrDirectoryExists(file_name, copy_params)) {
4238  THROW_DB_EXCEPTION("File or directory \"" + file_path.string() +
4239  "\" does not exist.")
4240  }
4241  } else {
4242  // check for regular file
4243  if (!shared::file_or_glob_path_exists(file_path.string())) {
4244  THROW_DB_EXCEPTION("File or directory \"" + file_path.string() +
4245  "\" does not exist.");
4246  }
4247  }
4248  }
4249  }
4250 
4251  try {
4253 #ifdef ENABLE_IMPORT_PARQUET
4255 #endif
4256  ) {
4257  import_export::Detector detector(file_path, copy_params);
4258  auto best_types = detector.getBestColumnTypes();
4259  std::vector<std::string> headers = detector.get_headers();
4260  copy_params = detector.get_copy_params();
4261 
4262  _return.copy_params = copyparams_to_thrift(copy_params);
4263  _return.row_set.row_desc.resize(best_types.size());
4264  for (size_t col_idx = 0; col_idx < best_types.size(); col_idx++) {
4265  TColumnType col;
4266  auto& ti = best_types[col_idx];
4267  col.col_type.precision = ti.get_precision();
4268  col.col_type.scale = ti.get_scale();
4269  col.col_type.comp_param = ti.get_comp_param();
4270  if (ti.is_geometry()) {
4271  // set this so encoding_to_thrift does the right thing
4272  ti.set_compression(copy_params.geo_coords_encoding);
4273  // fill in these directly
4274  col.col_type.precision = static_cast<int>(copy_params.geo_coords_type);
4275  col.col_type.scale = copy_params.geo_coords_srid;
4276  col.col_type.comp_param = copy_params.geo_coords_comp_param;
4277  }
4278  col.col_type.type = type_to_thrift(ti);
4279  col.col_type.encoding = encoding_to_thrift(ti);
4280  if (ti.is_array()) {
4281  col.col_type.is_array = true;
4282  }
4283  if (copy_params.sanitize_column_names) {
4284  col.col_name = ImportHelpers::sanitize_name(headers[col_idx]);
4285  } else {
4286  col.col_name = headers[col_idx];
4287  }
4288  col.is_reserved_keyword = ImportHelpers::is_reserved_name(col.col_name);
4289  _return.row_set.row_desc[col_idx] = col;
4290  }
4291  auto sample_data =
4292  detector.get_sample_rows(import_export::Detector::kDefaultSampleRowsCount);
4293 
4294  TRow sample_row;
4295  for (auto row : sample_data) {
4296  sample_row.cols.clear();
4297  for (const auto& s : row) {
4298  TDatum td;
4299  td.val.str_val = s;
4300  td.is_null = s.empty();
4301  sample_row.cols.push_back(td);
4302  }
4303  _return.row_set.rows.push_back(sample_row);
4304  }
4305  } else if (copy_params.source_type == import_export::SourceType::kGeoFile ||
4307  check_geospatial_files(file_path, copy_params);
4308  std::list<ColumnDescriptor> cds = import_export::Importer::gdalToColumnDescriptors(
4309  file_path.string(), is_raster, Geospatial::kGeoColumnName, copy_params);
4310  for (auto cd : cds) {
4311  if (copy_params.sanitize_column_names) {
4312  cd.columnName = ImportHelpers::sanitize_name(cd.columnName);
4313  }
4314  _return.row_set.row_desc.push_back(populateThriftColumnType(nullptr, &cd));
4315  }
4316  if (!is_raster) {
4317  // @TODO(se) support for raster?
4318  std::map<std::string, std::vector<std::string>> sample_data;
4320  file_path.string(),
4322  sample_data,
4324  copy_params);
4325  if (sample_data.size() > 0) {
4326  for (size_t i = 0; i < sample_data.begin()->second.size(); i++) {
4327  TRow sample_row;
4328  for (auto cd : cds) {
4329  TDatum td;
4330  td.val.str_val = sample_data[cd.sourceName].at(i);
4331  td.is_null = td.val.str_val.empty();
4332  sample_row.cols.push_back(td);
4333  }
4334  _return.row_set.rows.push_back(sample_row);
4335  }
4336  }
4337  }
4338  _return.copy_params = copyparams_to_thrift(copy_params);
4339  }
4340  } catch (const std::exception& e) {
4341  THROW_DB_EXCEPTION("detect_column_types error: " + std::string(e.what()));
4342  }
4343 }
boost::filesystem::path import_path_
Definition: DBHandler.h:629
void add_vsi_archive_prefix(std::string &path)
Definition: DBHandler.cpp:4036
bool is_a_supported_archive_file(const std::string &path)
Definition: DBHandler.cpp:4102
TDatumType::type type_to_thrift(const SQLTypeInfo &type_info)
bool path_is_relative(const std::string &path)
Definition: DBHandler.cpp:4070
TCopyParams copyparams_to_thrift(const import_export::CopyParams &cp)
Definition: DBHandler.cpp:3875
static bool gdalFileExists(const std::string &path, const CopyParams &copy_params)
Definition: Importer.cpp:5136
const std::string kGeoColumnName
Definition: ColumnNames.h:23
bool is_reserved_name(const std::string &name)
std::optional< std::string > regex_path_filter
Definition: CopyParams.h:84
TColumnType populateThriftColumnType(const Catalog_Namespace::Catalog *cat, const ColumnDescriptor *cd)
Definition: DBHandler.cpp:2262
void check_geospatial_files(const boost::filesystem::path file_path, const import_export::CopyParams &copy_params)
Definition: DBHandler.cpp:4960
static void readMetadataSampleGDAL(const std::string &fileName, const std::string &geoColumnName, std::map< std::string, std::vector< std::string >> &metadata, int rowLimit, const CopyParams &copy_params)
Definition: Importer.cpp:4718
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
void validate_import_file_path_if_local(const std::string &file_path)
Definition: DBHandler.cpp:4156
std::string sanitize_name(const std::string &name)
import_export::SourceType source_type
Definition: CopyParams.h:57
static constexpr size_t kDefaultSampleRowsCount
Definition: Importer.h:731
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1448
static bool gdalFileOrDirectoryExists(const std::string &path, const CopyParams &copy_params)
Definition: Importer.cpp:5141
bool file_or_glob_path_exists(const std::string &path)
void check_read_only(const std::string &str)
Definition: DBHandler.cpp:500
std::string find_first_geo_file_in_archive(const std::string &archive_path, const import_export::CopyParams &copy_params)
Definition: DBHandler.cpp:4115
void add_vsi_network_prefix(std::string &path)
Definition: DBHandler.cpp:4007
#define CHECK(condition)
Definition: Logger.h:222
static const std::list< ColumnDescriptor > gdalToColumnDescriptors(const std::string &fileName, const bool is_raster, const std::string &geoColumnName, const CopyParams &copy_params)
Definition: Importer.cpp:4915
std::vector< std::string > local_glob_filter_sort_files(const std::string &file_path, const FilePathOptions &options, const bool recurse)
import_export::CopyParams thrift_to_copyparams(const TCopyParams &cp)
Definition: DBHandler.cpp:3680
std::optional< std::string > file_sort_order_by
Definition: CopyParams.h:85
void add_vsi_geo_prefix(std::string &path)
Definition: DBHandler.cpp:4029
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1737
#define STDLOG(...)
Definition: QueryState.h:235
std::optional< std::string > file_sort_regex
Definition: CopyParams.h:86
TEncodingType::type encoding_to_thrift(const SQLTypeInfo &type_info)
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
EncodingType geo_coords_encoding
Definition: CopyParams.h:75
bool is_local_file(const std::string &file_path)
Definition: DBHandler.cpp:4150

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::disconnect ( const TSessionId &  session)
override

Definition at line 677 of file DBHandler.cpp.

References disconnect_impl(), get_session_it_unsafe(), getConnectionInfo(), logger::INFO, LOG, sessions_mutex_, STDLOG, and toString().

677  {
678  auto stdlog = STDLOG();
679  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
680 
682  auto session_it = get_session_it_unsafe(session, write_lock);
683  stdlog.setSessionInfo(session_it->second);
684  const auto dbname = session_it->second->getCatalog().getCurrentDB().dbName;
685 
686  LOG(INFO) << "User " << session_it->second->get_currentUser().userLoggable()
687  << " disconnected from database " << dbname
688  << " with public_session_id: " << session_it->second->get_public_session_id();
689 
690  disconnect_impl(session_it, write_lock);
691 }
#define LOG(tag)
Definition: Logger.h:216
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void disconnect_impl(const SessionMap::iterator &session_it, heavyai::unique_lock< heavyai::shared_mutex > &write_lock)
Definition: DBHandler.cpp:693
std::unique_lock< T > unique_lock
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1448
heavyai::shared_mutex sessions_mutex_
Definition: DBHandler.h:639
SessionMap::iterator get_session_it_unsafe(const TSessionId &session, SESSION_MAP_LOCK &lock)
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1737
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

void DBHandler::disconnect_impl ( const SessionMap::iterator &  session_it,
heavyai::unique_lock< heavyai::shared_mutex > &  write_lock 
)
private

Definition at line 693 of file DBHandler.cpp.

References LeafAggregator::disconnect(), leaf_aggregator_, LeafAggregator::leafCount(), render_group_assignment_map_, render_group_assignment_mutex_, render_handler_, and sessions_.

Referenced by disconnect(), and get_session_it_unsafe().

694  {
695  // session_it existence should already have been checked (i.e. called via
696  // get_session_it_unsafe(...))
697 
698  const auto session_id = session_it->second->get_session_id();
699  std::exception_ptr leaf_exception = nullptr;
700  try {
701  if (leaf_aggregator_.leafCount() > 0) {
702  leaf_aggregator_.disconnect(session_id);
703  }
704  } catch (...) {
705  leaf_exception = std::current_exception();
706  }
707 
708  {
709  std::lock_guard<std::mutex> lock(render_group_assignment_mutex_);
710  render_group_assignment_map_.erase(session_id);
711  }
712 
713  sessions_.erase(session_it);
714  write_lock.unlock();
715 
716  if (render_handler_) {
717  render_handler_->disconnect(session_id);
718  }
719 }
std::mutex render_group_assignment_mutex_
Definition: DBHandler.h:1108
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:625
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void disconnect(const TSessionId session)
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:645
size_t leafCount() const
RenderGroupAnalyzerSessionMap render_group_assignment_map_
Definition: DBHandler.h:1107
SessionMap sessions_
Definition: DBHandler.h:951

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::emergency_shutdown ( )

Definition at line 7322 of file DBHandler.cpp.

References calcite_.

Referenced by shutdown().

7322  {
7323  if (calcite_) {
7324  calcite_->close_calcite_server(false);
7325  }
7326 }
std::shared_ptr< Calcite > calcite_
Definition: DBHandler.h:648

+ Here is the caller graph for this function:

void DBHandler::execute_distributed_copy_statement ( Parser::CopyTableStmt copy_stmt,
const Catalog_Namespace::SessionInfo session_info 
)
private

Definition at line 6639 of file DBHandler.cpp.

Referenced by sql_execute_impl().

6641  {
6642 }

+ Here is the caller graph for this function:

void DBHandler::execute_next_render_step ( TRenderStepResult &  _return,
const TPendingRenderQuery &  pending_render,
const TRenderAggDataMap &  merged_data 
)
override

Definition at line 7069 of file DBHandler.cpp.

References measure< TimeT >::execution(), logger::INFO, LOG, render_handler_, and THROW_DB_EXCEPTION.

7071  {
7072  if (!render_handler_) {
7073  THROW_DB_EXCEPTION("Backend rendering is disabled.");
7074  }
7075 
7076  LOG(INFO) << "execute_next_render_step: id:" << pending_render.id;
7077  auto time_ms = measure<>::execution([&]() {
7078  try {
7079  render_handler_->execute_next_render_step(_return, pending_render, merged_data);
7080  } catch (std::exception& e) {
7081  THROW_DB_EXCEPTION(e.what());
7082  }
7083  });
7084  LOG(INFO) << "execute_next_render_step-COMPLETED id: " << pending_render.id
7085  << ", time: " << time_ms << "ms ";
7086 }
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:216
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:645
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

void DBHandler::execute_query_step ( TStepResult &  _return,
const TPendingQuery &  pending_query,
const TSubqueryId  subquery_id,
const std::string &  start_time_str 
)
override

Definition at line 6835 of file DBHandler.cpp.

References measure< TimeT >::execution(), logger::INFO, leaf_handler_, LOG, and THROW_DB_EXCEPTION.

6838  {
6839  if (!leaf_handler_) {
6840  THROW_DB_EXCEPTION("Distributed support is disabled.");
6841  }
6842  LOG(INFO) << "execute_query_step : id:" << pending_query.id;
6843  auto time_ms = measure<>::execution([&]() {
6844  try {
6845  leaf_handler_->execute_query_step(
6846  _return, pending_query, subquery_id, start_time_str);
6847  } catch (std::exception& e) {
6848  THROW_DB_EXCEPTION(e.what());
6849  }
6850  });
6851  LOG(INFO) << "execute_query_step-COMPLETED " << time_ms << "ms";
6852 }
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:216
std::unique_ptr< HeavyDBLeafHandler > leaf_handler_
Definition: DBHandler.h:647
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

std::vector< PushedDownFilterInfo > DBHandler::execute_rel_alg ( ExecutionResult _return,
QueryStateProxy  query_state_proxy,
const std::string &  query_ra,
const bool  column_format,
const ExecutorDeviceType  executor_device_type,
const int32_t  first_n,
const int32_t  at_most_n,
const bool  just_validate,
const bool  find_push_down_candidates,
const ExplainInfo explain_info,
const std::optional< size_t >  executor_index = std::nullopt 
) const
private

Definition at line 5979 of file DBHandler.cpp.

References allow_loop_joins_, allow_multifrag_, cat(), query_state::QueryStateProxy::createTimer(), Default, measure< TimeT >::execution(), ExecutionResult::Explanation, g_dynamic_watchdog_time_limit, g_enable_columnar_output, g_enable_dynamic_watchdog, g_enable_runtime_query_interrupt, g_enable_watchdog, g_pending_query_interrupt_freq, g_running_query_interrupt_freq, query_state::QueryState::getConstSessionInfo(), ExecutionResult::getExecutionTime(), Executor::getExecutor(), ExecutionResult::getPushedDownFilterInfo(), query_state::QueryStateProxy::getQueryState(), ExecutionResult::getRows(), SystemParameters::gpu_input_mem_limit, lockmgr::TableSchemaLockMgr::instance(), lockmgr::TableDataLockMgr::instance(), intel_jit_profile_, ExplainInfo::isCalciteExplain(), ExplainInfo::isJustExplain(), ExplainInfo::isOptimizedExplain(), ExplainInfo::isPlanExplain(), jit_debug_, Optimized, ExecutionResult::QueryResult, ExecutionResult::setExecutionTime(), ExecutionResult::setResultType(), system_parameters_, Executor::UNITARY_EXECUTOR_ID, and VLOG.

Referenced by execute_rel_alg_with_filter_push_down(), get_tables_meta_impl(), sql_execute_impl(), and validate_rel_alg().

5990  {
5991  query_state::Timer timer = query_state_proxy.createTimer(__func__);
5992 
5993  VLOG(1) << "Table Schema Locks:\n" << lockmgr::TableSchemaLockMgr::instance();
5994  VLOG(1) << "Table Data Locks:\n" << lockmgr::TableDataLockMgr::instance();
5995 
5996  auto& cat = query_state_proxy.getQueryState().getConstSessionInfo()->getCatalog();
5997  auto executor = Executor::getExecutor(
5998  executor_index ? *executor_index : Executor::UNITARY_EXECUTOR_ID,
5999  jit_debug_ ? "/tmp" : "",
6000  jit_debug_ ? "mapdquery" : "",
6002  RelAlgExecutor ra_executor(executor.get(),
6003  cat,
6004  query_ra,
6005  query_state_proxy.getQueryState().shared_from_this());
6006  CompilationOptions co = {executor_device_type,
6007  /*hoist_literals=*/true,
6010  /*allow_lazy_fetch=*/true,
6011  /*filter_on_deleted_column=*/true,
6012  explain_info.isOptimizedExplain()
6016  auto validate_or_explain_query =
6017  explain_info.isJustExplain() || explain_info.isCalciteExplain() || just_validate;
6019  false,
6021  explain_info.isJustExplain(),
6022  allow_loop_joins_ || just_validate,
6024  jit_debug_,
6025  just_validate,
6028  find_push_down_candidates,
6029  explain_info.isCalciteExplain(),
6031  g_enable_runtime_query_interrupt && !validate_or_explain_query &&
6032  !query_state_proxy.getQueryState()
6034  ->get_session_id()
6035  .empty(),
6038  auto execution_time_ms = _return.getExecutionTime() + measure<>::execution([&]() {
6039  _return = ra_executor.executeRelAlgQuery(
6040  co, eo, explain_info.isPlanExplain(), nullptr);
6041  });
6042  // reduce execution time by the time spent during queue waiting
6043  const auto rs = _return.getRows();
6044  if (rs) {
6045  execution_time_ms -= rs->getQueueTime();
6046  }
6047  _return.setExecutionTime(execution_time_ms);
6048  VLOG(1) << cat.getDataMgr().getSystemMemoryUsage();
6049  const auto& filter_push_down_info = _return.getPushedDownFilterInfo();
6050  if (!filter_push_down_info.empty()) {
6051  return filter_push_down_info;
6052  }
6053  if (explain_info.isJustExplain()) {
6055  } else if (!explain_info.isCalciteExplain()) {
6057  }
6058  return {};
6059 }
double g_running_query_interrupt_freq
Definition: Execute.cpp:129
static TableSchemaLockMgr & instance()
Definition: LockMgr.h:40
std::string cat(Ts &&...args)
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
unsigned g_pending_query_interrupt_freq
Definition: Execute.cpp:128
const bool jit_debug_
Definition: DBHandler.h:633
bool g_enable_dynamic_watchdog
Definition: Execute.cpp:80
Timer createTimer(char const *event_name)
Definition: QueryState.cpp:138
static TableDataLockMgr & instance()
Definition: LockMgr.h:78
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:468
bool g_enable_columnar_output
Definition: Execute.cpp:99
QueryState & getQueryState()
Definition: QueryState.h:181
const std::shared_ptr< ResultSet > & getRows() const
bool g_enable_watchdog
SystemParameters & system_parameters_
Definition: DBHandler.h:643
bool isOptimizedExplain() const
Definition: ParserWrapper.h:57
void setResultType(RType type)
bool isJustExplain() const
Definition: ParserWrapper.h:40
const bool allow_loop_joins_
Definition: DBHandler.h:637
const bool intel_jit_profile_
Definition: DBHandler.h:634
const std::vector< PushedDownFilterInfo > & getPushedDownFilterInfo() const
void setExecutionTime(int64_t execution_time_ms)
bool isCalciteExplain() const
Definition: ParserWrapper.h:58
bool allow_multifrag_
Definition: DBHandler.h:635
unsigned g_dynamic_watchdog_time_limit
Definition: Execute.cpp:85
std::shared_ptr< Catalog_Namespace::SessionInfo const > getConstSessionInfo() const
Definition: QueryState.cpp:88
int64_t getExecutionTime() const
bool g_enable_runtime_query_interrupt
Definition: Execute.cpp:125
#define VLOG(n)
Definition: Logger.h:316
bool isPlanExplain() const
Definition: ParserWrapper.h:59
static const ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:376

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::execute_rel_alg_with_filter_push_down ( ExecutionResult _return,
QueryStateProxy  query_state_proxy,
std::string &  query_ra,
const bool  column_format,
const ExecutorDeviceType  executor_device_type,
const int32_t  first_n,
const int32_t  at_most_n,
const bool  just_explain,
const bool  just_calcite_explain,
const std::vector< PushedDownFilterInfo > &  filter_push_down_requests 
)
private

Definition at line 6589 of file DBHandler.cpp.

References ExecutionResult::addExecutionTime(), execute_rel_alg(), ExecutionResult::Explanation, query_state::QueryStateProxy::getQueryState(), query_state::QueryState::getQueryStr(), ExplainInfo::IR, parse_to_ra(), system_parameters_, and ExecutionResult::updateResultSet().

Referenced by sql_execute_impl().

6599  {
6600  // collecting the selected filters' info to be sent to Calcite:
6601  std::vector<TFilterPushDownInfo> filter_push_down_info;
6602  for (const auto& req : filter_push_down_requests) {
6603  TFilterPushDownInfo filter_push_down_info_for_request;
6604  filter_push_down_info_for_request.input_prev = req.input_prev;
6605  filter_push_down_info_for_request.input_start = req.input_start;
6606  filter_push_down_info_for_request.input_next = req.input_next;
6607  filter_push_down_info.push_back(filter_push_down_info_for_request);
6608  }
6609  // deriving the new relational algebra plan with respect to the pushed down filters
6610  _return.addExecutionTime(measure<>::execution([&]() {
6611  query_ra = parse_to_ra(query_state_proxy,
6612  query_state_proxy.getQueryState().getQueryStr(),
6613  filter_push_down_info,
6614  false,
6616  .first.plan_result;
6617  }));
6618 
6619  if (is_calcite_explain) {
6620  // return the new ra as the result
6621  _return.updateResultSet(query_ra, ExecutionResult::Explanation);
6622  return;
6623  }
6624 
6625  // execute the new relational algebra plan:
6626  auto explain_info = ExplainInfo(ExplainInfo::ExplainType::IR);
6627  execute_rel_alg(_return,
6628  query_state_proxy,
6629  query_ra,
6630  column_format,
6631  executor_device_type,
6632  first_n,
6633  at_most_n,
6634  /*just_validate=*/false,
6635  /*find_push_down_candidates=*/false,
6636  explain_info);
6637 }
std::vector< PushedDownFilterInfo > execute_rel_alg(ExecutionResult &_return, QueryStateProxy, const std::string &query_ra, const bool column_format, const ExecutorDeviceType executor_device_type, const int32_t first_n, const int32_t at_most_n, const bool just_validate, const bool find_push_down_candidates, const ExplainInfo &explain_info, const std::optional< size_t > executor_index=std::nullopt) const
Definition: DBHandler.cpp:5979
std::string const & getQueryStr() const
Definition: QueryState.h:157
void addExecutionTime(int64_t execution_time_ms)
QueryState & getQueryState()
Definition: QueryState.h:181
void updateResultSet(const std::string &query_ra, RType type, bool success=true)
SystemParameters & system_parameters_
Definition: DBHandler.h:643
std::pair< TPlanResult, lockmgr::LockedTableDescriptors > parse_to_ra(QueryStateProxy, const std::string &query_str, const std::vector< TFilterPushDownInfo > &filter_push_down_info, const bool acquire_locks, const SystemParameters &system_parameters, bool check_privileges=true)
Definition: DBHandler.cpp:6644

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::executeDdl ( TQueryResult &  _return,
const std::string &  query_ra,
std::shared_ptr< Catalog_Namespace::SessionInfo const >  session_ptr 
)
private

Definition at line 7849 of file DBHandler.cpp.

References alterSession(), alterSystemClear(), DdlCommandExecutor::commandStr(), convertResultSet(), ExecutionResult::empty(), DdlCommandExecutor::execute(), measure< TimeT >::execution(), getQueries(), ExecutionResult::getRows(), DdlCommandExecutor::getSessionParameter(), DdlCommandExecutor::getTargetQuerySessionToKill(), getUserSessions(), interruptQuery(), DdlCommandExecutor::isAlterSessionSet(), DdlCommandExecutor::isAlterSystemClear(), DdlCommandExecutor::isKillQuery(), DdlCommandExecutor::isShowQueries(), DdlCommandExecutor::isShowUserSessions(), read_only_, run_benchmark_import::result, and DdlCommandExecutor::returnCacheType().

Referenced by sql_execute_impl().

7852  {
7853  DdlCommandExecutor executor = DdlCommandExecutor(query_ra, session_ptr);
7854  std::string commandStr = executor.commandStr();
7855 
7856  if (executor.isKillQuery()) {
7857  interruptQuery(*session_ptr, executor.getTargetQuerySessionToKill());
7858  } else {
7860  int64_t execution_time_ms;
7861  if (executor.isShowQueries()) {
7862  // getQueries still requires Thrift cannot be nested into DdlCommandExecutor
7863  _return.execution_time_ms +=
7864  measure<>::execution([&]() { result = getQueries(session_ptr); });
7865  } else if (executor.isShowUserSessions()) {
7866  // getUserSessions still requires Thrift cannot be nested into DdlCommandExecutor
7867  _return.execution_time_ms +=
7868  measure<>::execution([&]() { result = getUserSessions(session_ptr); });
7869  } else if (executor.isAlterSystemClear()) {
7870  alterSystemClear(session_ptr->get_session_id(),
7871  result,
7872  executor.returnCacheType(),
7873  execution_time_ms);
7874  _return.execution_time_ms += execution_time_ms;
7875 
7876  } else if (executor.isAlterSessionSet()) {
7877  alterSession(session_ptr->get_session_id(),
7878  result,
7879  executor.getSessionParameter(),
7880  execution_time_ms);
7881  _return.execution_time_ms += execution_time_ms;
7882  } else {
7883  _return.execution_time_ms +=
7884  measure<>::execution([&]() { result = executor.execute(read_only_); });
7885  }
7886 
7887  if (!result.empty()) {
7888  // reduce execution time by the time spent during queue waiting
7889  _return.execution_time_ms -= result.getRows()->getQueueTime();
7890  convertResultSet(result, *session_ptr, commandStr, _return);
7891  }
7892  }
7893 }
const std::string getTargetQuerySessionToKill() const
const std::string commandStr() const
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
std::pair< std::string, std::string > getSessionParameter() const
void convertResultSet(ExecutionResult &result, const Catalog_Namespace::SessionInfo &session_info, const std::string &query_state_str, TQueryResult &_return)
Definition: DBHandler.cpp:7498
bool isAlterSystemClear() const
const std::shared_ptr< ResultSet > & getRows() const
bool isShowUserSessions() const
void alterSession(const std::string &sesson_id, ExecutionResult &result, const std::pair< std::string, std::string > &session_parameter, int64_t &execution_time_ms)
Definition: DBHandler.cpp:7824
bool isAlterSessionSet() const
ExecutionResult execute(bool read_only_mode)
const bool read_only_
Definition: DBHandler.h:636
std::string returnCacheType() const
void interruptQuery(const Catalog_Namespace::SessionInfo &session_info, const std::string &target_session)
Definition: DBHandler.cpp:7707
ExecutionResult getQueries(std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
Definition: DBHandler.cpp:7571
ExecutionResult getUserSessions(std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
Definition: DBHandler.cpp:7527
void alterSystemClear(const std::string &sesson_id, ExecutionResult &result, const std::string &cache_type, int64_t &execution_time_ms)
Definition: DBHandler.cpp:7808

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::executeDdl ( ExecutionResult _return,
const std::string &  query_ra,
std::shared_ptr< Catalog_Namespace::SessionInfo const >  session_ptr 
)
private

Definition at line 7895 of file DBHandler.cpp.

References alterSession(), alterSystemClear(), ExecutionResult::CalciteDdl, DdlCommandExecutor::commandStr(), DdlCommandExecutor::execute(), measure< TimeT >::execution(), getQueries(), ExecutionResult::getResultType(), DdlCommandExecutor::getSessionParameter(), DdlCommandExecutor::getTargetQuerySessionToKill(), getUserSessions(), interruptQuery(), DdlCommandExecutor::isAlterSessionSet(), DdlCommandExecutor::isAlterSystemClear(), DdlCommandExecutor::isKillQuery(), DdlCommandExecutor::isShowQueries(), DdlCommandExecutor::isShowUserSessions(), ExecutionResult::QueryResult, read_only_, DdlCommandExecutor::returnCacheType(), ExecutionResult::setExecutionTime(), and ExecutionResult::setResultType().

7898  {
7899  DdlCommandExecutor executor = DdlCommandExecutor(query_ra, session_ptr);
7900  std::string commandStr = executor.commandStr();
7901 
7902  if (executor.isKillQuery()) {
7903  interruptQuery(*session_ptr, executor.getTargetQuerySessionToKill());
7904  } else {
7905  int64_t execution_time_ms;
7906  if (executor.isShowQueries()) {
7907  // getQueries still requires Thrift cannot be nested into DdlCommandExecutor
7908  execution_time_ms =
7909  measure<>::execution([&]() { _return = getQueries(session_ptr); });
7910  } else if (executor.isShowUserSessions()) {
7911  // getUserSessions still requires Thrift cannot be nested into DdlCommandExecutor
7912  execution_time_ms =
7913  measure<>::execution([&]() { _return = getUserSessions(session_ptr); });
7914  } else if (executor.isAlterSystemClear()) {
7915  alterSystemClear(session_ptr->get_session_id(),
7916  _return,
7917  executor.returnCacheType(),
7918  execution_time_ms);
7919  } else if (executor.isAlterSessionSet()) {
7920  alterSession(session_ptr->get_session_id(),
7921  _return,
7922  executor.getSessionParameter(),
7923  execution_time_ms);
7924  } else {
7925  execution_time_ms =
7926  measure<>::execution([&]() { _return = executor.execute(read_only_); });
7927  }
7928  _return.setExecutionTime(execution_time_ms);
7929  }
7930  if (_return.getResultType() == ExecutionResult::QueryResult) {
7931  // ResultType defaults to QueryResult => which can limit
7932  // the number of lines output via ConvertRow... use CalciteDdl instead
7934  }
7935 }
RType getResultType() const
const std::string getTargetQuerySessionToKill() const
const std::string commandStr() const
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
std::pair< std::string, std::string > getSessionParameter() const
bool isAlterSystemClear() const
bool isShowUserSessions() const
void alterSession(const std::string &sesson_id, ExecutionResult &result, const std::pair< std::string, std::string > &session_parameter, int64_t &execution_time_ms)
Definition: DBHandler.cpp:7824
void setResultType(RType type)
bool isAlterSessionSet() const
ExecutionResult execute(bool read_only_mode)
const bool read_only_
Definition: DBHandler.h:636
std::string returnCacheType() const
void setExecutionTime(int64_t execution_time_ms)
void interruptQuery(const Catalog_Namespace::SessionInfo &session_info, const std::string &target_session)
Definition: DBHandler.cpp:7707
ExecutionResult getQueries(std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
Definition: DBHandler.cpp:7571
ExecutionResult getUserSessions(std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
Definition: DBHandler.cpp:7527
void alterSystemClear(const std::string &sesson_id, ExecutionResult &result, const std::string &cache_type, int64_t &execution_time_ms)
Definition: DBHandler.cpp:7808

+ Here is the call graph for this function:

template<>
void DBHandler::expire_idle_sessions_unsafe ( heavyai::unique_lock< heavyai::shared_mutex > &  write_lock)

Definition at line 194 of file DBHandler.cpp.

References check_session_exp_unsafe(), LeafAggregator::disconnect(), anonymous_namespace{DBHandler.cpp}::get_session_from_map(), logger::INFO, leaf_aggregator_, LeafAggregator::leafCount(), LOG, render_handler_, and sessions_.

195  {
196  std::vector<std::string> expired_sessions;
197  for (auto session_pair : sessions_) {
198  auto session_it = get_session_from_map(session_pair.first, sessions_);
199  try {
200  check_session_exp_unsafe(session_it);
201  } catch (const ForceDisconnect& e) {
202  expired_sessions.emplace_back(session_it->second->get_session_id());
203  }
204  }
205 
206  for (auto session_id : expired_sessions) {
207  if (leaf_aggregator_.leafCount() > 0) {
208  try {
209  leaf_aggregator_.disconnect(session_id);
210  } catch (TDBException& toe) {
211  LOG(INFO) << " Problem disconnecting from leaves : " << toe.what();
212  } catch (std::exception& e) {
213  LOG(INFO)
214  << " Problem disconnecting from leaves, check leaf logs for additonal info";
215  }
216  }
217  sessions_.erase(session_id);
218  }
219  if (render_handler_) {
220  write_lock.unlock();
221  for (auto session_id : expired_sessions) {
222  // NOTE: the render disconnect is done after the session lock is released to
223  // avoid a deadlock. See: https://heavyai.atlassian.net/browse/BE-3324
224  // This out-of-scope solution is a compromise for now until a better session
225  // handling/locking mechanism is developed for the renderer. Note as well that the
226  // session_id cannot be immediately reused. If a render request were to slip in
227  // after the lock is released and before the render disconnect could cause a
228  // problem.
229  render_handler_->disconnect(session_id);
230  }
231  }
232 }
#define LOG(tag)
Definition: Logger.h:216
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:625
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void disconnect(const TSessionId session)
SessionMap::iterator get_session_from_map(const TSessionId &session, SessionMap &session_map)
Definition: DBHandler.cpp:147
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:645
size_t leafCount() const
void check_session_exp_unsafe(const SessionMap::iterator &session_it)
Definition: DBHandler.cpp:5869
SessionMap sessions_
Definition: DBHandler.h:951

+ Here is the call graph for this function:

template<typename SESSION_MAP_LOCK >
void DBHandler::expire_idle_sessions_unsafe ( SESSION_MAP_LOCK &  lock)
private

Referenced by connect_impl().

+ Here is the caller graph for this function:

std::unordered_map< std::string, std::unordered_set< std::string > > DBHandler::fill_column_names_by_table ( std::vector< std::string > &  table_names,
query_state::StdLog stdlog 
)
private

Definition at line 1717 of file DBHandler.cpp.

References get_table_details_impl().

Referenced by get_token_based_completions().

1718  {
1719  std::unordered_map<std::string, std::unordered_set<std::string>> column_names_by_table;
1720  for (auto it = table_names.begin(); it != table_names.end();) {
1721  TTableDetails table_details;
1722  try {
1723  get_table_details_impl(table_details, stdlog, *it, false, false);
1724  } catch (const TDBException& e) {
1725  // Remove the corrupted Table/View name from the list for further processing.
1726  it = table_names.erase(it);
1727  continue;
1728  }
1729  for (const auto& column_type : table_details.row_desc) {
1730  column_names_by_table[*it].emplace(column_type.col_name);
1731  }
1732  ++it;
1733  }
1734  return column_names_by_table;
1735 }
void get_table_details_impl(TTableDetails &_return, query_state::StdLog &stdlog, const std::string &table_name, const bool get_system, const bool get_physical, const std::string &database_name={})
Definition: DBHandler.cpp:2421

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::fillGeoColumns ( const TSessionId &  session,
const Catalog_Namespace::Catalog catalog,
std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &  import_buffers,
const ColumnDescriptor cd,
size_t &  col_idx,
size_t  num_rows,
const std::string &  table_name,
bool  assign_render_groups 
)
private

Definition at line 3004 of file DBHandler.cpp.

References CHECK, CHECK_EQ, ColumnDescriptor::columnName, ColumnDescriptor::columnType, g_enable_assign_render_groups, SQLTypeInfo::get_type(), Geospatial::GeoTypesFactory::getGeoColumns(), logger::INFO, IS_GEO_POLY, LOG, LOG_IF, render_group_assignment_map_, render_group_assignment_mutex_, import_export::Importer::set_geo_physical_import_buffer_columnar(), and THROW_DB_EXCEPTION.

Referenced by fillMissingBuffers(), load_table(), and loadTableBinaryColumnarInternal().

3012  {
3013  auto geo_col_idx = col_idx - 1;
3014  const auto wkt_or_wkb_hex_column = import_buffers[geo_col_idx]->getGeoStringBuffer();
3015  std::vector<std::vector<double>> coords_column, bounds_column;
3016  std::vector<std::vector<int>> ring_sizes_column, poly_rings_column;
3017  std::vector<int> render_groups_column;
3018  SQLTypeInfo ti = cd->columnType;
3019  if (num_rows != wkt_or_wkb_hex_column->size() ||
3020  !Geospatial::GeoTypesFactory::getGeoColumns(wkt_or_wkb_hex_column,
3021  ti,
3022  coords_column,
3023  bounds_column,
3024  ring_sizes_column,
3025  poly_rings_column,
3026  true)) {
3027  std::ostringstream oss;
3028  oss << "Invalid geometry in column " << cd->columnName;
3029  THROW_DB_EXCEPTION(oss.str());
3030  }
3031 
3032  // start or continue assigning render groups for poly columns?
3033  if (IS_GEO_POLY(cd->columnType.get_type()) && assign_render_groups &&
3035  // get RGA to use
3036  import_export::RenderGroupAnalyzer* render_group_analyzer{};
3037  {
3038  // mutex the map access
3039  std::lock_guard<std::mutex> lock(render_group_assignment_mutex_);
3040 
3041  // emplace new RGA or fetch existing RGA from map
3042  auto [itr_table, emplaced_table] = render_group_assignment_map_.try_emplace(
3043  session, RenderGroupAssignmentTableMap());
3044  LOG_IF(INFO, emplaced_table)
3045  << "load_table_binary_columnar_polys: Creating Render Group Assignment "
3046  "Persistent Data for Session '"
3047  << session << "'";
3048  auto [itr_column, emplaced_column] =
3049  itr_table->second.try_emplace(table_name, RenderGroupAssignmentColumnMap());
3050  LOG_IF(INFO, emplaced_column)
3051  << "load_table_binary_columnar_polys: Creating Render Group Assignment "
3052  "Persistent Data for Table '"
3053  << table_name << "'";
3054  auto [itr_analyzer, emplaced_analyzer] = itr_column->second.try_emplace(
3055  cd->columnName, std::make_unique<import_export::RenderGroupAnalyzer>());
3056  LOG_IF(INFO, emplaced_analyzer)
3057  << "load_table_binary_columnar_polys: Creating Render Group Assignment "
3058  "Persistent Data for Column '"
3059  << cd->columnName << "'";
3060  render_group_analyzer = itr_analyzer->second.get();
3061  CHECK(render_group_analyzer);
3062 
3063  // seed new RGA from existing table/column, to handle appends
3064  if (emplaced_analyzer) {
3065  LOG(INFO) << "load_table_binary_columnar_polys: Seeding Render Groups from "
3066  "existing table...";
3067  render_group_analyzer->seedFromExistingTableContents(
3068  catalog, table_name, cd->columnName);
3069  LOG(INFO) << "load_table_binary_columnar_polys: Done";
3070  }
3071  }
3072 
3073  // assign render groups for this set of bounds
3074  LOG(INFO) << "load_table_binary_columnar_polys: Assigning Render Groups...";
3075  render_groups_column.reserve(bounds_column.size());
3076  for (auto const& bounds : bounds_column) {
3077  CHECK_EQ(bounds.size(), 4u);
3078  int rg = render_group_analyzer->insertBoundsAndReturnRenderGroup(bounds);
3079  render_groups_column.push_back(rg);
3080  }
3081  LOG(INFO) << "load_table_binary_columnar_polys: Done";
3082  } else {
3083  // render groups all zero
3084  render_groups_column.resize(bounds_column.size(), 0);
3085  }
3086 
3087  // Populate physical columns, advance col_idx
3089  cd,
3090  import_buffers,
3091  col_idx,
3092  coords_column,
3093  bounds_column,
3094  ring_sizes_column,
3095  poly_rings_column,
3096  render_groups_column);
3097 }
#define CHECK_EQ(x, y)
Definition: Logger.h:230
std::unordered_map< std::string, RenderGroupAssignmentColumnMap > RenderGroupAssignmentTableMap
Definition: DBHandler.h:1104
std::mutex render_group_assignment_mutex_
Definition: DBHandler.h:1108
#define LOG(tag)
Definition: Logger.h:216
static void set_geo_physical_import_buffer_columnar(const Catalog_Namespace::Catalog &catalog, const ColumnDescriptor *cd, std::vector< std::unique_ptr< TypedImportBuffer >> &import_buffers, size_t &col_idx, std::vector< std::vector< double >> &coords_column, std::vector< std::vector< double >> &bounds_column, std::vector< std::vector< int >> &ring_sizes_column, std::vector< std::vector< int >> &poly_rings_column, std::vector< int > &render_groups_column)
Definition: Importer.cpp:1758
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:329
#define LOG_IF(severity, condition)
Definition: Logger.h:312
bool g_enable_assign_render_groups
static bool getGeoColumns(const std::string &wkt_or_wkb_hex, SQLTypeInfo &ti, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, const bool promote_poly_to_mpoly=false)
Definition: Types.cpp:937
#define CHECK(condition)
Definition: Logger.h:222
RenderGroupAnalyzerSessionMap render_group_assignment_map_
Definition: DBHandler.h:1107
SQLTypeInfo columnType
std::unordered_map< std::string, std::unique_ptr< import_export::RenderGroupAnalyzer >> RenderGroupAssignmentColumnMap
Definition: DBHandler.h:1102
std::string columnName
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
#define IS_GEO_POLY(T)
Definition: sqltypes.h:255

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::fillMissingBuffers ( const TSessionId &  session,
const Catalog_Namespace::Catalog catalog,
std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &  import_buffers,
const std::list< const ColumnDescriptor * > &  cds,
const std::vector< int > &  desc_id_to_column_id,
size_t  num_rows,
const std::string &  table_name,
bool  assign_render_groups 
)
private

Definition at line 3099 of file DBHandler.cpp.

References CHECK, and fillGeoColumns().

Referenced by load_table(), load_table_binary(), load_table_binary_arrow(), and loadTableBinaryColumnarInternal().

3107  {
3108  size_t skip_physical_cols = 0;
3109  size_t col_idx = 0, import_idx = 0;
3110  for (const auto& cd : cds) {
3111  if (skip_physical_cols > 0) {
3112  CHECK(cd->isGeoPhyCol);
3113  skip_physical_cols--;
3114  continue;
3115  } else if (cd->columnType.is_geometry()) {
3116  skip_physical_cols = cd->columnType.get_physical_cols();
3117  }
3118  if (desc_id_to_column_id[import_idx] == -1) {
3119  import_buffers[col_idx]->addDefaultValues(cd, num_rows);
3120  col_idx++;
3121  if (cd->columnType.is_geometry()) {
3122  fillGeoColumns(session,
3123  catalog,
3124  import_buffers,
3125  cd,
3126  col_idx,
3127  num_rows,
3128  table_name,
3129  assign_render_groups);
3130  }
3131  } else {
3132  col_idx++;
3133  col_idx += skip_physical_cols;
3134  }
3135  import_idx++;
3136  }
3137 }
void fillGeoColumns(const TSessionId &session, const Catalog_Namespace::Catalog &catalog, std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &import_buffers, const ColumnDescriptor *cd, size_t &col_idx, size_t num_rows, const std::string &table_name, bool assign_render_groups)
Definition: DBHandler.cpp:3004
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

TRowDescriptor DBHandler::fixup_row_descriptor ( const TRowDescriptor &  row_desc,
const Catalog_Namespace::Catalog cat 
)
private

Definition at line 6146 of file DBHandler.cpp.

References DictDescriptor::dictNBits, and Catalog_Namespace::Catalog::getMetadataForDict().

Referenced by get_table_details_impl(), and sql_validate().

6147  {
6148  TRowDescriptor fixedup_row_desc;
6149  for (const TColumnType& col_desc : row_desc) {
6150  auto fixedup_col_desc = col_desc;
6151  if (col_desc.col_type.encoding == TEncodingType::DICT &&
6152  col_desc.col_type.comp_param > 0) {
6153  const auto dd = cat.getMetadataForDict(col_desc.col_type.comp_param, false);
6154  fixedup_col_desc.col_type.comp_param = dd->dictNBits;
6155  }
6156  fixedup_row_desc.push_back(fixedup_col_desc);
6157  }
6158 
6159  return fixedup_row_desc;
6160 }
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1960

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::get_all_effective_roles_for_user ( std::vector< std::string > &  _return,
const TSessionId &  session,
const std::string &  granteeName 
)
override

Definition at line 2196 of file DBHandler.cpp.

References get_session_ptr(), getAllRolesForUserImpl(), and STDLOG.

2198  {
2199  auto stdlog = STDLOG(get_session_ptr(sessionId));
2200  auto session_ptr = stdlog.getConstSessionInfo();
2201  getAllRolesForUserImpl(session_ptr, roles, sessionId, granteeName, /*effective=*/true);
2202 }
void getAllRolesForUserImpl(std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr, std::vector< std::string > &roles, const TSessionId &sessionId, const std::string &granteeName, bool effective)
Definition: DBHandler.cpp:2152
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

void DBHandler::get_all_files_in_archive ( std::vector< std::string > &  _return,
const TSessionId &  session,
const std::string &  archive_path,
const TCopyParams &  copy_params 
)
override

Definition at line 5722 of file DBHandler.cpp.

References anonymous_namespace{DBHandler.cpp}::add_vsi_archive_prefix(), anonymous_namespace{DBHandler.cpp}::add_vsi_network_prefix(), import_export::Importer::gdalFileExists(), import_export::Importer::gdalGetAllFilesInArchive(), get_session_ptr(), getConnectionInfo(), import_path_, anonymous_namespace{DBHandler.cpp}::is_a_supported_archive_file(), anonymous_namespace{DBHandler.cpp}::path_is_relative(), STDLOG, thrift_to_copyparams(), THROW_DB_EXCEPTION, toString(), and anonymous_namespace{DBHandler.cpp}::validate_import_file_path_if_local().

5725  {
5726  auto stdlog =
5727  STDLOG(get_session_ptr(session), "get_all_files_in_archive", archive_path_in);
5728  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
5729 
5730  std::string archive_path(archive_path_in);
5731  if (path_is_relative(archive_path)) {
5732  // assume relative paths are relative to data_path / import / <session>
5733  auto file_path = import_path_ / picosha2::hash256_hex_string(session) /
5734  boost::filesystem::path(archive_path).filename();
5735  archive_path = file_path.string();
5736  }
5737  validate_import_file_path_if_local(archive_path);
5738 
5739  if (is_a_supported_archive_file(archive_path)) {
5740  // find the archive file
5741  add_vsi_network_prefix(archive_path);
5742  if (!import_export::Importer::gdalFileExists(archive_path,
5743  thrift_to_copyparams(copy_params))) {
5744  THROW_DB_EXCEPTION("Archive does not exist: " + archive_path_in);
5745  }
5746  // find all files in archive
5747  add_vsi_archive_prefix(archive_path);
5749  archive_path, thrift_to_copyparams(copy_params));
5750  // prepend them all with original path
5751  for (auto& s : _return) {
5752  s = archive_path_in + '/' + s;
5753  }
5754  }
5755 }
boost::filesystem::path import_path_
Definition: DBHandler.h:629
void add_vsi_archive_prefix(std::string &path)
Definition: DBHandler.cpp:4036
bool is_a_supported_archive_file(const std::string &path)
Definition: DBHandler.cpp:4102
bool path_is_relative(const std::string &path)
Definition: DBHandler.cpp:4070
static bool gdalFileExists(const std::string &path, const CopyParams &copy_params)
Definition: Importer.cpp:5136
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
void validate_import_file_path_if_local(const std::string &file_path)
Definition: DBHandler.cpp:4156
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1448
void add_vsi_network_prefix(std::string &path)
Definition: DBHandler.cpp:4007
static std::vector< std::string > gdalGetAllFilesInArchive(const std::string &archive_path, const CopyParams &copy_params)
Definition: Importer.cpp:5213
import_export::CopyParams thrift_to_copyparams(const TCopyParams &cp)
Definition: DBHandler.cpp:3680
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1737
#define STDLOG(...)
Definition: QueryState.h:235
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

void DBHandler::get_all_roles_for_user ( std::vector< std::string > &  _return,
const TSessionId &  session,
const std::string &  granteeName 
)
override

Definition at line 2186 of file DBHandler.cpp.

References get_session_ptr(), getAllRolesForUserImpl(), and STDLOG.

2188  {
2189  // WARNING: This function only returns directly granted roles.
2190  // See also: get_all_effective_roles_for_user() for all of a user's roles.
2191  auto stdlog = STDLOG(get_session_ptr(sessionId));
2192  auto session_ptr = stdlog.getConstSessionInfo();
2193  getAllRolesForUserImpl(session_ptr, roles, sessionId, granteeName, /*effective=*/false);
2194 }
void getAllRolesForUserImpl(std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr, std::vector< std::string > &roles, const TSessionId &sessionId, const std::string &granteeName, bool effective)
Definition: DBHandler.cpp:2152
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

void DBHandler::get_completion_hints ( std::vector< TCompletionHint > &  hints,
const TSessionId &  session,
const std::string &  sql,
const int  cursor 
)
override

Definition at line 1610 of file DBHandler.cpp.

References anonymous_namespace{DBHandler.cpp}::extract_projection_tokens_for_completion(), get_completion_hints_unsorted(), get_session_ptr(), get_uc_compatible_table_names_by_column(), gpu_enabled::sort(), STDLOG, and to_upper().

1613  {
1614  auto stdlog = STDLOG(get_session_ptr(session));
1615  std::vector<std::string> visible_tables; // Tables allowed for the given session.
1616  get_completion_hints_unsorted(hints, visible_tables, stdlog, sql, cursor);
1617  const auto proj_tokens = extract_projection_tokens_for_completion(sql);
1618  auto compatible_table_names = get_uc_compatible_table_names_by_column(
1619  proj_tokens.uc_column_names, visible_tables, stdlog);
1620  // Add the table qualifiers explicitly specified by the user.
1621  compatible_table_names.insert(proj_tokens.uc_column_table_qualifiers.begin(),
1622  proj_tokens.uc_column_table_qualifiers.end());
1623  // Sort the hints by category, from COLUMN (most specific) to KEYWORD.
1624  std::sort(
1625  hints.begin(),
1626  hints.end(),
1627  [&compatible_table_names](const TCompletionHint& lhs, const TCompletionHint& rhs) {
1628  if (lhs.type == TCompletionHintType::TABLE &&
1629  rhs.type == TCompletionHintType::TABLE) {
1630  // Between two tables, one which is compatible with the specified
1631  // projections and one which isn't, pick the one which is compatible.
1632  if (compatible_table_names.find(to_upper(lhs.hints.back())) !=
1633  compatible_table_names.end() &&
1634  compatible_table_names.find(to_upper(rhs.hints.back())) ==
1635  compatible_table_names.end()) {
1636  return true;
1637  }
1638  }
1639  return lhs.type < rhs.type;
1640  });
1641 }
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
ProjectionTokensForCompletion extract_projection_tokens_for_completion(const std::string &sql)
Definition: DBHandler.cpp:1586
void get_completion_hints_unsorted(std::vector< TCompletionHint > &hints, std::vector< std::string > &visible_tables, query_state::StdLog &stdlog, const std::string &sql, const int cursor)
Definition: DBHandler.cpp:1643
std::string to_upper(const std::string &str)
std::unordered_set< std::string > get_uc_compatible_table_names_by_column(const std::unordered_set< std::string > &uc_column_names, std::vector< std::string > &table_names, query_state::StdLog &stdlog)
Definition: DBHandler.cpp:1742
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

void DBHandler::get_completion_hints_unsorted ( std::vector< TCompletionHint > &  hints,
std::vector< std::string > &  visible_tables,
query_state::StdLog stdlog,
const std::string &  sql,
const int  cursor 
)
private

Definition at line 1643 of file DBHandler.cpp.

References calcite_, logger::ERROR, GET_PHYSICAL_TABLES_AND_VIEWS, get_tables_impl(), get_token_based_completions(), query_state::StdLog::getConstSessionInfo(), just_whitelisted_keyword_hints(), and LOG.

Referenced by get_completion_hints().

1647  {
1648  const auto& session_info = *stdlog.getConstSessionInfo();
1649  try {
1650  get_tables_impl(visible_tables, session_info, GET_PHYSICAL_TABLES_AND_VIEWS);
1651 
1652  // Filter out keywords suggested by Calcite which we don't support.
1654  calcite_->getCompletionHints(session_info, visible_tables, sql, cursor));
1655  } catch (const std::exception& e) {
1656  TDBException ex;
1657  ex.error_msg = std::string(e.what());
1658  LOG(ERROR) << ex.error_msg;
1659  throw ex;
1660  }
1661  boost::regex from_expr{R"(\s+from\s+)", boost::regex::extended | boost::regex::icase};
1662  const size_t length_to_cursor =
1663  cursor < 0 ? sql.size() : std::min(sql.size(), static_cast<size_t>(cursor));
1664  // Trust hints from Calcite after the FROM keyword.
1665  if (boost::regex_search(sql.cbegin(), sql.cbegin() + length_to_cursor, from_expr)) {
1666  return;
1667  }
1668  // Before FROM, the query is too incomplete for context-sensitive completions.
1669  get_token_based_completions(hints, stdlog, visible_tables, sql, cursor);
1670 }
#define LOG(tag)
Definition: Logger.h:216
std::vector< TCompletionHint > just_whitelisted_keyword_hints(const std::vector< TCompletionHint > &hints)
void get_tables_impl(std::vector< std::string > &table_names, const Catalog_Namespace::SessionInfo &, const GetTablesType get_tables_type, const std::string &database_name={})
Definition: DBHandler.cpp:2562
void get_token_based_completions(std::vector< TCompletionHint > &hints, query_state::StdLog &stdlog, std::vector< std::string > &visible_tables, const std::string &sql, const int cursor)
Definition: DBHandler.cpp:1672
std::shared_ptr< Calcite > calcite_
Definition: DBHandler.h:648
std::shared_ptr< Catalog_Namespace::SessionInfo const > getConstSessionInfo() const
Definition: QueryState.cpp:159

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::shared_ptr< const Catalog_Namespace::SessionInfo > DBHandler::get_const_session_ptr ( const TSessionId &  session)
private

Definition at line 5896 of file DBHandler.cpp.

References get_session_ptr().

5897  {
5898  return get_session_ptr(session);
5899 }
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919

+ Here is the call graph for this function:

void DBHandler::get_custom_expressions ( std::vector< TCustomExpression > &  _return,
const TSessionId &  session 
)
override

Definition at line 4462 of file DBHandler.cpp.

References anonymous_namespace{DBHandler.cpp}::create_thrift_obj_from_custom_expr(), custom_expressions_mutex_, get_session_ptr(), getConnectionInfo(), STDLOG, and toString().

4463  {
4464  auto stdlog = STDLOG(get_session_ptr(session));
4465  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4466 
4467  auto session_ptr = stdlog.getConstSessionInfo();
4468  auto& catalog = session_ptr->getCatalog();
4470  auto custom_expressions =
4471  catalog.getCustomExpressionsForUser(session_ptr->get_currentUser());
4472  for (const auto& custom_expression : custom_expressions) {
4473  _return.emplace_back(create_thrift_obj_from_custom_expr(*custom_expression, catalog));
4474  }
4475 }
heavyai::shared_lock< heavyai::shared_mutex > read_lock
std::shared_lock< T > shared_lock
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1448
heavyai::shared_mutex custom_expressions_mutex_
Definition: DBHandler.h:1109
TCustomExpression create_thrift_obj_from_custom_expr(const CustomExpression &custom_expr, const Catalog &catalog)
Definition: DBHandler.cpp:4422
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1737
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

void DBHandler::get_dashboard ( TDashboard &  _return,
const TSessionId &  session,
const int32_t  dashboard_id 
)
override

Definition at line 4511 of file DBHandler.cpp.

References cat(), get_dashboard_impl(), get_session_ptr(), getConnectionInfo(), is_allowed_on_dashboard(), STDLOG, THROW_DB_EXCEPTION, to_string(), toString(), Catalog_Namespace::UserMetadata::userName, and AccessPrivileges::VIEW_DASHBOARD.

4513  {
4514  auto stdlog = STDLOG(get_session_ptr(session));
4515  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4516  auto session_ptr = stdlog.getConstSessionInfo();
4517  auto const& cat = session_ptr->getCatalog();
4519  auto dash = cat.getMetadataForDashboard(dashboard_id);
4520  if (!dash) {
4521  THROW_DB_EXCEPTION("Dashboard with dashboard id " + std::to_string(dashboard_id) +
4522  " doesn't exist");
4523  }
4525  *session_ptr, dash->dashboardId, AccessPrivileges::VIEW_DASHBOARD)) {
4526  THROW_DB_EXCEPTION("User has no view privileges for the dashboard with id " +
4527  std::to_string(dashboard_id));
4528  }
4529  user_meta.userName = "";
4530  SysCatalog::instance().getMetadataForUserById(dash->userId, user_meta);
4531  dashboard = get_dashboard_impl(session_ptr, user_meta, dash);
4532 }
static const AccessPrivileges VIEW_DASHBOARD
Definition: DBObject.h:171
std::string cat(Ts &&...args)
std::string to_string(char const *&&v)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
TDashboard get_dashboard_impl(const std::shared_ptr< Catalog_Namespace::SessionInfo const > &session_ptr, Catalog_Namespace::UserMetadata &user_meta, const DashboardDescriptor *dash, const bool populate_state=true)
Definition: DBHandler.cpp:4554
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1448
static bool is_allowed_on_dashboard(const Catalog_Namespace::SessionInfo &session_info, int32_t dashboard_id, AccessPrivileges requestedPermissions)
Definition: DBHandler.cpp:4386
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1737
#define STDLOG(...)
Definition: QueryState.h:235
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

void DBHandler::get_dashboard_grantees ( std::vector< TDashboardGrantees > &  _return,
const TSessionId &  session,
const int32_t  dashboard_id 
)
override

Definition at line 4884 of file DBHandler.cpp.

References cat(), DashboardPrivileges::CREATE_DASHBOARD, DashboardDBObjectType, DashboardPrivileges::DELETE_DASHBOARD, DashboardPrivileges::EDIT_DASHBOARD, get_session_ptr(), getConnectionInfo(), STDLOG, THROW_DB_EXCEPTION, to_string(), toString(), Catalog_Namespace::UserMetadata::userId, Catalog_Namespace::UserMetadata::userName, and DashboardPrivileges::VIEW_DASHBOARD.

4887  {
4888  auto stdlog = STDLOG(get_session_ptr(session));
4889  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4890  auto session_ptr = stdlog.getConstSessionInfo();
4891  auto const& cat = session_ptr->getCatalog();
4893  auto dash = cat.getMetadataForDashboard(dashboard_id);
4894  if (!dash) {
4895  THROW_DB_EXCEPTION("Dashboard id " + std::to_string(dashboard_id) +
4896  " does not exist");
4897  } else if (session_ptr->get_currentUser().userId != dash->userId &&
4898  !session_ptr->get_currentUser().isSuper) {
4900  "User should be either owner of dashboard or super user to access grantees");
4901  }
4902  std::vector<ObjectRoleDescriptor*> objectsList;
4903  objectsList = SysCatalog::instance().getMetadataForObject(
4904  cat.getCurrentDB().dbId,
4905  static_cast<int>(DBObjectType::DashboardDBObjectType),
4906  dashboard_id); // By default objecttypecan be only dashabaords
4907  user_meta.userId = -1;
4908  user_meta.userName = "";
4909  SysCatalog::instance().getMetadataForUserById(dash->userId, user_meta);
4910  for (auto object : objectsList) {
4911  if (user_meta.userName == object->roleName) {
4912  // Mask owner
4913  continue;
4914  }
4915  TDashboardGrantees grantee;
4916  TDashboardPermissions perm;
4917  grantee.name = object->roleName;
4918  grantee.is_user = object->roleType;
4919  perm.create_ = object->privs.hasPermission(DashboardPrivileges::CREATE_DASHBOARD);
4920  perm.delete_ = object->privs.hasPermission(DashboardPrivileges::DELETE_DASHBOARD);
4921  perm.edit_ = object->privs.hasPermission(DashboardPrivileges::EDIT_DASHBOARD);
4922  perm.view_ = object->privs.hasPermission(DashboardPrivileges::VIEW_DASHBOARD);
4923  grantee.permissions = perm;
4924  dashboard_grantees.push_back(grantee);
4925  }
4926 }
std::string cat(Ts &&...args)
std::string to_string(char const *&&v)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1448
static const int32_t EDIT_DASHBOARD
Definition: DBObject.h:104
static const int32_t DELETE_DASHBOARD
Definition: DBObject.h:102
static const int32_t VIEW_DASHBOARD
Definition: DBObject.h:103
static const int32_t CREATE_DASHBOARD
Definition: DBObject.h:101
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1737
#define STDLOG(...)
Definition: QueryState.h:235
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

TDashboard DBHandler::get_dashboard_impl ( const std::shared_ptr< Catalog_Namespace::SessionInfo const > &  session_ptr,
Catalog_Namespace::UserMetadata user_meta,
const DashboardDescriptor dash,
const bool  populate_state = true 
)
private

Definition at line 4554 of file DBHandler.cpp.

References cat(), DashboardPrivileges::CREATE_DASHBOARD, DashboardDBObjectType, DashboardDescriptor::dashboardId, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, DashboardPrivileges::DELETE_DASHBOARD, DashboardPrivileges::EDIT_DASHBOARD, DBObject::getPrivileges(), AccessPrivileges::hasPermission(), DashboardDescriptor::imageHash, DashboardDescriptor::updateTime, DashboardDescriptor::user, DashboardDescriptor::userId, Catalog_Namespace::UserMetadata::userName, and DashboardPrivileges::VIEW_DASHBOARD.

Referenced by get_dashboard(), and get_dashboards().

4558  {
4559  auto const& cat = session_ptr->getCatalog();
4560  SysCatalog::instance().getMetadataForUserById(dash->userId, user_meta);
4561  auto objects_list = SysCatalog::instance().getMetadataForObject(
4562  cat.getCurrentDB().dbId,
4563  static_cast<int>(DBObjectType::DashboardDBObjectType),
4564  dash->dashboardId);
4565  TDashboard dashboard;
4566  dashboard.dashboard_name = dash->dashboardName;
4567  if (populate_state) {
4568  dashboard.dashboard_state = dash->dashboardState;
4569  }
4570  dashboard.image_hash = dash->imageHash;
4571  dashboard.update_time = dash->updateTime;
4572  dashboard.dashboard_metadata = dash->dashboardMetadata;
4573  dashboard.dashboard_id = dash->dashboardId;
4574  dashboard.dashboard_owner = dash->user;
4575  TDashboardPermissions perms;
4576  // Super user has all permissions.
4577  if (session_ptr->get_currentUser().isSuper) {
4578  perms.create_ = true;
4579  perms.delete_ = true;
4580  perms.edit_ = true;
4581  perms.view_ = true;
4582  } else {
4583  // Collect all grants on current user
4584  // add them to the permissions.
4585  auto obj_to_find =
4586  DBObject(dashboard.dashboard_id, DBObjectType::DashboardDBObjectType);
4587  obj_to_find.loadKey(session_ptr->getCatalog());
4588  std::vector<std::string> grantees =
4589  SysCatalog::instance().getRoles(true,
4590  session_ptr->get_currentUser().isSuper,
4591  session_ptr->get_currentUser().userName);
4592  for (const auto& grantee : grantees) {
4593  DBObject* object_found;
4594  auto* gr = SysCatalog::instance().getGrantee(grantee);
4595  if (gr && (object_found = gr->findDbObject(obj_to_find.getObjectKey(), true))) {
4596  const auto obj_privs = object_found->getPrivileges();
4597  perms.create_ |= obj_privs.hasPermission(DashboardPrivileges::CREATE_DASHBOARD);
4598  perms.delete_ |= obj_privs.hasPermission(DashboardPrivileges::DELETE_DASHBOARD);
4599  perms.edit_ |= obj_privs.hasPermission(DashboardPrivileges::EDIT_DASHBOARD);
4600  perms.view_ |= obj_privs.hasPermission(DashboardPrivileges::VIEW_DASHBOARD);
4601  }
4602  }
4603  }
4604  dashboard.dashboard_permissions = perms;
4605  if (objects_list.empty() ||
4606  (objects_list.size() == 1 && objects_list[0]->roleName == user_meta.userName)) {
4607  dashboard.is_dash_shared = false;
4608  } else {
4609  dashboard.is_dash_shared = true;
4610  }
4611  return dashboard;
4612 }
std::string cat(Ts &&...args)
static const int32_t EDIT_DASHBOARD
Definition: DBObject.h:104
static const int32_t DELETE_DASHBOARD
Definition: DBObject.h:102
const AccessPrivileges & getPrivileges() const
Definition: DBObject.h:226
static const int32_t VIEW_DASHBOARD
Definition: DBObject.h:103
static const int32_t CREATE_DASHBOARD
Definition: DBObject.h:101
bool hasPermission(int permission) const
Definition: DBObject.h:141

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::get_dashboards ( std::vector< TDashboard > &  _return,
const TSessionId &  session 
)
override

Definition at line 4534 of file DBHandler.cpp.

References cat(), get_dashboard_impl(), get_session_ptr(), getConnectionInfo(), is_allowed_on_dashboard(), STDLOG, toString(), Catalog_Namespace::UserMetadata::userName, and AccessPrivileges::VIEW_DASHBOARD.

4535  {
4536  auto stdlog = STDLOG(get_session_ptr(session));
4537  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4538  auto session_ptr = stdlog.getConstSessionInfo();
4539  auto const& cat = session_ptr->getCatalog();
4541  const auto dashes = cat.getAllDashboardsMetadata();
4542  user_meta.userName = "";
4543  for (const auto dash : dashes) {
4545  *session_ptr, dash->dashboardId, AccessPrivileges::VIEW_DASHBOARD)) {
4546  // dashboardState is intentionally not populated here
4547  // for payload reasons
4548  // use get_dashboard call to get state
4549  dashboards.push_back(get_dashboard_impl(session_ptr, user_meta, dash, false));
4550  }
4551  }
4552 }
static const AccessPrivileges VIEW_DASHBOARD
Definition: DBObject.h:171
std::string cat(Ts &&...args)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
TDashboard get_dashboard_impl(const std::shared_ptr< Catalog_Namespace::SessionInfo const > &session_ptr, Catalog_Namespace::UserMetadata &user_meta, const DashboardDescriptor *dash, const bool populate_state=true)
Definition: DBHandler.cpp:4554
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1448
static bool is_allowed_on_dashboard(const Catalog_Namespace::SessionInfo &session_info, int32_t dashboard_id, AccessPrivileges requestedPermissions)
Definition: DBHandler.cpp:4386
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1737
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

void DBHandler::get_databases ( std::vector< TDBInfo > &  _return,
const TSessionId &  session 
)
override

Definition at line 2885 of file DBHandler.cpp.

References get_session_ptr(), getConnectionInfo(), STDLOG, and toString().

2885  {
2886  auto stdlog = STDLOG(get_session_ptr(session));
2887  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
2888  auto session_ptr = stdlog.getConstSessionInfo();
2889  const auto& user = session_ptr->get_currentUser();
2891  SysCatalog::instance().getDatabaseListForUser(user);
2892  for (auto& db : dbs) {
2893  TDBInfo dbinfo;
2894  dbinfo.db_name = std::move(db.dbName);
2895  dbinfo.db_owner = std::move(db.dbOwnerName);
2896  dbinfos.push_back(std::move(dbinfo));
2897  }
2898 }
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1448
std::list< DBSummary > DBSummaryList
Definition: SysCatalog.h:145
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1737
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

void DBHandler::get_db_object_privs ( std::vector< TDBObject > &  _return,
const TSessionId &  session,
const std::string &  objectName,
const TDBObjectType::type  type 
)
override

Definition at line 2068 of file DBHandler.cpp.

References AccessPrivileges::ALL_TABLE, cat(), DashboardDBObjectType, DatabaseDBObjectType, get_session_ptr(), DBObject::getObjectKey(), DBObject::loadKey(), serialize_db_object(), ServerDBObjectType, STDLOG, TableDBObjectType, THROW_DB_EXCEPTION, to_string(), and ViewDBObjectType.

2071  {
2072  auto stdlog = STDLOG(get_session_ptr(sessionId));
2073  auto session_ptr = stdlog.getConstSessionInfo();
2074  DBObjectType object_type;
2075  switch (type) {
2077  object_type = DBObjectType::DatabaseDBObjectType;
2078  break;
2080  object_type = DBObjectType::TableDBObjectType;
2081  break;
2084  break;
2086  object_type = DBObjectType::ViewDBObjectType;
2087  break;
2089  object_type = DBObjectType::ServerDBObjectType;
2090  break;
2091  default:
2092  THROW_DB_EXCEPTION("Failed to get object privileges for " + objectName +
2093  ": unknown object type (" + std::to_string(type) + ").");
2094  }
2095  DBObject object_to_find(objectName, object_type);
2096 
2097  // TODO(adb): Use DatabaseLock to protect method
2098  try {
2099  if (object_type == DashboardDBObjectType) {
2100  if (objectName == "") {
2101  object_to_find = DBObject(-1, object_type);
2102  } else {
2103  object_to_find = DBObject(std::stoi(objectName), object_type);
2104  }
2105  } else if ((object_type == TableDBObjectType || object_type == ViewDBObjectType) &&
2106  !objectName.empty()) {
2107  // special handling for view / table
2108  auto const& cat = session_ptr->getCatalog();
2109  auto td = cat.getMetadataForTable(objectName, false);
2110  if (td) {
2111  object_type = td->isView ? ViewDBObjectType : TableDBObjectType;
2112  object_to_find = DBObject(objectName, object_type);
2113  }
2114  }
2115  object_to_find.loadKey(session_ptr->getCatalog());
2116  } catch (const std::exception&) {
2117  THROW_DB_EXCEPTION("Object with name " + objectName + " does not exist.");
2118  }
2119 
2120  // object type on database level
2121  DBObject object_to_find_dblevel("", object_type);
2122  object_to_find_dblevel.loadKey(session_ptr->getCatalog());
2123  // if user is superuser respond with a full priv
2124  if (session_ptr->get_currentUser().isSuper) {
2125  // using ALL_TABLE here to set max permissions
2126  DBObject dbObj{object_to_find.getObjectKey(),
2128  session_ptr->get_currentUser().userId};
2129  dbObj.setName("super");
2130  TDBObjects.push_back(
2131  serialize_db_object(session_ptr->get_currentUser().userName, dbObj));
2132  };
2133 
2134  std::vector<std::string> grantees =
2135  SysCatalog::instance().getRoles(true,
2136  session_ptr->get_currentUser().isSuper,
2137  session_ptr->get_currentUser().userName);
2138  for (const auto& grantee : grantees) {
2139  DBObject* object_found;
2140  auto* gr = SysCatalog::instance().getGrantee(grantee);
2141  if (gr && (object_found = gr->findDbObject(object_to_find.getObjectKey(), true))) {
2142  TDBObjects.push_back(serialize_db_object(grantee, *object_found));
2143  }
2144  // check object permissions on Database level
2145  if (gr &&
2146  (object_found = gr->findDbObject(object_to_find_dblevel.getObjectKey(), true))) {
2147  TDBObjects.push_back(serialize_db_object(grantee, *object_found));
2148  }
2149  }
2150 }
std::string cat(Ts &&...args)
DBObjectKey getObjectKey() const
Definition: DBObject.h:221
DBObjectType
Definition: DBObject.h:40
std::string to_string(char const *&&v)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
static TDBObject serialize_db_object(const std::string &roleName, const DBObject &inObject)
Definition: DBHandler.cpp:1829
static const AccessPrivileges ALL_TABLE
Definition: DBObject.h:157
#define STDLOG(...)
Definition: QueryState.h:235
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

void DBHandler::get_db_objects_for_grantee ( std::vector< TDBObject > &  _return,
const TSessionId &  session,
const std::string &  roleName 
)
override

Definition at line 2041 of file DBHandler.cpp.

References get_session_ptr(), serialize_db_object(), STDLOG, and THROW_DB_EXCEPTION.

2043  {
2044  auto stdlog = STDLOG(get_session_ptr(sessionId));
2045  auto session_ptr = stdlog.getConstSessionInfo();
2046  auto const& user = session_ptr->get_currentUser();
2047  if (!user.isSuper &&
2048  !SysCatalog::instance().isRoleGrantedToGrantee(user.userName, roleName, false)) {
2049  return;
2050  }
2051  auto* rl = SysCatalog::instance().getGrantee(roleName);
2052  if (rl) {
2053  auto dbId = session_ptr->getCatalog().getCurrentDB().dbId;
2054  for (auto& dbObject : *rl->getDbObjects(true)) {
2055  if (dbObject.first.dbId != dbId) {
2056  // TODO (max): it doesn't scale well in case we have many DBs (not a typical
2057  // usecase for now, though)
2058  continue;
2059  }
2060  TDBObject tdbObject = serialize_db_object(roleName, *dbObject.second);
2061  TDBObjectsForRole.push_back(tdbObject);
2062  }
2063  } else {
2064  THROW_DB_EXCEPTION("User or role " + roleName + " does not exist.");
2065  }
2066 }
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
static TDBObject serialize_db_object(const std::string &roleName, const DBObject &inObject)
Definition: DBHandler.cpp:1829
#define STDLOG(...)
Definition: QueryState.h:235
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

void DBHandler::get_device_parameters ( std::map< std::string, std::string > &  _return,
const TSessionId &  session 
)
override

Definition at line 7341 of file DBHandler.cpp.

References cpu_mode_only_, EXPOSE_THRIFT_MAP, and get_session_copy().

7342  {
7343  const auto session_info = get_session_copy(session);
7344  auto params = ::get_device_parameters(cpu_mode_only_);
7345  for (auto item : params) {
7346  _return.insert(item);
7347  }
7348  EXPOSE_THRIFT_MAP(TDeviceType);
7349  EXPOSE_THRIFT_MAP(TDatumType);
7350  EXPOSE_THRIFT_MAP(TEncodingType);
7351  EXPOSE_THRIFT_MAP(TExtArgumentType);
7352  EXPOSE_THRIFT_MAP(TOutputBufferSizeType);
7353 }
#define EXPOSE_THRIFT_MAP(TYPENAME)
Definition: DBHandler.cpp:7330
Catalog_Namespace::SessionInfo get_session_copy(const TSessionId &session)
Definition: DBHandler.cpp:5901
void get_device_parameters(std::map< std::string, std::string > &_return, const TSessionId &session) override
Definition: DBHandler.cpp:7341
bool cpu_mode_only_
Definition: DBHandler.h:638

+ Here is the call graph for this function:

void DBHandler::get_first_geo_file_in_archive ( std::string &  _return,
const TSessionId &  session,
const std::string &  archive_path,
const TCopyParams &  copy_params 
)
override

Definition at line 5679 of file DBHandler.cpp.

References anonymous_namespace{DBHandler.cpp}::add_vsi_archive_prefix(), anonymous_namespace{DBHandler.cpp}::add_vsi_network_prefix(), anonymous_namespace{DBHandler.cpp}::find_first_geo_file_in_archive(), import_export::Importer::gdalFileExists(), get_session_ptr(), getConnectionInfo(), import_path_, anonymous_namespace{DBHandler.cpp}::is_a_supported_archive_file(), anonymous_namespace{DBHandler.cpp}::path_is_relative(), STDLOG, thrift_to_copyparams(), THROW_DB_EXCEPTION, toString(), and anonymous_namespace{DBHandler.cpp}::validate_import_file_path_if_local().

5682  {
5683  auto stdlog =
5684  STDLOG(get_session_ptr(session), "get_first_geo_file_in_archive", archive_path_in);
5685  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
5686 
5687  std::string archive_path(archive_path_in);
5688 
5689  if (path_is_relative(archive_path)) {
5690  // assume relative paths are relative to data_path / import / <session>
5691  auto file_path = import_path_ / picosha2::hash256_hex_string(session) /
5692  boost::filesystem::path(archive_path).filename();
5693  archive_path = file_path.string();
5694  }
5695  validate_import_file_path_if_local(archive_path);
5696 
5697  if (is_a_supported_archive_file(archive_path)) {
5698  // find the archive file
5699  add_vsi_network_prefix(archive_path);
5700  if (!import_export::Importer::gdalFileExists(archive_path,
5701  thrift_to_copyparams(copy_params))) {
5702  THROW_DB_EXCEPTION("Archive does not exist: " + archive_path_in);
5703  }
5704  // find geo file in archive
5705  add_vsi_archive_prefix(archive_path);
5706  std::string geo_file =
5707  find_first_geo_file_in_archive(archive_path, thrift_to_copyparams(copy_params));
5708  // what did we get?
5709  if (geo_file.size()) {
5710  // prepend it with the original path
5711  _return = archive_path_in + std::string("/") + geo_file;
5712  } else {
5713  // just return the original path
5714  _return = archive_path_in;
5715  }
5716  } else {
5717  // just return the original path
5718  _return = archive_path_in;
5719  }
5720 }
boost::filesystem::path import_path_
Definition: DBHandler.h:629
void add_vsi_archive_prefix(std::string &path)
Definition: DBHandler.cpp:4036
bool is_a_supported_archive_file(const std::string &path)
Definition: DBHandler.cpp:4102
bool path_is_relative(const std::string &path)
Definition: DBHandler.cpp:4070
static bool gdalFileExists(const std::string &path, const CopyParams &copy_params)
Definition: Importer.cpp:5136
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5919
void validate_import_file_path_if_local(const std::string &file_path)
Definition: DBHandler.cpp:4156
std::string toString(const Executor::ExtModuleKinds &kind)
Definition: Execute.h:1448
std::string find_first_geo_file_in_archive(const std::string &archive_path, const import_export::CopyParams &copy_params)
Definition: DBHandler.cpp:4115
void add_vsi_network_prefix(std::string &path)
Definition: DBHandler.cpp:4007
import_export::CopyParams thrift_to_copyparams(const TCopyParams &cp)
Definition: DBHandler.cpp:3680
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1737
#define STDLOG(...)
Definition: QueryState.h:235
#define THROW_DB_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

void DBHandler::get_function_details ( std::vector< TUserDefinedFunction > &  _return,
const TSessionId &  session,
const std::vector< std::string > &  udf_names 
)
override

Definition at line 7456 of file DBHandler.cpp.

References ExtensionFunctionsWhitelist::get_ext_funcs(), and ThriftSerializers::to_thrift().

7458  {
7459  for (const std::string& udf_name : udf_names) {
7460  for (auto udf :