OmniSciDB  8fa3bf436f
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
DBHandler Class Reference

#include <DBHandler.h>

+ Inheritance diagram for DBHandler:
+ Collaboration diagram for DBHandler:

Classes

struct  GeoCopyFromSessions
 
struct  GeoCopyFromState
 

Public Member Functions

 DBHandler (const std::vector< LeafHostInfo > &db_leaves, const std::vector< LeafHostInfo > &string_leaves, const std::string &base_data_path, const bool allow_multifrag, const bool jit_debug, const bool intel_jit_profile, const bool read_only, const bool allow_loop_joins, const bool enable_rendering, const bool renderer_use_vulkan_driver, const bool enable_auto_clear_render_mem, const int render_oom_retry_threshold, const size_t render_mem_bytes, const size_t max_concurrent_render_sessions, const size_t reserved_gpu_mem, const bool render_compositor_use_last_gpu, const size_t num_reader_threads, const AuthMetadata &authMetadata, SystemParameters &system_parameters, const bool legacy_syntax, const int idle_session_duration, const int max_session_duration, const bool enable_runtime_udf_registration, const std::string &udf_filename, const std::string &clang_path, const std::vector< std::string > &clang_options, const DiskCacheConfig &disk_cache_config, const bool is_new_db)
 
void initialize (const bool is_new_db)
 
 ~DBHandler () override
 
void krb5_connect (TKrb5Session &session, const std::string &token, const std::string &dbname) override
 
void connect (TSessionId &session, const std::string &username, const std::string &passwd, const std::string &dbname) override
 
void disconnect (const TSessionId &session) override
 
void switch_database (const TSessionId &session, const std::string &dbname) override
 
void clone_session (TSessionId &session2, const TSessionId &session1) override
 
void get_server_status (TServerStatus &_return, const TSessionId &session) override
 
void get_status (std::vector< TServerStatus > &_return, const TSessionId &session) override
 
void get_hardware_info (TClusterHardwareInfo &_return, const TSessionId &session) override
 
bool hasTableAccessPrivileges (const TableDescriptor *td, const Catalog_Namespace::SessionInfo &session_info)
 
void get_tables (std::vector< std::string > &_return, const TSessionId &session) override
 
void get_physical_tables (std::vector< std::string > &_return, const TSessionId &session) override
 
void get_views (std::vector< std::string > &_return, const TSessionId &session) override
 
void get_tables_meta (std::vector< TTableMeta > &_return, const TSessionId &session) override
 
void get_table_details (TTableDetails &_return, const TSessionId &session, const std::string &table_name) override
 
void get_internal_table_details (TTableDetails &_return, const TSessionId &session, const std::string &table_name) override
 
void get_users (std::vector< std::string > &_return, const TSessionId &session) override
 
void get_databases (std::vector< TDBInfo > &_return, const TSessionId &session) override
 
void get_version (std::string &_return) override
 
void start_heap_profile (const TSessionId &session) override
 
void stop_heap_profile (const TSessionId &session) override
 
void get_heap_profile (std::string &_return, const TSessionId &session) override
 
void get_memory (std::vector< TNodeMemoryInfo > &_return, const TSessionId &session, const std::string &memory_level) override
 
void clear_cpu_memory (const TSessionId &session) override
 
void clear_gpu_memory (const TSessionId &session) override
 
void set_cur_session (const TSessionId &parent_session, const TSessionId &leaf_session, const std::string &start_time_str, const std::string &label) override
 
void invalidate_cur_session (const TSessionId &parent_session, const TSessionId &leaf_session, const std::string &start_time_str, const std::string &label) override
 
void set_table_epoch (const TSessionId &session, const int db_id, const int table_id, const int new_epoch) override
 
void set_table_epoch_by_name (const TSessionId &session, const std::string &table_name, const int new_epoch) override
 
int32_t get_table_epoch (const TSessionId &session, const int32_t db_id, const int32_t table_id) override
 
int32_t get_table_epoch_by_name (const TSessionId &session, const std::string &table_name) override
 
void get_table_epochs (std::vector< TTableEpochInfo > &_return, const TSessionId &session, const int32_t db_id, const int32_t table_id) override
 
void set_table_epochs (const TSessionId &session, const int32_t db_id, const std::vector< TTableEpochInfo > &table_epochs) override
 
void get_session_info (TSessionInfo &_return, const TSessionId &session) override
 
void sql_execute (ExecutionResult &_return, const TSessionId &session, const std::string &query, const bool column_format, const int32_t first_n, const int32_t at_most_n)
 
void sql_execute (TQueryResult &_return, const TSessionId &session, const std::string &query, const bool column_format, const std::string &nonce, const int32_t first_n, const int32_t at_most_n) override
 
void get_completion_hints (std::vector< TCompletionHint > &hints, const TSessionId &session, const std::string &sql, const int cursor) override
 
void sql_execute_df (TDataFrame &_return, const TSessionId &session, const std::string &query, const TDeviceType::type device_type, const int32_t device_id, const int32_t first_n, const TArrowTransport::type transport_method) override
 
void sql_execute_gdf (TDataFrame &_return, const TSessionId &session, const std::string &query, const int32_t device_id, const int32_t first_n) override
 
void deallocate_df (const TSessionId &session, const TDataFrame &df, const TDeviceType::type device_type, const int32_t device_id) override
 
void interrupt (const TSessionId &query_session, const TSessionId &interrupt_session) override
 
void sql_validate (TRowDescriptor &_return, const TSessionId &session, const std::string &query) override
 
void set_execution_mode (const TSessionId &session, const TExecuteMode::type mode) override
 
void render_vega (TRenderResult &_return, const TSessionId &session, const int64_t widget_id, const std::string &vega_json, const int32_t compression_level, const std::string &nonce) override
 
void get_result_row_for_pixel (TPixelTableRowResult &_return, const TSessionId &session, const int64_t widget_id, const TPixel &pixel, const std::map< std::string, std::vector< std::string >> &table_col_names, const bool column_format, const int32_t pixel_radius, const std::string &nonce) override
 
int32_t create_custom_expression (const TSessionId &session, const TCustomExpression &custom_expression) override
 
void get_custom_expressions (std::vector< TCustomExpression > &_return, const TSessionId &session) override
 
void update_custom_expression (const TSessionId &session, const int32_t id, const std::string &expression_json) override
 
void delete_custom_expressions (const TSessionId &session, const std::vector< int32_t > &custom_expression_ids, const bool do_soft_delete) override
 
void get_dashboard (TDashboard &_return, const TSessionId &session, const int32_t dashboard_id) override
 
void get_dashboards (std::vector< TDashboard > &_return, const TSessionId &session) override
 
int32_t create_dashboard (const TSessionId &session, const std::string &dashboard_name, const std::string &dashboard_state, const std::string &image_hash, const std::string &dashboard_metadata) override
 
void replace_dashboard (const TSessionId &session, const int32_t dashboard_id, const std::string &dashboard_name, const std::string &dashboard_owner, const std::string &dashboard_state, const std::string &image_hash, const std::string &dashboard_metadata) override
 
void delete_dashboard (const TSessionId &session, const int32_t dashboard_id) override
 
void share_dashboards (const TSessionId &session, const std::vector< int32_t > &dashboard_ids, const std::vector< std::string > &groups, const TDashboardPermissions &permissions) override
 
void delete_dashboards (const TSessionId &session, const std::vector< int32_t > &dashboard_ids) override
 
void share_dashboard (const TSessionId &session, const int32_t dashboard_id, const std::vector< std::string > &groups, const std::vector< std::string > &objects, const TDashboardPermissions &permissions, const bool grant_role) override
 
void unshare_dashboards (const TSessionId &session, const std::vector< int32_t > &dashboard_ids, const std::vector< std::string > &groups, const TDashboardPermissions &permissions) override
 
void unshare_dashboard (const TSessionId &session, const int32_t dashboard_id, const std::vector< std::string > &groups, const std::vector< std::string > &objects, const TDashboardPermissions &permissions) override
 
void get_dashboard_grantees (std::vector< TDashboardGrantees > &_return, const TSessionId &session, const int32_t dashboard_id) override
 
void get_link_view (TFrontendView &_return, const TSessionId &session, const std::string &link) override
 
void create_link (std::string &_return, const TSessionId &session, const std::string &view_state, const std::string &view_metadata) override
 
void load_table_binary (const TSessionId &session, const std::string &table_name, const std::vector< TRow > &rows, const std::vector< std::string > &column_names) override
 
std::unique_ptr
< lockmgr::AbstractLockContainer
< const TableDescriptor * > > 
prepare_columnar_loader (const Catalog_Namespace::SessionInfo &session_info, const std::string &table_name, size_t num_cols, std::unique_ptr< import_export::Loader > *loader, std::vector< std::unique_ptr< import_export::TypedImportBuffer >> *import_buffers, const std::vector< std::string > &column_names)
 
void load_table_binary_columnar (const TSessionId &session, const std::string &table_name, const std::vector< TColumn > &cols, const std::vector< std::string > &column_names) override
 
void load_table_binary_columnar_polys (const TSessionId &session, const std::string &table_name, const std::vector< TColumn > &cols, const std::vector< std::string > &column_names, const bool assign_render_groups) override
 
void load_table_binary_arrow (const TSessionId &session, const std::string &table_name, const std::string &arrow_stream, const bool use_column_names) override
 
void load_table (const TSessionId &session, const std::string &table_name, const std::vector< TStringRow > &rows, const std::vector< std::string > &column_names) override
 
void detect_column_types (TDetectResult &_return, const TSessionId &session, const std::string &file_name, const TCopyParams &copy_params) override
 
void create_table (const TSessionId &session, const std::string &table_name, const TRowDescriptor &row_desc, const TFileType::type file_type, const TCreateParams &create_params) override
 
void import_table (const TSessionId &session, const std::string &table_name, const std::string &file_name, const TCopyParams &copy_params) override
 
void import_geo_table (const TSessionId &session, const std::string &table_name, const std::string &file_name, const TCopyParams &copy_params, const TRowDescriptor &row_desc, const TCreateParams &create_params) override
 
void import_table_status (TImportStatus &_return, const TSessionId &session, const std::string &import_id) override
 
void get_first_geo_file_in_archive (std::string &_return, const TSessionId &session, const std::string &archive_path, const TCopyParams &copy_params) override
 
void get_all_files_in_archive (std::vector< std::string > &_return, const TSessionId &session, const std::string &archive_path, const TCopyParams &copy_params) override
 
void get_layers_in_geo_file (std::vector< TGeoFileLayerInfo > &_return, const TSessionId &session, const std::string &file_name, const TCopyParams &copy_params) override
 
int64_t query_get_outer_fragment_count (const TSessionId &session, const std::string &select_query) override
 
void check_table_consistency (TTableMeta &_return, const TSessionId &session, const int32_t table_id) override
 
void start_query (TPendingQuery &_return, const TSessionId &leaf_session, const TSessionId &parent_session, const std::string &query_ra, const std::string &start_time_str, const bool just_explain, const std::vector< int64_t > &outer_fragment_indices) override
 
void execute_query_step (TStepResult &_return, const TPendingQuery &pending_query, const TSubqueryId subquery_id, const std::string &start_time_str) override
 
void broadcast_serialized_rows (const TSerializedRows &serialized_rows, const TRowDescriptor &row_desc, const TQueryId query_id, const TSubqueryId subquery_id, const bool is_final_subquery_result) override
 
void start_render_query (TPendingRenderQuery &_return, const TSessionId &session, const int64_t widget_id, const int16_t node_idx, const std::string &vega_json) override
 
void execute_next_render_step (TRenderStepResult &_return, const TPendingRenderQuery &pending_render, const TRenderAggDataMap &merged_data) override
 
void insert_data (const TSessionId &session, const TInsertData &insert_data) override
 
void checkpoint (const TSessionId &session, const int32_t table_id) override
 
void get_roles (std::vector< std::string > &_return, const TSessionId &session) override
 
bool has_role (const TSessionId &sessionId, const std::string &granteeName, const std::string &roleName) override
 
bool has_object_privilege (const TSessionId &sessionId, const std::string &granteeName, const std::string &objectName, const TDBObjectType::type object_type, const TDBObjectPermissions &permissions) override
 
void get_db_objects_for_grantee (std::vector< TDBObject > &_return, const TSessionId &session, const std::string &roleName) override
 
void get_db_object_privs (std::vector< TDBObject > &_return, const TSessionId &session, const std::string &objectName, const TDBObjectType::type type) override
 
void get_all_roles_for_user (std::vector< std::string > &_return, const TSessionId &session, const std::string &granteeName) override
 
std::vector< std::string > get_valid_groups (const TSessionId &session, int32_t dashboard_id, std::vector< std::string > groups)
 
void set_license_key (TLicenseInfo &_return, const TSessionId &session, const std::string &key, const std::string &nonce) override
 
void get_license_claims (TLicenseInfo &_return, const TSessionId &session, const std::string &nonce) override
 
void get_device_parameters (std::map< std::string, std::string > &_return, const TSessionId &session) override
 
void register_runtime_extension_functions (const TSessionId &session, const std::vector< TUserDefinedFunction > &udfs, const std::vector< TUserDefinedTableFunction > &udtfs, const std::map< std::string, std::string > &device_ir_map) override
 
void shutdown ()
 
void emergency_shutdown ()
 
TSessionId getInvalidSessionId () const
 
void internal_connect (TSessionId &session, const std::string &username, const std::string &dbname)
 
bool isAggregator () const
 
template<typename... ARGS>
std::shared_ptr
< query_state::QueryState
create_query_state (ARGS &&...args)
 
Catalog_Namespace::SessionInfo get_session_copy (const TSessionId &session)
 
std::shared_ptr
< Catalog_Namespace::SessionInfo
get_session_copy_ptr (const TSessionId &session)
 
void get_tables_meta_impl (std::vector< TTableMeta > &_return, QueryStateProxy query_state_proxy, const Catalog_Namespace::SessionInfo &session_info, const bool with_table_locks=true)
 
template<>
SessionMap::iterator get_session_it_unsafe (const TSessionId &session, mapd_shared_lock< mapd_shared_mutex > &read_lock)
 
template<>
SessionMap::iterator get_session_it_unsafe (const TSessionId &session, mapd_unique_lock< mapd_shared_mutex > &write_lock)
 
template<>
void expire_idle_sessions_unsafe (mapd_unique_lock< mapd_shared_mutex > &write_lock)
 

Static Public Member Functions

static size_t max_bytes_for_thrift ()
 
static void parser_with_error_handler (const std::string &query_str, std::list< std::unique_ptr< Parser::Stmt >> &parse_trees)
 

Public Attributes

std::shared_ptr
< Data_Namespace::DataMgr
data_mgr_
 
LeafAggregator leaf_aggregator_
 
std::vector< LeafHostInfodb_leaves_
 
std::vector< LeafHostInfostring_leaves_
 
const std::string base_data_path_
 
boost::filesystem::path import_path_
 
ExecutorDeviceType executor_device_type_
 
std::default_random_engine random_gen_
 
std::uniform_int_distribution
< int64_t > 
session_id_dist_
 
const bool jit_debug_
 
const bool intel_jit_profile_
 
bool allow_multifrag_
 
const bool read_only_
 
const bool allow_loop_joins_
 
bool cpu_mode_only_
 
mapd_shared_mutex sessions_mutex_
 
std::mutex render_mutex_
 
int64_t start_time_
 
const AuthMetadataauthMetadata_
 
SystemParameterssystem_parameters_
 
std::unique_ptr< RenderHandlerrender_handler_
 
std::unique_ptr< MapDAggHandleragg_handler_
 
std::unique_ptr< MapDLeafHandlerleaf_handler_
 
std::shared_ptr< Calcitecalcite_
 
const bool legacy_syntax_
 
std::unique_ptr
< QueryDispatchQueue
dispatch_queue_
 

Private Types

enum  AssignRenderGroupsMode { AssignRenderGroupsMode::kNone, AssignRenderGroupsMode::kAssign, AssignRenderGroupsMode::kCleanUp }
 
using RenderGroupAssignmentColumnMap = std::unordered_map< std::string, std::unique_ptr< import_export::RenderGroupAnalyzer >>
 
using RenderGroupAssignmentTableMap = std::unordered_map< std::string, RenderGroupAssignmentColumnMap >
 
using RenderGroupAnalyzerSessionMap = std::unordered_map< TSessionId, RenderGroupAssignmentTableMap >
 

Private Member Functions

std::shared_ptr
< Catalog_Namespace::SessionInfo
create_new_session (TSessionId &session, const std::string &dbname, const Catalog_Namespace::UserMetadata &user_meta, std::shared_ptr< Catalog_Namespace::Catalog > cat)
 
void connect_impl (TSessionId &session, const std::string &passwd, const std::string &dbname, const Catalog_Namespace::UserMetadata &user_meta, std::shared_ptr< Catalog_Namespace::Catalog > cat, query_state::StdLog &stdlog)
 
void disconnect_impl (const SessionMap::iterator &session_it, mapd_unique_lock< mapd_shared_mutex > &write_lock)
 
void check_table_load_privileges (const TSessionId &session, const std::string &table_name)
 
void check_table_load_privileges (const Catalog_Namespace::SessionInfo &session_info, const std::string &table_name)
 
void get_tables_impl (std::vector< std::string > &table_names, const Catalog_Namespace::SessionInfo &, const GetTablesType get_tables_type)
 
void get_table_details_impl (TTableDetails &_return, query_state::StdLog &stdlog, const std::string &table_name, const bool get_system, const bool get_physical)
 
void check_read_only (const std::string &str)
 
void check_session_exp_unsafe (const SessionMap::iterator &session_it)
 
void validateGroups (const std::vector< std::string > &groups)
 
void validateDashboardIdsForSharing (const Catalog_Namespace::SessionInfo &session_info, const std::vector< int32_t > &dashboard_ids)
 
void shareOrUnshareDashboards (const TSessionId &session, const std::vector< int32_t > &dashboard_ids, const std::vector< std::string > &groups, const TDashboardPermissions &permissions, const bool do_share)
 
std::shared_ptr< const
Catalog_Namespace::SessionInfo
get_const_session_ptr (const TSessionId &session)
 
std::shared_ptr
< Catalog_Namespace::SessionInfo
get_session_ptr (const TSessionId &session_id)
 
template<typename SESSION_MAP_LOCK >
SessionMap::iterator get_session_it_unsafe (const TSessionId &session, SESSION_MAP_LOCK &lock)
 
template<typename SESSION_MAP_LOCK >
void expire_idle_sessions_unsafe (SESSION_MAP_LOCK &lock)
 
std::pair< TPlanResult,
lockmgr::LockedTableDescriptors
parse_to_ra (QueryStateProxy, const std::string &query_str, const std::vector< TFilterPushDownInfo > &filter_push_down_info, const bool acquire_locks, const SystemParameters &system_parameters, bool check_privileges=true)
 
void sql_execute_local (TQueryResult &_return, const QueryStateProxy &query_state_proxy, const std::shared_ptr< Catalog_Namespace::SessionInfo > session_ptr, const std::string &query_str, const bool column_format, const std::string &nonce, const int32_t first_n, const int32_t at_most_n, const bool use_calcite)
 
int64_t process_geo_copy_from (const TSessionId &session_id)
 
void sql_execute_impl (ExecutionResult &_return, QueryStateProxy, const bool column_format, const ExecutorDeviceType executor_device_type, const int32_t first_n, const int32_t at_most_n, const bool use_calcite)
 
bool user_can_access_table (const Catalog_Namespace::SessionInfo &, const TableDescriptor *td, const AccessPrivileges acess_priv)
 
void execute_distributed_copy_statement (Parser::CopyTableStmt *, const Catalog_Namespace::SessionInfo &session_info)
 
TQueryResult validate_rel_alg (const std::string &query_ra, QueryStateProxy)
 
std::vector< PushedDownFilterInfoexecute_rel_alg (ExecutionResult &_return, QueryStateProxy, const std::string &query_ra, const bool column_format, const ExecutorDeviceType executor_device_type, const int32_t first_n, const int32_t at_most_n, const bool just_validate, const bool find_push_down_candidates, const ExplainInfo &explain_info, const std::optional< size_t > executor_index=std::nullopt) const
 
void execute_rel_alg_with_filter_push_down (ExecutionResult &_return, QueryStateProxy, std::string &query_ra, const bool column_format, const ExecutorDeviceType executor_device_type, const int32_t first_n, const int32_t at_most_n, const bool just_explain, const bool just_calcite_explain, const std::vector< PushedDownFilterInfo > &filter_push_down_requests)
 
void execute_rel_alg_df (TDataFrame &_return, const std::string &query_ra, QueryStateProxy query_state_proxy, const Catalog_Namespace::SessionInfo &session_info, const ExecutorDeviceType device_type, const size_t device_id, const int32_t first_n, const TArrowTransport::type transport_method) const
 
void executeDdl (TQueryResult &_return, const std::string &query_ra, std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
 
void executeDdl (ExecutionResult &_return, const std::string &query_ra, std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
 
TColumnType populateThriftColumnType (const Catalog_Namespace::Catalog *cat, const ColumnDescriptor *cd)
 
TRowDescriptor fixup_row_descriptor (const TRowDescriptor &row_desc, const Catalog_Namespace::Catalog &cat)
 
void set_execution_mode_nolock (Catalog_Namespace::SessionInfo *session_ptr, const TExecuteMode::type mode)
 
char unescape_char (std::string str)
 
import_export::CopyParams thrift_to_copyparams (const TCopyParams &cp)
 
TCopyParams copyparams_to_thrift (const import_export::CopyParams &cp)
 
void check_geospatial_files (const boost::filesystem::path file_path, const import_export::CopyParams &copy_params)
 
void render_rel_alg (TRenderResult &_return, const std::string &query_ra, const std::string &query_str, const Catalog_Namespace::SessionInfo &session_info, const std::string &render_type, const bool is_projection_query)
 
TColumnType create_geo_column (const TDatumType::type type, const std::string &name, const bool is_array)
 
void convertResultSet (ExecutionResult &result, const Catalog_Namespace::SessionInfo &session_info, const std::string &query_state_str, TQueryResult &_return)
 
std::vector< TargetMetaInfogetTargetMetaInfo (const std::vector< std::shared_ptr< Analyzer::TargetEntry >> &targets) const
 
std::vector< std::string > getTargetNames (const std::vector< TargetMetaInfo > &targets) const
 
std::vector< std::string > getTargetNames (const std::vector< std::shared_ptr< Analyzer::TargetEntry >> &targets) const
 
void get_completion_hints_unsorted (std::vector< TCompletionHint > &hints, std::vector< std::string > &visible_tables, query_state::StdLog &stdlog, const std::string &sql, const int cursor)
 
void get_token_based_completions (std::vector< TCompletionHint > &hints, query_state::StdLog &stdlog, std::vector< std::string > &visible_tables, const std::string &sql, const int cursor)
 
std::unordered_map
< std::string,
std::unordered_set
< std::string > > 
fill_column_names_by_table (std::vector< std::string > &table_names, query_state::StdLog &stdlog)
 
ConnectionInfo getConnectionInfo () const
 
TDashboard get_dashboard_impl (const std::shared_ptr< Catalog_Namespace::SessionInfo const > &session_ptr, Catalog_Namespace::UserMetadata &user_meta, const DashboardDescriptor *dash, const bool populate_state=true)
 
std::unordered_set< std::string > get_uc_compatible_table_names_by_column (const std::unordered_set< std::string > &uc_column_names, std::vector< std::string > &table_names, query_state::StdLog &stdlog)
 
std::unique_ptr
< lockmgr::AbstractLockContainer
< const TableDescriptor * > > 
prepare_loader_generic (const Catalog_Namespace::SessionInfo &session_info, const std::string &table_name, size_t num_cols, std::unique_ptr< import_export::Loader > *loader, std::vector< std::unique_ptr< import_export::TypedImportBuffer >> *import_buffers, const std::vector< std::string > &column_names, std::string load_type)
 
void check_and_invalidate_sessions (Parser::DDLStmt *ddl)
 
template<typename STMT_TYPE >
void invalidate_sessions (std::string &name, STMT_TYPE *stmt)
 
std::string const createInMemoryCalciteSession (const std::shared_ptr< Catalog_Namespace::Catalog > &catalog_ptr)
 
bool isInMemoryCalciteSession (const Catalog_Namespace::UserMetadata user_meta)
 
void removeInMemoryCalciteSession (const std::string &session_id)
 
ExecutionResult getUserSessions (std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
 
ExecutionResult getQueries (std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
 
void interruptQuery (const Catalog_Namespace::SessionInfo &session_info, const std::string &target_session)
 
void load_table_binary_columnar_internal (const TSessionId &session, const std::string &table_name, const std::vector< TColumn > &cols, const std::vector< std::string > &column_names, const AssignRenderGroupsMode assign_render_groups_mode)
 

Static Private Member Functions

static void value_to_thrift_column (const TargetValue &tv, const SQLTypeInfo &ti, TColumn &column)
 
static TDatum value_to_thrift (const TargetValue &tv, const SQLTypeInfo &ti)
 
static std::string apply_copy_to_shim (const std::string &query_str)
 
static void convertData (TQueryResult &_return, ExecutionResult &result, const QueryStateProxy &query_state_proxy, const std::string &query_str, const bool column_format, const int32_t first_n, const int32_t at_most_n)
 
static void convertExplain (TQueryResult &_return, const ResultSet &results, const bool column_format)
 
static void convertResult (TQueryResult &_return, const ResultSet &results, const bool column_format)
 
static void convertRows (TQueryResult &_return, QueryStateProxy query_state_proxy, const std::vector< TargetMetaInfo > &targets, const ResultSet &results, const bool column_format, const int32_t first_n, const int32_t at_most_n)
 
static void createSimpleResult (TQueryResult &_return, const ResultSet &results, const bool column_format, const std::string label)
 
static bool has_database_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permissions)
 
static bool has_table_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permission)
 
static bool has_dashboard_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permissions)
 
static bool has_view_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permissions)
 
static bool has_server_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permissions)
 

Private Attributes

std::atomic< bool > initialized_ {false}
 
query_state::QueryStates query_states_
 
SessionMap sessions_
 
bool super_user_rights_
 
const int idle_session_duration_
 
const int max_session_duration_
 
const bool runtime_udf_registration_enabled_
 
const bool enable_rendering_
 
const bool renderer_use_vulkan_driver_
 
const bool enable_auto_clear_render_mem_
 
const int render_oom_retry_threshold_
 
const size_t max_concurrent_render_sessions_
 
const size_t reserved_gpu_mem_
 
const bool render_compositor_use_last_gpu_
 
const size_t render_mem_bytes_
 
const size_t num_reader_threads_
 
const DiskCacheConfigdisk_cache_config_
 
const std::string & udf_filename_
 
const std::string & clang_path_
 
const std::vector< std::string > & clang_options_
 
GeoCopyFromSessions geo_copy_from_sessions
 
std::mutex handle_to_dev_ptr_mutex_
 
std::unordered_map
< std::string, std::string > 
ipc_handle_to_dev_ptr_
 
std::map< const std::string,
const PermissionFuncPtr
permissionFuncMap_
 
RenderGroupAnalyzerSessionMap render_group_assignment_map_
 
std::mutex render_group_assignment_mutex_
 
mapd_shared_mutex custom_expressions_mutex_
 

Friends

class RenderHandler::Impl
 
class MapDAggHandler
 
class MapDLeafHandler
 
void run_warmup_queries (mapd::shared_ptr< DBHandler > handler, std::string base_path, std::string query_file_path)
 

Detailed Description

Definition at line 157 of file DBHandler.h.

Member Typedef Documentation

using DBHandler::RenderGroupAnalyzerSessionMap = std::unordered_map<TSessionId, RenderGroupAssignmentTableMap>
private

Definition at line 1006 of file DBHandler.h.

using DBHandler::RenderGroupAssignmentColumnMap = std::unordered_map<std::string, std::unique_ptr<import_export::RenderGroupAnalyzer>>
private

Definition at line 1002 of file DBHandler.h.

using DBHandler::RenderGroupAssignmentTableMap = std::unordered_map<std::string, RenderGroupAssignmentColumnMap>
private

Definition at line 1004 of file DBHandler.h.

Member Enumeration Documentation

Enumerator
kNone 
kAssign 
kCleanUp 

Definition at line 991 of file DBHandler.h.

991 { kNone, kAssign, kCleanUp };

Constructor & Destructor Documentation

DBHandler::DBHandler ( const std::vector< LeafHostInfo > &  db_leaves,
const std::vector< LeafHostInfo > &  string_leaves,
const std::string &  base_data_path,
const bool  allow_multifrag,
const bool  jit_debug,
const bool  intel_jit_profile,
const bool  read_only,
const bool  allow_loop_joins,
const bool  enable_rendering,
const bool  renderer_use_vulkan_driver,
const bool  enable_auto_clear_render_mem,
const int  render_oom_retry_threshold,
const size_t  render_mem_bytes,
const size_t  max_concurrent_render_sessions,
const size_t  reserved_gpu_mem,
const bool  render_compositor_use_last_gpu,
const size_t  num_reader_threads,
const AuthMetadata authMetadata,
SystemParameters system_parameters,
const bool  legacy_syntax,
const int  idle_session_duration,
const int  max_session_duration,
const bool  enable_runtime_udf_registration,
const std::string &  udf_filename,
const std::string &  clang_path,
const std::vector< std::string > &  clang_options,
const DiskCacheConfig disk_cache_config,
const bool  is_new_db 
)

Definition at line 214 of file DBHandler.cpp.

245  : leaf_aggregator_(db_leaves)
246  , db_leaves_(db_leaves)
247  , string_leaves_(string_leaves)
248  , base_data_path_(base_data_path)
249  , random_gen_(std::random_device{}())
251  , jit_debug_(jit_debug)
252  , intel_jit_profile_(intel_jit_profile)
253  , allow_multifrag_(allow_multifrag)
254  , read_only_(read_only)
255  , allow_loop_joins_(allow_loop_joins)
256  , authMetadata_(authMetadata)
257  , system_parameters_(system_parameters)
258  , legacy_syntax_(legacy_syntax)
259  , dispatch_queue_(
260  std::make_unique<QueryDispatchQueue>(system_parameters.num_executors))
261  , super_user_rights_(false)
262  , idle_session_duration_(idle_session_duration * 60)
263  , max_session_duration_(max_session_duration * 60)
264  , runtime_udf_registration_enabled_(enable_runtime_udf_registration)
265 
266  , enable_rendering_(enable_rendering)
267  , renderer_use_vulkan_driver_(renderer_use_vulkan_driver)
268  , enable_auto_clear_render_mem_(enable_auto_clear_render_mem)
269  , render_oom_retry_threshold_(render_oom_retry_threshold)
270  , max_concurrent_render_sessions_(max_concurrent_render_sessions)
271  , reserved_gpu_mem_(reserved_gpu_mem)
272  , render_compositor_use_last_gpu_(render_compositor_use_last_gpu)
273  , render_mem_bytes_(render_mem_bytes)
274  , num_reader_threads_(num_reader_threads)
275 #ifdef ENABLE_GEOS
276  , libgeos_so_filename_(libgeos_so_filename)
277 #endif
278  , disk_cache_config_(disk_cache_config)
279  , udf_filename_(udf_filename)
280  , clang_path_(clang_path)
281  , clang_options_(clang_options)
282 
283 {
284  LOG(INFO) << "OmniSci Server " << MAPD_RELEASE;
285  initialize(is_new_db);
286 }
std::vector< LeafHostInfo > string_leaves_
Definition: DBHandler.h:568
const std::vector< std::string > & clang_options_
Definition: DBHandler.h:891
std::unique_ptr< QueryDispatchQueue > dispatch_queue_
Definition: DBHandler.h:591
const bool runtime_udf_registration_enabled_
Definition: DBHandler.h:874
const std::string & udf_filename_
Definition: DBHandler.h:889
const int render_oom_retry_threshold_
Definition: DBHandler.h:879
#define LOG(tag)
Definition: Logger.h:194
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:566
#define INT32_MAX
const std::string base_data_path_
Definition: DBHandler.h:569
void initialize(const bool is_new_db)
Definition: DBHandler.cpp:288
const bool jit_debug_
Definition: DBHandler.h:574
const size_t render_mem_bytes_
Definition: DBHandler.h:883
const bool renderer_use_vulkan_driver_
Definition: DBHandler.h:877
const size_t reserved_gpu_mem_
Definition: DBHandler.h:881
const bool render_compositor_use_last_gpu_
Definition: DBHandler.h:882
const int max_session_duration_
Definition: DBHandler.h:872
std::vector< LeafHostInfo > db_leaves_
Definition: DBHandler.h:567
const std::string & clang_path_
Definition: DBHandler.h:890
SystemParameters & system_parameters_
Definition: DBHandler.h:584
const size_t num_reader_threads_
Definition: DBHandler.h:884
const DiskCacheConfig & disk_cache_config_
Definition: DBHandler.h:888
const bool enable_auto_clear_render_mem_
Definition: DBHandler.h:878
const bool allow_loop_joins_
Definition: DBHandler.h:578
const bool enable_rendering_
Definition: DBHandler.h:876
const bool intel_jit_profile_
Definition: DBHandler.h:575
bool super_user_rights_
Definition: DBHandler.h:868
const bool read_only_
Definition: DBHandler.h:577
const bool legacy_syntax_
Definition: DBHandler.h:589
const int idle_session_duration_
Definition: DBHandler.h:871
const size_t max_concurrent_render_sessions_
Definition: DBHandler.h:880
bool allow_multifrag_
Definition: DBHandler.h:576
static const std::string MAPD_RELEASE
Definition: release.h:43
const AuthMetadata & authMetadata_
Definition: DBHandler.h:583
std::default_random_engine random_gen_
Definition: DBHandler.h:572
std::uniform_int_distribution< int64_t > session_id_dist_
Definition: DBHandler.h:573
DBHandler::~DBHandler ( )
override

Definition at line 470 of file DBHandler.cpp.

470 {}

Member Function Documentation

std::string DBHandler::apply_copy_to_shim ( const std::string &  query_str)
staticprivate

Definition at line 1483 of file DBHandler.cpp.

References apply_shim(), and run_benchmark_import::result.

Referenced by sql_execute_impl().

1483  {
1484  auto result = query_str;
1485  {
1486  // boost::regex copy_to{R"(COPY\s\((.*)\)\sTO\s(.*))", boost::regex::extended |
1487  // boost::regex::icase};
1488  boost::regex copy_to{R"(COPY\s*\(([^#])(.+)\)\s+TO\s)",
1489  boost::regex::extended | boost::regex::icase};
1490  apply_shim(result, copy_to, [](std::string& result, const boost::smatch& what) {
1491  result.replace(
1492  what.position(), what.length(), "COPY (#~#" + what[1] + what[2] + "#~#) TO ");
1493  });
1494  }
1495  return result;
1496 }
void apply_shim(std::string &result, const boost::regex &reg_expr, const std::function< void(std::string &, const boost::smatch &)> &shim_fn)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::broadcast_serialized_rows ( const TSerializedRows &  serialized_rows,
const TRowDescriptor &  row_desc,
const TQueryId  query_id,
const TSubqueryId  subquery_id,
const bool  is_final_subquery_result 
)
override

Definition at line 6347 of file DBHandler.cpp.

References measure< TimeT >::execution(), logger::INFO, leaf_handler_, LOG, and THROW_MAPD_EXCEPTION.

6351  {
6352  if (!leaf_handler_) {
6353  THROW_MAPD_EXCEPTION("Distributed support is disabled.");
6354  }
6355  LOG(INFO) << "BROADCAST-SERIALIZED-ROWS id:" << query_id;
6356  auto time_ms = measure<>::execution([&]() {
6357  try {
6358  leaf_handler_->broadcast_serialized_rows(
6359  serialized_rows, row_desc, query_id, subquery_id, is_final_subquery_result);
6360  } catch (std::exception& e) {
6361  THROW_MAPD_EXCEPTION(std::string("Exception: ") + e.what());
6362  }
6363  });
6364  LOG(INFO) << "BROADCAST-SERIALIZED-ROWS COMPLETED " << time_ms << "ms";
6365 }
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:194
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
std::unique_ptr< MapDLeafHandler > leaf_handler_
Definition: DBHandler.h:587

+ Here is the call graph for this function:

void DBHandler::check_and_invalidate_sessions ( Parser::DDLStmt ddl)
private

Definition at line 5785 of file DBHandler.cpp.

References invalidate_sessions().

Referenced by sql_execute_impl().

5785  {
5786  const auto drop_db_stmt = dynamic_cast<Parser::DropDBStmt*>(ddl);
5787  if (drop_db_stmt) {
5788  invalidate_sessions(*drop_db_stmt->getDatabaseName(), drop_db_stmt);
5789  return;
5790  }
5791  const auto rename_db_stmt = dynamic_cast<Parser::RenameDatabaseStmt*>(ddl);
5792  if (rename_db_stmt) {
5793  invalidate_sessions(*rename_db_stmt->getPreviousDatabaseName(), rename_db_stmt);
5794  return;
5795  }
5796  const auto drop_user_stmt = dynamic_cast<Parser::DropUserStmt*>(ddl);
5797  if (drop_user_stmt) {
5798  invalidate_sessions(*drop_user_stmt->getUserName(), drop_user_stmt);
5799  return;
5800  }
5801  const auto rename_user_stmt = dynamic_cast<Parser::RenameUserStmt*>(ddl);
5802  if (rename_user_stmt) {
5803  invalidate_sessions(*rename_user_stmt->getOldUserName(), rename_user_stmt);
5804  return;
5805  }
5806 }
void invalidate_sessions(std::string &name, STMT_TYPE *stmt)
Definition: DBHandler.h:948

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::check_geospatial_files ( const boost::filesystem::path  file_path,
const import_export::CopyParams copy_params 
)
private

Definition at line 4431 of file DBHandler.cpp.

References import_export::Importer::gdalFileExists().

Referenced by detect_column_types(), and import_geo_table().

4432  {
4433  const std::list<std::string> shp_ext{".shp", ".shx", ".dbf"};
4434  if (std::find(shp_ext.begin(),
4435  shp_ext.end(),
4436  boost::algorithm::to_lower_copy(file_path.extension().string())) !=
4437  shp_ext.end()) {
4438  for (auto ext : shp_ext) {
4439  auto aux_file = file_path;
4441  aux_file.replace_extension(boost::algorithm::to_upper_copy(ext)).string(),
4442  copy_params) &&
4444  aux_file.replace_extension(ext).string(), copy_params)) {
4445  throw std::runtime_error("required file for shapefile does not exist: " +
4446  aux_file.filename().string());
4447  }
4448  }
4449  }
4450 }
static bool gdalFileExists(const std::string &path, const CopyParams &copy_params)
Definition: Importer.cpp:4737

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::check_read_only ( const std::string &  str)
private

Definition at line 488 of file DBHandler.cpp.

References read_only_, and THROW_MAPD_EXCEPTION.

Referenced by create_custom_expression(), create_dashboard(), create_table(), delete_custom_expressions(), delete_dashboards(), detect_column_types(), import_geo_table(), import_table(), prepare_loader_generic(), replace_dashboard(), set_license_key(), shareOrUnshareDashboards(), and update_custom_expression().

488  {
489  if (DBHandler::read_only_) {
490  THROW_MAPD_EXCEPTION(str + " disabled: server running in read-only mode.");
491  }
492 }
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
const bool read_only_
Definition: DBHandler.h:577

+ Here is the caller graph for this function:

void DBHandler::check_session_exp_unsafe ( const SessionMap::iterator &  session_it)
private

Definition at line 5338 of file DBHandler.cpp.

References idle_session_duration_, logger::INFO, isInMemoryCalciteSession(), LOG, max_session_duration_, and run_benchmark_import::start_time.

Referenced by expire_idle_sessions_unsafe(), and get_session_it_unsafe().

5338  {
5339  if (session_it->second.use_count() > 2 ||
5340  isInMemoryCalciteSession(session_it->second->get_currentUser())) {
5341  // SessionInfo is being used in more than one active operation. Original copy + one
5342  // stored in StdLog. Skip the checks.
5343  return;
5344  }
5345  time_t last_used_time = session_it->second->get_last_used_time();
5346  time_t start_time = session_it->second->get_start_time();
5347  const auto current_session_duration = time(0) - last_used_time;
5348  if (current_session_duration > idle_session_duration_) {
5349  LOG(INFO) << "Session " << session_it->second->get_public_session_id()
5350  << " idle duration " << current_session_duration
5351  << " seconds exceeds maximum idle duration " << idle_session_duration_
5352  << " seconds. Invalidating session.";
5353  throw ForceDisconnect("Idle Session Timeout. User should re-authenticate.");
5354  }
5355  const auto total_session_duration = time(0) - start_time;
5356  if (total_session_duration > max_session_duration_) {
5357  LOG(INFO) << "Session " << session_it->second->get_public_session_id()
5358  << " total duration " << total_session_duration
5359  << " seconds exceeds maximum total session duration "
5360  << max_session_duration_ << " seconds. Invalidating session.";
5361  throw ForceDisconnect("Maximum active Session Timeout. User should re-authenticate.");
5362  }
5363 }
#define LOG(tag)
Definition: Logger.h:194
const int max_session_duration_
Definition: DBHandler.h:872
const int idle_session_duration_
Definition: DBHandler.h:871
bool isInMemoryCalciteSession(const Catalog_Namespace::UserMetadata user_meta)
Definition: DBHandler.cpp:520

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::check_table_consistency ( TTableMeta &  _return,
const TSessionId &  session,
const int32_t  table_id 
)
override

Definition at line 6284 of file DBHandler.cpp.

References get_session_ptr(), leaf_handler_, STDLOG, and THROW_MAPD_EXCEPTION.

6286  {
6287  auto stdlog = STDLOG(get_session_ptr(session));
6288  if (!leaf_handler_) {
6289  THROW_MAPD_EXCEPTION("Distributed support is disabled.");
6290  }
6291  try {
6292  leaf_handler_->check_table_consistency(_return, session, table_id);
6293  } catch (std::exception& e) {
6294  THROW_MAPD_EXCEPTION(std::string("Exception: ") + e.what());
6295  }
6296 }
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
std::unique_ptr< MapDLeafHandler > leaf_handler_
Definition: DBHandler.h:587
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

void DBHandler::check_table_load_privileges ( const TSessionId &  session,
const std::string &  table_name 
)
private

Definition at line 5422 of file DBHandler.cpp.

References get_session_copy().

Referenced by import_geo_table(), import_table(), and prepare_loader_generic().

5423  {
5424  const auto session_info = get_session_copy(session);
5425  check_table_load_privileges(session_info, table_name);
5426 }
void check_table_load_privileges(const TSessionId &session, const std::string &table_name)
Definition: DBHandler.cpp:5422
Catalog_Namespace::SessionInfo get_session_copy(const TSessionId &session)
Definition: DBHandler.cpp:5370

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::check_table_load_privileges ( const Catalog_Namespace::SessionInfo session_info,
const std::string &  table_name 
)
private

Definition at line 5405 of file DBHandler.cpp.

References cat(), Catalog_Namespace::SessionInfo::get_currentUser(), Catalog_Namespace::SessionInfo::getCatalog(), AccessPrivileges::INSERT_INTO_TABLE, DBObject::loadKey(), DBObject::setPrivileges(), TableDBObjectType, and THROW_MAPD_EXCEPTION.

5407  {
5408  auto user_metadata = session_info.get_currentUser();
5409  auto& cat = session_info.getCatalog();
5410  DBObject dbObject(table_name, TableDBObjectType);
5411  dbObject.loadKey(cat);
5412  dbObject.setPrivileges(AccessPrivileges::INSERT_INTO_TABLE);
5413  std::vector<DBObject> privObjects;
5414  privObjects.push_back(dbObject);
5415  if (!SysCatalog::instance().checkPrivileges(user_metadata, privObjects)) {
5416  THROW_MAPD_EXCEPTION("Violation of access privileges: user " +
5417  user_metadata.userLoggable() +
5418  " has no insert privileges for table " + table_name + ".");
5419  }
5420 }
std::string cat(Ts &&...args)
static const AccessPrivileges INSERT_INTO_TABLE
Definition: DBObject.h:163
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
Catalog & getCatalog() const
Definition: SessionInfo.h:66
const UserMetadata & get_currentUser() const
Definition: SessionInfo.h:72

+ Here is the call graph for this function:

void DBHandler::checkpoint ( const TSessionId &  session,
const int32_t  table_id 
)
override

Definition at line 6505 of file DBHandler.cpp.

References cat(), get_session_ptr(), and STDLOG.

6505  {
6506  auto stdlog = STDLOG(get_session_ptr(session));
6507  auto session_ptr = stdlog.getConstSessionInfo();
6508  auto& cat = session_ptr->getCatalog();
6509  cat.checkpoint(table_id);
6510 }
std::string cat(Ts &&...args)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

void DBHandler::clear_cpu_memory ( const TSessionId &  session)
override

Definition at line 2571 of file DBHandler.cpp.

References LeafAggregator::clear_leaf_cpu_memory(), Executor::clearMemory(), Data_Namespace::CPU_LEVEL, get_session_ptr(), getConnectionInfo(), leaf_aggregator_, LeafAggregator::leafCount(), render_handler_, STDLOG, THROW_MAPD_EXCEPTION, and toString().

2571  {
2572  auto stdlog = STDLOG(get_session_ptr(session));
2573  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
2574  auto session_ptr = stdlog.getConstSessionInfo();
2575  if (!session_ptr->get_currentUser().isSuper) {
2576  THROW_MAPD_EXCEPTION("Superuser privilege is required to run clear_cpu_memory");
2577  }
2578  try {
2580  } catch (const std::exception& e) {
2581  THROW_MAPD_EXCEPTION(e.what());
2582  }
2583  if (render_handler_) {
2584  render_handler_->clear_cpu_memory();
2585  }
2586 
2587  if (leaf_aggregator_.leafCount() > 0) {
2589  }
2590 }
std::string toString(const ExtArgumentType &sig_type)
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:566
static void clearMemory(const Data_Namespace::MemoryLevel memory_level)
Definition: Execute.cpp:185
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
void clear_leaf_cpu_memory(const TSessionId session)
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:585
size_t leafCount() const
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1695
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

void DBHandler::clear_gpu_memory ( const TSessionId &  session)
override

Definition at line 2550 of file DBHandler.cpp.

References LeafAggregator::clear_leaf_gpu_memory(), Executor::clearMemory(), get_session_ptr(), getConnectionInfo(), Data_Namespace::GPU_LEVEL, leaf_aggregator_, LeafAggregator::leafCount(), render_handler_, STDLOG, THROW_MAPD_EXCEPTION, and toString().

2550  {
2551  auto stdlog = STDLOG(get_session_ptr(session));
2552  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
2553  auto session_ptr = stdlog.getConstSessionInfo();
2554  if (!session_ptr->get_currentUser().isSuper) {
2555  THROW_MAPD_EXCEPTION("Superuser privilege is required to run clear_gpu_memory");
2556  }
2557  try {
2559  } catch (const std::exception& e) {
2560  THROW_MAPD_EXCEPTION(e.what());
2561  }
2562  if (render_handler_) {
2563  render_handler_->clear_gpu_memory();
2564  }
2565 
2566  if (leaf_aggregator_.leafCount() > 0) {
2568  }
2569 }
std::string toString(const ExtArgumentType &sig_type)
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:566
void clear_leaf_gpu_memory(const TSessionId session)
static void clearMemory(const Data_Namespace::MemoryLevel memory_level)
Definition: Execute.cpp:185
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:585
size_t leafCount() const
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1695
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

void DBHandler::clone_session ( TSessionId &  session2,
const TSessionId &  session1 
)
override

Definition at line 736 of file DBHandler.cpp.

References cat(), LeafAggregator::clone_session(), create_new_session(), get_session_it_unsafe(), get_session_ptr(), getConnectionInfo(), leaf_aggregator_, LeafAggregator::leafCount(), sessions_mutex_, STDLOG, THROW_MAPD_EXCEPTION, and toString().

736  {
737  auto stdlog = STDLOG(get_session_ptr(session1));
738  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
739  mapd_unique_lock<mapd_shared_mutex> write_lock(sessions_mutex_);
740  auto session_it = get_session_it_unsafe(session1, write_lock);
741 
742  try {
743  const Catalog_Namespace::UserMetadata& user_meta =
744  session_it->second->get_currentUser();
745  std::shared_ptr<Catalog> cat = session_it->second->get_catalog_ptr();
746  auto session2_ptr = create_new_session(session2, cat->name(), user_meta, cat);
747  if (leaf_aggregator_.leafCount() > 0) {
748  leaf_aggregator_.clone_session(session1, session2);
749  return;
750  }
751  } catch (std::exception& e) {
752  THROW_MAPD_EXCEPTION(e.what());
753  }
754 }
std::string cat(Ts &&...args)
std::string toString(const ExtArgumentType &sig_type)
void clone_session(const TSessionId session1, const TSessionId session2)
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:566
std::shared_ptr< Catalog_Namespace::SessionInfo > create_new_session(TSessionId &session, const std::string &dbname, const Catalog_Namespace::UserMetadata &user_meta, std::shared_ptr< Catalog_Namespace::Catalog > cat)
Definition: DBHandler.cpp:615
mapd_shared_mutex sessions_mutex_
Definition: DBHandler.h:580
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
size_t leafCount() const
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
SessionMap::iterator get_session_it_unsafe(const TSessionId &session, SESSION_MAP_LOCK &lock)
mapd_unique_lock< mapd_shared_mutex > write_lock
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1695
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

void DBHandler::connect ( TSessionId &  session,
const std::string &  username,
const std::string &  passwd,
const std::string &  dbname 
)
override

Definition at line 573 of file DBHandler.cpp.

References AccessPrivileges::ACCESS, cat(), connect_impl(), DatabaseDBObjectType, get_session_ptr(), getConnectionInfo(), DBObject::loadKey(), Catalog_Namespace::UserMetadata::restriction, DBObject::setPrivileges(), STDLOG, super_user_rights_, THROW_MAPD_EXCEPTION, toString(), and Catalog_Namespace::UserMetadata::userLoggable().

576  {
577  auto stdlog = STDLOG(); // session_info set by connect_impl()
578  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
579  std::string username2 = username; // login() may reset username given as argument
580  std::string dbname2 = dbname; // login() may reset dbname given as argument
582  std::shared_ptr<Catalog> cat = nullptr;
583  try {
584  cat = SysCatalog::instance().login(
585  dbname2, username2, passwd, user_meta, !super_user_rights_);
586  } catch (std::exception& e) {
587  stdlog.appendNameValuePairs("user", username, "db", dbname, "exception", e.what());
588  THROW_MAPD_EXCEPTION(e.what());
589  }
590 
591  DBObject dbObject(dbname2, DatabaseDBObjectType);
592  dbObject.loadKey(*cat);
593  dbObject.setPrivileges(AccessPrivileges::ACCESS);
594  std::vector<DBObject> dbObjects;
595  dbObjects.push_back(dbObject);
596  if (!SysCatalog::instance().checkPrivileges(user_meta, dbObjects)) {
597  stdlog.appendNameValuePairs(
598  "user", username, "db", dbname, "exception", "Missing Privileges");
599  THROW_MAPD_EXCEPTION("Unauthorized Access: user " + user_meta.userLoggable() +
600  " is not allowed to access database " + dbname2 + ".");
601  }
602  connect_impl(session, passwd, dbname2, user_meta, cat, stdlog);
603 
604  // Restriction is returned as part of the users metadata on login but
605  // is per session so transfering it over here
606  // Currently only SAML can even set a Restriction
607  auto restriction = std::make_shared<Restriction>(user_meta.restriction);
608  auto login_session = get_session_ptr(session);
609  login_session->set_restriction(restriction);
610 
611  // if pki auth session will come back encrypted with user pubkey
612  SysCatalog::instance().check_for_session_encryption(passwd, session);
613 }
std::string cat(Ts &&...args)
std::string toString(const ExtArgumentType &sig_type)
void connect_impl(TSessionId &session, const std::string &passwd, const std::string &dbname, const Catalog_Namespace::UserMetadata &user_meta, std::shared_ptr< Catalog_Namespace::Catalog > cat, query_state::StdLog &stdlog)
Definition: DBHandler.cpp:633
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
bool super_user_rights_
Definition: DBHandler.h:868
static const AccessPrivileges ACCESS
Definition: DBObject.h:155
std::string userLoggable() const
Definition: SysCatalog.cpp:125
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1695
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

void DBHandler::connect_impl ( TSessionId &  session,
const std::string &  passwd,
const std::string &  dbname,
const Catalog_Namespace::UserMetadata user_meta,
std::shared_ptr< Catalog_Namespace::Catalog cat,
query_state::StdLog stdlog 
)
private

Definition at line 633 of file DBHandler.cpp.

References query_state::StdLog::appendNameValuePairs(), LeafAggregator::connect(), create_new_session(), expire_idle_sessions_unsafe(), getConnectionInfo(), query_state::StdLog::getConstSessionInfo(), join(), leaf_aggregator_, LeafAggregator::leafCount(), SystemParameters::num_sessions, sessions_, sessions_mutex_, query_state::StdLog::setSessionInfo(), super_user_rights_, system_parameters_, THROW_MAPD_EXCEPTION, toString(), and Catalog_Namespace::UserMetadata::userName.

Referenced by connect(), and internal_connect().

638  {
639  // TODO(sy): Is there any reason to have dbname as a parameter
640  // here when the cat parameter already provides cat->name()?
641  // Should dbname and cat->name() ever differ?
642  {
643  mapd_unique_lock<mapd_shared_mutex> write_lock(sessions_mutex_);
646  sessions_.size() + 1 > static_cast<size_t>(system_parameters_.num_sessions)) {
647  THROW_MAPD_EXCEPTION("Too many active sessions");
648  }
649  }
650  {
651  mapd_lock_guard<mapd_shared_mutex> write_lock(sessions_mutex_);
652  auto session_ptr = create_new_session(session, dbname, user_meta, cat);
653  stdlog.setSessionInfo(session_ptr);
654  session_ptr->set_connection_info(getConnectionInfo().toString());
655  if (!super_user_rights_) { // no need to connect to leaf_aggregator_ at this time
656  // while doing warmup
657  if (leaf_aggregator_.leafCount() > 0) {
658  leaf_aggregator_.connect(*session_ptr, user_meta.userName, passwd, dbname);
659  return;
660  }
661  }
662  }
663  auto const roles =
664  stdlog.getConstSessionInfo()->get_currentUser().isSuper
665  ? std::vector<std::string>{{"super"}}
666  : SysCatalog::instance().getRoles(
667  false, false, stdlog.getConstSessionInfo()->get_currentUser().userName);
668  stdlog.appendNameValuePairs("roles", boost::algorithm::join(roles, ","));
669 }
std::string cat(Ts &&...args)
std::string toString(const ExtArgumentType &sig_type)
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:566
std::shared_ptr< Catalog_Namespace::SessionInfo > create_new_session(TSessionId &session, const std::string &dbname, const Catalog_Namespace::UserMetadata &user_meta, std::shared_ptr< Catalog_Namespace::Catalog > cat)
Definition: DBHandler.cpp:615
std::string join(T const &container, std::string const &delim)
void connect(const Catalog_Namespace::SessionInfo &parent_session_info, const std::string &user, const std::string &passwd, const std::string &dbname)
void setSessionInfo(std::shared_ptr< Catalog_Namespace::SessionInfo >)
Definition: QueryState.cpp:267
mapd_shared_mutex sessions_mutex_
Definition: DBHandler.h:580
void expire_idle_sessions_unsafe(SESSION_MAP_LOCK &lock)
SystemParameters & system_parameters_
Definition: DBHandler.h:584
size_t leafCount() const
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
void appendNameValuePairs(Pairs &&...pairs)
Definition: QueryState.h:307
bool super_user_rights_
Definition: DBHandler.h:868
mapd_unique_lock< mapd_shared_mutex > write_lock
std::shared_ptr< Catalog_Namespace::SessionInfo const > getConstSessionInfo() const
Definition: QueryState.cpp:141
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1695
SessionMap sessions_
Definition: DBHandler.h:866

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::convertData ( TQueryResult &  _return,
ExecutionResult result,
const QueryStateProxy query_state_proxy,
const std::string &  query_str,
const bool  column_format,
const int32_t  first_n,
const int32_t  at_most_n 
)
staticprivate

Definition at line 1155 of file DBHandler.cpp.

References ExecutionResult::CalciteDdl, convertExplain(), convertResult(), convertRows(), ExecutionResult::empty(), ExecutionResult::Explaination, ExecutionResult::getExecutionTime(), ExecutionResult::getResultType(), ExecutionResult::getRows(), ExecutionResult::getTargetsMeta(), ExecutionResult::QueryResult, and ExecutionResult::SimpleResult.

Referenced by get_tables_meta_impl(), sql_execute_local(), and validate_rel_alg().

1161  {
1162  _return.execution_time_ms += result.getExecutionTime();
1163  if (result.empty()) {
1164  return;
1165  }
1166 
1167  switch (result.getResultType()) {
1169  convertRows(_return,
1170  query_state_proxy,
1171  result.getTargetsMeta(),
1172  *result.getRows(),
1173  column_format,
1174  first_n,
1175  at_most_n);
1176  break;
1178  convertResult(_return, *result.getRows(), true);
1179  break;
1181  convertExplain(_return, *result.getRows(), true);
1182  break;
1184  convertRows(_return,
1185  query_state_proxy,
1186  result.getTargetsMeta(),
1187  *result.getRows(),
1188  column_format,
1189  -1,
1190  -1);
1191  break;
1192  }
1193 }
RType getResultType() const
static void convertExplain(TQueryResult &_return, const ResultSet &results, const bool column_format)
Definition: DBHandler.cpp:5758
static void convertResult(TQueryResult &_return, const ResultSet &results, const bool column_format)
Definition: DBHandler.cpp:5764
const std::vector< TargetMetaInfo > & getTargetsMeta() const
const std::shared_ptr< ResultSet > & getRows() const
static void convertRows(TQueryResult &_return, QueryStateProxy query_state_proxy, const std::vector< TargetMetaInfo > &targets, const ResultSet &results, const bool column_format, const int32_t first_n, const int32_t at_most_n)
Definition: DBHandler.cpp:5650
int64_t getExecutionTime() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::convertExplain ( TQueryResult &  _return,
const ResultSet results,
const bool  column_format 
)
staticprivate

Definition at line 5758 of file DBHandler.cpp.

References createSimpleResult().

Referenced by convertData().

5760  {
5761  createSimpleResult(_return, results, column_format, "Explanation");
5762 }
static void createSimpleResult(TQueryResult &_return, const ResultSet &results, const bool column_format, const std::string label)
Definition: DBHandler.cpp:5721

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::convertResult ( TQueryResult &  _return,
const ResultSet results,
const bool  column_format 
)
staticprivate

Definition at line 5764 of file DBHandler.cpp.

References createSimpleResult().

Referenced by convertData().

5766  {
5767  createSimpleResult(_return, results, column_format, "Result");
5768 }
static void createSimpleResult(TQueryResult &_return, const ResultSet &results, const bool column_format, const std::string label)
Definition: DBHandler.cpp:5721

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::convertResultSet ( ExecutionResult result,
const Catalog_Namespace::SessionInfo session_info,
const std::string &  query_state_str,
TQueryResult &  _return 
)
private

Definition at line 6775 of file DBHandler.cpp.

References convertRows(), create_query_state(), Catalog_Namespace::SessionInfo::get_session_id(), get_session_ptr(), ExecutionResult::getDataPtr(), and ExecutionResult::getTargetsMeta().

Referenced by executeDdl().

6778  {
6779  // Stuff ResultSet into _return (which is a TQueryResult)
6780  // calls convertRows, but after some setup using session_info
6781 
6782  auto session_ptr = get_session_ptr(session_info.get_session_id());
6783  auto qs = create_query_state(session_ptr, query_state_str);
6784  QueryStateProxy qsp = qs->createQueryStateProxy();
6785 
6786  // omnisql only accepts column format as being 'VALID",
6787  // assume that omnisci_server should only return column format
6788  int32_t nRows = result.getDataPtr()->rowCount();
6789 
6790  convertRows(_return,
6791  qsp,
6792  result.getTargetsMeta(),
6793  *result.getDataPtr(),
6794  /*column_format=*/true,
6795  /*first_n=*/nRows,
6796  /*at_most_n=*/nRows);
6797 }
std::shared_ptr< query_state::QueryState > create_query_state(ARGS &&...args)
Definition: DBHandler.h:594
const ResultSetPtr & getDataPtr() const
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
const std::vector< TargetMetaInfo > & getTargetsMeta() const
std::string get_session_id() const
Definition: SessionInfo.h:77
static void convertRows(TQueryResult &_return, QueryStateProxy query_state_proxy, const std::vector< TargetMetaInfo > &targets, const ResultSet &results, const bool column_format, const int32_t first_n, const int32_t at_most_n)
Definition: DBHandler.cpp:5650

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::convertRows ( TQueryResult &  _return,
QueryStateProxy  query_state_proxy,
const std::vector< TargetMetaInfo > &  targets,
const ResultSet results,
const bool  column_format,
const int32_t  first_n,
const int32_t  at_most_n 
)
staticprivate

Definition at line 5650 of file DBHandler.cpp.

References query_state::QueryStateProxy::createTimer(), i, ThriftSerializers::target_meta_infos_to_thrift(), THROW_MAPD_EXCEPTION, to_string(), value_to_thrift(), and value_to_thrift_column().

Referenced by convertData(), and convertResultSet().

5656  {
5657  query_state::Timer timer = query_state_proxy.createTimer(__func__);
5658  _return.row_set.row_desc = ThriftSerializers::target_meta_infos_to_thrift(targets);
5659  int32_t fetched{0};
5660  if (column_format) {
5661  _return.row_set.is_columnar = true;
5662  std::vector<TColumn> tcolumns(results.colCount());
5663  while (first_n == -1 || fetched < first_n) {
5664  const auto crt_row = results.getNextRow(true, true);
5665  if (crt_row.empty()) {
5666  break;
5667  }
5668  ++fetched;
5669  if (at_most_n >= 0 && fetched > at_most_n) {
5670  THROW_MAPD_EXCEPTION("The result contains more rows than the specified cap of " +
5671  std::to_string(at_most_n));
5672  }
5673  for (size_t i = 0; i < results.colCount(); ++i) {
5674  const auto agg_result = crt_row[i];
5675  value_to_thrift_column(agg_result, targets[i].get_type_info(), tcolumns[i]);
5676  }
5677  }
5678  for (size_t i = 0; i < results.colCount(); ++i) {
5679  _return.row_set.columns.push_back(tcolumns[i]);
5680  }
5681  } else {
5682  _return.row_set.is_columnar = false;
5683  while (first_n == -1 || fetched < first_n) {
5684  const auto crt_row = results.getNextRow(true, true);
5685  if (crt_row.empty()) {
5686  break;
5687  }
5688  ++fetched;
5689  if (at_most_n >= 0 && fetched > at_most_n) {
5690  THROW_MAPD_EXCEPTION("The result contains more rows than the specified cap of " +
5691  std::to_string(at_most_n));
5692  }
5693  TRow trow;
5694  trow.cols.reserve(results.colCount());
5695  for (size_t i = 0; i < results.colCount(); ++i) {
5696  const auto agg_result = crt_row[i];
5697  trow.cols.push_back(value_to_thrift(agg_result, targets[i].get_type_info()));
5698  }
5699  _return.row_set.rows.push_back(trow);
5700  }
5701  }
5702 }
TRowDescriptor target_meta_infos_to_thrift(const std::vector< TargetMetaInfo > &targets)
static void value_to_thrift_column(const TargetValue &tv, const SQLTypeInfo &ti, TColumn &column)
Definition: DBHandler.cpp:890
Timer createTimer(char const *event_name)
Definition: QueryState.cpp:124
std::string to_string(char const *&&v)
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
static TDatum value_to_thrift(const TargetValue &tv, const SQLTypeInfo &ti)
Definition: DBHandler.cpp:1013

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

TCopyParams DBHandler::copyparams_to_thrift ( const import_export::CopyParams cp)
private

Definition at line 3486 of file DBHandler.cpp.

References import_export::CopyParams::array_begin, import_export::CopyParams::array_delim, import_export::CopyParams::array_end, import_export::AUTODETECT, CHECK, import_export::CopyParams::delimiter, import_export::CopyParams::escape, import_export::CopyParams::file_type, import_export::CopyParams::geo_assign_render_groups, import_export::CopyParams::geo_coords_comp_param, import_export::CopyParams::geo_coords_encoding, import_export::CopyParams::geo_coords_srid, import_export::CopyParams::geo_coords_type, import_export::CopyParams::geo_explode_collections, import_export::CopyParams::geo_layer_name, GEOGRAPHY, GEOMETRY, import_export::HAS_HEADER, import_export::CopyParams::has_header, kENCODING_GEOINT, kGEOGRAPHY, kGEOMETRY, import_export::CopyParams::line_delim, import_export::NO_HEADER, import_export::CopyParams::null_str, import_export::POLYGON, POLYGON, import_export::CopyParams::quote, import_export::CopyParams::quoted, import_export::CopyParams::s3_access_key, import_export::CopyParams::s3_endpoint, import_export::CopyParams::s3_region, import_export::CopyParams::s3_secret_key, import_export::CopyParams::sanitize_column_names, import_export::CopyParams::source_srid, and import_export::CopyParams::threads.

Referenced by detect_column_types(), and process_geo_copy_from().

3486  {
3487  TCopyParams copy_params;
3488  copy_params.delimiter = cp.delimiter;
3489  copy_params.null_str = cp.null_str;
3490  switch (cp.has_header) {
3492  copy_params.has_header = TImportHeaderRow::AUTODETECT;
3493  break;
3495  copy_params.has_header = TImportHeaderRow::NO_HEADER;
3496  break;
3498  copy_params.has_header = TImportHeaderRow::HAS_HEADER;
3499  break;
3500  default:
3501  CHECK(false);
3502  break;
3503  }
3504  copy_params.quoted = cp.quoted;
3505  copy_params.quote = cp.quote;
3506  copy_params.escape = cp.escape;
3507  copy_params.line_delim = cp.line_delim;
3508  copy_params.array_delim = cp.array_delim;
3509  copy_params.array_begin = cp.array_begin;
3510  copy_params.array_end = cp.array_end;
3511  copy_params.threads = cp.threads;
3512  copy_params.s3_access_key = cp.s3_access_key;
3513  copy_params.s3_secret_key = cp.s3_secret_key;
3514  copy_params.s3_region = cp.s3_region;
3515  copy_params.s3_endpoint = cp.s3_endpoint;
3516  switch (cp.file_type) {
3518  copy_params.file_type = TFileType::POLYGON;
3519  break;
3520  default:
3521  copy_params.file_type = TFileType::DELIMITED;
3522  break;
3523  }
3524  switch (cp.geo_coords_encoding) {
3525  case kENCODING_GEOINT:
3526  copy_params.geo_coords_encoding = TEncodingType::GEOINT;
3527  break;
3528  default:
3529  copy_params.geo_coords_encoding = TEncodingType::NONE;
3530  break;
3531  }
3532  copy_params.geo_coords_comp_param = cp.geo_coords_comp_param;
3533  switch (cp.geo_coords_type) {
3534  case kGEOGRAPHY:
3535  copy_params.geo_coords_type = TDatumType::GEOGRAPHY;
3536  break;
3537  case kGEOMETRY:
3538  copy_params.geo_coords_type = TDatumType::GEOMETRY;
3539  break;
3540  default:
3541  CHECK(false);
3542  break;
3543  }
3544  copy_params.geo_coords_srid = cp.geo_coords_srid;
3545  copy_params.sanitize_column_names = cp.sanitize_column_names;
3546  copy_params.geo_layer_name = cp.geo_layer_name;
3547  copy_params.geo_assign_render_groups = cp.geo_assign_render_groups;
3548  copy_params.geo_explode_collections = cp.geo_explode_collections;
3549  copy_params.source_srid = cp.source_srid;
3550  return copy_params;
3551 }
std::string s3_secret_key
Definition: CopyParams.h:63
#define GEOMETRY
ImportHeaderRow has_header
Definition: CopyParams.h:48
std::string geo_layer_name
Definition: CopyParams.h:79
#define CHECK(condition)
Definition: Logger.h:203
#define GEOGRAPHY
#define POLYGON
std::string s3_access_key
Definition: CopyParams.h:62
EncodingType geo_coords_encoding
Definition: CopyParams.h:74

+ Here is the caller graph for this function:

int32_t DBHandler::create_custom_expression ( const TSessionId &  session,
const TCustomExpression &  custom_expression 
)
override

Definition at line 3946 of file DBHandler.cpp.

References CHECK, check_read_only(), anonymous_namespace{DBHandler.cpp}::create_custom_expr_from_thrift_obj(), custom_expressions_mutex_, get_session_ptr(), getConnectionInfo(), STDLOG, THROW_MAPD_EXCEPTION, and toString().

3947  {
3948  auto stdlog = STDLOG(get_session_ptr(session));
3949  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
3950  check_read_only("create_custom_expression");
3951 
3952  auto session_ptr = stdlog.getConstSessionInfo();
3953  if (!session_ptr->get_currentUser().isSuper) {
3954  THROW_MAPD_EXCEPTION("Custom expressions can only be created by super users.")
3955  }
3956  auto& catalog = session_ptr->getCatalog();
3957  CHECK(t_custom_expr.data_source_type == TDataSourceType::type::TABLE)
3958  << "Unexpected data source type: "
3959  << static_cast<int>(t_custom_expr.data_source_type);
3960  if (catalog.getMetadataForTable(t_custom_expr.data_source_id, false) == nullptr) {
3961  THROW_MAPD_EXCEPTION("Custom expression references a table that does not exist.")
3962  }
3963  mapd_unique_lock<mapd_shared_mutex> write_lock(custom_expressions_mutex_);
3964  return catalog.createCustomExpression(
3965  create_custom_expr_from_thrift_obj(t_custom_expr));
3966 }
std::string toString(const ExtArgumentType &sig_type)
mapd_shared_mutex custom_expressions_mutex_
Definition: DBHandler.h:1009
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
void check_read_only(const std::string &str)
Definition: DBHandler.cpp:488
std::unique_ptr< Catalog_Namespace::CustomExpression > create_custom_expr_from_thrift_obj(const TCustomExpression &t_custom_expr)
Definition: DBHandler.cpp:3918
#define CHECK(condition)
Definition: Logger.h:203
mapd_unique_lock< mapd_shared_mutex > write_lock
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1695
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

int32_t DBHandler::create_dashboard ( const TSessionId &  session,
const std::string &  dashboard_name,
const std::string &  dashboard_state,
const std::string &  image_hash,
const std::string &  dashboard_metadata 
)
override

Definition at line 4119 of file DBHandler.cpp.

References cat(), check_read_only(), AccessPrivileges::CREATE_DASHBOARD, DashboardDBObjectType, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, get_session_ptr(), getConnectionInfo(), DashboardDescriptor::imageHash, STDLOG, THROW_MAPD_EXCEPTION, to_string(), toString(), DashboardDescriptor::user, and DashboardDescriptor::userId.

4123  {
4124  auto stdlog = STDLOG(get_session_ptr(session));
4125  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4126  auto session_ptr = stdlog.getConstSessionInfo();
4127  check_read_only("create_dashboard");
4128  auto& cat = session_ptr->getCatalog();
4129 
4130  if (!session_ptr->checkDBAccessPrivileges(DBObjectType::DashboardDBObjectType,
4132  THROW_MAPD_EXCEPTION("Not enough privileges to create a dashboard.");
4133  }
4134 
4135  auto dash = cat.getMetadataForDashboard(
4136  std::to_string(session_ptr->get_currentUser().userId), dashboard_name);
4137  if (dash) {
4138  THROW_MAPD_EXCEPTION("Dashboard with name: " + dashboard_name + " already exists.");
4139  }
4140 
4142  dd.dashboardName = dashboard_name;
4143  dd.dashboardState = dashboard_state;
4144  dd.imageHash = image_hash;
4145  dd.dashboardMetadata = dashboard_metadata;
4146  dd.userId = session_ptr->get_currentUser().userId;
4147  dd.user = session_ptr->get_currentUser().userName;
4148 
4149  try {
4150  auto id = cat.createDashboard(dd);
4151  // TODO: transactionally unsafe
4152  SysCatalog::instance().createDBObject(
4153  session_ptr->get_currentUser(), dashboard_name, DashboardDBObjectType, cat, id);
4154  return id;
4155  } catch (const std::exception& e) {
4156  THROW_MAPD_EXCEPTION(std::string("Exception: ") + e.what());
4157  }
4158 }
std::string cat(Ts &&...args)
std::string toString(const ExtArgumentType &sig_type)
static const AccessPrivileges CREATE_DASHBOARD
Definition: DBObject.h:172
std::string to_string(char const *&&v)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
void check_read_only(const std::string &str)
Definition: DBHandler.cpp:488
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1695
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

TColumnType DBHandler::create_geo_column ( const TDatumType::type  type,
const std::string &  name,
const bool  is_array 
)
private

Definition at line 4421 of file DBHandler.cpp.

References setup::name, and run_benchmark_import::type.

4423  {
4424  TColumnType ct;
4425  ct.col_name = name;
4426  ct.col_type.type = type;
4427  ct.col_type.is_array = is_array;
4428  return ct;
4429 }
string name
Definition: setup.in.py:72
void DBHandler::create_link ( std::string &  _return,
const TSessionId &  session,
const std::string &  view_state,
const std::string &  view_metadata 
)
override

Definition at line 4399 of file DBHandler.cpp.

References cat(), get_session_ptr(), getConnectionInfo(), STDLOG, THROW_MAPD_EXCEPTION, toString(), LinkDescriptor::userId, LinkDescriptor::viewMetadata, and LinkDescriptor::viewState.

4402  {
4403  auto stdlog = STDLOG(get_session_ptr(session));
4404  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4405  auto session_ptr = stdlog.getConstSessionInfo();
4406  // check_read_only("create_link");
4407  auto& cat = session_ptr->getCatalog();
4408 
4409  LinkDescriptor ld;
4410  ld.userId = session_ptr->get_currentUser().userId;
4411  ld.viewState = view_state;
4412  ld.viewMetadata = view_metadata;
4413 
4414  try {
4415  _return = cat.createLink(ld, 6);
4416  } catch (const std::exception& e) {
4417  THROW_MAPD_EXCEPTION(std::string("Exception: ") + e.what());
4418  }
4419 }
std::string cat(Ts &&...args)
std::string toString(const ExtArgumentType &sig_type)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1695
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

std::shared_ptr< Catalog_Namespace::SessionInfo > DBHandler::create_new_session ( TSessionId &  session,
const std::string &  dbname,
const Catalog_Namespace::UserMetadata user_meta,
std::shared_ptr< Catalog_Namespace::Catalog cat 
)
private

Definition at line 615 of file DBHandler.cpp.

References CHECK, executor_device_type_, generate_random_string(), logger::INFO, LOG, sessions_, and Catalog_Namespace::UserMetadata::userLoggable().

Referenced by clone_session(), and connect_impl().

619  {
620  do {
621  session = generate_random_string(32);
622  } while (sessions_.find(session) != sessions_.end());
623  std::pair<SessionMap::iterator, bool> emplace_retval =
624  sessions_.emplace(session,
625  std::make_shared<Catalog_Namespace::SessionInfo>(
626  cat, user_meta, executor_device_type_, session));
627  CHECK(emplace_retval.second);
628  auto& session_ptr = emplace_retval.first->second;
629  LOG(INFO) << "User " << user_meta.userLoggable() << " connected to database " << dbname;
630  return session_ptr;
631 }
std::string cat(Ts &&...args)
#define LOG(tag)
Definition: Logger.h:194
std::string generate_random_string(const size_t len)
ExecutorDeviceType executor_device_type_
Definition: DBHandler.h:571
#define CHECK(condition)
Definition: Logger.h:203
std::string userLoggable() const
Definition: SysCatalog.cpp:125
SessionMap sessions_
Definition: DBHandler.h:866

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename... ARGS>
std::shared_ptr<query_state::QueryState> DBHandler::create_query_state ( ARGS &&...  args)
inline

Definition at line 594 of file DBHandler.h.

References run_benchmark_import::args.

Referenced by convertResultSet(), get_table_details_impl(), get_tables_meta(), sql_execute(), sql_execute_df(), and sql_validate().

594  {
595  return query_states_.create(std::forward<ARGS>(args)...);
596  }
CircleBuffer::value_type create(ARGS &&...args)
Definition: QueryState.h:189
query_state::QueryStates query_states_
Definition: DBHandler.h:865

+ Here is the caller graph for this function:

void DBHandler::create_table ( const TSessionId &  session,
const std::string &  table_name,
const TRowDescriptor &  row_desc,
const TFileType::type  file_type,
const TCreateParams &  create_params 
)
override

Definition at line 4452 of file DBHandler.cpp.

References check_read_only(), DECIMAL, getConnectionInfo(), ImportHelpers::is_reserved_name(), join(), kENCODING_DICT, kENCODING_FIXED, kENCODING_GEOINT, kENCODING_NONE, LINESTRING, MULTIPOLYGON, POINT, POLYGON, ImportHelpers::sanitize_name(), sql_execute(), STDLOG, thrift_to_encoding(), thrift_to_encoding_name(), thrift_to_name(), THROW_MAPD_EXCEPTION, to_string(), and toString().

Referenced by import_geo_table().

4456  {
4457  auto stdlog = STDLOG("table_name", table_name);
4458  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4459  check_read_only("create_table");
4460 
4461  if (ImportHelpers::is_reserved_name(table_name)) {
4462  THROW_MAPD_EXCEPTION("Invalid table name (reserved keyword): " + table_name);
4463  } else if (table_name != ImportHelpers::sanitize_name(table_name)) {
4464  THROW_MAPD_EXCEPTION("Invalid characters in table name: " + table_name);
4465  }
4466 
4467  auto rds = rd;
4468 
4469  // no longer need to manually add the poly column for a TFileType::POLYGON table
4470  // a column of the correct geo type has already been added
4471  // @TODO simon.eves rename TFileType::POLYGON to TFileType::GEO or something!
4472 
4473  std::string stmt{"CREATE TABLE " + table_name};
4474  std::vector<std::string> col_stmts;
4475 
4476  for (auto col : rds) {
4477  if (ImportHelpers::is_reserved_name(col.col_name)) {
4478  THROW_MAPD_EXCEPTION("Invalid column name (reserved keyword): " + col.col_name);
4479  } else if (col.col_name != ImportHelpers::sanitize_name(col.col_name)) {
4480  THROW_MAPD_EXCEPTION("Invalid characters in column name: " + col.col_name);
4481  }
4482  if (col.col_type.type == TDatumType::INTERVAL_DAY_TIME ||
4483  col.col_type.type == TDatumType::INTERVAL_YEAR_MONTH) {
4484  THROW_MAPD_EXCEPTION("Unsupported type: " + thrift_to_name(col.col_type) +
4485  " for column: " + col.col_name);
4486  }
4487 
4488  if (col.col_type.type == TDatumType::DECIMAL) {
4489  // if no precision or scale passed in set to default 14,7
4490  if (col.col_type.precision == 0 && col.col_type.scale == 0) {
4491  col.col_type.precision = 14;
4492  col.col_type.scale = 7;
4493  }
4494  }
4495 
4496  std::string col_stmt;
4497  col_stmt.append(col.col_name + " " + thrift_to_name(col.col_type));
4498 
4499  // As of 2016-06-27 the Immerse v1 frontend does not explicitly set the
4500  // `nullable` argument, leading this to default to false. Uncomment for v2.
4501  // if (!col.col_type.nullable) col_stmt.append(" NOT NULL");
4502 
4503  if (thrift_to_encoding(col.col_type.encoding) != kENCODING_NONE) {
4504  col_stmt.append(" ENCODING " + thrift_to_encoding_name(col.col_type));
4505  if (thrift_to_encoding(col.col_type.encoding) == kENCODING_DICT ||
4506  thrift_to_encoding(col.col_type.encoding) == kENCODING_FIXED ||
4507  thrift_to_encoding(col.col_type.encoding) == kENCODING_GEOINT) {
4508  col_stmt.append("(" + std::to_string(col.col_type.comp_param) + ")");
4509  }
4510  } else if (col.col_type.type == TDatumType::STR) {
4511  // non DICT encoded strings
4512  col_stmt.append(" ENCODING NONE");
4513  } else if (col.col_type.type == TDatumType::POINT ||
4514  col.col_type.type == TDatumType::LINESTRING ||
4515  col.col_type.type == TDatumType::POLYGON ||
4516  col.col_type.type == TDatumType::MULTIPOLYGON) {
4517  // non encoded compressable geo
4518  if (col.col_type.scale == 4326) {
4519  col_stmt.append(" ENCODING NONE");
4520  }
4521  }
4522  col_stmts.push_back(col_stmt);
4523  }
4524 
4525  stmt.append(" (" + boost::algorithm::join(col_stmts, ", ") + ")");
4526 
4527  if (create_params.is_replicated) {
4528  stmt.append(" WITH (PARTITIONS = 'REPLICATED')");
4529  }
4530 
4531  stmt.append(";");
4532 
4533  TQueryResult ret;
4534  sql_execute(ret, session, stmt, true, "", -1, -1);
4535 }
#define LINESTRING
std::string toString(const ExtArgumentType &sig_type)
EncodingType thrift_to_encoding(const TEncodingType::type tEncodingType)
std::string join(T const &container, std::string const &delim)
std::string to_string(char const *&&v)
#define MULTIPOLYGON
#define POINT
bool is_reserved_name(const std::string &name)
void sql_execute(ExecutionResult &_return, const TSessionId &session, const std::string &query, const bool column_format, const int32_t first_n, const int32_t at_most_n)
Definition: DBHandler.cpp:1274
std::string sanitize_name(const std::string &name)
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
std::string thrift_to_encoding_name(const TTypeInfo &ti)
std::string thrift_to_name(const TTypeInfo &ti)
void check_read_only(const std::string &str)
Definition: DBHandler.cpp:488
#define DECIMAL
#define POLYGON
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1695
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string const DBHandler::createInMemoryCalciteSession ( const std::shared_ptr< Catalog_Namespace::Catalog > &  catalog_ptr)
private

Definition at line 494 of file DBHandler.cpp.

References calcite_, CHECK, executor_device_type_, generate_random_string(), sessions_, and sessions_mutex_.

Referenced by parse_to_ra().

495  {
496  // We would create an in memory session for calcite with super user privileges which
497  // would be used for getting all tables metadata when a user runs the query. The
498  // session would be under the name of a proxy user/password which would only persist
499  // till server's lifetime or execution of calcite query(in memory) whichever is the
500  // earliest.
501  mapd_lock_guard<mapd_shared_mutex> write_lock(sessions_mutex_);
502  std::string session_id;
503  do {
504  session_id = generate_random_string(64);
505  } while (sessions_.find(session_id) != sessions_.end());
506  Catalog_Namespace::UserMetadata user_meta(-1,
507  calcite_->getInternalSessionProxyUserName(),
508  calcite_->getInternalSessionProxyPassword(),
509  true,
510  -1,
511  true);
512  const auto emplace_ret =
513  sessions_.emplace(session_id,
514  std::make_shared<Catalog_Namespace::SessionInfo>(
515  catalog_ptr, user_meta, executor_device_type_, session_id));
516  CHECK(emplace_ret.second);
517  return session_id;
518 }
mapd_shared_mutex sessions_mutex_
Definition: DBHandler.h:580
std::string generate_random_string(const size_t len)
ExecutorDeviceType executor_device_type_
Definition: DBHandler.h:571
std::shared_ptr< Calcite > calcite_
Definition: DBHandler.h:588
#define CHECK(condition)
Definition: Logger.h:203
mapd_unique_lock< mapd_shared_mutex > write_lock
SessionMap sessions_
Definition: DBHandler.h:866

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::createSimpleResult ( TQueryResult &  _return,
const ResultSet results,
const bool  column_format,
const std::string  label 
)
staticprivate

Definition at line 5721 of file DBHandler.cpp.

References CHECK, CHECK_EQ, and run_benchmark_import::label.

Referenced by convertExplain(), and convertResult().

5724  {
5725  CHECK_EQ(size_t(1), results.rowCount());
5726  TColumnType proj_info;
5727  proj_info.col_name = label;
5728  proj_info.col_type.type = TDatumType::STR;
5729  proj_info.col_type.nullable = false;
5730  proj_info.col_type.is_array = false;
5731  _return.row_set.row_desc.push_back(proj_info);
5732  const auto crt_row = results.getNextRow(true, true);
5733  const auto tv = crt_row[0];
5734  CHECK(results.getNextRow(true, true).empty());
5735  const auto scalar_tv = boost::get<ScalarTargetValue>(&tv);
5736  CHECK(scalar_tv);
5737  const auto s_n = boost::get<NullableString>(scalar_tv);
5738  CHECK(s_n);
5739  const auto s = boost::get<std::string>(s_n);
5740  CHECK(s);
5741  if (column_format) {
5742  TColumn tcol;
5743  tcol.data.str_col.push_back(*s);
5744  tcol.nulls.push_back(false);
5745  _return.row_set.is_columnar = true;
5746  _return.row_set.columns.push_back(tcol);
5747  } else {
5748  TDatum explanation;
5749  explanation.val.str_val = *s;
5750  explanation.is_null = false;
5751  TRow trow;
5752  trow.cols.push_back(explanation);
5753  _return.row_set.is_columnar = false;
5754  _return.row_set.rows.push_back(trow);
5755  }
5756 }
#define CHECK_EQ(x, y)
Definition: Logger.h:211
#define CHECK(condition)
Definition: Logger.h:203

+ Here is the caller graph for this function:

void DBHandler::deallocate_df ( const TSessionId &  session,
const TDataFrame &  df,
const TDeviceType::type  device_type,
const int32_t  device_id 
)
override

Definition at line 1453 of file DBHandler.cpp.

References CPU, data_mgr_, ArrowResultSet::deallocateArrowResultBuffer(), logger::ERROR, get_session_ptr(), GPU, handle_to_dev_ptr_mutex_, ipc_handle_to_dev_ptr_, LOG, run_benchmark_import::result, and STDLOG.

1456  {
1457  auto stdlog = STDLOG(get_session_ptr(session));
1458  std::string serialized_cuda_handle = "";
1459  if (device_type == TDeviceType::GPU) {
1460  std::lock_guard<std::mutex> map_lock(handle_to_dev_ptr_mutex_);
1461  if (ipc_handle_to_dev_ptr_.count(df.df_handle) != size_t(1)) {
1462  TOmniSciException ex;
1463  ex.error_msg = std::string(
1464  "Exception: current data frame handle is not bookkept or been inserted "
1465  "twice");
1466  LOG(ERROR) << ex.error_msg;
1467  throw ex;
1468  }
1469  serialized_cuda_handle = ipc_handle_to_dev_ptr_[df.df_handle];
1470  ipc_handle_to_dev_ptr_.erase(df.df_handle);
1471  }
1472  std::vector<char> sm_handle(df.sm_handle.begin(), df.sm_handle.end());
1473  std::vector<char> df_handle(df.df_handle.begin(), df.df_handle.end());
1475  sm_handle, df.sm_size, df_handle, df.df_size, serialized_cuda_handle};
1477  result,
1478  device_type == TDeviceType::CPU ? ExecutorDeviceType::CPU : ExecutorDeviceType::GPU,
1479  device_id,
1480  data_mgr_);
1481 }
std::mutex handle_to_dev_ptr_mutex_
Definition: DBHandler.h:927
#define LOG(tag)
Definition: Logger.h:194
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
static void deallocateArrowResultBuffer(const ArrowResult &result, const ExecutorDeviceType device_type, const size_t device_id, std::shared_ptr< Data_Namespace::DataMgr > &data_mgr)
std::shared_ptr< Data_Namespace::DataMgr > data_mgr_
Definition: DBHandler.h:564
std::unordered_map< std::string, std::string > ipc_handle_to_dev_ptr_
Definition: DBHandler.h:928
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

void DBHandler::delete_custom_expressions ( const TSessionId &  session,
const std::vector< int32_t > &  custom_expression_ids,
const bool  do_soft_delete 
)
override

Definition at line 3999 of file DBHandler.cpp.

References check_read_only(), custom_expressions_mutex_, get_session_ptr(), getConnectionInfo(), STDLOG, THROW_MAPD_EXCEPTION, and toString().

4002  {
4003  auto stdlog = STDLOG(get_session_ptr(session));
4004  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4005  check_read_only("delete_custom_expressions");
4006 
4007  auto session_ptr = stdlog.getConstSessionInfo();
4008  if (!session_ptr->get_currentUser().isSuper) {
4009  THROW_MAPD_EXCEPTION("Custom expressions can only be deleted by super users.")
4010  }
4011  auto& catalog = session_ptr->getCatalog();
4012  mapd_unique_lock<mapd_shared_mutex> write_lock(custom_expressions_mutex_);
4013  catalog.deleteCustomExpressions(custom_expression_ids, do_soft_delete);
4014 }
std::string toString(const ExtArgumentType &sig_type)
mapd_shared_mutex custom_expressions_mutex_
Definition: DBHandler.h:1009
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
void check_read_only(const std::string &str)
Definition: DBHandler.cpp:488
mapd_unique_lock< mapd_shared_mutex > write_lock
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1695
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

void DBHandler::delete_dashboard ( const TSessionId &  session,
const int32_t  dashboard_id 
)
override

Definition at line 4200 of file DBHandler.cpp.

References delete_dashboards().

4200  {
4201  delete_dashboards(session, {dashboard_id});
4202 }
void delete_dashboards(const TSessionId &session, const std::vector< int32_t > &dashboard_ids) override
Definition: DBHandler.cpp:4204

+ Here is the call graph for this function:

void DBHandler::delete_dashboards ( const TSessionId &  session,
const std::vector< int32_t > &  dashboard_ids 
)
override

Definition at line 4204 of file DBHandler.cpp.

References cat(), check_read_only(), get_session_ptr(), getConnectionInfo(), STDLOG, THROW_MAPD_EXCEPTION, and toString().

Referenced by delete_dashboard().

4205  {
4206  auto stdlog = STDLOG(get_session_ptr(session));
4207  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4208  auto session_ptr = stdlog.getConstSessionInfo();
4209  check_read_only("delete_dashboards");
4210  auto& cat = session_ptr->getCatalog();
4211  // Checks will be performed in catalog
4212  try {
4213  cat.deleteMetadataForDashboards(dashboard_ids, session_ptr->get_currentUser());
4214  } catch (const std::exception& e) {
4215  THROW_MAPD_EXCEPTION(std::string("Exception: ") + e.what());
4216  }
4217 }
std::string cat(Ts &&...args)
std::string toString(const ExtArgumentType &sig_type)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
void check_read_only(const std::string &str)
Definition: DBHandler.cpp:488
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1695
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::detect_column_types ( TDetectResult &  _return,
const TSessionId &  session,
const std::string &  file_name,
const TCopyParams &  copy_params 
)
override

Definition at line 3712 of file DBHandler.cpp.

References anonymous_namespace{DBHandler.cpp}::add_vsi_archive_prefix(), anonymous_namespace{DBHandler.cpp}::add_vsi_geo_prefix(), anonymous_namespace{DBHandler.cpp}::add_vsi_network_prefix(), import_export::Detector::best_encodings, import_export::Detector::best_sqltypes, check_geospatial_files(), check_read_only(), copyparams_to_thrift(), import_export::DELIMITED, encoding_to_thrift(), import_export::CopyParams::file_type, anonymous_namespace{DBHandler.cpp}::find_first_geo_file_in_archive(), import_export::Importer::gdalFileExists(), import_export::Importer::gdalFileOrDirectoryExists(), import_export::Importer::gdalToColumnDescriptors(), import_export::CopyParams::geo_coords_comp_param, import_export::CopyParams::geo_coords_encoding, import_export::CopyParams::geo_coords_srid, import_export::CopyParams::geo_coords_type, import_export::DataStreamSink::get_copy_params(), import_export::Detector::get_headers(), import_export::Detector::get_sample_rows(), get_session_ptr(), getConnectionInfo(), i, import_path_, anonymous_namespace{DBHandler.cpp}::is_a_supported_archive_file(), anonymous_namespace{DBHandler.cpp}::is_a_supported_geo_file(), IS_GEO, ImportHelpers::is_reserved_name(), OMNISCI_GEO_PREFIX, anonymous_namespace{DBHandler.cpp}::path_is_relative(), import_export::POLYGON, populateThriftColumnType(), import_export::Importer::readMetadataSampleGDAL(), import_export::CopyParams::sanitize_column_names, ImportHelpers::sanitize_name(), SQLTypeInfo::set_compression(), STDLOG, t, thrift_to_copyparams(), THROW_MAPD_EXCEPTION, toString(), type_to_thrift(), and anonymous_namespace{DBHandler.cpp}::validate_import_file_path_if_local().

Referenced by import_geo_table().

3715  {
3716  auto stdlog = STDLOG(get_session_ptr(session));
3717  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
3718  check_read_only("detect_column_types");
3719 
3721 
3722  std::string file_name{file_name_in};
3723  if (path_is_relative(file_name)) {
3724  // assume relative paths are relative to data_path / mapd_import / <session>
3725  auto file_path = import_path_ / picosha2::hash256_hex_string(session) /
3726  boost::filesystem::path(file_name).filename();
3727  file_name = file_path.string();
3728  }
3730 
3731  // if it's a geo table, handle alternative paths (S3, HTTP, archive etc.)
3732  if (copy_params.file_type == import_export::FileType::POLYGON) {
3733  if (is_a_supported_geo_file(file_name, true)) {
3734  // prepare to detect geo file directly
3735  add_vsi_network_prefix(file_name);
3736  add_vsi_geo_prefix(file_name);
3737  } else if (is_a_supported_archive_file(file_name)) {
3738  // find the archive file
3739  add_vsi_network_prefix(file_name);
3740  if (!import_export::Importer::gdalFileExists(file_name, copy_params)) {
3741  THROW_MAPD_EXCEPTION("Archive does not exist: " + file_name_in);
3742  }
3743  // find geo file in archive
3744  add_vsi_archive_prefix(file_name);
3745  std::string geo_file = find_first_geo_file_in_archive(file_name, copy_params);
3746  // prepare to detect that geo file
3747  if (geo_file.size()) {
3748  file_name = file_name + std::string("/") + geo_file;
3749  }
3750  } else {
3751  THROW_MAPD_EXCEPTION("File is not a supported geo or geo archive format: " +
3752  file_name_in);
3753  }
3754  }
3755 
3756  auto file_path = boost::filesystem::path(file_name);
3757  // can be a s3 url
3758  if (!boost::istarts_with(file_name, "s3://")) {
3759  if (!boost::filesystem::path(file_name).is_absolute()) {
3760  file_path = import_path_ / picosha2::hash256_hex_string(session) /
3761  boost::filesystem::path(file_name).filename();
3762  file_name = file_path.string();
3763  }
3764 
3765  if (copy_params.file_type == import_export::FileType::POLYGON) {
3766  // check for geo file
3767  if (!import_export::Importer::gdalFileOrDirectoryExists(file_name, copy_params)) {
3768  THROW_MAPD_EXCEPTION("File does not exist: " + file_path.string());
3769  }
3770  } else {
3771  // check for regular file
3772  if (!boost::filesystem::exists(file_path)) {
3773  THROW_MAPD_EXCEPTION("File does not exist: " + file_path.string());
3774  }
3775  }
3776  }
3777 
3778  try {
3780 #ifdef ENABLE_IMPORT_PARQUET
3781  || (copy_params.file_type == import_export::FileType::PARQUET)
3782 #endif
3783  ) {
3784  import_export::Detector detector(file_path, copy_params);
3785  std::vector<SQLTypes> best_types = detector.best_sqltypes;
3786  std::vector<EncodingType> best_encodings = detector.best_encodings;
3787  std::vector<std::string> headers = detector.get_headers();
3788  copy_params = detector.get_copy_params();
3789 
3790  _return.copy_params = copyparams_to_thrift(copy_params);
3791  _return.row_set.row_desc.resize(best_types.size());
3792  for (size_t col_idx = 0; col_idx < best_types.size(); col_idx++) {
3793  TColumnType col;
3794  SQLTypes t = best_types[col_idx];
3795  EncodingType encodingType = best_encodings[col_idx];
3796  SQLTypeInfo ti(t, false, encodingType);
3797  if (IS_GEO(t)) {
3798  // set this so encoding_to_thrift does the right thing
3799  ti.set_compression(copy_params.geo_coords_encoding);
3800  // fill in these directly
3801  col.col_type.precision = static_cast<int>(copy_params.geo_coords_type);
3802  col.col_type.scale = copy_params.geo_coords_srid;
3803  col.col_type.comp_param = copy_params.geo_coords_comp_param;
3804  }
3805  col.col_type.type = type_to_thrift(ti);
3806  col.col_type.encoding = encoding_to_thrift(ti);
3807  if (copy_params.sanitize_column_names) {
3808  col.col_name = ImportHelpers::sanitize_name(headers[col_idx]);
3809  } else {
3810  col.col_name = headers[col_idx];
3811  }
3812  col.is_reserved_keyword = ImportHelpers::is_reserved_name(col.col_name);
3813  _return.row_set.row_desc[col_idx] = col;
3814  }
3815  size_t num_samples = 100;
3816  auto sample_data = detector.get_sample_rows(num_samples);
3817 
3818  TRow sample_row;
3819  for (auto row : sample_data) {
3820  sample_row.cols.clear();
3821  for (const auto& s : row) {
3822  TDatum td;
3823  td.val.str_val = s;
3824  td.is_null = s.empty();
3825  sample_row.cols.push_back(td);
3826  }
3827  _return.row_set.rows.push_back(sample_row);
3828  }
3829  } else if (copy_params.file_type == import_export::FileType::POLYGON) {
3830  // @TODO simon.eves get this from somewhere!
3831  const std::string geoColumnName(OMNISCI_GEO_PREFIX);
3832 
3833  check_geospatial_files(file_path, copy_params);
3834  std::list<ColumnDescriptor> cds = import_export::Importer::gdalToColumnDescriptors(
3835  file_path.string(), geoColumnName, copy_params);
3836  for (auto cd : cds) {
3837  if (copy_params.sanitize_column_names) {
3838  cd.columnName = ImportHelpers::sanitize_name(cd.columnName);
3839  }
3840  _return.row_set.row_desc.push_back(populateThriftColumnType(nullptr, &cd));
3841  }
3842  std::map<std::string, std::vector<std::string>> sample_data;
3844  file_path.string(), geoColumnName, sample_data, 100, copy_params);
3845  if (sample_data.size() > 0) {
3846  for (size_t i = 0; i < sample_data.begin()->second.size(); i++) {
3847  TRow sample_row;
3848  for (auto cd : cds) {
3849  TDatum td;
3850  td.val.str_val = sample_data[cd.sourceName].at(i);
3851  td.is_null = td.val.str_val.empty();
3852  sample_row.cols.push_back(td);
3853  }
3854  _return.row_set.rows.push_back(sample_row);
3855  }
3856  }
3857  _return.copy_params = copyparams_to_thrift(copy_params);
3858  }
3859  } catch (const std::exception& e) {
3860  THROW_MAPD_EXCEPTION("detect_column_types error: " + std::string(e.what()));
3861  }
3862 }
boost::filesystem::path import_path_
Definition: DBHandler.h:570
void add_vsi_archive_prefix(std::string &path)
Definition: DBHandler.cpp:3583
bool is_a_supported_archive_file(const std::string &path)
Definition: DBHandler.cpp:3651
std::string toString(const ExtArgumentType &sig_type)
TDatumType::type type_to_thrift(const SQLTypeInfo &type_info)
SQLTypes
Definition: sqltypes.h:37
bool path_is_relative(const std::string &path)
Definition: DBHandler.cpp:3617
TCopyParams copyparams_to_thrift(const import_export::CopyParams &cp)
Definition: DBHandler.cpp:3486
static bool gdalFileExists(const std::string &path, const CopyParams &copy_params)
Definition: Importer.cpp:4737
EncodingType
Definition: sqltypes.h:227
bool is_reserved_name(const std::string &name)
TColumnType populateThriftColumnType(const Catalog_Namespace::Catalog *cat, const ColumnDescriptor *cd)
Definition: DBHandler.cpp:2201
void check_geospatial_files(const boost::filesystem::path file_path, const import_export::CopyParams &copy_params)
Definition: DBHandler.cpp:4431
static const std::list< ColumnDescriptor > gdalToColumnDescriptors(const std::string &fileName, const std::string &geoColumnName, const CopyParams &copy_params)
Definition: Importer.cpp:4614
static void readMetadataSampleGDAL(const std::string &fileName, const std::string &geoColumnName, std::map< std::string, std::vector< std::string >> &metadata, int rowLimit, const CopyParams &copy_params)
Definition: Importer.cpp:4480
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
void validate_import_file_path_if_local(const std::string &file_path)
Definition: DBHandler.cpp:3705
std::string sanitize_name(const std::string &name)
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
const std::string OMNISCI_GEO_PREFIX
Definition: Transforms.h:22
static bool gdalFileOrDirectoryExists(const std::string &path, const CopyParams &copy_params)
Definition: Importer.cpp:4742
void check_read_only(const std::string &str)
Definition: DBHandler.cpp:488
std::string find_first_geo_file_in_archive(const std::string &archive_path, const import_export::CopyParams &copy_params)
Definition: DBHandler.cpp:3664
void add_vsi_network_prefix(std::string &path)
Definition: DBHandler.cpp:3554
char * t
import_export::CopyParams thrift_to_copyparams(const TCopyParams &cp)
Definition: DBHandler.cpp:3366
#define IS_GEO(T)
Definition: sqltypes.h:245
void add_vsi_geo_prefix(std::string &path)
Definition: DBHandler.cpp:3576
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1695
#define STDLOG(...)
Definition: QueryState.h:229
TEncodingType::type encoding_to_thrift(const SQLTypeInfo &type_info)
EncodingType geo_coords_encoding
Definition: CopyParams.h:74
bool is_a_supported_geo_file(const std::string &path, bool include_gz)
Definition: DBHandler.cpp:3633

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::disconnect ( const TSessionId &  session)
override

Definition at line 671 of file DBHandler.cpp.

References disconnect_impl(), get_session_it_unsafe(), getConnectionInfo(), logger::INFO, LOG, sessions_mutex_, STDLOG, and toString().

671  {
672  auto stdlog = STDLOG();
673  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
674 
675  mapd_unique_lock<mapd_shared_mutex> write_lock(sessions_mutex_);
676  auto session_it = get_session_it_unsafe(session, write_lock);
677  stdlog.setSessionInfo(session_it->second);
678  const auto dbname = session_it->second->getCatalog().getCurrentDB().dbName;
679 
680  LOG(INFO) << "User " << session_it->second->get_currentUser().userLoggable()
681  << " disconnected from database " << dbname
682  << " with public_session_id: " << session_it->second->get_public_session_id();
683 
684  disconnect_impl(session_it, write_lock);
685 }
std::string toString(const ExtArgumentType &sig_type)
#define LOG(tag)
Definition: Logger.h:194
mapd_shared_mutex sessions_mutex_
Definition: DBHandler.h:580
SessionMap::iterator get_session_it_unsafe(const TSessionId &session, SESSION_MAP_LOCK &lock)
void disconnect_impl(const SessionMap::iterator &session_it, mapd_unique_lock< mapd_shared_mutex > &write_lock)
Definition: DBHandler.cpp:687
mapd_unique_lock< mapd_shared_mutex > write_lock
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1695
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

void DBHandler::disconnect_impl ( const SessionMap::iterator &  session_it,
mapd_unique_lock< mapd_shared_mutex > &  write_lock 
)
private

Definition at line 687 of file DBHandler.cpp.

References LeafAggregator::disconnect(), leaf_aggregator_, LeafAggregator::leafCount(), render_group_assignment_map_, render_group_assignment_mutex_, render_handler_, and sessions_.

Referenced by disconnect(), and get_session_it_unsafe().

688  {
689  // session_it existence should already have been checked (i.e. called via
690  // get_session_it_unsafe(...))
691 
692  const auto session_id = session_it->second->get_session_id();
693  std::exception_ptr leaf_exception = nullptr;
694  try {
695  if (leaf_aggregator_.leafCount() > 0) {
696  leaf_aggregator_.disconnect(session_id);
697  }
698  } catch (...) {
699  leaf_exception = std::current_exception();
700  }
701 
702  {
703  std::lock_guard<std::mutex> lock(render_group_assignment_mutex_);
704  render_group_assignment_map_.erase(session_id);
705  }
706 
707  sessions_.erase(session_it);
708  write_lock.unlock();
709 
710  if (render_handler_) {
711  render_handler_->disconnect(session_id);
712  }
713 }
std::mutex render_group_assignment_mutex_
Definition: DBHandler.h:1008
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:566
void disconnect(const TSessionId session)
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:585
size_t leafCount() const
RenderGroupAnalyzerSessionMap render_group_assignment_map_
Definition: DBHandler.h:1007
mapd_unique_lock< mapd_shared_mutex > write_lock
SessionMap sessions_
Definition: DBHandler.h:866

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::emergency_shutdown ( )

Definition at line 6678 of file DBHandler.cpp.

References calcite_.

Referenced by shutdown().

6678  {
6679  if (calcite_) {
6680  calcite_->close_calcite_server(false);
6681  }
6682 }
std::shared_ptr< Calcite > calcite_
Definition: DBHandler.h:588

+ Here is the caller graph for this function:

void DBHandler::execute_distributed_copy_statement ( Parser::CopyTableStmt copy_stmt,
const Catalog_Namespace::SessionInfo session_info 
)
private

Definition at line 6158 of file DBHandler.cpp.

References Parser::CopyTableStmt::execute(), and leaf_aggregator_.

Referenced by sql_execute_impl().

6160  {
6161  auto importer_factory = [&session_info, this](
6162  const Catalog& catalog,
6163  const TableDescriptor* td,
6164  const std::string& file_path,
6165  const import_export::CopyParams& copy_params) {
6166  return std::make_unique<import_export::Importer>(
6167  new DistributedLoader(session_info, td, &leaf_aggregator_),
6168  file_path,
6169  copy_params);
6170  };
6171  copy_stmt->execute(session_info, importer_factory);
6172 }
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:102
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:566
void execute(const Catalog_Namespace::SessionInfo &session) override

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::execute_next_render_step ( TRenderStepResult &  _return,
const TPendingRenderQuery &  pending_render,
const TRenderAggDataMap &  merged_data 
)
override

Definition at line 6486 of file DBHandler.cpp.

References measure< TimeT >::execution(), logger::INFO, LOG, render_handler_, and THROW_MAPD_EXCEPTION.

6488  {
6489  if (!render_handler_) {
6490  THROW_MAPD_EXCEPTION("Backend rendering is disabled.");
6491  }
6492 
6493  LOG(INFO) << "execute_next_render_step: id:" << pending_render.id;
6494  auto time_ms = measure<>::execution([&]() {
6495  try {
6496  render_handler_->execute_next_render_step(_return, pending_render, merged_data);
6497  } catch (std::exception& e) {
6498  THROW_MAPD_EXCEPTION(std::string("Exception: ") + e.what());
6499  }
6500  });
6501  LOG(INFO) << "execute_next_render_step-COMPLETED id: " << pending_render.id
6502  << ", time: " << time_ms << "ms ";
6503 }
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:194
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:585
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110

+ Here is the call graph for this function:

void DBHandler::execute_query_step ( TStepResult &  _return,
const TPendingQuery &  pending_query,
const TSubqueryId  subquery_id,
const std::string &  start_time_str 
)
override

Definition at line 6328 of file DBHandler.cpp.

References measure< TimeT >::execution(), logger::INFO, leaf_handler_, LOG, and THROW_MAPD_EXCEPTION.

6331  {
6332  if (!leaf_handler_) {
6333  THROW_MAPD_EXCEPTION("Distributed support is disabled.");
6334  }
6335  LOG(INFO) << "execute_query_step : id:" << pending_query.id;
6336  auto time_ms = measure<>::execution([&]() {
6337  try {
6338  leaf_handler_->execute_query_step(
6339  _return, pending_query, subquery_id, start_time_str);
6340  } catch (std::exception& e) {
6341  THROW_MAPD_EXCEPTION(std::string("Exception: ") + e.what());
6342  }
6343  });
6344  LOG(INFO) << "execute_query_step-COMPLETED " << time_ms << "ms";
6345 }
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:194
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
std::unique_ptr< MapDLeafHandler > leaf_handler_
Definition: DBHandler.h:587

+ Here is the call graph for this function:

std::vector< PushedDownFilterInfo > DBHandler::execute_rel_alg ( ExecutionResult _return,
QueryStateProxy  query_state_proxy,
const std::string &  query_ra,
const bool  column_format,
const ExecutorDeviceType  executor_device_type,
const int32_t  first_n,
const int32_t  at_most_n,
const bool  just_validate,
const bool  find_push_down_candidates,
const ExplainInfo explain_info,
const std::optional< size_t >  executor_index = std::nullopt 
) const
private

Definition at line 5448 of file DBHandler.cpp.

References allow_loop_joins_, allow_multifrag_, cat(), CPU, query_state::QueryStateProxy::createTimer(), Default, measure< TimeT >::execution(), ExplainInfo::explain_optimized, ExplainInfo::explain_plan, ExecutionResult::Explaination, g_dynamic_watchdog_time_limit, g_enable_columnar_output, g_enable_dynamic_watchdog, g_enable_runtime_query_interrupt, g_enable_watchdog, g_pending_query_interrupt_freq, g_running_query_interrupt_freq, query_state::QueryState::getConstSessionInfo(), ExecutionResult::getExecutionTime(), Executor::getExecutor(), RelAlgExecutor::getParsedQueryHints(), ExecutionResult::getPushedDownFilterInfo(), query_state::QueryStateProxy::getQueryState(), ExecutionResult::getRows(), SystemParameters::gpu_input_mem_limit, lockmgr::TableSchemaLockMgr::instance(), lockmgr::TableDataLockMgr::instance(), intel_jit_profile_, RegisteredQueryHint::isHintRegistered(), jit_debug_, ExplainInfo::justCalciteExplain(), ExplainInfo::justExplain(), kCpuMode, Optimized, ExecutionResult::QueryResult, ExecutionResult::setExecutionTime(), ExecutionResult::setResultType(), system_parameters_, Executor::UNITARY_EXECUTOR_ID, and VLOG.

Referenced by execute_rel_alg_with_filter_push_down(), get_tables_meta_impl(), sql_execute_impl(), and validate_rel_alg().

5459  {
5460  query_state::Timer timer = query_state_proxy.createTimer(__func__);
5461 
5462  VLOG(1) << "Table Schema Locks:\n" << lockmgr::TableSchemaLockMgr::instance();
5463  VLOG(1) << "Table Data Locks:\n" << lockmgr::TableDataLockMgr::instance();
5464 
5465  const auto& cat = query_state_proxy.getQueryState().getConstSessionInfo()->getCatalog();
5466  auto executor = Executor::getExecutor(
5467  executor_index ? *executor_index : Executor::UNITARY_EXECUTOR_ID,
5468  jit_debug_ ? "/tmp" : "",
5469  jit_debug_ ? "mapdquery" : "",
5471  RelAlgExecutor ra_executor(executor.get(),
5472  cat,
5473  query_ra,
5474  query_state_proxy.getQueryState().shared_from_this());
5475  // handle hints
5476  const auto& query_hints = ra_executor.getParsedQueryHints();
5477  const bool cpu_mode_enabled = query_hints.isHintRegistered(QueryHint::kCpuMode);
5478  CompilationOptions co = {
5479  cpu_mode_enabled ? ExecutorDeviceType::CPU : executor_device_type,
5480  /*hoist_literals=*/true,
5483  /*allow_lazy_fetch=*/true,
5484  /*filter_on_deleted_column=*/true,
5488  auto validate_or_explain_query =
5489  explain_info.justExplain() || explain_info.justCalciteExplain() || just_validate;
5492  explain_info.justExplain(),
5493  allow_loop_joins_ || just_validate,
5495  jit_debug_,
5496  just_validate,
5499  find_push_down_candidates,
5500  explain_info.justCalciteExplain(),
5502  g_enable_runtime_query_interrupt && !validate_or_explain_query &&
5503  !query_state_proxy.getQueryState()
5505  ->get_session_id()
5506  .empty(),
5509  auto execution_time_ms = _return.getExecutionTime() + measure<>::execution([&]() {
5510  _return = ra_executor.executeRelAlgQuery(
5511  co, eo, explain_info.explain_plan, nullptr);
5512  });
5513  // reduce execution time by the time spent during queue waiting
5514  const auto rs = _return.getRows();
5515  if (rs) {
5516  execution_time_ms -= rs->getQueueTime();
5517  }
5518  _return.setExecutionTime(execution_time_ms);
5519  VLOG(1) << cat.getDataMgr().getSystemMemoryUsage();
5520  const auto& filter_push_down_info = _return.getPushedDownFilterInfo();
5521  if (!filter_push_down_info.empty()) {
5522  return filter_push_down_info;
5523  }
5524  if (explain_info.justExplain()) {
5526  } else if (!explain_info.justCalciteExplain()) {
5528  }
5529  return {};
5530 }
double g_running_query_interrupt_freq
Definition: Execute.cpp:118
static TableSchemaLockMgr & instance()
Definition: LockMgr.h:40
bool g_enable_watchdog
std::string cat(Ts &&...args)
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
unsigned g_pending_query_interrupt_freq
Definition: Execute.cpp:117
const bool jit_debug_
Definition: DBHandler.h:574
bool g_enable_dynamic_watchdog
Definition: Execute.cpp:77
Timer createTimer(char const *event_name)
Definition: QueryState.cpp:124
static TableDataLockMgr & instance()
Definition: LockMgr.h:76
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:163
bool g_enable_columnar_output
Definition: Execute.cpp:93
bool justExplain() const
Definition: ParserWrapper.h:39
bool explain_plan
Definition: ParserWrapper.h:34
QueryState & getQueryState()
Definition: QueryState.h:175
const std::shared_ptr< ResultSet > & getRows() const
SystemParameters & system_parameters_
Definition: DBHandler.h:584
void setResultType(RType type)
const bool allow_loop_joins_
Definition: DBHandler.h:578
const bool intel_jit_profile_
Definition: DBHandler.h:575
RegisteredQueryHint getParsedQueryHints()
const std::vector< PushedDownFilterInfo > & getPushedDownFilterInfo() const
const bool isHintRegistered(const QueryHint hint) const
Definition: QueryHint.h:203
bool explain_optimized
Definition: ParserWrapper.h:33
void setExecutionTime(int64_t execution_time_ms)
bool allow_multifrag_
Definition: DBHandler.h:576
unsigned g_dynamic_watchdog_time_limit
Definition: Execute.cpp:80
bool justCalciteExplain() const
Definition: ParserWrapper.h:41
std::shared_ptr< Catalog_Namespace::SessionInfo const > getConstSessionInfo() const
Definition: QueryState.cpp:79
int64_t getExecutionTime() const
bool g_enable_runtime_query_interrupt
Definition: Execute.cpp:114
#define VLOG(n)
Definition: Logger.h:297
static const ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:366

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::execute_rel_alg_df ( TDataFrame &  _return,
const std::string &  query_ra,
QueryStateProxy  query_state_proxy,
const Catalog_Namespace::SessionInfo session_info,
const ExecutorDeviceType  device_type,
const size_t  device_id,
const int32_t  first_n,
const TArrowTransport::type  transport_method 
) const
private

Definition at line 5532 of file DBHandler.cpp.

References allow_loop_joins_, allow_multifrag_, cat(), CHECK, CPU, data_mgr_, Default, measure< TimeT >::execution(), g_dynamic_watchdog_time_limit, g_enable_columnar_output, g_enable_dynamic_watchdog, g_enable_runtime_query_interrupt, g_enable_watchdog, g_pending_query_interrupt_freq, g_running_query_interrupt_freq, Catalog_Namespace::SessionInfo::get_executor_device_type(), Catalog_Namespace::SessionInfo::getCatalog(), query_state::QueryState::getConstSessionInfo(), Executor::getExecutor(), RelAlgExecutor::getParsedQueryHints(), query_state::QueryStateProxy::getQueryState(), getTargetNames(), GPU, SystemParameters::gpu_input_mem_limit, handle_to_dev_ptr_mutex_, intel_jit_profile_, ipc_handle_to_dev_ptr_, RegisteredQueryHint::isHintRegistered(), jit_debug_, kCpuMode, system_parameters_, and Executor::UNITARY_EXECUTOR_ID.

Referenced by sql_execute_df().

5539  {
5540  const auto& cat = session_info.getCatalog();
5541  CHECK(device_type == ExecutorDeviceType::CPU ||
5544  jit_debug_ ? "/tmp" : "",
5545  jit_debug_ ? "mapdquery" : "",
5547  RelAlgExecutor ra_executor(executor.get(),
5548  cat,
5549  query_ra,
5550  query_state_proxy.getQueryState().shared_from_this());
5551  const auto& query_hints = ra_executor.getParsedQueryHints();
5552  const bool cpu_mode_enabled = query_hints.isHintRegistered(QueryHint::kCpuMode);
5553  CompilationOptions co = {cpu_mode_enabled ? ExecutorDeviceType::CPU : device_type,
5554  /*hoist_literals=*/true,
5557  /*allow_lazy_fetch=*/true,
5558  /*filter_on_deleted_column=*/true,
5561  ExecutionOptions eo = {
5564  false,
5567  jit_debug_,
5568  false,
5571  false,
5572  false,
5574  g_enable_runtime_query_interrupt && !query_state_proxy.getQueryState()
5576  ->get_session_id()
5577  .empty(),
5580  ExecutionResult result{std::make_shared<ResultSet>(std::vector<TargetInfo>{},
5583  nullptr,
5584  nullptr,
5585  0,
5586  0),
5587  {}};
5588  _return.execution_time_ms += measure<>::execution(
5589  [&]() { result = ra_executor.executeRelAlgQuery(co, eo, false, nullptr); });
5590  _return.execution_time_ms -= result.getRows()->getQueueTime();
5591  const auto rs = result.getRows();
5592  const auto converter =
5593  std::make_unique<ArrowResultSetConverter>(rs,
5594  data_mgr_,
5595  device_type,
5596  device_id,
5597  getTargetNames(result.getTargetsMeta()),
5598  first_n,
5599  ArrowTransport(transport_method));
5600  ArrowResult arrow_result;
5601  _return.arrow_conversion_time_ms +=
5602  measure<>::execution([&] { arrow_result = converter->getArrowResult(); });
5603  _return.sm_handle =
5604  std::string(arrow_result.sm_handle.begin(), arrow_result.sm_handle.end());
5605  _return.sm_size = arrow_result.sm_size;
5606  _return.df_handle =
5607  std::string(arrow_result.df_handle.begin(), arrow_result.df_handle.end());
5608  _return.df_buffer =
5609  std::string(arrow_result.df_buffer.begin(), arrow_result.df_buffer.end());
5610  if (device_type == ExecutorDeviceType::GPU) {
5611  std::lock_guard<std::mutex> map_lock(handle_to_dev_ptr_mutex_);
5612  CHECK(!ipc_handle_to_dev_ptr_.count(_return.df_handle));
5613  ipc_handle_to_dev_ptr_.insert(
5614  std::make_pair(_return.df_handle, arrow_result.serialized_cuda_handle));
5615  }
5616  _return.df_size = arrow_result.df_size;
5617 }
double g_running_query_interrupt_freq
Definition: Execute.cpp:118
bool g_enable_watchdog
std::string cat(Ts &&...args)
std::mutex handle_to_dev_ptr_mutex_
Definition: DBHandler.h:927
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
unsigned g_pending_query_interrupt_freq
Definition: Execute.cpp:117
ArrowTransport
std::vector< std::string > getTargetNames(const std::vector< TargetMetaInfo > &targets) const
Definition: DBHandler.cpp:5641
const bool jit_debug_
Definition: DBHandler.h:574
bool g_enable_dynamic_watchdog
Definition: Execute.cpp:77
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:163
bool g_enable_columnar_output
Definition: Execute.cpp:93
QueryState & getQueryState()
Definition: QueryState.h:175
SystemParameters & system_parameters_
Definition: DBHandler.h:584
const bool allow_loop_joins_
Definition: DBHandler.h:578
const bool intel_jit_profile_
Definition: DBHandler.h:575
Catalog & getCatalog() const
Definition: SessionInfo.h:66
RegisteredQueryHint getParsedQueryHints()
std::shared_ptr< Data_Namespace::DataMgr > data_mgr_
Definition: DBHandler.h:564
const bool isHintRegistered(const QueryHint hint) const
Definition: QueryHint.h:203
#define CHECK(condition)
Definition: Logger.h:203
std::unordered_map< std::string, std::string > ipc_handle_to_dev_ptr_
Definition: DBHandler.h:928
bool allow_multifrag_
Definition: DBHandler.h:576
unsigned g_dynamic_watchdog_time_limit
Definition: Execute.cpp:80
std::shared_ptr< Catalog_Namespace::SessionInfo const > getConstSessionInfo() const
Definition: QueryState.cpp:79
const ExecutorDeviceType get_executor_device_type() const
Definition: SessionInfo.h:73
bool g_enable_runtime_query_interrupt
Definition: Execute.cpp:114
static const ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:366

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::execute_rel_alg_with_filter_push_down ( ExecutionResult _return,
QueryStateProxy  query_state_proxy,
std::string &  query_ra,
const bool  column_format,
const ExecutorDeviceType  executor_device_type,
const int32_t  first_n,
const int32_t  at_most_n,
const bool  just_explain,
const bool  just_calcite_explain,
const std::vector< PushedDownFilterInfo > &  filter_push_down_requests 
)
private

Definition at line 6107 of file DBHandler.cpp.

References ExecutionResult::addExecutionTime(), ExplainInfo::defaults(), execute_rel_alg(), ExecutionResult::Explaination, query_state::QueryStateProxy::getQueryState(), query_state::QueryState::getQueryStr(), parse_to_ra(), system_parameters_, and ExecutionResult::updateResultSet().

Referenced by sql_execute_impl().

6117  {
6118  // collecting the selected filters' info to be sent to Calcite:
6119  std::vector<TFilterPushDownInfo> filter_push_down_info;
6120  for (const auto& req : filter_push_down_requests) {
6121  TFilterPushDownInfo filter_push_down_info_for_request;
6122  filter_push_down_info_for_request.input_prev = req.input_prev;
6123  filter_push_down_info_for_request.input_start = req.input_start;
6124  filter_push_down_info_for_request.input_next = req.input_next;
6125  filter_push_down_info.push_back(filter_push_down_info_for_request);
6126  }
6127  // deriving the new relational algebra plan with respect to the pushed down filters
6128  _return.addExecutionTime(measure<>::execution([&]() {
6129  query_ra = parse_to_ra(query_state_proxy,
6130  query_state_proxy.getQueryState().getQueryStr(),
6131  filter_push_down_info,
6132  false,
6134  .first.plan_result;
6135  }));
6136 
6137  if (just_calcite_explain) {
6138  // return the new ra as the result
6140  return;
6141  }
6142 
6143  // execute the new relational algebra plan:
6144  auto explain_info = ExplainInfo::defaults();
6145  explain_info.explain = just_explain;
6146  execute_rel_alg(_return,
6147  query_state_proxy,
6148  query_ra,
6149  column_format,
6150  executor_device_type,
6151  first_n,
6152  at_most_n,
6153  /*just_validate=*/false,
6154  /*find_push_down_candidates=*/false,
6155  explain_info);
6156 }
static ExplainInfo defaults()
Definition: ParserWrapper.h:37
std::vector< PushedDownFilterInfo > execute_rel_alg(ExecutionResult &_return, QueryStateProxy, const std::string &query_ra, const bool column_format, const ExecutorDeviceType executor_device_type, const int32_t first_n, const int32_t at_most_n, const bool just_validate, const bool find_push_down_candidates, const ExplainInfo &explain_info, const std::optional< size_t > executor_index=std::nullopt) const
Definition: DBHandler.cpp:5448
std::string const & getQueryStr() const
Definition: QueryState.h:153
void addExecutionTime(int64_t execution_time_ms)
QueryState & getQueryState()
Definition: QueryState.h:175
void updateResultSet(const std::string &query_ra, RType type, bool success=true)
SystemParameters & system_parameters_
Definition: DBHandler.h:584
std::pair< TPlanResult, lockmgr::LockedTableDescriptors > parse_to_ra(QueryStateProxy, const std::string &query_str, const std::vector< TFilterPushDownInfo > &filter_push_down_info, const bool acquire_locks, const SystemParameters &system_parameters, bool check_privileges=true)
Definition: DBHandler.cpp:6174

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::executeDdl ( TQueryResult &  _return,
const std::string &  query_ra,
std::shared_ptr< Catalog_Namespace::SessionInfo const session_ptr 
)
private

Definition at line 6991 of file DBHandler.cpp.

References DdlCommandExecutor::commandStr(), convertResultSet(), ExecutionResult::empty(), DdlCommandExecutor::execute(), measure< TimeT >::execution(), getQueries(), ExecutionResult::getRows(), DdlCommandExecutor::getTargetQuerySessionToKill(), getUserSessions(), interruptQuery(), DdlCommandExecutor::isKillQuery(), DdlCommandExecutor::isShowQueries(), DdlCommandExecutor::isShowUserSessions(), and run_benchmark_import::result.

Referenced by sql_execute_impl().

6994  {
6995  DdlCommandExecutor executor = DdlCommandExecutor(query_ra, session_ptr);
6996  std::string commandStr = executor.commandStr();
6997 
6998  if (executor.isKillQuery()) {
6999  interruptQuery(*session_ptr, executor.getTargetQuerySessionToKill());
7000  } else {
7002 
7003  if (executor.isShowQueries()) {
7004  // getQueries still requires Thrift cannot be nested into DdlCommandExecutor
7005  _return.execution_time_ms +=
7006  measure<>::execution([&]() { result = getQueries(session_ptr); });
7007  } else if (executor.isShowUserSessions()) {
7008  // getUserSessions still requires Thrift cannot be nested into DdlCommandExecutor
7009  _return.execution_time_ms +=
7010  measure<>::execution([&]() { result = getUserSessions(session_ptr); });
7011  } else {
7012  _return.execution_time_ms +=
7013  measure<>::execution([&]() { result = executor.execute(); });
7014  }
7015 
7016  if (!result.empty()) {
7017  // reduce execution time by the time spent during queue waiting
7018  _return.execution_time_ms -= result.getRows()->getQueueTime();
7019 
7020  convertResultSet(result, *session_ptr, commandStr, _return);
7021  }
7022  }
7023 }
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
void convertResultSet(ExecutionResult &result, const Catalog_Namespace::SessionInfo &session_info, const std::string &query_state_str, TQueryResult &_return)
Definition: DBHandler.cpp:6775
const std::string getTargetQuerySessionToKill()
const std::string commandStr()
const std::shared_ptr< ResultSet > & getRows() const
ExecutionResult execute()
void interruptQuery(const Catalog_Namespace::SessionInfo &session_info, const std::string &target_session)
Definition: DBHandler.cpp:6924
ExecutionResult getQueries(std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
Definition: DBHandler.cpp:6848
ExecutionResult getUserSessions(std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
Definition: DBHandler.cpp:6804

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::executeDdl ( ExecutionResult _return,
const std::string &  query_ra,
std::shared_ptr< Catalog_Namespace::SessionInfo const session_ptr 
)
private

Definition at line 7025 of file DBHandler.cpp.

References ExecutionResult::CalciteDdl, DdlCommandExecutor::commandStr(), DdlCommandExecutor::execute(), measure< TimeT >::execution(), getQueries(), DdlCommandExecutor::getTargetQuerySessionToKill(), getUserSessions(), interruptQuery(), DdlCommandExecutor::isKillQuery(), DdlCommandExecutor::isShowQueries(), DdlCommandExecutor::isShowUserSessions(), ExecutionResult::setExecutionTime(), and ExecutionResult::setResultType().

7028  {
7029  DdlCommandExecutor executor = DdlCommandExecutor(query_ra, session_ptr);
7030  std::string commandStr = executor.commandStr();
7031  if (executor.isKillQuery()) {
7032  interruptQuery(*session_ptr, executor.getTargetQuerySessionToKill());
7033  } else {
7034  int64_t execution_time_ms;
7035  if (executor.isShowQueries()) {
7036  // getQueries still requires Thrift cannot be nested into DdlCommandExecutor
7037  execution_time_ms =
7038  measure<>::execution([&]() { _return = getQueries(session_ptr); });
7039  } else if (executor.isShowUserSessions()) {
7040  // getUserSessions still requires Thrift cannot be nested into DdlCommandExecutor
7041  execution_time_ms =
7042  measure<>::execution([&]() { _return = getUserSessions(session_ptr); });
7043  } else {
7044  execution_time_ms = measure<>::execution([&]() { _return = executor.execute(); });
7045  }
7046  _return.setExecutionTime(execution_time_ms);
7047  }
7049 }
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
const std::string getTargetQuerySessionToKill()
const std::string commandStr()
void setResultType(RType type)
ExecutionResult execute()
void setExecutionTime(int64_t execution_time_ms)
void interruptQuery(const Catalog_Namespace::SessionInfo &session_info, const std::string &target_session)
Definition: DBHandler.cpp:6924
ExecutionResult getQueries(std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
Definition: DBHandler.cpp:6848
ExecutionResult getUserSessions(std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
Definition: DBHandler.cpp:6804

+ Here is the call graph for this function:

template<>
void DBHandler::expire_idle_sessions_unsafe ( mapd_unique_lock< mapd_shared_mutex > &  write_lock)

Definition at line 170 of file DBHandler.cpp.

References check_session_exp_unsafe(), LeafAggregator::disconnect(), anonymous_namespace{DBHandler.cpp}::get_session_from_map(), logger::INFO, leaf_aggregator_, LeafAggregator::leafCount(), LOG, render_handler_, and sessions_.

171  {
172  std::vector<std::string> expired_sessions;
173  for (auto session_pair : sessions_) {
174  auto session_it = get_session_from_map(session_pair.first, sessions_);
175  try {
176  check_session_exp_unsafe(session_it);
177  } catch (const ForceDisconnect& e) {
178  expired_sessions.emplace_back(session_it->second->get_session_id());
179  }
180  }
181 
182  for (auto session_id : expired_sessions) {
183  if (leaf_aggregator_.leafCount() > 0) {
184  try {
185  leaf_aggregator_.disconnect(session_id);
186  } catch (TOmniSciException& toe) {
187  LOG(INFO) << " Problem disconnecting from leaves : " << toe.what();
188  } catch (std::exception& e) {
189  LOG(INFO)
190  << " Problem disconnecting from leaves, check leaf logs for additonal info";
191  }
192  }
193  sessions_.erase(session_id);
194  }
195  if (render_handler_) {
196  write_lock.unlock();
197  for (auto session_id : expired_sessions) {
198  // NOTE: the render disconnect is done after the session lock is released to
199  // avoid a deadlock. See: https://omnisci.atlassian.net/browse/BE-3324
200  // This out-of-scope solution is a compromise for now until a better session
201  // handling/locking mechanism is developed for the renderer. Note as well that the
202  // session_id cannot be immediately reused. If a render request were to slip in
203  // after the lock is released and before the render disconnect could cause a
204  // problem.
205  render_handler_->disconnect(session_id);
206  }
207  }
208 }
#define LOG(tag)
Definition: Logger.h:194
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:566
void disconnect(const TSessionId session)
SessionMap::iterator get_session_from_map(const TSessionId &session, SessionMap &session_map)
Definition: DBHandler.cpp:123
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:585
size_t leafCount() const
void check_session_exp_unsafe(const SessionMap::iterator &session_it)
Definition: DBHandler.cpp:5338
mapd_unique_lock< mapd_shared_mutex > write_lock
SessionMap sessions_
Definition: DBHandler.h:866

+ Here is the call graph for this function:

template<typename SESSION_MAP_LOCK >
void DBHandler::expire_idle_sessions_unsafe ( SESSION_MAP_LOCK &  lock)
private

Referenced by connect_impl().

+ Here is the caller graph for this function:

std::unordered_map< std::string, std::unordered_set< std::string > > DBHandler::fill_column_names_by_table ( std::vector< std::string > &  table_names,
query_state::StdLog stdlog 
)
private

Definition at line 1675 of file DBHandler.cpp.

References get_table_details_impl().

Referenced by get_token_based_completions().

1676  {
1677  std::unordered_map<std::string, std::unordered_set<std::string>> column_names_by_table;
1678  for (auto it = table_names.begin(); it != table_names.end();) {
1679  TTableDetails table_details;
1680  try {
1681  get_table_details_impl(table_details, stdlog, *it, false, false);
1682  } catch (const TOmniSciException& e) {
1683  // Remove the corrupted Table/View name from the list for further processing.
1684  it = table_names.erase(it);
1685  continue;
1686  }
1687  for (const auto& column_type : table_details.row_desc) {
1688  column_names_by_table[*it].emplace(column_type.col_name);
1689  }
1690  ++it;
1691  }
1692  return column_names_by_table;
1693 }
void get_table_details_impl(TTableDetails &_return, query_state::StdLog &stdlog, const std::string &table_name, const bool get_system, const bool get_physical)
Definition: DBHandler.cpp:2261

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

TRowDescriptor DBHandler::fixup_row_descriptor ( const TRowDescriptor &  row_desc,
const Catalog_Namespace::Catalog cat 
)
private

Definition at line 5704 of file DBHandler.cpp.

References DictDescriptor::dictNBits, and Catalog_Namespace::Catalog::getMetadataForDict().

Referenced by get_table_details_impl(), and sql_validate().

5705  {
5706  TRowDescriptor fixedup_row_desc;
5707  for (const TColumnType& col_desc : row_desc) {
5708  auto fixedup_col_desc = col_desc;
5709  if (col_desc.col_type.encoding == TEncodingType::DICT &&
5710  col_desc.col_type.comp_param > 0) {
5711  const auto dd = cat.getMetadataForDict(col_desc.col_type.comp_param, false);
5712  fixedup_col_desc.col_type.comp_param = dd->dictNBits;
5713  }
5714  fixedup_row_desc.push_back(fixedup_col_desc);
5715  }
5716 
5717  return fixedup_row_desc;
5718 }
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1494

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::get_all_files_in_archive ( std::vector< std::string > &  _return,
const TSessionId &  session,
const std::string &  archive_path,
const TCopyParams &  copy_params 
)
override

Definition at line 5188 of file DBHandler.cpp.

References anonymous_namespace{DBHandler.cpp}::add_vsi_archive_prefix(), anonymous_namespace{DBHandler.cpp}::add_vsi_network_prefix(), import_export::Importer::gdalFileExists(), import_export::Importer::gdalGetAllFilesInArchive(), get_session_ptr(), getConnectionInfo(), import_path_, anonymous_namespace{DBHandler.cpp}::is_a_supported_archive_file(), anonymous_namespace{DBHandler.cpp}::path_is_relative(), STDLOG, thrift_to_copyparams(), THROW_MAPD_EXCEPTION, toString(), and anonymous_namespace{DBHandler.cpp}::validate_import_file_path_if_local().

5191  {
5192  auto stdlog =
5193  STDLOG(get_session_ptr(session), "get_all_files_in_archive", archive_path_in);
5194  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
5195 
5196  std::string archive_path(archive_path_in);
5197  if (path_is_relative(archive_path)) {
5198  // assume relative paths are relative to data_path / mapd_import / <session>
5199  auto file_path = import_path_ / picosha2::hash256_hex_string(session) /
5200  boost::filesystem::path(archive_path).filename();
5201  archive_path = file_path.string();
5202  }
5203  validate_import_file_path_if_local(archive_path);
5204 
5205  if (is_a_supported_archive_file(archive_path)) {
5206  // find the archive file
5207  add_vsi_network_prefix(archive_path);
5208  if (!import_export::Importer::gdalFileExists(archive_path,
5209  thrift_to_copyparams(copy_params))) {
5210  THROW_MAPD_EXCEPTION("Archive does not exist: " + archive_path_in);
5211  }
5212  // find all files in archive
5213  add_vsi_archive_prefix(archive_path);
5215  archive_path, thrift_to_copyparams(copy_params));
5216  // prepend them all with original path
5217  for (auto& s : _return) {
5218  s = archive_path_in + '/' + s;
5219  }
5220  }
5221 }
boost::filesystem::path import_path_
Definition: DBHandler.h:570
void add_vsi_archive_prefix(std::string &path)
Definition: DBHandler.cpp:3583
bool is_a_supported_archive_file(const std::string &path)
Definition: DBHandler.cpp:3651
std::string toString(const ExtArgumentType &sig_type)
bool path_is_relative(const std::string &path)
Definition: DBHandler.cpp:3617
static bool gdalFileExists(const std::string &path, const CopyParams &copy_params)
Definition: Importer.cpp:4737
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
void validate_import_file_path_if_local(const std::string &file_path)
Definition: DBHandler.cpp:3705
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
void add_vsi_network_prefix(std::string &path)
Definition: DBHandler.cpp:3554
static std::vector< std::string > gdalGetAllFilesInArchive(const std::string &archive_path, const CopyParams &copy_params)
Definition: Importer.cpp:4814
import_export::CopyParams thrift_to_copyparams(const TCopyParams &cp)
Definition: DBHandler.cpp:3366
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1695
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

void DBHandler::get_all_roles_for_user ( std::vector< std::string > &  _return,
const TSessionId &  session,
const std::string &  granteeName 
)
override

Definition at line 2109 of file DBHandler.cpp.

References CHECK, get_session_ptr(), STDLOG, and THROW_MAPD_EXCEPTION.

2111  {
2112  auto stdlog = STDLOG(get_session_ptr(sessionId));
2113  auto session_ptr = stdlog.getConstSessionInfo();
2114  auto* grantee = SysCatalog::instance().getGrantee(granteeName);
2115  if (grantee) {
2116  if (session_ptr->get_currentUser().isSuper) {
2117  roles = grantee->getRoles();
2118  } else if (grantee->isUser()) {
2119  if (session_ptr->get_currentUser().userName == granteeName) {
2120  roles = grantee->getRoles();
2121  } else {
2123  "Only a superuser is authorized to request list of roles granted to "
2124  "another "
2125  "user.");
2126  }
2127  } else {
2128  CHECK(!grantee->isUser());
2129  // granteeName is actually a roleName here and we can check a role
2130  // only if it is granted to us
2131  if (SysCatalog::instance().isRoleGrantedToGrantee(
2132  session_ptr->get_currentUser().userName, granteeName, false)) {
2133  roles = grantee->getRoles();
2134  } else {
2135  THROW_MAPD_EXCEPTION("A user can check only roles granted to him.");
2136  }
2137  }
2138  } else {
2139  THROW_MAPD_EXCEPTION("Grantee " + granteeName + " does not exist.");
2140  }
2141 }
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
#define CHECK(condition)
Definition: Logger.h:203
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

void DBHandler::get_completion_hints ( std::vector< TCompletionHint > &  hints,
const TSessionId &  session,
const std::string &  sql,
const int  cursor 
)
override

Definition at line 1569 of file DBHandler.cpp.

References anonymous_namespace{DBHandler.cpp}::extract_projection_tokens_for_completion(), get_completion_hints_unsorted(), get_session_ptr(), get_uc_compatible_table_names_by_column(), gpu_enabled::sort(), STDLOG, and to_upper().

1572  {
1573  auto stdlog = STDLOG(get_session_ptr(session));
1574  std::vector<std::string> visible_tables; // Tables allowed for the given session.
1575  get_completion_hints_unsorted(hints, visible_tables, stdlog, sql, cursor);
1576  const auto proj_tokens = extract_projection_tokens_for_completion(sql);
1577  auto compatible_table_names = get_uc_compatible_table_names_by_column(
1578  proj_tokens.uc_column_names, visible_tables, stdlog);
1579  // Add the table qualifiers explicitly specified by the user.
1580  compatible_table_names.insert(proj_tokens.uc_column_table_qualifiers.begin(),
1581  proj_tokens.uc_column_table_qualifiers.end());
1582  // Sort the hints by category, from COLUMN (most specific) to KEYWORD.
1583  std::sort(
1584  hints.begin(),
1585  hints.end(),
1586  [&compatible_table_names](const TCompletionHint& lhs, const TCompletionHint& rhs) {
1587  if (lhs.type == TCompletionHintType::TABLE &&
1588  rhs.type == TCompletionHintType::TABLE) {
1589  // Between two tables, one which is compatible with the specified
1590  // projections and one which isn't, pick the one which is compatible.
1591  if (compatible_table_names.find(to_upper(lhs.hints.back())) !=
1592  compatible_table_names.end() &&
1593  compatible_table_names.find(to_upper(rhs.hints.back())) ==
1594  compatible_table_names.end()) {
1595  return true;
1596  }
1597  }
1598  return lhs.type < rhs.type;
1599  });
1600 }
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
ProjectionTokensForCompletion extract_projection_tokens_for_completion(const std::string &sql)
Definition: DBHandler.cpp:1545
void get_completion_hints_unsorted(std::vector< TCompletionHint > &hints, std::vector< std::string > &visible_tables, query_state::StdLog &stdlog, const std::string &sql, const int cursor)
Definition: DBHandler.cpp:1602
std::string to_upper(const std::string &str)
std::unordered_set< std::string > get_uc_compatible_table_names_by_column(const std::unordered_set< std::string > &uc_column_names, std::vector< std::string > &table_names, query_state::StdLog &stdlog)
Definition: DBHandler.cpp:1700
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

void DBHandler::get_completion_hints_unsorted ( std::vector< TCompletionHint > &  hints,
std::vector< std::string > &  visible_tables,
query_state::StdLog stdlog,
const std::string &  sql,
const int  cursor 
)
private

Definition at line 1602 of file DBHandler.cpp.

References calcite_, logger::ERROR, GET_PHYSICAL_TABLES_AND_VIEWS, get_tables_impl(), get_token_based_completions(), query_state::StdLog::getConstSessionInfo(), just_whitelisted_keyword_hints(), and LOG.

Referenced by get_completion_hints().

1606  {
1607  const auto& session_info = *stdlog.getConstSessionInfo();
1608  try {
1609  get_tables_impl(visible_tables, session_info, GET_PHYSICAL_TABLES_AND_VIEWS);
1610  // Filter out keywords suggested by Calcite which we don't support.
1612  calcite_->getCompletionHints(session_info, visible_tables, sql, cursor));
1613  } catch (const std::exception& e) {
1614  TOmniSciException ex;
1615  ex.error_msg = "Exception: " + std::string(e.what());
1616  LOG(ERROR) << ex.error_msg;
1617  throw ex;
1618  }
1619  boost::regex from_expr{R"(\s+from\s+)", boost::regex::extended | boost::regex::icase};
1620  const size_t length_to_cursor =
1621  cursor < 0 ? sql.size() : std::min(sql.size(), static_cast<size_t>(cursor));
1622  // Trust hints from Calcite after the FROM keyword.
1623  if (boost::regex_search(sql.cbegin(), sql.cbegin() + length_to_cursor, from_expr)) {
1624  return;
1625  }
1626  // Before FROM, the query is too incomplete for context-sensitive completions.
1627  get_token_based_completions(hints, stdlog, visible_tables, sql, cursor);
1628 }
#define LOG(tag)
Definition: Logger.h:194
std::vector< TCompletionHint > just_whitelisted_keyword_hints(const std::vector< TCompletionHint > &hints)
void get_token_based_completions(std::vector< TCompletionHint > &hints, query_state::StdLog &stdlog, std::vector< std::string > &visible_tables, const std::string &sql, const int cursor)
Definition: DBHandler.cpp:1630
std::shared_ptr< Calcite > calcite_
Definition: DBHandler.h:588
void get_tables_impl(std::vector< std::string > &table_names, const Catalog_Namespace::SessionInfo &, const GetTablesType get_tables_type)
Definition: DBHandler.cpp:2387
std::shared_ptr< Catalog_Namespace::SessionInfo const > getConstSessionInfo() const
Definition: QueryState.cpp:141

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::shared_ptr< const Catalog_Namespace::SessionInfo > DBHandler::get_const_session_ptr ( const TSessionId &  session)
private

Definition at line 5365 of file DBHandler.cpp.

References get_session_ptr().

5366  {
5367  return get_session_ptr(session);
5368 }
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388

+ Here is the call graph for this function:

void DBHandler::get_custom_expressions ( std::vector< TCustomExpression > &  _return,
const TSessionId &  session 
)
override

Definition at line 3968 of file DBHandler.cpp.

References anonymous_namespace{DBHandler.cpp}::create_thrift_obj_from_custom_expr(), custom_expressions_mutex_, get_session_ptr(), getConnectionInfo(), STDLOG, and toString().

3969  {
3970  auto stdlog = STDLOG(get_session_ptr(session));
3971  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
3972 
3973  auto session_ptr = stdlog.getConstSessionInfo();
3974  auto& catalog = session_ptr->getCatalog();
3975  mapd_shared_lock<mapd_shared_mutex> read_lock(custom_expressions_mutex_);
3976  auto custom_expressions =
3977  catalog.getCustomExpressionsForUser(session_ptr->get_currentUser());
3978  for (const auto& custom_expression : custom_expressions) {
3979  _return.emplace_back(create_thrift_obj_from_custom_expr(*custom_expression));
3980  }
3981 }
std::string toString(const ExtArgumentType &sig_type)
mapd_shared_mutex custom_expressions_mutex_
Definition: DBHandler.h:1009
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
TCustomExpression create_thrift_obj_from_custom_expr(const CustomExpression &custom_expr)
Definition: DBHandler.cpp:3930
mapd_shared_lock< mapd_shared_mutex > read_lock
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1695
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

void DBHandler::get_dashboard ( TDashboard &  _return,
const TSessionId &  session,
const int32_t  dashboard_id 
)
override

Definition at line 4017 of file DBHandler.cpp.

References cat(), get_dashboard_impl(), get_session_ptr(), getConnectionInfo(), is_allowed_on_dashboard(), STDLOG, THROW_MAPD_EXCEPTION, to_string(), toString(), Catalog_Namespace::UserMetadata::userName, and AccessPrivileges::VIEW_DASHBOARD.

4019  {
4020  auto stdlog = STDLOG(get_session_ptr(session));
4021  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4022  auto session_ptr = stdlog.getConstSessionInfo();
4023  auto const& cat = session_ptr->getCatalog();
4025  auto dash = cat.getMetadataForDashboard(dashboard_id);
4026  if (!dash) {
4027  THROW_MAPD_EXCEPTION("Dashboard with dashboard id " + std::to_string(dashboard_id) +
4028  " doesn't exist");
4029  }
4031  *session_ptr, dash->dashboardId, AccessPrivileges::VIEW_DASHBOARD)) {
4032  THROW_MAPD_EXCEPTION("User has no view privileges for the dashboard with id " +
4033  std::to_string(dashboard_id));
4034  }
4035  user_meta.userName = "";
4036  SysCatalog::instance().getMetadataForUserById(dash->userId, user_meta);
4037  dashboard = get_dashboard_impl(session_ptr, user_meta, dash);
4038 }
static const AccessPrivileges VIEW_DASHBOARD
Definition: DBObject.h:173
std::string cat(Ts &&...args)
std::string toString(const ExtArgumentType &sig_type)
std::string to_string(char const *&&v)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
TDashboard get_dashboard_impl(const std::shared_ptr< Catalog_Namespace::SessionInfo const > &session_ptr, Catalog_Namespace::UserMetadata &user_meta, const DashboardDescriptor *dash, const bool populate_state=true)
Definition: DBHandler.cpp:4060
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
static bool is_allowed_on_dashboard(const Catalog_Namespace::SessionInfo &session_info, int32_t dashboard_id, AccessPrivileges requestedPermissions)
Definition: DBHandler.cpp:3901
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1695
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

void DBHandler::get_dashboard_grantees ( std::vector< TDashboardGrantees > &  _return,
const TSessionId &  session,
const int32_t  dashboard_id 
)
override

Definition at line 4355 of file DBHandler.cpp.

References cat(), DashboardPrivileges::CREATE_DASHBOARD, DashboardDBObjectType, DashboardPrivileges::DELETE_DASHBOARD, DashboardPrivileges::EDIT_DASHBOARD, get_session_ptr(), getConnectionInfo(), STDLOG, THROW_MAPD_EXCEPTION, to_string(), toString(), Catalog_Namespace::UserMetadata::userId, Catalog_Namespace::UserMetadata::userName, and DashboardPrivileges::VIEW_DASHBOARD.

4358  {
4359  auto stdlog = STDLOG(get_session_ptr(session));
4360  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4361  auto session_ptr = stdlog.getConstSessionInfo();
4362  auto const& cat = session_ptr->getCatalog();
4364  auto dash = cat.getMetadataForDashboard(dashboard_id);
4365  if (!dash) {
4366  THROW_MAPD_EXCEPTION("Exception: Dashboard id " + std::to_string(dashboard_id) +
4367  " does not exist");
4368  } else if (session_ptr->get_currentUser().userId != dash->userId &&
4369  !session_ptr->get_currentUser().isSuper) {
4371  "User should be either owner of dashboard or super user to access grantees");
4372  }
4373  std::vector<ObjectRoleDescriptor*> objectsList;
4374  objectsList = SysCatalog::instance().getMetadataForObject(
4375  cat.getCurrentDB().dbId,
4376  static_cast<int>(DBObjectType::DashboardDBObjectType),
4377  dashboard_id); // By default objecttypecan be only dashabaords
4378  user_meta.userId = -1;
4379  user_meta.userName = "";
4380  SysCatalog::instance().getMetadataForUserById(dash->userId, user_meta);
4381  for (auto object : objectsList) {
4382  if (user_meta.userName == object->roleName) {
4383  // Mask owner
4384  continue;
4385  }
4386  TDashboardGrantees grantee;
4387  TDashboardPermissions perm;
4388  grantee.name = object->roleName;
4389  grantee.is_user = object->roleType;
4390  perm.create_ = object->privs.hasPermission(DashboardPrivileges::CREATE_DASHBOARD);
4391  perm.delete_ = object->privs.hasPermission(DashboardPrivileges::DELETE_DASHBOARD);
4392  perm.edit_ = object->privs.hasPermission(DashboardPrivileges::EDIT_DASHBOARD);
4393  perm.view_ = object->privs.hasPermission(DashboardPrivileges::VIEW_DASHBOARD);
4394  grantee.permissions = perm;
4395  dashboard_grantees.push_back(grantee);
4396  }
4397 }
std::string cat(Ts &&...args)
std::string toString(const ExtArgumentType &sig_type)
std::string to_string(char const *&&v)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
static const int32_t EDIT_DASHBOARD
Definition: DBObject.h:106
static const int32_t DELETE_DASHBOARD
Definition: DBObject.h:104
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
static const int32_t VIEW_DASHBOARD
Definition: DBObject.h:105
static const int32_t CREATE_DASHBOARD
Definition: DBObject.h:103
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1695
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

TDashboard DBHandler::get_dashboard_impl ( const std::shared_ptr< Catalog_Namespace::SessionInfo const > &  session_ptr,
Catalog_Namespace::UserMetadata user_meta,
const DashboardDescriptor dash,
const bool  populate_state = true 
)
private

Definition at line 4060 of file DBHandler.cpp.

References cat(), DashboardPrivileges::CREATE_DASHBOARD, DashboardDBObjectType, DashboardDescriptor::dashboardId, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, DashboardPrivileges::DELETE_DASHBOARD, DashboardPrivileges::EDIT_DASHBOARD, DBObject::getPrivileges(), AccessPrivileges::hasPermission(), DashboardDescriptor::imageHash, DashboardDescriptor::updateTime, DashboardDescriptor::user, DashboardDescriptor::userId, Catalog_Namespace::UserMetadata::userName, and DashboardPrivileges::VIEW_DASHBOARD.

Referenced by get_dashboard(), and get_dashboards().

4064  {
4065  auto const& cat = session_ptr->getCatalog();
4066  SysCatalog::instance().getMetadataForUserById(dash->userId, user_meta);
4067  auto objects_list = SysCatalog::instance().getMetadataForObject(
4068  cat.getCurrentDB().dbId,
4069  static_cast<int>(DBObjectType::DashboardDBObjectType),
4070  dash->dashboardId);
4071  TDashboard dashboard;
4072  dashboard.dashboard_name = dash->dashboardName;
4073  if (populate_state)
4074  dashboard.dashboard_state = dash->dashboardState;
4075  dashboard.image_hash = dash->imageHash;
4076  dashboard.update_time = dash->updateTime;
4077  dashboard.dashboard_metadata = dash->dashboardMetadata;
4078  dashboard.dashboard_id = dash->dashboardId;
4079  dashboard.dashboard_owner = dash->user;
4080  TDashboardPermissions perms;
4081  // Super user has all permissions.
4082  if (session_ptr->get_currentUser().isSuper) {
4083  perms.create_ = true;
4084  perms.delete_ = true;
4085  perms.edit_ = true;
4086  perms.view_ = true;
4087  } else {
4088  // Collect all grants on current user
4089  // add them to the permissions.
4090  auto obj_to_find =
4091  DBObject(dashboard.dashboard_id, DBObjectType::DashboardDBObjectType);
4092  obj_to_find.loadKey(session_ptr->getCatalog());
4093  std::vector<std::string> grantees =
4094  SysCatalog::instance().getRoles(true,
4095  session_ptr->get_currentUser().isSuper,
4096  session_ptr->get_currentUser().userName);
4097  for (const auto& grantee : grantees) {
4098  DBObject* object_found;
4099  auto* gr = SysCatalog::instance().getGrantee(grantee);
4100  if (gr && (object_found = gr->findDbObject(obj_to_find.getObjectKey(), true))) {
4101  const auto obj_privs = object_found->getPrivileges();
4102  perms.create_ |= obj_privs.hasPermission(DashboardPrivileges::CREATE_DASHBOARD);
4103  perms.delete_ |= obj_privs.hasPermission(DashboardPrivileges::DELETE_DASHBOARD);
4104  perms.edit_ |= obj_privs.hasPermission(DashboardPrivileges::EDIT_DASHBOARD);
4105  perms.view_ |= obj_privs.hasPermission(DashboardPrivileges::VIEW_DASHBOARD);
4106  }
4107  }
4108  }
4109  dashboard.dashboard_permissions = perms;
4110  if (objects_list.empty() ||
4111  (objects_list.size() == 1 && objects_list[0]->roleName == user_meta.userName)) {
4112  dashboard.is_dash_shared = false;
4113  } else {
4114  dashboard.is_dash_shared = true;
4115  }
4116  return dashboard;
4117 }
std::string cat(Ts &&...args)
static const int32_t EDIT_DASHBOARD
Definition: DBObject.h:106
static const int32_t DELETE_DASHBOARD
Definition: DBObject.h:104
const AccessPrivileges & getPrivileges() const
Definition: DBObject.h:218
static const int32_t VIEW_DASHBOARD
Definition: DBObject.h:105
static const int32_t CREATE_DASHBOARD
Definition: DBObject.h:103
bool hasPermission(int permission) const
Definition: DBObject.h:143

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::get_dashboards ( std::vector< TDashboard > &  _return,
const TSessionId &  session 
)
override

Definition at line 4040 of file DBHandler.cpp.

References cat(), get_dashboard_impl(), get_session_ptr(), getConnectionInfo(), is_allowed_on_dashboard(), STDLOG, toString(), Catalog_Namespace::UserMetadata::userName, and AccessPrivileges::VIEW_DASHBOARD.

4041  {
4042  auto stdlog = STDLOG(get_session_ptr(session));
4043  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4044  auto session_ptr = stdlog.getConstSessionInfo();
4045  auto const& cat = session_ptr->getCatalog();
4047  const auto dashes = cat.getAllDashboardsMetadata();
4048  user_meta.userName = "";
4049  for (const auto dash : dashes) {
4051  *session_ptr, dash->dashboardId, AccessPrivileges::VIEW_DASHBOARD)) {
4052  // dashboardState is intentionally not populated here
4053  // for payload reasons
4054  // use get_dashboard call to get state
4055  dashboards.push_back(get_dashboard_impl(session_ptr, user_meta, dash, false));
4056  }
4057  }
4058 }
static const AccessPrivileges VIEW_DASHBOARD
Definition: DBObject.h:173
std::string cat(Ts &&...args)
std::string toString(const ExtArgumentType &sig_type)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
TDashboard get_dashboard_impl(const std::shared_ptr< Catalog_Namespace::SessionInfo const > &session_ptr, Catalog_Namespace::UserMetadata &user_meta, const DashboardDescriptor *dash, const bool populate_state=true)
Definition: DBHandler.cpp:4060
static bool is_allowed_on_dashboard(const Catalog_Namespace::SessionInfo &session_info, int32_t dashboard_id, AccessPrivileges requestedPermissions)
Definition: DBHandler.cpp:3901
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1695
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

void DBHandler::get_databases ( std::vector< TDBInfo > &  _return,
const TSessionId &  session 
)
override

Definition at line 2675 of file DBHandler.cpp.

References get_session_ptr(), getConnectionInfo(), STDLOG, and toString().

2675  {
2676  auto stdlog = STDLOG(get_session_ptr(session));
2677  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
2678  auto session_ptr = stdlog.getConstSessionInfo();
2679  const auto& user = session_ptr->get_currentUser();
2681  SysCatalog::instance().getDatabaseListForUser(user);
2682  for (auto& db : dbs) {
2683  TDBInfo dbinfo;
2684  dbinfo.db_name = std::move(db.dbName);
2685  dbinfo.db_owner = std::move(db.dbOwnerName);
2686  dbinfos.push_back(std::move(dbinfo));
2687  }
2688 }
std::string toString(const ExtArgumentType &sig_type)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
std::list< DBSummary > DBSummaryList
Definition: SysCatalog.h:129
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1695
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

void DBHandler::get_db_object_privs ( std::vector< TDBObject > &  _return,
const TSessionId &  session,
const std::string &  objectName,
const TDBObjectType::type  type 
)
override

Definition at line 2025 of file DBHandler.cpp.

References AccessPrivileges::ALL_TABLE, cat(), DashboardDBObjectType, DatabaseDBObjectType, get_session_ptr(), DBObject::getObjectKey(), DBObject::loadKey(), serialize_db_object(), ServerDBObjectType, STDLOG, TableDBObjectType, THROW_MAPD_EXCEPTION, to_string(), and ViewDBObjectType.

2028  {
2029  auto stdlog = STDLOG(get_session_ptr(sessionId));
2030  auto session_ptr = stdlog.getConstSessionInfo();
2031  DBObjectType object_type;
2032  switch (type) {
2034  object_type = DBObjectType::DatabaseDBObjectType;
2035  break;
2037  object_type = DBObjectType::TableDBObjectType;
2038  break;
2041  break;
2043  object_type = DBObjectType::ViewDBObjectType;
2044  break;
2046  object_type = DBObjectType::ServerDBObjectType;
2047  break;
2048  default:
2049  THROW_MAPD_EXCEPTION("Failed to get object privileges for " + objectName +
2050  ": unknown object type (" + std::to_string(type) + ").");
2051  }
2052  DBObject object_to_find(objectName, object_type);
2053 
2054  // TODO(adb): Use DatabaseLock to protect method
2055  try {
2056  if (object_type == DashboardDBObjectType) {
2057  if (objectName == "") {
2058  object_to_find = DBObject(-1, object_type);
2059  } else {
2060  object_to_find = DBObject(std::stoi(objectName), object_type);
2061  }
2062  } else if ((object_type == TableDBObjectType || object_type == ViewDBObjectType) &&
2063  !objectName.empty()) {
2064  // special handling for view / table
2065  auto const& cat = session_ptr->getCatalog();
2066  auto td = cat.getMetadataForTable(objectName, false);
2067  if (td) {
2068  object_type = td->isView ? ViewDBObjectType : TableDBObjectType;
2069  object_to_find = DBObject(objectName, object_type);
2070  }
2071  }
2072  object_to_find.loadKey(session_ptr->getCatalog());
2073  } catch (const std::exception&) {
2074  THROW_MAPD_EXCEPTION("Object with name " + objectName + " does not exist.");
2075  }
2076 
2077  // object type on database level
2078  DBObject object_to_find_dblevel("", object_type);
2079  object_to_find_dblevel.loadKey(session_ptr->getCatalog());
2080  // if user is superuser respond with a full priv
2081  if (session_ptr->get_currentUser().isSuper) {
2082  // using ALL_TABLE here to set max permissions
2083  DBObject dbObj{object_to_find.getObjectKey(),
2085  session_ptr->get_currentUser().userId};
2086  dbObj.setName("super");
2087  TDBObjects.push_back(
2088  serialize_db_object(session_ptr->get_currentUser().userName, dbObj));
2089  };
2090 
2091  std::vector<std::string> grantees =
2092  SysCatalog::instance().getRoles(true,
2093  session_ptr->get_currentUser().isSuper,
2094  session_ptr->get_currentUser().userName);
2095  for (const auto& grantee : grantees) {
2096  DBObject* object_found;
2097  auto* gr = SysCatalog::instance().getGrantee(grantee);
2098  if (gr && (object_found = gr->findDbObject(object_to_find.getObjectKey(), true))) {
2099  TDBObjects.push_back(serialize_db_object(grantee, *object_found));
2100  }
2101  // check object permissions on Database level
2102  if (gr &&
2103  (object_found = gr->findDbObject(object_to_find_dblevel.getObjectKey(), true))) {
2104  TDBObjects.push_back(serialize_db_object(grantee, *object_found));
2105  }
2106  }
2107 }
std::string cat(Ts &&...args)
DBObjectKey getObjectKey() const
Definition: DBObject.h:213
DBObjectType
Definition: DBObject.h:42
std::string to_string(char const *&&v)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
static TDBObject serialize_db_object(const std::string &roleName, const DBObject &inObject)
Definition: DBHandler.cpp:1786
static const AccessPrivileges ALL_TABLE
Definition: DBObject.h:159
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

void DBHandler::get_db_objects_for_grantee ( std::vector< TDBObject > &  _return,
const TSessionId &  session,
const std::string &  roleName 
)
override

Definition at line 1998 of file DBHandler.cpp.

References get_session_ptr(), serialize_db_object(), STDLOG, and THROW_MAPD_EXCEPTION.

2000  {
2001  auto stdlog = STDLOG(get_session_ptr(sessionId));
2002  auto session_ptr = stdlog.getConstSessionInfo();
2003  auto const& user = session_ptr->get_currentUser();
2004  if (!user.isSuper &&
2005  !SysCatalog::instance().isRoleGrantedToGrantee(user.userName, roleName, false)) {
2006  return;
2007  }
2008  auto* rl = SysCatalog::instance().getGrantee(roleName);
2009  if (rl) {
2010  auto dbId = session_ptr->getCatalog().getCurrentDB().dbId;
2011  for (auto& dbObject : *rl->getDbObjects(true)) {
2012  if (dbObject.first.dbId != dbId) {
2013  // TODO (max): it doesn't scale well in case we have many DBs (not a typical
2014  // usecase for now, though)
2015  continue;
2016  }
2017  TDBObject tdbObject = serialize_db_object(roleName, *dbObject.second);
2018  TDBObjectsForRole.push_back(tdbObject);
2019  }
2020  } else {
2021  THROW_MAPD_EXCEPTION("User or role " + roleName + " does not exist.");
2022  }
2023 }
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
static TDBObject serialize_db_object(const std::string &roleName, const DBObject &inObject)
Definition: DBHandler.cpp:1786
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

void DBHandler::get_device_parameters ( std::map< std::string, std::string > &  _return,
const TSessionId &  session 
)
override

Definition at line 6697 of file DBHandler.cpp.

References cpu_mode_only_, EXPOSE_THRIFT_MAP, and get_session_copy().

6698  {
6699  const auto session_info = get_session_copy(session);
6700  auto params = ::get_device_parameters(cpu_mode_only_);
6701  for (auto item : params) {
6702  _return.insert(item);
6703  }
6704  EXPOSE_THRIFT_MAP(TDeviceType);
6705  EXPOSE_THRIFT_MAP(TDatumType);
6706  EXPOSE_THRIFT_MAP(TEncodingType);
6707  EXPOSE_THRIFT_MAP(TExtArgumentType);
6708  EXPOSE_THRIFT_MAP(TOutputBufferSizeType);
6709 }
#define EXPOSE_THRIFT_MAP(TYPENAME)
Definition: DBHandler.cpp:6686
Catalog_Namespace::SessionInfo get_session_copy(const TSessionId &session)
Definition: DBHandler.cpp:5370
void get_device_parameters(std::map< std::string, std::string > &_return, const TSessionId &session) override
Definition: DBHandler.cpp:6697
bool cpu_mode_only_
Definition: DBHandler.h:579

+ Here is the call graph for this function:

void DBHandler::get_first_geo_file_in_archive ( std::string &  _return,
const TSessionId &  session,
const std::string &  archive_path,
const TCopyParams &  copy_params 
)
override

Definition at line 5145 of file DBHandler.cpp.

References anonymous_namespace{DBHandler.cpp}::add_vsi_archive_prefix(), anonymous_namespace{DBHandler.cpp}::add_vsi_network_prefix(), anonymous_namespace{DBHandler.cpp}::find_first_geo_file_in_archive(), import_export::Importer::gdalFileExists(), get_session_ptr(), getConnectionInfo(), import_path_, anonymous_namespace{DBHandler.cpp}::is_a_supported_archive_file(), anonymous_namespace{DBHandler.cpp}::path_is_relative(), STDLOG, thrift_to_copyparams(), THROW_MAPD_EXCEPTION, toString(), and anonymous_namespace{DBHandler.cpp}::validate_import_file_path_if_local().

5148  {
5149  auto stdlog =
5150  STDLOG(get_session_ptr(session), "get_first_geo_file_in_archive", archive_path_in);
5151  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
5152 
5153  std::string archive_path(archive_path_in);
5154 
5155  if (path_is_relative(archive_path)) {
5156  // assume relative paths are relative to data_path / mapd_import / <session>
5157  auto file_path = import_path_ / picosha2::hash256_hex_string(session) /
5158  boost::filesystem::path(archive_path).filename();
5159  archive_path = file_path.string();
5160  }
5161  validate_import_file_path_if_local(archive_path);
5162 
5163  if (is_a_supported_archive_file(archive_path)) {
5164  // find the archive file
5165  add_vsi_network_prefix(archive_path);
5166  if (!import_export::Importer::gdalFileExists(archive_path,
5167  thrift_to_copyparams(copy_params))) {
5168  THROW_MAPD_EXCEPTION("Archive does not exist: " + archive_path_in);
5169  }
5170  // find geo file in archive
5171  add_vsi_archive_prefix(archive_path);
5172  std::string geo_file =
5173  find_first_geo_file_in_archive(archive_path, thrift_to_copyparams(copy_params));
5174  // what did we get?
5175  if (geo_file.size()) {
5176  // prepend it with the original path
5177  _return = archive_path_in + std::string("/") + geo_file;
5178  } else {
5179  // just return the original path
5180  _return = archive_path_in;
5181  }
5182  } else {
5183  // just return the original path
5184  _return = archive_path_in;
5185  }
5186 }
boost::filesystem::path import_path_
Definition: DBHandler.h:570
void add_vsi_archive_prefix(std::string &path)
Definition: DBHandler.cpp:3583
bool is_a_supported_archive_file(const std::string &path)
Definition: DBHandler.cpp:3651
std::string toString(const ExtArgumentType &sig_type)
bool path_is_relative(const std::string &path)
Definition: DBHandler.cpp:3617
static bool gdalFileExists(const std::string &path, const CopyParams &copy_params)
Definition: Importer.cpp:4737
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
void validate_import_file_path_if_local(const std::string &file_path)
Definition: DBHandler.cpp:3705
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
std::string find_first_geo_file_in_archive(const std::string &archive_path, const import_export::CopyParams &copy_params)
Definition: DBHandler.cpp:3664
void add_vsi_network_prefix(std::string &path)
Definition: DBHandler.cpp:3554
import_export::CopyParams thrift_to_copyparams(const TCopyParams &cp)
Definition: DBHandler.cpp:3366
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1695
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

void DBHandler::get_hardware_info ( TClusterHardwareInfo &  _return,
const TSessionId &  session 
)
override

Definition at line 838 of file DBHandler.cpp.

References data_mgr_, get_session_ptr(), getConnectionInfo(), LeafAggregator::getHardwareInfo(), leaf_aggregator_, LeafAggregator::leafCount(), STDLOG, and toString().

839  {
840  auto stdlog = STDLOG(get_session_ptr(session));
841  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
842  THardwareInfo ret;
843  const auto cuda_mgr = data_mgr_->getCudaMgr();
844  if (cuda_mgr) {
845  ret.num_gpu_hw = cuda_mgr->getDeviceCount();
846  ret.start_gpu = cuda_mgr->getStartGpu();
847  if (ret.start_gpu >= 0) {
848  ret.num_gpu_allocated = cuda_mgr->getDeviceCount() - cuda_mgr->getStartGpu();
849  // ^ This will break as soon as we allow non contiguous GPU allocations to MapD
850  }
851  for (int16_t device_id = 0; device_id < ret.num_gpu_hw; device_id++) {
852  TGpuSpecification gpu_spec;
853  auto deviceProperties = cuda_mgr->getDeviceProperties(device_id);
854  gpu_spec.num_sm = deviceProperties->numMPs;
855  gpu_spec.clock_frequency_kHz = deviceProperties->clockKhz;
856  gpu_spec.memory = deviceProperties->globalMem;
857  gpu_spec.compute_capability_major = deviceProperties->computeMajor;
858  gpu_spec.compute_capability_minor = deviceProperties->computeMinor;
859  ret.gpu_info.push_back(gpu_spec);
860  }
861  }
862 
863  // start hardware/OS dependent code
864  ret.num_cpu_hw = std::thread::hardware_concurrency();
865  // ^ This might return diffrent results in case of hyper threading
866  // end hardware/OS dependent code
867 
868  _return.hardware_info.push_back(ret);
869  if (leaf_aggregator_.leafCount() > 0) {
870  ret.host_name = "aggregator";
871  TClusterHardwareInfo leaf_hardware = leaf_aggregator_.getHardwareInfo(session);
872  _return.hardware_info.insert(_return.hardware_info.end(),
873  leaf_hardware.hardware_info.begin(),
874  leaf_hardware.hardware_info.end());
875  }
876 }
std::string toString(const ExtArgumentType &sig_type)
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:566
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
size_t leafCount() const
TClusterHardwareInfo getHardwareInfo(TSessionId session)
std::shared_ptr< Data_Namespace::DataMgr > data_mgr_
Definition: DBHandler.h:564
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1695
#define STDLOG(...)
Definition: QueryState.h:229

+ Here is the call graph for this function:

void DBHandler::get_heap_profile ( std::string &  _return,
const TSessionId &  session 
)
override

Definition at line 5323 of file DBHandler.cpp.

References get_session_ptr(), STDLOG, and THROW_MAPD_EXCEPTION.

5323  {
5324  auto stdlog = STDLOG(get_session_ptr(session));
5325 #ifdef HAVE_PROFILER
5326  if (!IsHeapProfilerRunning()) {
5327  THROW_MAPD_EXCEPTION("Profiler not running");
5328  }
5329  auto profile_buff = GetHeapProfile();
5330  profile = profile_buff;
5331  free(profile_buff);
5332 #else
5333  THROW_MAPD_EXCEPTION("Profiler not enabled");
5334 #endif // HAVE_PROFILER
5335 }
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5388
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:110
#define STDLOG(...)
Definition: QueryState.h:229