OmniSciDB  d2f719934e
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
DBHandler Class Reference

#include <DBHandler.h>

+ Inheritance diagram for DBHandler:
+ Collaboration diagram for DBHandler:

Classes

struct  DeferredCopyFromSessions
 
struct  DeferredCopyFromState
 

Public Member Functions

 DBHandler (const std::vector< LeafHostInfo > &db_leaves, const std::vector< LeafHostInfo > &string_leaves, const std::string &base_data_path, const bool allow_multifrag, const bool jit_debug, const bool intel_jit_profile, const bool read_only, const bool allow_loop_joins, const bool enable_rendering, const bool renderer_use_vulkan_driver, const bool renderer_prefer_igpu, const unsigned renderer_vulkan_timeout_ms, const bool enable_auto_clear_render_mem, const int render_oom_retry_threshold, const size_t render_mem_bytes, const size_t max_concurrent_render_sessions, const size_t reserved_gpu_mem, const bool render_compositor_use_last_gpu, const size_t num_reader_threads, const AuthMetadata &authMetadata, SystemParameters &system_parameters, const bool legacy_syntax, const int idle_session_duration, const int max_session_duration, const bool enable_runtime_udf_registration, const std::string &udf_filename, const std::string &clang_path, const std::vector< std::string > &clang_options, const File_Namespace::DiskCacheConfig &disk_cache_config, const bool is_new_db)
 
void initialize (const bool is_new_db)
 
 ~DBHandler () override
 
void krb5_connect (TKrb5Session &session, const std::string &token, const std::string &dbname) override
 
void connect (TSessionId &session, const std::string &username, const std::string &passwd, const std::string &dbname) override
 
void disconnect (const TSessionId &session) override
 
void switch_database (const TSessionId &session, const std::string &dbname) override
 
void clone_session (TSessionId &session2, const TSessionId &session1) override
 
void get_server_status (TServerStatus &_return, const TSessionId &session) override
 
void get_status (std::vector< TServerStatus > &_return, const TSessionId &session) override
 
void get_hardware_info (TClusterHardwareInfo &_return, const TSessionId &session) override
 
bool hasTableAccessPrivileges (const TableDescriptor *td, const Catalog_Namespace::SessionInfo &session_info)
 
void get_tables (std::vector< std::string > &_return, const TSessionId &session) override
 
void get_tables_for_database (std::vector< std::string > &_return, const TSessionId &session, const std::string &database_name) override
 
void get_physical_tables (std::vector< std::string > &_return, const TSessionId &session) override
 
void get_views (std::vector< std::string > &_return, const TSessionId &session) override
 
void get_tables_meta (std::vector< TTableMeta > &_return, const TSessionId &session) override
 
void get_table_details (TTableDetails &_return, const TSessionId &session, const std::string &table_name) override
 
void get_table_details_for_database (TTableDetails &_return, const TSessionId &session, const std::string &table_name, const std::string &database_name) override
 
void get_internal_table_details (TTableDetails &_return, const TSessionId &session, const std::string &table_name) override
 
void get_internal_table_details_for_database (TTableDetails &_return, const TSessionId &session, const std::string &table_name, const std::string &database_name) override
 
void get_users (std::vector< std::string > &_return, const TSessionId &session) override
 
void get_databases (std::vector< TDBInfo > &_return, const TSessionId &session) override
 
void get_version (std::string &_return) override
 
void start_heap_profile (const TSessionId &session) override
 
void stop_heap_profile (const TSessionId &session) override
 
void get_heap_profile (std::string &_return, const TSessionId &session) override
 
void get_memory (std::vector< TNodeMemoryInfo > &_return, const TSessionId &session, const std::string &memory_level) override
 
void clear_cpu_memory (const TSessionId &session) override
 
void clear_gpu_memory (const TSessionId &session) override
 
void clearRenderMemory (const TSessionId &session)
 
void set_cur_session (const TSessionId &parent_session, const TSessionId &leaf_session, const std::string &start_time_str, const std::string &label, bool for_running_query_kernel) override
 
void invalidate_cur_session (const TSessionId &parent_session, const TSessionId &leaf_session, const std::string &start_time_str, const std::string &label, bool for_running_query_kernel) override
 
void set_table_epoch (const TSessionId &session, const int db_id, const int table_id, const int new_epoch) override
 
void set_table_epoch_by_name (const TSessionId &session, const std::string &table_name, const int new_epoch) override
 
int32_t get_table_epoch (const TSessionId &session, const int32_t db_id, const int32_t table_id) override
 
int32_t get_table_epoch_by_name (const TSessionId &session, const std::string &table_name) override
 
void get_table_epochs (std::vector< TTableEpochInfo > &_return, const TSessionId &session, const int32_t db_id, const int32_t table_id) override
 
void set_table_epochs (const TSessionId &session, const int32_t db_id, const std::vector< TTableEpochInfo > &table_epochs) override
 
void get_session_info (TSessionInfo &_return, const TSessionId &session) override
 
void sql_execute (ExecutionResult &_return, const TSessionId &session, const std::string &query, const bool column_format, const int32_t first_n, const int32_t at_most_n)
 
void sql_execute (TQueryResult &_return, const TSessionId &session, const std::string &query, const bool column_format, const std::string &nonce, const int32_t first_n, const int32_t at_most_n) override
 
void get_completion_hints (std::vector< TCompletionHint > &hints, const TSessionId &session, const std::string &sql, const int cursor) override
 
void sql_execute_df (TDataFrame &_return, const TSessionId &session, const std::string &query, const TDeviceType::type device_type, const int32_t device_id, const int32_t first_n, const TArrowTransport::type transport_method) override
 
void sql_execute_gdf (TDataFrame &_return, const TSessionId &session, const std::string &query, const int32_t device_id, const int32_t first_n) override
 
void deallocate_df (const TSessionId &session, const TDataFrame &df, const TDeviceType::type device_type, const int32_t device_id) override
 
void interrupt (const TSessionId &query_session, const TSessionId &interrupt_session) override
 
void sql_validate (TRowDescriptor &_return, const TSessionId &session, const std::string &query) override
 
void set_execution_mode (const TSessionId &session, const TExecuteMode::type mode) override
 
void render_vega (TRenderResult &_return, const TSessionId &session, const int64_t widget_id, const std::string &vega_json, const int32_t compression_level, const std::string &nonce) override
 
void get_result_row_for_pixel (TPixelTableRowResult &_return, const TSessionId &session, const int64_t widget_id, const TPixel &pixel, const std::map< std::string, std::vector< std::string >> &table_col_names, const bool column_format, const int32_t pixel_radius, const std::string &nonce) override
 
int32_t create_custom_expression (const TSessionId &session, const TCustomExpression &custom_expression) override
 
void get_custom_expressions (std::vector< TCustomExpression > &_return, const TSessionId &session) override
 
void update_custom_expression (const TSessionId &session, const int32_t id, const std::string &expression_json) override
 
void delete_custom_expressions (const TSessionId &session, const std::vector< int32_t > &custom_expression_ids, const bool do_soft_delete) override
 
void get_dashboard (TDashboard &_return, const TSessionId &session, const int32_t dashboard_id) override
 
void get_dashboards (std::vector< TDashboard > &_return, const TSessionId &session) override
 
int32_t create_dashboard (const TSessionId &session, const std::string &dashboard_name, const std::string &dashboard_state, const std::string &image_hash, const std::string &dashboard_metadata) override
 
void replace_dashboard (const TSessionId &session, const int32_t dashboard_id, const std::string &dashboard_name, const std::string &dashboard_owner, const std::string &dashboard_state, const std::string &image_hash, const std::string &dashboard_metadata) override
 
void delete_dashboard (const TSessionId &session, const int32_t dashboard_id) override
 
void share_dashboards (const TSessionId &session, const std::vector< int32_t > &dashboard_ids, const std::vector< std::string > &groups, const TDashboardPermissions &permissions) override
 
void delete_dashboards (const TSessionId &session, const std::vector< int32_t > &dashboard_ids) override
 
void share_dashboard (const TSessionId &session, const int32_t dashboard_id, const std::vector< std::string > &groups, const std::vector< std::string > &objects, const TDashboardPermissions &permissions, const bool grant_role) override
 
void unshare_dashboards (const TSessionId &session, const std::vector< int32_t > &dashboard_ids, const std::vector< std::string > &groups, const TDashboardPermissions &permissions) override
 
void unshare_dashboard (const TSessionId &session, const int32_t dashboard_id, const std::vector< std::string > &groups, const std::vector< std::string > &objects, const TDashboardPermissions &permissions) override
 
void get_dashboard_grantees (std::vector< TDashboardGrantees > &_return, const TSessionId &session, const int32_t dashboard_id) override
 
void get_link_view (TFrontendView &_return, const TSessionId &session, const std::string &link) override
 
void create_link (std::string &_return, const TSessionId &session, const std::string &view_state, const std::string &view_metadata) override
 
void load_table_binary (const TSessionId &session, const std::string &table_name, const std::vector< TRow > &rows, const std::vector< std::string > &column_names) override
 
std::unique_ptr
< lockmgr::AbstractLockContainer
< const TableDescriptor * > > 
prepare_columnar_loader (const Catalog_Namespace::SessionInfo &session_info, const std::string &table_name, size_t num_cols, std::unique_ptr< import_export::Loader > *loader, std::vector< std::unique_ptr< import_export::TypedImportBuffer >> *import_buffers, const std::vector< std::string > &column_names)
 
void load_table_binary_columnar (const TSessionId &session, const std::string &table_name, const std::vector< TColumn > &cols, const std::vector< std::string > &column_names) override
 
void load_table_binary_columnar_polys (const TSessionId &session, const std::string &table_name, const std::vector< TColumn > &cols, const std::vector< std::string > &column_names, const bool assign_render_groups) override
 
void load_table_binary_arrow (const TSessionId &session, const std::string &table_name, const std::string &arrow_stream, const bool use_column_names) override
 
void load_table (const TSessionId &session, const std::string &table_name, const std::vector< TStringRow > &rows, const std::vector< std::string > &column_names) override
 
void detect_column_types (TDetectResult &_return, const TSessionId &session, const std::string &file_name, const TCopyParams &copy_params) override
 
void create_table (const TSessionId &session, const std::string &table_name, const TRowDescriptor &row_desc, const TFileType::type file_type, const TCreateParams &create_params) override
 
void import_table (const TSessionId &session, const std::string &table_name, const std::string &file_name, const TCopyParams &copy_params) override
 
void import_geo_table (const TSessionId &session, const std::string &table_name, const std::string &file_name, const TCopyParams &copy_params, const TRowDescriptor &row_desc, const TCreateParams &create_params) override
 
void import_table_status (TImportStatus &_return, const TSessionId &session, const std::string &import_id) override
 
void get_first_geo_file_in_archive (std::string &_return, const TSessionId &session, const std::string &archive_path, const TCopyParams &copy_params) override
 
void get_all_files_in_archive (std::vector< std::string > &_return, const TSessionId &session, const std::string &archive_path, const TCopyParams &copy_params) override
 
void get_layers_in_geo_file (std::vector< TGeoFileLayerInfo > &_return, const TSessionId &session, const std::string &file_name, const TCopyParams &copy_params) override
 
int64_t query_get_outer_fragment_count (const TSessionId &session, const std::string &select_query) override
 
void check_table_consistency (TTableMeta &_return, const TSessionId &session, const int32_t table_id) override
 
void start_query (TPendingQuery &_return, const TSessionId &leaf_session, const TSessionId &parent_session, const std::string &query_ra, const std::string &start_time_str, const bool just_explain, const std::vector< int64_t > &outer_fragment_indices) override
 
void execute_query_step (TStepResult &_return, const TPendingQuery &pending_query, const TSubqueryId subquery_id, const std::string &start_time_str) override
 
void broadcast_serialized_rows (const TSerializedRows &serialized_rows, const TRowDescriptor &row_desc, const TQueryId query_id, const TSubqueryId subquery_id, const bool is_final_subquery_result) override
 
void start_render_query (TPendingRenderQuery &_return, const TSessionId &session, const int64_t widget_id, const int16_t node_idx, const std::string &vega_json) override
 
void execute_next_render_step (TRenderStepResult &_return, const TPendingRenderQuery &pending_render, const TRenderAggDataMap &merged_data) override
 
void insert_data (const TSessionId &session, const TInsertData &insert_data) override
 
void checkpoint (const TSessionId &session, const int32_t table_id) override
 
void get_roles (std::vector< std::string > &_return, const TSessionId &session) override
 
bool has_role (const TSessionId &sessionId, const std::string &granteeName, const std::string &roleName) override
 
bool has_object_privilege (const TSessionId &sessionId, const std::string &granteeName, const std::string &objectName, const TDBObjectType::type object_type, const TDBObjectPermissions &permissions) override
 
void get_db_objects_for_grantee (std::vector< TDBObject > &_return, const TSessionId &session, const std::string &roleName) override
 
void get_db_object_privs (std::vector< TDBObject > &_return, const TSessionId &session, const std::string &objectName, const TDBObjectType::type type) override
 
void get_all_roles_for_user (std::vector< std::string > &_return, const TSessionId &session, const std::string &granteeName) override
 
void get_all_effective_roles_for_user (std::vector< std::string > &_return, const TSessionId &session, const std::string &granteeName) override
 
std::vector< std::string > get_valid_groups (const TSessionId &session, int32_t dashboard_id, std::vector< std::string > groups)
 
void set_license_key (TLicenseInfo &_return, const TSessionId &session, const std::string &key, const std::string &nonce) override
 
void get_license_claims (TLicenseInfo &_return, const TSessionId &session, const std::string &nonce) override
 
void get_device_parameters (std::map< std::string, std::string > &_return, const TSessionId &session) override
 
void register_runtime_extension_functions (const TSessionId &session, const std::vector< TUserDefinedFunction > &udfs, const std::vector< TUserDefinedTableFunction > &udtfs, const std::map< std::string, std::string > &device_ir_map) override
 
void shutdown ()
 
void emergency_shutdown ()
 
TSessionId getInvalidSessionId () const
 
void internal_connect (TSessionId &session, const std::string &username, const std::string &dbname)
 
bool isAggregator () const
 
template<typename... ARGS>
std::shared_ptr
< query_state::QueryState
create_query_state (ARGS &&...args)
 
Catalog_Namespace::SessionInfo get_session_copy (const TSessionId &session)
 
std::shared_ptr
< Catalog_Namespace::SessionInfo
get_session_copy_ptr (const TSessionId &session)
 
void get_tables_meta_impl (std::vector< TTableMeta > &_return, QueryStateProxy query_state_proxy, const Catalog_Namespace::SessionInfo &session_info, const bool with_table_locks=true)
 
void resizeDispatchQueue (size_t queue_size)
 
template<>
SessionMap::iterator get_session_it_unsafe (const TSessionId &session, mapd_shared_lock< mapd_shared_mutex > &read_lock)
 
template<>
SessionMap::iterator get_session_it_unsafe (const TSessionId &session, mapd_unique_lock< mapd_shared_mutex > &write_lock)
 
template<>
void expire_idle_sessions_unsafe (mapd_unique_lock< mapd_shared_mutex > &write_lock)
 

Static Public Member Functions

static size_t max_bytes_for_thrift ()
 
static void parser_with_error_handler (const std::string &query_str, std::list< std::unique_ptr< Parser::Stmt >> &parse_trees)
 

Public Attributes

std::shared_ptr
< Data_Namespace::DataMgr
data_mgr_
 
LeafAggregator leaf_aggregator_
 
std::vector< LeafHostInfodb_leaves_
 
std::vector< LeafHostInfostring_leaves_
 
const std::string base_data_path_
 
boost::filesystem::path import_path_
 
ExecutorDeviceType executor_device_type_
 
std::default_random_engine random_gen_
 
std::uniform_int_distribution
< int64_t > 
session_id_dist_
 
const bool jit_debug_
 
const bool intel_jit_profile_
 
bool allow_multifrag_
 
const bool read_only_
 
const bool allow_loop_joins_
 
bool cpu_mode_only_
 
mapd_shared_mutex sessions_mutex_
 
std::mutex render_mutex_
 
int64_t start_time_
 
const AuthMetadataauthMetadata_
 
SystemParameterssystem_parameters_
 
std::unique_ptr< RenderHandlerrender_handler_
 
std::unique_ptr< MapDAggHandleragg_handler_
 
std::unique_ptr< MapDLeafHandlerleaf_handler_
 
std::shared_ptr< Calcitecalcite_
 
const bool legacy_syntax_
 
std::unique_ptr
< QueryDispatchQueue
dispatch_queue_
 

Protected Member Functions

std::shared_ptr
< Catalog_Namespace::SessionInfo
get_session_ptr (const TSessionId &session_id)
 
ConnectionInfo getConnectionInfo () const
 

Private Types

enum  AssignRenderGroupsMode { AssignRenderGroupsMode::kNone, AssignRenderGroupsMode::kAssign, AssignRenderGroupsMode::kCleanUp }
 
using RenderGroupAssignmentColumnMap = std::unordered_map< std::string, std::unique_ptr< import_export::RenderGroupAnalyzer >>
 
using RenderGroupAssignmentTableMap = std::unordered_map< std::string, RenderGroupAssignmentColumnMap >
 
using RenderGroupAnalyzerSessionMap = std::unordered_map< TSessionId, RenderGroupAssignmentTableMap >
 

Private Member Functions

std::shared_ptr
< Catalog_Namespace::SessionInfo
create_new_session (TSessionId &session, const std::string &dbname, const Catalog_Namespace::UserMetadata &user_meta, std::shared_ptr< Catalog_Namespace::Catalog > cat)
 
void connect_impl (TSessionId &session, const std::string &passwd, const std::string &dbname, const Catalog_Namespace::UserMetadata &user_meta, std::shared_ptr< Catalog_Namespace::Catalog > cat, query_state::StdLog &stdlog)
 
void disconnect_impl (const SessionMap::iterator &session_it, mapd_unique_lock< mapd_shared_mutex > &write_lock)
 
void check_table_load_privileges (const TSessionId &session, const std::string &table_name)
 
void check_table_load_privileges (const Catalog_Namespace::SessionInfo &session_info, const std::string &table_name)
 
void get_tables_impl (std::vector< std::string > &table_names, const Catalog_Namespace::SessionInfo &, const GetTablesType get_tables_type, const std::string &database_name={})
 
void get_table_details_impl (TTableDetails &_return, query_state::StdLog &stdlog, const std::string &table_name, const bool get_system, const bool get_physical, const std::string &database_name={})
 
void getAllRolesForUserImpl (std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr, std::vector< std::string > &roles, const TSessionId &sessionId, const std::string &granteeName, bool effective)
 
void check_read_only (const std::string &str)
 
void check_session_exp_unsafe (const SessionMap::iterator &session_it)
 
void validateGroups (const std::vector< std::string > &groups)
 
void validateDashboardIdsForSharing (const Catalog_Namespace::SessionInfo &session_info, const std::vector< int32_t > &dashboard_ids)
 
void shareOrUnshareDashboards (const TSessionId &session, const std::vector< int32_t > &dashboard_ids, const std::vector< std::string > &groups, const TDashboardPermissions &permissions, const bool do_share)
 
std::shared_ptr< const
Catalog_Namespace::SessionInfo
get_const_session_ptr (const TSessionId &session)
 
template<typename SESSION_MAP_LOCK >
SessionMap::iterator get_session_it_unsafe (const TSessionId &session, SESSION_MAP_LOCK &lock)
 
template<typename SESSION_MAP_LOCK >
void expire_idle_sessions_unsafe (SESSION_MAP_LOCK &lock)
 
std::pair< TPlanResult,
lockmgr::LockedTableDescriptors
parse_to_ra (QueryStateProxy, const std::string &query_str, const std::vector< TFilterPushDownInfo > &filter_push_down_info, const bool acquire_locks, const SystemParameters &system_parameters, bool check_privileges=true)
 
void sql_execute_local (TQueryResult &_return, const QueryStateProxy &query_state_proxy, const std::shared_ptr< Catalog_Namespace::SessionInfo > session_ptr, const std::string &query_str, const bool column_format, const std::string &nonce, const int32_t first_n, const int32_t at_most_n, const bool use_calcite)
 
int64_t process_deferred_copy_from (const TSessionId &session_id)
 
void sql_execute_impl (ExecutionResult &_return, QueryStateProxy, const bool column_format, const ExecutorDeviceType executor_device_type, const int32_t first_n, const int32_t at_most_n, const bool use_calcite)
 
bool user_can_access_table (const Catalog_Namespace::SessionInfo &, const TableDescriptor *td, const AccessPrivileges acess_priv)
 
void execute_distributed_copy_statement (Parser::CopyTableStmt *, const Catalog_Namespace::SessionInfo &session_info)
 
TQueryResult validate_rel_alg (const std::string &query_ra, QueryStateProxy)
 
std::vector< PushedDownFilterInfoexecute_rel_alg (ExecutionResult &_return, QueryStateProxy, const std::string &query_ra, const bool column_format, const ExecutorDeviceType executor_device_type, const int32_t first_n, const int32_t at_most_n, const bool just_validate, const bool find_push_down_candidates, const ExplainInfo &explain_info, const std::optional< size_t > executor_index=std::nullopt) const
 
void execute_rel_alg_with_filter_push_down (ExecutionResult &_return, QueryStateProxy, std::string &query_ra, const bool column_format, const ExecutorDeviceType executor_device_type, const int32_t first_n, const int32_t at_most_n, const bool just_explain, const bool just_calcite_explain, const std::vector< PushedDownFilterInfo > &filter_push_down_requests)
 
void execute_rel_alg_df (TDataFrame &_return, const std::string &query_ra, QueryStateProxy query_state_proxy, const Catalog_Namespace::SessionInfo &session_info, const ExecutorDeviceType executor_device_type, const ExecutorDeviceType results_device_type, const size_t device_id, const int32_t first_n, const TArrowTransport::type transport_method) const
 
void executeDdl (TQueryResult &_return, const std::string &query_ra, std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
 
void executeDdl (ExecutionResult &_return, const std::string &query_ra, std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
 
TColumnType populateThriftColumnType (const Catalog_Namespace::Catalog *cat, const ColumnDescriptor *cd)
 
TRowDescriptor fixup_row_descriptor (const TRowDescriptor &row_desc, const Catalog_Namespace::Catalog &cat)
 
void set_execution_mode_nolock (Catalog_Namespace::SessionInfo *session_ptr, const TExecuteMode::type mode)
 
char unescape_char (std::string str)
 
import_export::CopyParams thrift_to_copyparams (const TCopyParams &cp)
 
TCopyParams copyparams_to_thrift (const import_export::CopyParams &cp)
 
void check_geospatial_files (const boost::filesystem::path file_path, const import_export::CopyParams &copy_params)
 
void render_rel_alg (TRenderResult &_return, const std::string &query_ra, const std::string &query_str, const Catalog_Namespace::SessionInfo &session_info, const std::string &render_type, const bool is_projection_query)
 
TColumnType create_geo_column (const TDatumType::type type, const std::string &name, const bool is_array)
 
void convertResultSet (ExecutionResult &result, const Catalog_Namespace::SessionInfo &session_info, const std::string &query_state_str, TQueryResult &_return)
 
std::vector< TargetMetaInfogetTargetMetaInfo (const std::vector< std::shared_ptr< Analyzer::TargetEntry >> &targets) const
 
std::vector< std::string > getTargetNames (const std::vector< TargetMetaInfo > &targets) const
 
std::vector< std::string > getTargetNames (const std::vector< std::shared_ptr< Analyzer::TargetEntry >> &targets) const
 
void get_completion_hints_unsorted (std::vector< TCompletionHint > &hints, std::vector< std::string > &visible_tables, query_state::StdLog &stdlog, const std::string &sql, const int cursor)
 
void get_token_based_completions (std::vector< TCompletionHint > &hints, query_state::StdLog &stdlog, std::vector< std::string > &visible_tables, const std::string &sql, const int cursor)
 
std::unordered_map
< std::string,
std::unordered_set
< std::string > > 
fill_column_names_by_table (std::vector< std::string > &table_names, query_state::StdLog &stdlog)
 
TDashboard get_dashboard_impl (const std::shared_ptr< Catalog_Namespace::SessionInfo const > &session_ptr, Catalog_Namespace::UserMetadata &user_meta, const DashboardDescriptor *dash, const bool populate_state=true)
 
std::unordered_set< std::string > get_uc_compatible_table_names_by_column (const std::unordered_set< std::string > &uc_column_names, std::vector< std::string > &table_names, query_state::StdLog &stdlog)
 
std::unique_ptr
< lockmgr::AbstractLockContainer
< const TableDescriptor * > > 
prepare_loader_generic (const Catalog_Namespace::SessionInfo &session_info, const std::string &table_name, size_t num_cols, std::unique_ptr< import_export::Loader > *loader, std::vector< std::unique_ptr< import_export::TypedImportBuffer >> *import_buffers, const std::vector< std::string > &column_names, std::string load_type)
 
void fillGeoColumns (const TSessionId &session, const Catalog_Namespace::Catalog &catalog, std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &import_buffers, const ColumnDescriptor *cd, size_t &col_idx, size_t num_rows, const std::string &table_name, bool assign_render_groups)
 
void fillMissingBuffers (const TSessionId &session, const Catalog_Namespace::Catalog &catalog, std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &import_buffers, const std::list< const ColumnDescriptor * > &cds, const std::vector< int > &desc_id_to_column_id, size_t num_rows, const std::string &table_name, bool assign_render_groups)
 
void check_and_invalidate_sessions (Parser::DDLStmt *ddl)
 
template<typename STMT_TYPE >
void invalidate_sessions (std::string &name, STMT_TYPE *stmt)
 
std::string const createInMemoryCalciteSession (const std::shared_ptr< Catalog_Namespace::Catalog > &catalog_ptr)
 
bool isInMemoryCalciteSession (const Catalog_Namespace::UserMetadata user_meta)
 
void removeInMemoryCalciteSession (const std::string &session_id)
 
ExecutionResult getUserSessions (std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
 
ExecutionResult getQueries (std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
 
void get_queries_info (std::vector< TQueryInfo > &_return, const TSessionId &session)
 
void interruptQuery (const Catalog_Namespace::SessionInfo &session_info, const std::string &target_session)
 
void load_table_binary_columnar_internal (const TSessionId &session, const std::string &table_name, const std::vector< TColumn > &cols, const std::vector< std::string > &column_names, const AssignRenderGroupsMode assign_render_groups_mode)
 
TRole::type getServerRole () const
 
void import_geo_table_internal (const TSessionId &session, const std::string &table_name, const std::string &file_name, const TCopyParams &copy_params, const TRowDescriptor &row_desc, const TCreateParams &create_params)
 

Static Private Member Functions

static void value_to_thrift_column (const TargetValue &tv, const SQLTypeInfo &ti, TColumn &column)
 
static TDatum value_to_thrift (const TargetValue &tv, const SQLTypeInfo &ti)
 
static void convertData (TQueryResult &_return, ExecutionResult &result, const QueryStateProxy &query_state_proxy, const std::string &query_str, const bool column_format, const int32_t first_n, const int32_t at_most_n)
 
static void convertExplain (TQueryResult &_return, const ResultSet &results, const bool column_format)
 
static void convertResult (TQueryResult &_return, const ResultSet &results, const bool column_format)
 
static void convertRows (TQueryResult &_return, QueryStateProxy query_state_proxy, const std::vector< TargetMetaInfo > &targets, const ResultSet &results, const bool column_format, const int32_t first_n, const int32_t at_most_n)
 
static void createSimpleResult (TQueryResult &_return, const ResultSet &results, const bool column_format, const std::string label)
 
static bool has_database_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permissions)
 
static bool has_table_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permission)
 
static bool has_dashboard_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permissions)
 
static bool has_view_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permissions)
 
static bool has_server_permission (const AccessPrivileges &privs, const TDBObjectPermissions &permissions)
 

Private Attributes

std::atomic< bool > initialized_ {false}
 
query_state::QueryStates query_states_
 
SessionMap sessions_
 
bool super_user_rights_
 
const int idle_session_duration_
 
const int max_session_duration_
 
const bool runtime_udf_registration_enabled_
 
const bool enable_rendering_
 
const bool renderer_use_vulkan_driver_
 
const bool renderer_prefer_igpu_
 
const unsigned renderer_vulkan_timeout_
 
const bool enable_auto_clear_render_mem_
 
const int render_oom_retry_threshold_
 
const size_t max_concurrent_render_sessions_
 
const size_t reserved_gpu_mem_
 
const bool render_compositor_use_last_gpu_
 
const size_t render_mem_bytes_
 
const size_t num_reader_threads_
 
const
File_Namespace::DiskCacheConfig
disk_cache_config_
 
const std::string & udf_filename_
 
const std::string & clang_path_
 
const std::vector< std::string > & clang_options_
 
DeferredCopyFromSessions deferred_copy_from_sessions
 
std::mutex handle_to_dev_ptr_mutex_
 
std::unordered_map
< std::string, std::string > 
ipc_handle_to_dev_ptr_
 
std::map< const std::string,
const PermissionFuncPtr
permissionFuncMap_
 
RenderGroupAnalyzerSessionMap render_group_assignment_map_
 
std::mutex render_group_assignment_mutex_
 
mapd_shared_mutex custom_expressions_mutex_
 

Friends

class RenderHandler::Impl
 
class MapDAggHandler
 
class MapDLeafHandler
 
void run_warmup_queries (std::shared_ptr< DBHandler > handler, std::string base_path, std::string query_file_path)
 

Detailed Description

Definition at line 159 of file DBHandler.h.

Member Typedef Documentation

using DBHandler::RenderGroupAnalyzerSessionMap = std::unordered_map<TSessionId, RenderGroupAssignmentTableMap>
private

Definition at line 1069 of file DBHandler.h.

using DBHandler::RenderGroupAssignmentColumnMap = std::unordered_map<std::string, std::unique_ptr<import_export::RenderGroupAnalyzer>>
private

Definition at line 1065 of file DBHandler.h.

using DBHandler::RenderGroupAssignmentTableMap = std::unordered_map<std::string, RenderGroupAssignmentColumnMap>
private

Definition at line 1067 of file DBHandler.h.

Member Enumeration Documentation

Enumerator
kNone 
kAssign 
kCleanUp 

Definition at line 1052 of file DBHandler.h.

1052 { kNone, kAssign, kCleanUp };

Constructor & Destructor Documentation

DBHandler::DBHandler ( const std::vector< LeafHostInfo > &  db_leaves,
const std::vector< LeafHostInfo > &  string_leaves,
const std::string &  base_data_path,
const bool  allow_multifrag,
const bool  jit_debug,
const bool  intel_jit_profile,
const bool  read_only,
const bool  allow_loop_joins,
const bool  enable_rendering,
const bool  renderer_use_vulkan_driver,
const bool  renderer_prefer_igpu,
const unsigned  renderer_vulkan_timeout_ms,
const bool  enable_auto_clear_render_mem,
const int  render_oom_retry_threshold,
const size_t  render_mem_bytes,
const size_t  max_concurrent_render_sessions,
const size_t  reserved_gpu_mem,
const bool  render_compositor_use_last_gpu,
const size_t  num_reader_threads,
const AuthMetadata authMetadata,
SystemParameters system_parameters,
const bool  legacy_syntax,
const int  idle_session_duration,
const int  max_session_duration,
const bool  enable_runtime_udf_registration,
const std::string &  udf_filename,
const std::string &  clang_path,
const std::vector< std::string > &  clang_options,
const File_Namespace::DiskCacheConfig disk_cache_config,
const bool  is_new_db 
)

Definition at line 232 of file DBHandler.cpp.

265  : leaf_aggregator_(db_leaves)
266  , db_leaves_(db_leaves)
267  , string_leaves_(string_leaves)
268  , base_data_path_(base_data_path)
269  , random_gen_(std::random_device{}())
271  , jit_debug_(jit_debug)
272  , intel_jit_profile_(intel_jit_profile)
273  , allow_multifrag_(allow_multifrag)
274  , read_only_(read_only)
275  , allow_loop_joins_(allow_loop_joins)
276  , authMetadata_(authMetadata)
277  , system_parameters_(system_parameters)
278  , legacy_syntax_(legacy_syntax)
279  , dispatch_queue_(
280  std::make_unique<QueryDispatchQueue>(system_parameters.num_executors))
281  , super_user_rights_(false)
282  , idle_session_duration_(idle_session_duration * 60)
283  , max_session_duration_(max_session_duration * 60)
284  , runtime_udf_registration_enabled_(enable_runtime_udf_registration)
285 
286  , enable_rendering_(enable_rendering)
287  , renderer_use_vulkan_driver_(renderer_use_vulkan_driver)
288  , renderer_prefer_igpu_(renderer_prefer_igpu)
289  , renderer_vulkan_timeout_(renderer_vulkan_timeout_ms)
290  , enable_auto_clear_render_mem_(enable_auto_clear_render_mem)
291  , render_oom_retry_threshold_(render_oom_retry_threshold)
292  , max_concurrent_render_sessions_(max_concurrent_render_sessions)
293  , reserved_gpu_mem_(reserved_gpu_mem)
294  , render_compositor_use_last_gpu_(render_compositor_use_last_gpu)
295  , render_mem_bytes_(render_mem_bytes)
296  , num_reader_threads_(num_reader_threads)
297 #ifdef ENABLE_GEOS
298  , libgeos_so_filename_(libgeos_so_filename)
299 #endif
300  , disk_cache_config_(disk_cache_config)
301  , udf_filename_(udf_filename)
302  , clang_path_(clang_path)
303  , clang_options_(clang_options)
304 
305 {
306  LOG(INFO) << "OmniSci Server " << MAPD_RELEASE;
307  initialize(is_new_db);
308 }
std::vector< LeafHostInfo > string_leaves_
Definition: DBHandler.h:590
const std::vector< std::string > & clang_options_
Definition: DBHandler.h:950
std::unique_ptr< QueryDispatchQueue > dispatch_queue_
Definition: DBHandler.h:613
const bool runtime_udf_registration_enabled_
Definition: DBHandler.h:931
const std::string & udf_filename_
Definition: DBHandler.h:948
const int render_oom_retry_threshold_
Definition: DBHandler.h:938
#define LOG(tag)
Definition: Logger.h:205
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:588
#define INT32_MAX
const unsigned renderer_vulkan_timeout_
Definition: DBHandler.h:936
const std::string base_data_path_
Definition: DBHandler.h:591
void initialize(const bool is_new_db)
Definition: DBHandler.cpp:310
const bool jit_debug_
Definition: DBHandler.h:596
const size_t render_mem_bytes_
Definition: DBHandler.h:942
const bool renderer_use_vulkan_driver_
Definition: DBHandler.h:934
const size_t reserved_gpu_mem_
Definition: DBHandler.h:940
const bool render_compositor_use_last_gpu_
Definition: DBHandler.h:941
const int max_session_duration_
Definition: DBHandler.h:929
std::vector< LeafHostInfo > db_leaves_
Definition: DBHandler.h:589
const File_Namespace::DiskCacheConfig & disk_cache_config_
Definition: DBHandler.h:947
const std::string & clang_path_
Definition: DBHandler.h:949
SystemParameters & system_parameters_
Definition: DBHandler.h:606
const size_t num_reader_threads_
Definition: DBHandler.h:943
const bool enable_auto_clear_render_mem_
Definition: DBHandler.h:937
const bool renderer_prefer_igpu_
Definition: DBHandler.h:935
const bool allow_loop_joins_
Definition: DBHandler.h:600
const bool enable_rendering_
Definition: DBHandler.h:933
const bool intel_jit_profile_
Definition: DBHandler.h:597
bool super_user_rights_
Definition: DBHandler.h:925
const bool read_only_
Definition: DBHandler.h:599
const bool legacy_syntax_
Definition: DBHandler.h:611
const int idle_session_duration_
Definition: DBHandler.h:928
const size_t max_concurrent_render_sessions_
Definition: DBHandler.h:939
bool allow_multifrag_
Definition: DBHandler.h:598
static const std::string MAPD_RELEASE
Definition: release.h:43
const AuthMetadata & authMetadata_
Definition: DBHandler.h:605
std::default_random_engine random_gen_
Definition: DBHandler.h:594
std::uniform_int_distribution< int64_t > session_id_dist_
Definition: DBHandler.h:595
DBHandler::~DBHandler ( )
override

Definition at line 496 of file DBHandler.cpp.

References shutdown().

496  {
497  shutdown();
498 }
void shutdown()
Definition: DBHandler.cpp:7224

+ Here is the call graph for this function:

Member Function Documentation

void DBHandler::broadcast_serialized_rows ( const TSerializedRows &  serialized_rows,
const TRowDescriptor &  row_desc,
const TQueryId  query_id,
const TSubqueryId  subquery_id,
const bool  is_final_subquery_result 
)
override

Definition at line 6882 of file DBHandler.cpp.

References measure< TimeT >::execution(), logger::INFO, leaf_handler_, LOG, logger::query_id(), and THROW_MAPD_EXCEPTION.

6886  {
6887  if (!leaf_handler_) {
6888  THROW_MAPD_EXCEPTION("Distributed support is disabled.");
6889  }
6890  LOG(INFO) << "BROADCAST-SERIALIZED-ROWS id:" << query_id;
6891  auto time_ms = measure<>::execution([&]() {
6892  try {
6893  leaf_handler_->broadcast_serialized_rows(
6894  serialized_rows, row_desc, query_id, subquery_id, is_final_subquery_result);
6895  } catch (std::exception& e) {
6896  THROW_MAPD_EXCEPTION(e.what());
6897  }
6898  });
6899  LOG(INFO) << "BROADCAST-SERIALIZED-ROWS COMPLETED " << time_ms << "ms";
6900 }
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:205
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
std::unique_ptr< MapDLeafHandler > leaf_handler_
Definition: DBHandler.h:609
QueryId query_id()
Definition: Logger.cpp:469

+ Here is the call graph for this function:

void DBHandler::check_and_invalidate_sessions ( Parser::DDLStmt ddl)
private

Definition at line 6191 of file DBHandler.cpp.

References invalidate_sessions().

Referenced by sql_execute_impl().

6191  {
6192  const auto drop_db_stmt = dynamic_cast<Parser::DropDBStmt*>(ddl);
6193  if (drop_db_stmt) {
6194  invalidate_sessions(*drop_db_stmt->getDatabaseName(), drop_db_stmt);
6195  return;
6196  }
6197  const auto rename_db_stmt = dynamic_cast<Parser::RenameDBStmt*>(ddl);
6198  if (rename_db_stmt) {
6199  invalidate_sessions(*rename_db_stmt->getPreviousDatabaseName(), rename_db_stmt);
6200  return;
6201  }
6202  const auto drop_user_stmt = dynamic_cast<Parser::DropUserStmt*>(ddl);
6203  if (drop_user_stmt) {
6204  invalidate_sessions(*drop_user_stmt->getUserName(), drop_user_stmt);
6205  return;
6206  }
6207  const auto rename_user_stmt = dynamic_cast<Parser::RenameUserStmt*>(ddl);
6208  if (rename_user_stmt) {
6209  invalidate_sessions(*rename_user_stmt->getOldUserName(), rename_user_stmt);
6210  return;
6211  }
6212 }
void invalidate_sessions(std::string &name, STMT_TYPE *stmt)
Definition: DBHandler.h:1007

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::check_geospatial_files ( const boost::filesystem::path  file_path,
const import_export::CopyParams copy_params 
)
private

Definition at line 4789 of file DBHandler.cpp.

References import_export::Importer::gdalFileExists().

Referenced by detect_column_types(), and import_geo_table_internal().

4790  {
4791  const std::list<std::string> shp_ext{".shp", ".shx", ".dbf"};
4792  if (std::find(shp_ext.begin(),
4793  shp_ext.end(),
4794  boost::algorithm::to_lower_copy(file_path.extension().string())) !=
4795  shp_ext.end()) {
4796  for (auto ext : shp_ext) {
4797  auto aux_file = file_path;
4799  aux_file.replace_extension(boost::algorithm::to_upper_copy(ext)).string(),
4800  copy_params) &&
4802  aux_file.replace_extension(ext).string(), copy_params)) {
4803  throw std::runtime_error("required file for shapefile does not exist: " +
4804  aux_file.filename().string());
4805  }
4806  }
4807  }
4808 }
static bool gdalFileExists(const std::string &path, const CopyParams &copy_params)
Definition: Importer.cpp:4982

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::check_read_only ( const std::string &  str)
private

Definition at line 516 of file DBHandler.cpp.

References read_only_, and THROW_MAPD_EXCEPTION.

Referenced by create_custom_expression(), create_dashboard(), create_table(), delete_custom_expressions(), delete_dashboards(), detect_column_types(), import_geo_table_internal(), import_table(), prepare_loader_generic(), replace_dashboard(), set_license_key(), shareOrUnshareDashboards(), sql_execute_impl(), and update_custom_expression().

516  {
517  if (DBHandler::read_only_) {
518  THROW_MAPD_EXCEPTION(str + " disabled: server running in read-only mode.");
519  }
520 }
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
const bool read_only_
Definition: DBHandler.h:599

+ Here is the caller graph for this function:

void DBHandler::check_session_exp_unsafe ( const SessionMap::iterator &  session_it)
private

Definition at line 5751 of file DBHandler.cpp.

References idle_session_duration_, logger::INFO, isInMemoryCalciteSession(), LOG, max_session_duration_, and run_benchmark_import::start_time.

Referenced by expire_idle_sessions_unsafe(), and get_session_it_unsafe().

5751  {
5752  if (session_it->second.use_count() > 2 ||
5753  isInMemoryCalciteSession(session_it->second->get_currentUser())) {
5754  // SessionInfo is being used in more than one active operation. Original copy + one
5755  // stored in StdLog. Skip the checks.
5756  return;
5757  }
5758  time_t last_used_time = session_it->second->get_last_used_time();
5759  time_t start_time = session_it->second->get_start_time();
5760  const auto current_session_duration = time(0) - last_used_time;
5761  if (current_session_duration > idle_session_duration_) {
5762  LOG(INFO) << "Session " << session_it->second->get_public_session_id()
5763  << " idle duration " << current_session_duration
5764  << " seconds exceeds maximum idle duration " << idle_session_duration_
5765  << " seconds. Invalidating session.";
5766  throw ForceDisconnect("Idle Session Timeout. User should re-authenticate.");
5767  }
5768  const auto total_session_duration = time(0) - start_time;
5769  if (total_session_duration > max_session_duration_) {
5770  LOG(INFO) << "Session " << session_it->second->get_public_session_id()
5771  << " total duration " << total_session_duration
5772  << " seconds exceeds maximum total session duration "
5773  << max_session_duration_ << " seconds. Invalidating session.";
5774  throw ForceDisconnect("Maximum active Session Timeout. User should re-authenticate.");
5775  }
5776 }
#define LOG(tag)
Definition: Logger.h:205
const int max_session_duration_
Definition: DBHandler.h:929
const int idle_session_duration_
Definition: DBHandler.h:928
bool isInMemoryCalciteSession(const Catalog_Namespace::UserMetadata user_meta)
Definition: DBHandler.cpp:549

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::check_table_consistency ( TTableMeta &  _return,
const TSessionId &  session,
const int32_t  table_id 
)
override

Definition at line 6819 of file DBHandler.cpp.

References get_session_ptr(), leaf_handler_, STDLOG, and THROW_MAPD_EXCEPTION.

6821  {
6822  auto stdlog = STDLOG(get_session_ptr(session));
6823  if (!leaf_handler_) {
6824  THROW_MAPD_EXCEPTION("Distributed support is disabled.");
6825  }
6826  try {
6827  leaf_handler_->check_table_consistency(_return, session, table_id);
6828  } catch (std::exception& e) {
6829  THROW_MAPD_EXCEPTION(e.what());
6830  }
6831 }
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5801
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
std::unique_ptr< MapDLeafHandler > leaf_handler_
Definition: DBHandler.h:609
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

void DBHandler::check_table_load_privileges ( const TSessionId &  session,
const std::string &  table_name 
)
private

Definition at line 5835 of file DBHandler.cpp.

References get_session_copy().

Referenced by import_geo_table_internal(), import_table(), and prepare_loader_generic().

5836  {
5837  const auto session_info = get_session_copy(session);
5838  check_table_load_privileges(session_info, table_name);
5839 }
void check_table_load_privileges(const TSessionId &session, const std::string &table_name)
Definition: DBHandler.cpp:5835
Catalog_Namespace::SessionInfo get_session_copy(const TSessionId &session)
Definition: DBHandler.cpp:5783

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::check_table_load_privileges ( const Catalog_Namespace::SessionInfo session_info,
const std::string &  table_name 
)
private

Definition at line 5818 of file DBHandler.cpp.

References cat(), Catalog_Namespace::SessionInfo::get_currentUser(), Catalog_Namespace::SessionInfo::getCatalog(), AccessPrivileges::INSERT_INTO_TABLE, DBObject::loadKey(), DBObject::setPrivileges(), TableDBObjectType, and THROW_MAPD_EXCEPTION.

5820  {
5821  auto user_metadata = session_info.get_currentUser();
5822  auto& cat = session_info.getCatalog();
5823  DBObject dbObject(table_name, TableDBObjectType);
5824  dbObject.loadKey(cat);
5825  dbObject.setPrivileges(AccessPrivileges::INSERT_INTO_TABLE);
5826  std::vector<DBObject> privObjects;
5827  privObjects.push_back(dbObject);
5828  if (!SysCatalog::instance().checkPrivileges(user_metadata, privObjects)) {
5829  THROW_MAPD_EXCEPTION("Violation of access privileges: user " +
5830  user_metadata.userLoggable() +
5831  " has no insert privileges for table " + table_name + ".");
5832  }
5833 }
std::string cat(Ts &&...args)
static const AccessPrivileges INSERT_INTO_TABLE
Definition: DBObject.h:163
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
Catalog & getCatalog() const
Definition: SessionInfo.h:67
const UserMetadata & get_currentUser() const
Definition: SessionInfo.h:73

+ Here is the call graph for this function:

void DBHandler::checkpoint ( const TSessionId &  session,
const int32_t  table_id 
)
override

Definition at line 7059 of file DBHandler.cpp.

References cat(), get_session_ptr(), and STDLOG.

7059  {
7060  auto stdlog = STDLOG(get_session_ptr(session));
7061  auto session_ptr = stdlog.getConstSessionInfo();
7062  auto& cat = session_ptr->getCatalog();
7063  cat.checkpoint(table_id);
7064 }
std::string cat(Ts &&...args)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5801
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

void DBHandler::clear_cpu_memory ( const TSessionId &  session)
override

Definition at line 2686 of file DBHandler.cpp.

References LeafAggregator::clear_leaf_cpu_memory(), Executor::clearMemory(), Data_Namespace::CPU_LEVEL, get_session_ptr(), getConnectionInfo(), leaf_aggregator_, LeafAggregator::leafCount(), render_handler_, STDLOG, THROW_MAPD_EXCEPTION, and toString().

Referenced by executeDdl().

2686  {
2687  auto stdlog = STDLOG(get_session_ptr(session));
2688  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
2689  auto session_ptr = stdlog.getConstSessionInfo();
2690  if (!session_ptr->get_currentUser().isSuper) {
2691  THROW_MAPD_EXCEPTION("Superuser privilege is required to run clear_cpu_memory");
2692  }
2693  try {
2695  } catch (const std::exception& e) {
2696  THROW_MAPD_EXCEPTION(e.what());
2697  }
2698  if (render_handler_) {
2699  render_handler_->clear_cpu_memory();
2700  }
2701 
2702  if (leaf_aggregator_.leafCount() > 0) {
2704  }
2705 }
std::string toString(const ExtArgumentType &sig_type)
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:588
static void clearMemory(const Data_Namespace::MemoryLevel memory_level)
Definition: Execute.cpp:202
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5801
void clear_leaf_cpu_memory(const TSessionId session)
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:607
size_t leafCount() const
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1747
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::clear_gpu_memory ( const TSessionId &  session)
override

Definition at line 2665 of file DBHandler.cpp.

References LeafAggregator::clear_leaf_gpu_memory(), Executor::clearMemory(), get_session_ptr(), getConnectionInfo(), Data_Namespace::GPU_LEVEL, leaf_aggregator_, LeafAggregator::leafCount(), render_handler_, STDLOG, THROW_MAPD_EXCEPTION, and toString().

Referenced by executeDdl().

2665  {
2666  auto stdlog = STDLOG(get_session_ptr(session));
2667  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
2668  auto session_ptr = stdlog.getConstSessionInfo();
2669  if (!session_ptr->get_currentUser().isSuper) {
2670  THROW_MAPD_EXCEPTION("Superuser privilege is required to run clear_gpu_memory");
2671  }
2672  try {
2674  } catch (const std::exception& e) {
2675  THROW_MAPD_EXCEPTION(e.what());
2676  }
2677  if (render_handler_) {
2678  render_handler_->clear_gpu_memory();
2679  }
2680 
2681  if (leaf_aggregator_.leafCount() > 0) {
2683  }
2684 }
std::string toString(const ExtArgumentType &sig_type)
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:588
void clear_leaf_gpu_memory(const TSessionId session)
static void clearMemory(const Data_Namespace::MemoryLevel memory_level)
Definition: Execute.cpp:202
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5801
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:607
size_t leafCount() const
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1747
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::clearRenderMemory ( const TSessionId &  session)

Definition at line 2707 of file DBHandler.cpp.

References get_session_ptr(), getConnectionInfo(), render_handler_, STDLOG, THROW_MAPD_EXCEPTION, and toString().

Referenced by executeDdl().

2707  {
2708  auto stdlog = STDLOG(get_session_ptr(session));
2709  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
2710  auto session_ptr = stdlog.getConstSessionInfo();
2711  if (!session_ptr->get_currentUser().isSuper) {
2712  THROW_MAPD_EXCEPTION("Superuser privilege is required to run clear_render_memory");
2713  }
2714  if (render_handler_) {
2715  render_handler_->clear_cpu_memory();
2716  render_handler_->clear_gpu_memory();
2717  }
2718 }
std::string toString(const ExtArgumentType &sig_type)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5801
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:607
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1747
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::clone_session ( TSessionId &  session2,
const TSessionId &  session1 
)
override

Definition at line 765 of file DBHandler.cpp.

References cat(), LeafAggregator::clone_session(), create_new_session(), get_session_it_unsafe(), get_session_ptr(), getConnectionInfo(), leaf_aggregator_, LeafAggregator::leafCount(), sessions_mutex_, STDLOG, THROW_MAPD_EXCEPTION, and toString().

765  {
766  auto stdlog = STDLOG(get_session_ptr(session1));
767  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
768  mapd_unique_lock<mapd_shared_mutex> write_lock(sessions_mutex_);
769  auto session_it = get_session_it_unsafe(session1, write_lock);
770 
771  try {
772  const Catalog_Namespace::UserMetadata& user_meta =
773  session_it->second->get_currentUser();
774  std::shared_ptr<Catalog> cat = session_it->second->get_catalog_ptr();
775  auto session2_ptr = create_new_session(session2, cat->name(), user_meta, cat);
776  if (leaf_aggregator_.leafCount() > 0) {
777  leaf_aggregator_.clone_session(session1, session2);
778  return;
779  }
780  } catch (std::exception& e) {
781  THROW_MAPD_EXCEPTION(e.what());
782  }
783 }
std::string cat(Ts &&...args)
std::string toString(const ExtArgumentType &sig_type)
void clone_session(const TSessionId session1, const TSessionId session2)
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:588
std::shared_ptr< Catalog_Namespace::SessionInfo > create_new_session(TSessionId &session, const std::string &dbname, const Catalog_Namespace::UserMetadata &user_meta, std::shared_ptr< Catalog_Namespace::Catalog > cat)
Definition: DBHandler.cpp:644
mapd_shared_mutex sessions_mutex_
Definition: DBHandler.h:602
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5801
size_t leafCount() const
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
SessionMap::iterator get_session_it_unsafe(const TSessionId &session, SESSION_MAP_LOCK &lock)
mapd_unique_lock< mapd_shared_mutex > write_lock
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1747
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

void DBHandler::connect ( TSessionId &  session,
const std::string &  username,
const std::string &  passwd,
const std::string &  dbname 
)
override

Definition at line 602 of file DBHandler.cpp.

References AccessPrivileges::ACCESS, cat(), connect_impl(), DatabaseDBObjectType, get_session_ptr(), getConnectionInfo(), DBObject::loadKey(), Catalog_Namespace::UserMetadata::restriction, DBObject::setPrivileges(), STDLOG, super_user_rights_, THROW_MAPD_EXCEPTION, toString(), and Catalog_Namespace::UserMetadata::userLoggable().

605  {
606  auto stdlog = STDLOG(); // session_info set by connect_impl()
607  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
608  std::string username2 = username; // login() may reset username given as argument
609  std::string dbname2 = dbname; // login() may reset dbname given as argument
611  std::shared_ptr<Catalog> cat = nullptr;
612  try {
613  cat = SysCatalog::instance().login(
614  dbname2, username2, passwd, user_meta, !super_user_rights_);
615  } catch (std::exception& e) {
616  stdlog.appendNameValuePairs("user", username, "db", dbname, "exception", e.what());
617  THROW_MAPD_EXCEPTION(e.what());
618  }
619 
620  DBObject dbObject(dbname2, DatabaseDBObjectType);
621  dbObject.loadKey(*cat);
622  dbObject.setPrivileges(AccessPrivileges::ACCESS);
623  std::vector<DBObject> dbObjects;
624  dbObjects.push_back(dbObject);
625  if (!SysCatalog::instance().checkPrivileges(user_meta, dbObjects)) {
626  stdlog.appendNameValuePairs(
627  "user", username, "db", dbname, "exception", "Missing Privileges");
628  THROW_MAPD_EXCEPTION("Unauthorized Access: user " + user_meta.userLoggable() +
629  " is not allowed to access database " + dbname2 + ".");
630  }
631  connect_impl(session, passwd, dbname2, user_meta, cat, stdlog);
632 
633  // Restriction is returned as part of the users metadata on login but
634  // is per session so transfering it over here
635  // Currently only SAML can even set a Restriction
636  auto restriction = std::make_shared<Restriction>(user_meta.restriction);
637  auto login_session = get_session_ptr(session);
638  login_session->set_restriction(restriction);
639 
640  // if pki auth session will come back encrypted with user pubkey
641  SysCatalog::instance().check_for_session_encryption(passwd, session);
642 }
std::string cat(Ts &&...args)
std::string toString(const ExtArgumentType &sig_type)
void connect_impl(TSessionId &session, const std::string &passwd, const std::string &dbname, const Catalog_Namespace::UserMetadata &user_meta, std::shared_ptr< Catalog_Namespace::Catalog > cat, query_state::StdLog &stdlog)
Definition: DBHandler.cpp:662
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5801
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
bool super_user_rights_
Definition: DBHandler.h:925
static const AccessPrivileges ACCESS
Definition: DBObject.h:155
std::string userLoggable() const
Definition: SysCatalog.cpp:127
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1747
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

void DBHandler::connect_impl ( TSessionId &  session,
const std::string &  passwd,
const std::string &  dbname,
const Catalog_Namespace::UserMetadata user_meta,
std::shared_ptr< Catalog_Namespace::Catalog cat,
query_state::StdLog stdlog 
)
private

Definition at line 662 of file DBHandler.cpp.

References query_state::StdLog::appendNameValuePairs(), LeafAggregator::connect(), create_new_session(), expire_idle_sessions_unsafe(), getConnectionInfo(), query_state::StdLog::getConstSessionInfo(), join(), leaf_aggregator_, LeafAggregator::leafCount(), SystemParameters::num_sessions, sessions_, sessions_mutex_, query_state::StdLog::setSessionInfo(), super_user_rights_, system_parameters_, THROW_MAPD_EXCEPTION, toString(), and Catalog_Namespace::UserMetadata::userName.

Referenced by connect(), and internal_connect().

667  {
668  // TODO(sy): Is there any reason to have dbname as a parameter
669  // here when the cat parameter already provides cat->name()?
670  // Should dbname and cat->name() ever differ?
671  {
672  mapd_unique_lock<mapd_shared_mutex> write_lock(sessions_mutex_);
675  sessions_.size() + 1 > static_cast<size_t>(system_parameters_.num_sessions)) {
676  THROW_MAPD_EXCEPTION("Too many active sessions");
677  }
678  }
679  {
680  mapd_lock_guard<mapd_shared_mutex> write_lock(sessions_mutex_);
681  auto session_ptr = create_new_session(session, dbname, user_meta, cat);
682  stdlog.setSessionInfo(session_ptr);
683  session_ptr->set_connection_info(getConnectionInfo().toString());
684  if (!super_user_rights_) { // no need to connect to leaf_aggregator_ at this time
685  // while doing warmup
686  if (leaf_aggregator_.leafCount() > 0) {
687  leaf_aggregator_.connect(*session_ptr, user_meta.userName, passwd, dbname);
688  return;
689  }
690  }
691  }
692  auto const roles =
693  stdlog.getConstSessionInfo()->get_currentUser().isSuper
694  ? std::vector<std::string>{{"super"}}
695  : SysCatalog::instance().getRoles(
696  false, false, stdlog.getConstSessionInfo()->get_currentUser().userName);
697  stdlog.appendNameValuePairs("roles", boost::algorithm::join(roles, ","));
698 }
std::string cat(Ts &&...args)
std::string toString(const ExtArgumentType &sig_type)
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:588
std::shared_ptr< Catalog_Namespace::SessionInfo > create_new_session(TSessionId &session, const std::string &dbname, const Catalog_Namespace::UserMetadata &user_meta, std::shared_ptr< Catalog_Namespace::Catalog > cat)
Definition: DBHandler.cpp:644
std::string join(T const &container, std::string const &delim)
void connect(const Catalog_Namespace::SessionInfo &parent_session_info, const std::string &user, const std::string &passwd, const std::string &dbname)
void setSessionInfo(std::shared_ptr< Catalog_Namespace::SessionInfo >)
Definition: QueryState.cpp:286
mapd_shared_mutex sessions_mutex_
Definition: DBHandler.h:602
void expire_idle_sessions_unsafe(SESSION_MAP_LOCK &lock)
SystemParameters & system_parameters_
Definition: DBHandler.h:606
size_t leafCount() const
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
void appendNameValuePairs(Pairs &&...pairs)
Definition: QueryState.h:313
bool super_user_rights_
Definition: DBHandler.h:925
mapd_unique_lock< mapd_shared_mutex > write_lock
std::shared_ptr< Catalog_Namespace::SessionInfo const > getConstSessionInfo() const
Definition: QueryState.cpp:159
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1747
SessionMap sessions_
Definition: DBHandler.h:923

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::convertData ( TQueryResult &  _return,
ExecutionResult result,
const QueryStateProxy query_state_proxy,
const std::string &  query_str,
const bool  column_format,
const int32_t  first_n,
const int32_t  at_most_n 
)
staticprivate

Definition at line 1221 of file DBHandler.cpp.

References ExecutionResult::CalciteDdl, convertExplain(), convertResult(), convertRows(), ExecutionResult::empty(), ExecutionResult::Explaination, ExecutionResult::getExecutionTime(), ExecutionResult::getResultType(), ExecutionResult::getRows(), ExecutionResult::getTargetsMeta(), ExecutionResult::QueryResult, and ExecutionResult::SimpleResult.

Referenced by get_tables_meta_impl(), sql_execute_local(), and validate_rel_alg().

1227  {
1228  _return.execution_time_ms += result.getExecutionTime();
1229  if (result.empty()) {
1230  return;
1231  }
1232 
1233  switch (result.getResultType()) {
1235  convertRows(_return,
1236  query_state_proxy,
1237  result.getTargetsMeta(),
1238  *result.getRows(),
1239  column_format,
1240  first_n,
1241  at_most_n);
1242  break;
1244  convertResult(_return, *result.getRows(), true);
1245  break;
1247  convertExplain(_return, *result.getRows(), true);
1248  break;
1250  convertRows(_return,
1251  query_state_proxy,
1252  result.getTargetsMeta(),
1253  *result.getRows(),
1254  column_format,
1255  -1,
1256  -1);
1257  break;
1258  }
1259 }
RType getResultType() const
static void convertExplain(TQueryResult &_return, const ResultSet &results, const bool column_format)
Definition: DBHandler.cpp:6164
static void convertResult(TQueryResult &_return, const ResultSet &results, const bool column_format)
Definition: DBHandler.cpp:6170
const std::vector< TargetMetaInfo > & getTargetsMeta() const
const std::shared_ptr< ResultSet > & getRows() const
static void convertRows(TQueryResult &_return, QueryStateProxy query_state_proxy, const std::vector< TargetMetaInfo > &targets, const ResultSet &results, const bool column_format, const int32_t first_n, const int32_t at_most_n)
Definition: DBHandler.cpp:6056
int64_t getExecutionTime() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::convertExplain ( TQueryResult &  _return,
const ResultSet results,
const bool  column_format 
)
staticprivate

Definition at line 6164 of file DBHandler.cpp.

References createSimpleResult().

Referenced by convertData().

6166  {
6167  createSimpleResult(_return, results, column_format, "Explanation");
6168 }
static void createSimpleResult(TQueryResult &_return, const ResultSet &results, const bool column_format, const std::string label)
Definition: DBHandler.cpp:6127

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::convertResult ( TQueryResult &  _return,
const ResultSet results,
const bool  column_format 
)
staticprivate

Definition at line 6170 of file DBHandler.cpp.

References createSimpleResult().

Referenced by convertData(), and executeDdl().

6172  {
6173  createSimpleResult(_return, results, column_format, "Result");
6174 }
static void createSimpleResult(TQueryResult &_return, const ResultSet &results, const bool column_format, const std::string label)
Definition: DBHandler.cpp:6127

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::convertResultSet ( ExecutionResult result,
const Catalog_Namespace::SessionInfo session_info,
const std::string &  query_state_str,
TQueryResult &  _return 
)
private

Definition at line 7344 of file DBHandler.cpp.

References CHECK, convertRows(), create_query_state(), Catalog_Namespace::SessionInfo::get_session_id(), get_session_ptr(), ExecutionResult::getDataPtr(), and ExecutionResult::getTargetsMeta().

Referenced by executeDdl().

7347  {
7348  // Stuff ResultSet into _return (which is a TQueryResult)
7349  // calls convertRows, but after some setup using session_info
7350 
7351  auto session_ptr = get_session_ptr(session_info.get_session_id());
7352  CHECK(session_ptr);
7353  auto qs = create_query_state(session_ptr, query_state_str);
7354  QueryStateProxy qsp = qs->createQueryStateProxy();
7355 
7356  // omnisql only accepts column format as being 'VALID",
7357  // assume that omnisci_server should only return column format
7358  int32_t nRows = result.getDataPtr()->rowCount();
7359 
7360  convertRows(_return,
7361  qsp,
7362  result.getTargetsMeta(),
7363  *result.getDataPtr(),
7364  /*column_format=*/true,
7365  /*first_n=*/nRows,
7366  /*at_most_n=*/nRows);
7367 }
std::shared_ptr< query_state::QueryState > create_query_state(ARGS &&...args)
Definition: DBHandler.h:616
const ResultSetPtr & getDataPtr() const
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5801
const std::vector< TargetMetaInfo > & getTargetsMeta() const
std::string get_session_id() const
Definition: SessionInfo.h:78
static void convertRows(TQueryResult &_return, QueryStateProxy query_state_proxy, const std::vector< TargetMetaInfo > &targets, const ResultSet &results, const bool column_format, const int32_t first_n, const int32_t at_most_n)
Definition: DBHandler.cpp:6056
#define CHECK(condition)
Definition: Logger.h:211

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::convertRows ( TQueryResult &  _return,
QueryStateProxy  query_state_proxy,
const std::vector< TargetMetaInfo > &  targets,
const ResultSet results,
const bool  column_format,
const int32_t  first_n,
const int32_t  at_most_n 
)
staticprivate

Definition at line 6056 of file DBHandler.cpp.

References query_state::QueryStateProxy::createTimer(), i, ThriftSerializers::target_meta_infos_to_thrift(), THROW_MAPD_EXCEPTION, to_string(), value_to_thrift(), and value_to_thrift_column().

Referenced by convertData(), and convertResultSet().

6062  {
6063  query_state::Timer timer = query_state_proxy.createTimer(__func__);
6064  _return.row_set.row_desc = ThriftSerializers::target_meta_infos_to_thrift(targets);
6065  int32_t fetched{0};
6066  if (column_format) {
6067  _return.row_set.is_columnar = true;
6068  std::vector<TColumn> tcolumns(results.colCount());
6069  while (first_n == -1 || fetched < first_n) {
6070  const auto crt_row = results.getNextRow(true, true);
6071  if (crt_row.empty()) {
6072  break;
6073  }
6074  ++fetched;
6075  if (at_most_n >= 0 && fetched > at_most_n) {
6076  THROW_MAPD_EXCEPTION("The result contains more rows than the specified cap of " +
6077  std::to_string(at_most_n));
6078  }
6079  for (size_t i = 0; i < results.colCount(); ++i) {
6080  const auto agg_result = crt_row[i];
6081  value_to_thrift_column(agg_result, targets[i].get_type_info(), tcolumns[i]);
6082  }
6083  }
6084  for (size_t i = 0; i < results.colCount(); ++i) {
6085  _return.row_set.columns.push_back(tcolumns[i]);
6086  }
6087  } else {
6088  _return.row_set.is_columnar = false;
6089  while (first_n == -1 || fetched < first_n) {
6090  const auto crt_row = results.getNextRow(true, true);
6091  if (crt_row.empty()) {
6092  break;
6093  }
6094  ++fetched;
6095  if (at_most_n >= 0 && fetched > at_most_n) {
6096  THROW_MAPD_EXCEPTION("The result contains more rows than the specified cap of " +
6097  std::to_string(at_most_n));
6098  }
6099  TRow trow;
6100  trow.cols.reserve(results.colCount());
6101  for (size_t i = 0; i < results.colCount(); ++i) {
6102  const auto agg_result = crt_row[i];
6103  trow.cols.push_back(value_to_thrift(agg_result, targets[i].get_type_info()));
6104  }
6105  _return.row_set.rows.push_back(trow);
6106  }
6107  }
6108 }
TRowDescriptor target_meta_infos_to_thrift(const std::vector< TargetMetaInfo > &targets)
static void value_to_thrift_column(const TargetValue &tv, const SQLTypeInfo &ti, TColumn &column)
Definition: DBHandler.cpp:956
Timer createTimer(char const *event_name)
Definition: QueryState.cpp:138
std::string to_string(char const *&&v)
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
static TDatum value_to_thrift(const TargetValue &tv, const SQLTypeInfo &ti)
Definition: DBHandler.cpp:1079

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

TCopyParams DBHandler::copyparams_to_thrift ( const import_export::CopyParams cp)
private

Definition at line 3744 of file DBHandler.cpp.

References import_export::CopyParams::array_begin, import_export::CopyParams::array_delim, import_export::CopyParams::array_end, CHECK, import_export::CopyParams::delimiter, DOUBLE, import_export::CopyParams::escape, FLOAT, import_export::CopyParams::geo_assign_render_groups, import_export::CopyParams::geo_coords_comp_param, import_export::CopyParams::geo_coords_encoding, import_export::CopyParams::geo_coords_srid, import_export::CopyParams::geo_coords_type, import_export::CopyParams::geo_explode_collections, import_export::CopyParams::geo_layer_name, GEOGRAPHY, GEOMETRY, import_export::CopyParams::has_header, import_export::kAuto, import_export::kAutoDetect, import_export::kDelimitedFile, import_export::kDouble, kENCODING_GEOINT, import_export::kFile, import_export::kFloat, import_export::kGeoFile, kGEOGRAPHY, kGEOMETRY, import_export::kHasHeader, import_export::kInt, import_export::kNoHeader, import_export::kNone, import_export::kParquetFile, import_export::kPoint, import_export::kRasterFile, import_export::kSmallInt, import_export::kWorld, import_export::CopyParams::line_delim, import_export::CopyParams::null_str, POINT, import_export::CopyParams::quote, import_export::CopyParams::quoted, import_export::CopyParams::raster_import_bands, import_export::CopyParams::raster_point_compute_angle, import_export::CopyParams::raster_point_transform, import_export::CopyParams::raster_point_type, import_export::CopyParams::raster_scanlines_per_thread, import_export::CopyParams::s3_access_key, import_export::CopyParams::s3_endpoint, import_export::CopyParams::s3_region, import_export::CopyParams::s3_secret_key, import_export::CopyParams::s3_session_token, import_export::CopyParams::sanitize_column_names, SMALLINT, import_export::CopyParams::source_srid, import_export::CopyParams::source_type, and import_export::CopyParams::threads.

Referenced by detect_column_types(), and process_deferred_copy_from().

3744  {
3745  TCopyParams copy_params;
3746  copy_params.delimiter = cp.delimiter;
3747  copy_params.null_str = cp.null_str;
3748  switch (cp.has_header) {
3750  copy_params.has_header = TImportHeaderRow::AUTODETECT;
3751  break;
3753  copy_params.has_header = TImportHeaderRow::NO_HEADER;
3754  break;
3756  copy_params.has_header = TImportHeaderRow::HAS_HEADER;
3757  break;
3758  default:
3759  CHECK(false);
3760  }
3761  copy_params.quoted = cp.quoted;
3762  copy_params.quote = cp.quote;
3763  copy_params.escape = cp.escape;
3764  copy_params.line_delim = cp.line_delim;
3765  copy_params.array_delim = cp.array_delim;
3766  copy_params.array_begin = cp.array_begin;
3767  copy_params.array_end = cp.array_end;
3768  copy_params.threads = cp.threads;
3769  copy_params.s3_access_key = cp.s3_access_key;
3770  copy_params.s3_secret_key = cp.s3_secret_key;
3771  copy_params.s3_session_token = cp.s3_session_token;
3772  copy_params.s3_region = cp.s3_region;
3773  copy_params.s3_endpoint = cp.s3_endpoint;
3774  switch (cp.source_type) {
3776  copy_params.file_type = TFileType::DELIMITED;
3777  break;
3779  copy_params.file_type = TFileType::GEO;
3780  break;
3782  copy_params.file_type = TFileType::PARQUET;
3783  break;
3785  copy_params.file_type = TFileType::RASTER;
3786  break;
3787  default:
3788  CHECK(false);
3789  }
3790  switch (cp.geo_coords_encoding) {
3791  case kENCODING_GEOINT:
3792  copy_params.geo_coords_encoding = TEncodingType::GEOINT;
3793  break;
3794  default:
3795  copy_params.geo_coords_encoding = TEncodingType::NONE;
3796  break;
3797  }
3798  copy_params.geo_coords_comp_param = cp.geo_coords_comp_param;
3799  switch (cp.geo_coords_type) {
3800  case kGEOGRAPHY:
3801  copy_params.geo_coords_type = TDatumType::GEOGRAPHY;
3802  break;
3803  case kGEOMETRY:
3804  copy_params.geo_coords_type = TDatumType::GEOMETRY;
3805  break;
3806  default:
3807  CHECK(false);
3808  }
3809  copy_params.geo_coords_srid = cp.geo_coords_srid;
3810  copy_params.sanitize_column_names = cp.sanitize_column_names;
3811  copy_params.geo_layer_name = cp.geo_layer_name;
3812  copy_params.geo_assign_render_groups = cp.geo_assign_render_groups;
3813  copy_params.geo_explode_collections = cp.geo_explode_collections;
3814  copy_params.source_srid = cp.source_srid;
3815  switch (cp.raster_point_type) {
3817  copy_params.raster_point_type = TRasterPointType::NONE;
3818  break;
3820  copy_params.raster_point_type = TRasterPointType::AUTO;
3821  break;
3823  copy_params.raster_point_type = TRasterPointType::SMALLINT;
3824  break;
3826  copy_params.raster_point_type = TRasterPointType::INT;
3827  break;
3829  copy_params.raster_point_type = TRasterPointType::FLOAT;
3830  break;
3832  copy_params.raster_point_type = TRasterPointType::DOUBLE;
3833  break;
3835  copy_params.raster_point_type = TRasterPointType::POINT;
3836  break;
3837  default:
3838  CHECK(false);
3839  }
3840  copy_params.raster_import_bands = cp.raster_import_bands;
3841  copy_params.raster_scanlines_per_thread = cp.raster_scanlines_per_thread;
3842  switch (cp.raster_point_transform) {
3844  copy_params.raster_point_transform = TRasterPointTransform::NONE;
3845  break;
3847  copy_params.raster_point_transform = TRasterPointTransform::AUTO;
3848  break;
3850  copy_params.raster_point_transform = TRasterPointTransform::FILE;
3851  break;
3853  copy_params.raster_point_transform = TRasterPointTransform::WORLD;
3854  break;
3855  default:
3856  CHECK(false);
3857  }
3858  copy_params.raster_point_compute_angle = cp.raster_point_compute_angle;
3859  return copy_params;
3860 }
std::string s3_secret_key
Definition: CopyParams.h:60
int32_t raster_scanlines_per_thread
Definition: CopyParams.h:85
#define GEOMETRY
#define SMALLINT
#define DOUBLE
#define POINT
ImportHeaderRow has_header
Definition: CopyParams.h:45
RasterPointType raster_point_type
Definition: CopyParams.h:83
import_export::SourceType source_type
Definition: CopyParams.h:56
std::string geo_layer_name
Definition: CopyParams.h:76
std::string s3_session_token
Definition: CopyParams.h:61
std::string raster_import_bands
Definition: CopyParams.h:84
#define CHECK(condition)
Definition: Logger.h:211
#define GEOGRAPHY
#define FLOAT
std::string s3_access_key
Definition: CopyParams.h:59
RasterPointTransform raster_point_transform
Definition: CopyParams.h:86
EncodingType geo_coords_encoding
Definition: CopyParams.h:71

+ Here is the caller graph for this function:

int32_t DBHandler::create_custom_expression ( const TSessionId &  session,
const TCustomExpression &  custom_expression 
)
override

Definition at line 4280 of file DBHandler.cpp.

References check_read_only(), anonymous_namespace{DBHandler.cpp}::create_custom_expr_from_thrift_obj(), custom_expressions_mutex_, get_session_ptr(), getConnectionInfo(), STDLOG, THROW_MAPD_EXCEPTION, and toString().

4281  {
4282  auto stdlog = STDLOG(get_session_ptr(session));
4283  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4284  check_read_only("create_custom_expression");
4285 
4286  auto session_ptr = stdlog.getConstSessionInfo();
4287  if (!session_ptr->get_currentUser().isSuper) {
4288  THROW_MAPD_EXCEPTION("Custom expressions can only be created by super users.")
4289  }
4290  auto& catalog = session_ptr->getCatalog();
4291  mapd_unique_lock<mapd_shared_mutex> write_lock(custom_expressions_mutex_);
4292  return catalog.createCustomExpression(
4293  create_custom_expr_from_thrift_obj(t_custom_expr, catalog));
4294 }
std::string toString(const ExtArgumentType &sig_type)
mapd_shared_mutex custom_expressions_mutex_
Definition: DBHandler.h:1072
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5801
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
void check_read_only(const std::string &str)
Definition: DBHandler.cpp:516
std::unique_ptr< Catalog_Namespace::CustomExpression > create_custom_expr_from_thrift_obj(const TCustomExpression &t_custom_expr, const Catalog &catalog)
Definition: DBHandler.cpp:4237
mapd_unique_lock< mapd_shared_mutex > write_lock
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1747
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

int32_t DBHandler::create_dashboard ( const TSessionId &  session,
const std::string &  dashboard_name,
const std::string &  dashboard_state,
const std::string &  image_hash,
const std::string &  dashboard_metadata 
)
override

Definition at line 4468 of file DBHandler.cpp.

References cat(), anonymous_namespace{DBHandler.cpp}::check_not_info_schema_db(), check_read_only(), AccessPrivileges::CREATE_DASHBOARD, DashboardDBObjectType, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, g_allow_system_dashboard_update, get_session_ptr(), getConnectionInfo(), DashboardDescriptor::imageHash, STDLOG, THROW_MAPD_EXCEPTION, to_string(), toString(), DashboardDescriptor::user, and DashboardDescriptor::userId.

4472  {
4473  auto stdlog = STDLOG(get_session_ptr(session));
4474  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4475  auto session_ptr = stdlog.getConstSessionInfo();
4476  check_read_only("create_dashboard");
4477  auto& cat = session_ptr->getCatalog();
4479  check_not_info_schema_db(cat.name(), true);
4480  }
4481 
4482  if (!session_ptr->checkDBAccessPrivileges(DBObjectType::DashboardDBObjectType,
4484  THROW_MAPD_EXCEPTION("Not enough privileges to create a dashboard.");
4485  }
4486 
4487  auto dash = cat.getMetadataForDashboard(
4488  std::to_string(session_ptr->get_currentUser().userId), dashboard_name);
4489  if (dash) {
4490  THROW_MAPD_EXCEPTION("Dashboard with name: " + dashboard_name + " already exists.");
4491  }
4492 
4494  dd.dashboardName = dashboard_name;
4495  dd.dashboardState = dashboard_state;
4496  dd.imageHash = image_hash;
4497  dd.dashboardMetadata = dashboard_metadata;
4498  dd.userId = session_ptr->get_currentUser().userId;
4499  dd.user = session_ptr->get_currentUser().userName;
4500 
4501  try {
4502  auto id = cat.createDashboard(dd);
4503  // TODO: transactionally unsafe
4504  SysCatalog::instance().createDBObject(
4505  session_ptr->get_currentUser(), dashboard_name, DashboardDBObjectType, cat, id);
4506  return id;
4507  } catch (const std::exception& e) {
4508  THROW_MAPD_EXCEPTION(e.what());
4509  }
4510 }
std::string cat(Ts &&...args)
std::string toString(const ExtArgumentType &sig_type)
static const AccessPrivileges CREATE_DASHBOARD
Definition: DBObject.h:172
std::string to_string(char const *&&v)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5801
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
void check_read_only(const std::string &str)
Definition: DBHandler.cpp:516
bool g_allow_system_dashboard_update
Definition: DBHandler.cpp:121
void check_not_info_schema_db(const std::string &db_name, bool throw_mapd_exception=false)
Definition: DBHandler.cpp:4454
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1747
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

TColumnType DBHandler::create_geo_column ( const TDatumType::type  type,
const std::string &  name,
const bool  is_array 
)
private

Definition at line 4779 of file DBHandler.cpp.

References setup::name, and run_benchmark_import::type.

4781  {
4782  TColumnType ct;
4783  ct.col_name = name;
4784  ct.col_type.type = type;
4785  ct.col_type.is_array = is_array;
4786  return ct;
4787 }
string name
Definition: setup.in.py:72
void DBHandler::create_link ( std::string &  _return,
const TSessionId &  session,
const std::string &  view_state,
const std::string &  view_metadata 
)
override

Definition at line 4757 of file DBHandler.cpp.

References cat(), get_session_ptr(), getConnectionInfo(), STDLOG, THROW_MAPD_EXCEPTION, toString(), LinkDescriptor::userId, LinkDescriptor::viewMetadata, and LinkDescriptor::viewState.

4760  {
4761  auto stdlog = STDLOG(get_session_ptr(session));
4762  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4763  auto session_ptr = stdlog.getConstSessionInfo();
4764  // check_read_only("create_link");
4765  auto& cat = session_ptr->getCatalog();
4766 
4767  LinkDescriptor ld;
4768  ld.userId = session_ptr->get_currentUser().userId;
4769  ld.viewState = view_state;
4770  ld.viewMetadata = view_metadata;
4771 
4772  try {
4773  _return = cat.createLink(ld, 6);
4774  } catch (const std::exception& e) {
4775  THROW_MAPD_EXCEPTION(e.what());
4776  }
4777 }
std::string cat(Ts &&...args)
std::string toString(const ExtArgumentType &sig_type)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5801
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1747
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

std::shared_ptr< Catalog_Namespace::SessionInfo > DBHandler::create_new_session ( TSessionId &  session,
const std::string &  dbname,
const Catalog_Namespace::UserMetadata user_meta,
std::shared_ptr< Catalog_Namespace::Catalog cat 
)
private

Definition at line 644 of file DBHandler.cpp.

References CHECK, executor_device_type_, generate_random_string(), logger::INFO, LOG, sessions_, and Catalog_Namespace::UserMetadata::userLoggable().

Referenced by clone_session(), and connect_impl().

648  {
649  do {
650  session = generate_random_string(32);
651  } while (sessions_.find(session) != sessions_.end());
652  std::pair<SessionMap::iterator, bool> emplace_retval =
653  sessions_.emplace(session,
654  std::make_shared<Catalog_Namespace::SessionInfo>(
655  cat, user_meta, executor_device_type_, session));
656  CHECK(emplace_retval.second);
657  auto& session_ptr = emplace_retval.first->second;
658  LOG(INFO) << "User " << user_meta.userLoggable() << " connected to database " << dbname;
659  return session_ptr;
660 }
std::string cat(Ts &&...args)
#define LOG(tag)
Definition: Logger.h:205
std::string generate_random_string(const size_t len)
ExecutorDeviceType executor_device_type_
Definition: DBHandler.h:593
#define CHECK(condition)
Definition: Logger.h:211
std::string userLoggable() const
Definition: SysCatalog.cpp:127
SessionMap sessions_
Definition: DBHandler.h:923

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename... ARGS>
std::shared_ptr<query_state::QueryState> DBHandler::create_query_state ( ARGS &&...  args)
inline

Definition at line 616 of file DBHandler.h.

References run_benchmark_import::args.

Referenced by convertResultSet(), get_table_details_impl(), get_tables_meta(), sql_execute(), sql_execute_df(), and sql_validate().

616  {
617  return query_states_.create(std::forward<ARGS>(args)...);
618  }
CircleBuffer::value_type create(ARGS &&...args)
Definition: QueryState.h:195
query_state::QueryStates query_states_
Definition: DBHandler.h:922

+ Here is the caller graph for this function:

void DBHandler::create_table ( const TSessionId &  session,
const std::string &  table_name,
const TRowDescriptor &  row_desc,
const TFileType::type  file_type,
const TCreateParams &  create_params 
)
override

Definition at line 4810 of file DBHandler.cpp.

References check_read_only(), DECIMAL, getConnectionInfo(), ImportHelpers::is_reserved_name(), join(), kENCODING_DICT, kENCODING_FIXED, kENCODING_GEOINT, kENCODING_NONE, LINESTRING, MULTIPOLYGON, POINT, POLYGON, ImportHelpers::sanitize_name(), sql_execute(), STDLOG, thrift_to_encoding(), thrift_to_encoding_name(), thrift_to_name(), THROW_MAPD_EXCEPTION, to_string(), and toString().

Referenced by import_geo_table_internal().

4814  {
4815  // @TODO(se) remove file_type which is unused
4816  auto stdlog = STDLOG("table_name", table_name);
4817  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4818  check_read_only("create_table");
4819 
4820  if (ImportHelpers::is_reserved_name(table_name)) {
4821  THROW_MAPD_EXCEPTION("Invalid table name (reserved keyword): " + table_name);
4822  } else if (table_name != ImportHelpers::sanitize_name(table_name)) {
4823  THROW_MAPD_EXCEPTION("Invalid characters in table name: " + table_name);
4824  }
4825 
4826  auto rds = rd;
4827 
4828  std::string stmt{"CREATE TABLE " + table_name};
4829  std::vector<std::string> col_stmts;
4830 
4831  for (auto col : rds) {
4832  if (ImportHelpers::is_reserved_name(col.col_name)) {
4833  THROW_MAPD_EXCEPTION("Invalid column name (reserved keyword): " + col.col_name);
4834  } else if (col.col_name != ImportHelpers::sanitize_name(col.col_name)) {
4835  THROW_MAPD_EXCEPTION("Invalid characters in column name: " + col.col_name);
4836  }
4837  if (col.col_type.type == TDatumType::INTERVAL_DAY_TIME ||
4838  col.col_type.type == TDatumType::INTERVAL_YEAR_MONTH) {
4839  THROW_MAPD_EXCEPTION("Unsupported type: " + thrift_to_name(col.col_type) +
4840  " for column: " + col.col_name);
4841  }
4842 
4843  if (col.col_type.type == TDatumType::DECIMAL) {
4844  // if no precision or scale passed in set to default 14,7
4845  if (col.col_type.precision == 0 && col.col_type.scale == 0) {
4846  col.col_type.precision = 14;
4847  col.col_type.scale = 7;
4848  }
4849  }
4850 
4851  std::string col_stmt;
4852  col_stmt.append(col.col_name + " " + thrift_to_name(col.col_type));
4853  if (col.__isset.default_value) {
4854  col_stmt.append(" DEFAULT " + col.default_value);
4855  }
4856 
4857  // As of 2016-06-27 the Immerse v1 frontend does not explicitly set the
4858  // `nullable` argument, leading this to default to false. Uncomment for v2.
4859  // if (!col.col_type.nullable) col_stmt.append(" NOT NULL");
4860 
4861  if (thrift_to_encoding(col.col_type.encoding) != kENCODING_NONE) {
4862  col_stmt.append(" ENCODING " + thrift_to_encoding_name(col.col_type));
4863  if (thrift_to_encoding(col.col_type.encoding) == kENCODING_DICT ||
4864  thrift_to_encoding(col.col_type.encoding) == kENCODING_FIXED ||
4865  thrift_to_encoding(col.col_type.encoding) == kENCODING_GEOINT) {
4866  col_stmt.append("(" + std::to_string(col.col_type.comp_param) + ")");
4867  }
4868  } else if (col.col_type.type == TDatumType::STR) {
4869  // non DICT encoded strings
4870  col_stmt.append(" ENCODING NONE");
4871  } else if (col.col_type.type == TDatumType::POINT ||
4872  col.col_type.type == TDatumType::LINESTRING ||
4873  col.col_type.type == TDatumType::POLYGON ||
4874  col.col_type.type == TDatumType::MULTIPOLYGON) {
4875  // non encoded compressable geo
4876  if (col.col_type.scale == 4326) {
4877  col_stmt.append(" ENCODING NONE");
4878  }
4879  }
4880  col_stmts.push_back(col_stmt);
4881  }
4882 
4883  stmt.append(" (" + boost::algorithm::join(col_stmts, ", ") + ")");
4884 
4885  if (create_params.is_replicated) {
4886  stmt.append(" WITH (PARTITIONS = 'REPLICATED')");
4887  }
4888 
4889  stmt.append(";");
4890 
4891  TQueryResult ret;
4892  sql_execute(ret, session, stmt, true, "", -1, -1);
4893 }
#define LINESTRING
std::string toString(const ExtArgumentType &sig_type)
EncodingType thrift_to_encoding(const TEncodingType::type tEncodingType)
std::string join(T const &container, std::string const &delim)
std::string to_string(char const *&&v)
#define MULTIPOLYGON
#define POINT
bool is_reserved_name(const std::string &name)
void sql_execute(ExecutionResult &_return, const TSessionId &session, const std::string &query, const bool column_format, const int32_t first_n, const int32_t at_most_n)
Definition: DBHandler.cpp:1339
std::string sanitize_name(const std::string &name)
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
std::string thrift_to_encoding_name(const TTypeInfo &ti)
std::string thrift_to_name(const TTypeInfo &ti)
void check_read_only(const std::string &str)
Definition: DBHandler.cpp:516
#define DECIMAL
#define POLYGON
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1747
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string const DBHandler::createInMemoryCalciteSession ( const std::shared_ptr< Catalog_Namespace::Catalog > &  catalog_ptr)
private

Definition at line 522 of file DBHandler.cpp.

References calcite_, CHECK, executor_device_type_, generate_random_string(), sessions_, and sessions_mutex_.

Referenced by parse_to_ra().

523  {
524  // We would create an in memory session for calcite with super user privileges which
525  // would be used for getting all tables metadata when a user runs the query. The
526  // session would be under the name of a proxy user/password which would only persist
527  // till server's lifetime or execution of calcite query(in memory) whichever is the
528  // earliest.
529  mapd_lock_guard<mapd_shared_mutex> write_lock(sessions_mutex_);
530  std::string session_id;
531  do {
532  session_id = generate_random_string(64);
533  } while (sessions_.find(session_id) != sessions_.end());
534  Catalog_Namespace::UserMetadata user_meta(-1,
535  calcite_->getInternalSessionProxyUserName(),
536  calcite_->getInternalSessionProxyPassword(),
537  true,
538  -1,
539  true,
540  false);
541  const auto emplace_ret =
542  sessions_.emplace(session_id,
543  std::make_shared<Catalog_Namespace::SessionInfo>(
544  catalog_ptr, user_meta, executor_device_type_, session_id));
545  CHECK(emplace_ret.second);
546  return session_id;
547 }
mapd_shared_mutex sessions_mutex_
Definition: DBHandler.h:602
std::string generate_random_string(const size_t len)
ExecutorDeviceType executor_device_type_
Definition: DBHandler.h:593
std::shared_ptr< Calcite > calcite_
Definition: DBHandler.h:610
#define CHECK(condition)
Definition: Logger.h:211
mapd_unique_lock< mapd_shared_mutex > write_lock
SessionMap sessions_
Definition: DBHandler.h:923

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::createSimpleResult ( TQueryResult &  _return,
const ResultSet results,
const bool  column_format,
const std::string  label 
)
staticprivate

Definition at line 6127 of file DBHandler.cpp.

References CHECK, CHECK_EQ, and run_benchmark_import::label.

Referenced by convertExplain(), and convertResult().

6130  {
6131  CHECK_EQ(size_t(1), results.rowCount());
6132  TColumnType proj_info;
6133  proj_info.col_name = label;
6134  proj_info.col_type.type = TDatumType::STR;
6135  proj_info.col_type.nullable = false;
6136  proj_info.col_type.is_array = false;
6137  _return.row_set.row_desc.push_back(proj_info);
6138  const auto crt_row = results.getNextRow(true, true);
6139  const auto tv = crt_row[0];
6140  CHECK(results.getNextRow(true, true).empty());
6141  const auto scalar_tv = boost::get<ScalarTargetValue>(&tv);
6142  CHECK(scalar_tv);
6143  const auto s_n = boost::get<NullableString>(scalar_tv);
6144  CHECK(s_n);
6145  const auto s = boost::get<std::string>(s_n);
6146  CHECK(s);
6147  if (column_format) {
6148  TColumn tcol;
6149  tcol.data.str_col.push_back(*s);
6150  tcol.nulls.push_back(false);
6151  _return.row_set.is_columnar = true;
6152  _return.row_set.columns.push_back(tcol);
6153  } else {
6154  TDatum explanation;
6155  explanation.val.str_val = *s;
6156  explanation.is_null = false;
6157  TRow trow;
6158  trow.cols.push_back(explanation);
6159  _return.row_set.is_columnar = false;
6160  _return.row_set.rows.push_back(trow);
6161  }
6162 }
#define CHECK_EQ(x, y)
Definition: Logger.h:219
#define CHECK(condition)
Definition: Logger.h:211

+ Here is the caller graph for this function:

void DBHandler::deallocate_df ( const TSessionId &  session,
const TDataFrame &  df,
const TDeviceType::type  device_type,
const int32_t  device_id 
)
override

Definition at line 1519 of file DBHandler.cpp.

References CPU, data_mgr_, ArrowResultSet::deallocateArrowResultBuffer(), logger::ERROR, get_session_ptr(), GPU, handle_to_dev_ptr_mutex_, ipc_handle_to_dev_ptr_, LOG, run_benchmark_import::result, and STDLOG.

1522  {
1523  auto stdlog = STDLOG(get_session_ptr(session));
1524  std::string serialized_cuda_handle = "";
1525  if (device_type == TDeviceType::GPU) {
1526  std::lock_guard<std::mutex> map_lock(handle_to_dev_ptr_mutex_);
1527  if (ipc_handle_to_dev_ptr_.count(df.df_handle) != size_t(1)) {
1528  TOmniSciException ex;
1529  ex.error_msg = std::string(
1530  "Current data frame handle is not bookkept or been inserted "
1531  "twice");
1532  LOG(ERROR) << ex.error_msg;
1533  throw ex;
1534  }
1535  serialized_cuda_handle = ipc_handle_to_dev_ptr_[df.df_handle];
1536  ipc_handle_to_dev_ptr_.erase(df.df_handle);
1537  }
1538  std::vector<char> sm_handle(df.sm_handle.begin(), df.sm_handle.end());
1539  std::vector<char> df_handle(df.df_handle.begin(), df.df_handle.end());
1541  sm_handle, df.sm_size, df_handle, df.df_size, serialized_cuda_handle};
1543  result,
1544  device_type == TDeviceType::CPU ? ExecutorDeviceType::CPU : ExecutorDeviceType::GPU,
1545  device_id,
1546  data_mgr_);
1547 }
std::mutex handle_to_dev_ptr_mutex_
Definition: DBHandler.h:986
#define LOG(tag)
Definition: Logger.h:205
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5801
static void deallocateArrowResultBuffer(const ArrowResult &result, const ExecutorDeviceType device_type, const size_t device_id, std::shared_ptr< Data_Namespace::DataMgr > &data_mgr)
std::shared_ptr< Data_Namespace::DataMgr > data_mgr_
Definition: DBHandler.h:586
std::unordered_map< std::string, std::string > ipc_handle_to_dev_ptr_
Definition: DBHandler.h:987
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

void DBHandler::delete_custom_expressions ( const TSessionId &  session,
const std::vector< int32_t > &  custom_expression_ids,
const bool  do_soft_delete 
)
override

Definition at line 4327 of file DBHandler.cpp.

References check_read_only(), custom_expressions_mutex_, get_session_ptr(), getConnectionInfo(), STDLOG, THROW_MAPD_EXCEPTION, and toString().

4330  {
4331  auto stdlog = STDLOG(get_session_ptr(session));
4332  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4333  check_read_only("delete_custom_expressions");
4334 
4335  auto session_ptr = stdlog.getConstSessionInfo();
4336  if (!session_ptr->get_currentUser().isSuper) {
4337  THROW_MAPD_EXCEPTION("Custom expressions can only be deleted by super users.")
4338  }
4339  auto& catalog = session_ptr->getCatalog();
4340  mapd_unique_lock<mapd_shared_mutex> write_lock(custom_expressions_mutex_);
4341  catalog.deleteCustomExpressions(custom_expression_ids, do_soft_delete);
4342 }
std::string toString(const ExtArgumentType &sig_type)
mapd_shared_mutex custom_expressions_mutex_
Definition: DBHandler.h:1072
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5801
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
void check_read_only(const std::string &str)
Definition: DBHandler.cpp:516
mapd_unique_lock< mapd_shared_mutex > write_lock
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1747
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

void DBHandler::delete_dashboard ( const TSessionId &  session,
const int32_t  dashboard_id 
)
override

Definition at line 4555 of file DBHandler.cpp.

References delete_dashboards().

4555  {
4556  delete_dashboards(session, {dashboard_id});
4557 }
void delete_dashboards(const TSessionId &session, const std::vector< int32_t > &dashboard_ids) override
Definition: DBHandler.cpp:4559

+ Here is the call graph for this function:

void DBHandler::delete_dashboards ( const TSessionId &  session,
const std::vector< int32_t > &  dashboard_ids 
)
override

Definition at line 4559 of file DBHandler.cpp.

References cat(), anonymous_namespace{DBHandler.cpp}::check_not_info_schema_db(), check_read_only(), g_allow_system_dashboard_update, get_session_ptr(), getConnectionInfo(), STDLOG, THROW_MAPD_EXCEPTION, and toString().

Referenced by delete_dashboard().

4560  {
4561  auto stdlog = STDLOG(get_session_ptr(session));
4562  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4563  auto session_ptr = stdlog.getConstSessionInfo();
4564  check_read_only("delete_dashboards");
4565  auto& cat = session_ptr->getCatalog();
4567  check_not_info_schema_db(cat.name(), true);
4568  }
4569  // Checks will be performed in catalog
4570  try {
4571  cat.deleteMetadataForDashboards(dashboard_ids, session_ptr->get_currentUser());
4572  } catch (const std::exception& e) {
4573  THROW_MAPD_EXCEPTION(e.what());
4574  }
4575 }
std::string cat(Ts &&...args)
std::string toString(const ExtArgumentType &sig_type)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5801
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
void check_read_only(const std::string &str)
Definition: DBHandler.cpp:516
bool g_allow_system_dashboard_update
Definition: DBHandler.cpp:121
void check_not_info_schema_db(const std::string &db_name, bool throw_mapd_exception=false)
Definition: DBHandler.cpp:4454
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1747
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::detect_column_types ( TDetectResult &  _return,
const TSessionId &  session,
const std::string &  file_name,
const TCopyParams &  copy_params 
)
override

Definition at line 4019 of file DBHandler.cpp.

References anonymous_namespace{DBHandler.cpp}::add_vsi_archive_prefix(), anonymous_namespace{DBHandler.cpp}::add_vsi_geo_prefix(), anonymous_namespace{DBHandler.cpp}::add_vsi_network_prefix(), import_export::Detector::best_encodings, import_export::Detector::best_sqltypes, check_geospatial_files(), check_read_only(), copyparams_to_thrift(), encoding_to_thrift(), anonymous_namespace{DBHandler.cpp}::find_first_geo_file_in_archive(), import_export::Importer::gdalFileExists(), import_export::Importer::gdalFileOrDirectoryExists(), import_export::Importer::gdalToColumnDescriptors(), import_export::CopyParams::geo_coords_comp_param, import_export::CopyParams::geo_coords_encoding, import_export::CopyParams::geo_coords_srid, import_export::CopyParams::geo_coords_type, import_export::DataStreamSink::get_copy_params(), import_export::Detector::get_headers(), import_export::Detector::get_sample_rows(), get_session_ptr(), getConnectionInfo(), i, import_path_, anonymous_namespace{DBHandler.cpp}::is_a_supported_archive_file(), IS_GEO, ImportHelpers::is_reserved_name(), import_export::kDelimitedFile, import_export::kGeoFile, import_export::kParquetFile, import_export::kRasterFile, OMNISCI_GEO_PREFIX, anonymous_namespace{DBHandler.cpp}::path_is_relative(), populateThriftColumnType(), import_export::Importer::readMetadataSampleGDAL(), import_export::CopyParams::sanitize_column_names, ImportHelpers::sanitize_name(), SQLTypeInfo::set_compression(), import_export::CopyParams::source_type, STDLOG, t, thrift_to_copyparams(), THROW_MAPD_EXCEPTION, toString(), type_to_thrift(), and anonymous_namespace{DBHandler.cpp}::validate_import_file_path_if_local().

Referenced by import_geo_table_internal().

4022  {
4023  auto stdlog = STDLOG(get_session_ptr(session));
4024  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4025  check_read_only("detect_column_types");
4026 
4028 
4029  std::string file_name{file_name_in};
4030  if (path_is_relative(file_name)) {
4031  // assume relative paths are relative to data_path / mapd_import / <session>
4032  auto file_path = import_path_ / picosha2::hash256_hex_string(session) /
4033  boost::filesystem::path(file_name).filename();
4034  file_name = file_path.string();
4035  }
4037 
4038  // if it's a geo or raster import, handle alternative paths (S3, HTTP, archive etc.)
4039  bool is_raster = false;
4040  if (copy_params.source_type == import_export::SourceType::kGeoFile) {
4041  if (is_a_supported_archive_file(file_name)) {
4042  // find the archive file
4043  add_vsi_network_prefix(file_name);
4044  if (!import_export::Importer::gdalFileExists(file_name, copy_params)) {
4045  THROW_MAPD_EXCEPTION("Archive does not exist: " + file_name_in);
4046  }
4047  // find geo file in archive
4048  add_vsi_archive_prefix(file_name);
4049  std::string geo_file = find_first_geo_file_in_archive(file_name, copy_params);
4050  // prepare to detect that geo file
4051  if (geo_file.size()) {
4052  file_name = file_name + std::string("/") + geo_file;
4053  }
4054  } else {
4055  // prepare to detect geo file directly
4056  add_vsi_network_prefix(file_name);
4057  add_vsi_geo_prefix(file_name);
4058  }
4059  } else if (copy_params.source_type == import_export::SourceType::kRasterFile) {
4060  // prepare to detect raster file directly
4061  add_vsi_network_prefix(file_name);
4062  add_vsi_geo_prefix(file_name);
4063  is_raster = true;
4064  }
4065 
4066  auto file_path = boost::filesystem::path(file_name);
4067  // can be a s3 url
4068  if (!boost::istarts_with(file_name, "s3://")) {
4069  if (!boost::filesystem::path(file_name).is_absolute()) {
4070  file_path = import_path_ / picosha2::hash256_hex_string(session) /
4071  boost::filesystem::path(file_name).filename();
4072  file_name = file_path.string();
4073  }
4074 
4075  if (copy_params.source_type == import_export::SourceType::kGeoFile ||
4077  // check for geo or raster file
4078  if (!import_export::Importer::gdalFileOrDirectoryExists(file_name, copy_params)) {
4079  THROW_MAPD_EXCEPTION("File does not exist: " + file_path.string());
4080  }
4081  } else {
4082  // check for regular file
4083  if (!boost::filesystem::exists(file_path)) {
4084  THROW_MAPD_EXCEPTION("File does not exist: " + file_path.string());
4085  }
4086  }
4087  }
4088 
4089  try {
4091 #ifdef ENABLE_IMPORT_PARQUET
4093 #endif
4094  ) {
4095  import_export::Detector detector(file_path, copy_params);
4096  std::vector<SQLTypes> best_types = detector.best_sqltypes;
4097  std::vector<EncodingType> best_encodings = detector.best_encodings;
4098  std::vector<std::string> headers = detector.get_headers();
4099  copy_params = detector.get_copy_params();
4100 
4101  _return.copy_params = copyparams_to_thrift(copy_params);
4102  _return.row_set.row_desc.resize(best_types.size());
4103  for (size_t col_idx = 0; col_idx < best_types.size(); col_idx++) {
4104  TColumnType col;
4105  SQLTypes t = best_types[col_idx];
4106  EncodingType encodingType = best_encodings[col_idx];
4107  SQLTypeInfo ti(t, false, encodingType);
4108  if (IS_GEO(t)) {
4109  // set this so encoding_to_thrift does the right thing
4110  ti.set_compression(copy_params.geo_coords_encoding);
4111  // fill in these directly
4112  col.col_type.precision = static_cast<int>(copy_params.geo_coords_type);
4113  col.col_type.scale = copy_params.geo_coords_srid;
4114  col.col_type.comp_param = copy_params.geo_coords_comp_param;
4115  }
4116  col.col_type.type = type_to_thrift(ti);
4117  col.col_type.encoding = encoding_to_thrift(ti);
4118  if (copy_params.sanitize_column_names) {
4119  col.col_name = ImportHelpers::sanitize_name(headers[col_idx]);
4120  } else {
4121  col.col_name = headers[col_idx];
4122  }
4123  col.is_reserved_keyword = ImportHelpers::is_reserved_name(col.col_name);
4124  _return.row_set.row_desc[col_idx] = col;
4125  }
4126  size_t num_samples = 100;
4127  auto sample_data = detector.get_sample_rows(num_samples);
4128 
4129  TRow sample_row;
4130  for (auto row : sample_data) {
4131  sample_row.cols.clear();
4132  for (const auto& s : row) {
4133  TDatum td;
4134  td.val.str_val = s;
4135  td.is_null = s.empty();
4136  sample_row.cols.push_back(td);
4137  }
4138  _return.row_set.rows.push_back(sample_row);
4139  }
4140  } else if (copy_params.source_type == import_export::SourceType::kGeoFile ||
4142  // @TODO simon.eves get this from somewhere!
4143  const std::string geoColumnName(OMNISCI_GEO_PREFIX);
4144 
4145  check_geospatial_files(file_path, copy_params);
4146  std::list<ColumnDescriptor> cds = import_export::Importer::gdalToColumnDescriptors(
4147  file_path.string(), is_raster, geoColumnName, copy_params);
4148  for (auto cd : cds) {
4149  if (copy_params.sanitize_column_names) {
4150  cd.columnName = ImportHelpers::sanitize_name(cd.columnName);
4151  }
4152  _return.row_set.row_desc.push_back(populateThriftColumnType(nullptr, &cd));
4153  }
4154  if (!is_raster) {
4155  // @TODO(se) support for raster?
4156  std::map<std::string, std::vector<std::string>> sample_data;
4158  file_path.string(), geoColumnName, sample_data, 100, copy_params);
4159  if (sample_data.size() > 0) {
4160  for (size_t i = 0; i < sample_data.begin()->second.size(); i++) {
4161  TRow sample_row;
4162  for (auto cd : cds) {
4163  TDatum td;
4164  td.val.str_val = sample_data[cd.sourceName].at(i);
4165  td.is_null = td.val.str_val.empty();
4166  sample_row.cols.push_back(td);
4167  }
4168  _return.row_set.rows.push_back(sample_row);
4169  }
4170  }
4171  }
4172  _return.copy_params = copyparams_to_thrift(copy_params);
4173  }
4174  } catch (const std::exception& e) {
4175  THROW_MAPD_EXCEPTION("detect_column_types error: " + std::string(e.what()));
4176  }
4177 }
boost::filesystem::path import_path_
Definition: DBHandler.h:592
void add_vsi_archive_prefix(std::string &path)
Definition: DBHandler.cpp:3892
bool is_a_supported_archive_file(const std::string &path)
Definition: DBHandler.cpp:3958
std::string toString(const ExtArgumentType &sig_type)
TDatumType::type type_to_thrift(const SQLTypeInfo &type_info)
SQLTypes
Definition: sqltypes.h:38
bool path_is_relative(const std::string &path)
Definition: DBHandler.cpp:3926
TCopyParams copyparams_to_thrift(const import_export::CopyParams &cp)
Definition: DBHandler.cpp:3744
static bool gdalFileExists(const std::string &path, const CopyParams &copy_params)
Definition: Importer.cpp:4982
EncodingType
Definition: sqltypes.h:233
bool is_reserved_name(const std::string &name)
TColumnType populateThriftColumnType(const Catalog_Namespace::Catalog *cat, const ColumnDescriptor *cd)
Definition: DBHandler.cpp:2272
void check_geospatial_files(const boost::filesystem::path file_path, const import_export::CopyParams &copy_params)
Definition: DBHandler.cpp:4789
static void readMetadataSampleGDAL(const std::string &fileName, const std::string &geoColumnName, std::map< std::string, std::vector< std::string >> &metadata, int rowLimit, const CopyParams &copy_params)
Definition: Importer.cpp:4604
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5801
void validate_import_file_path_if_local(const std::string &file_path)
Definition: DBHandler.cpp:4012
std::string sanitize_name(const std::string &name)
import_export::SourceType source_type
Definition: CopyParams.h:56
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
const std::string OMNISCI_GEO_PREFIX
Definition: Transforms.h:22
static bool gdalFileOrDirectoryExists(const std::string &path, const CopyParams &copy_params)
Definition: Importer.cpp:4987
void check_read_only(const std::string &str)
Definition: DBHandler.cpp:516
std::string find_first_geo_file_in_archive(const std::string &archive_path, const import_export::CopyParams &copy_params)
Definition: DBHandler.cpp:3971
void add_vsi_network_prefix(std::string &path)
Definition: DBHandler.cpp:3863
char * t
static const std::list< ColumnDescriptor > gdalToColumnDescriptors(const std::string &fileName, const bool is_raster, const std::string &geoColumnName, const CopyParams &copy_params)
Definition: Importer.cpp:4780
import_export::CopyParams thrift_to_copyparams(const TCopyParams &cp)
Definition: DBHandler.cpp:3564
#define IS_GEO(T)
Definition: sqltypes.h:251
void add_vsi_geo_prefix(std::string &path)
Definition: DBHandler.cpp:3885
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1747
#define STDLOG(...)
Definition: QueryState.h:235
TEncodingType::type encoding_to_thrift(const SQLTypeInfo &type_info)
EncodingType geo_coords_encoding
Definition: CopyParams.h:71

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::disconnect ( const TSessionId &  session)
override

Definition at line 700 of file DBHandler.cpp.

References disconnect_impl(), get_session_it_unsafe(), getConnectionInfo(), logger::INFO, LOG, sessions_mutex_, STDLOG, and toString().

700  {
701  auto stdlog = STDLOG();
702  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
703 
704  mapd_unique_lock<mapd_shared_mutex> write_lock(sessions_mutex_);
705  auto session_it = get_session_it_unsafe(session, write_lock);
706  stdlog.setSessionInfo(session_it->second);
707  const auto dbname = session_it->second->getCatalog().getCurrentDB().dbName;
708 
709  LOG(INFO) << "User " << session_it->second->get_currentUser().userLoggable()
710  << " disconnected from database " << dbname
711  << " with public_session_id: " << session_it->second->get_public_session_id();
712 
713  disconnect_impl(session_it, write_lock);
714 }
std::string toString(const ExtArgumentType &sig_type)
#define LOG(tag)
Definition: Logger.h:205
mapd_shared_mutex sessions_mutex_
Definition: DBHandler.h:602
SessionMap::iterator get_session_it_unsafe(const TSessionId &session, SESSION_MAP_LOCK &lock)
void disconnect_impl(const SessionMap::iterator &session_it, mapd_unique_lock< mapd_shared_mutex > &write_lock)
Definition: DBHandler.cpp:716
mapd_unique_lock< mapd_shared_mutex > write_lock
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1747
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

void DBHandler::disconnect_impl ( const SessionMap::iterator &  session_it,
mapd_unique_lock< mapd_shared_mutex > &  write_lock 
)
private

Definition at line 716 of file DBHandler.cpp.

References LeafAggregator::disconnect(), leaf_aggregator_, LeafAggregator::leafCount(), render_group_assignment_map_, render_group_assignment_mutex_, render_handler_, and sessions_.

Referenced by disconnect(), and get_session_it_unsafe().

717  {
718  // session_it existence should already have been checked (i.e. called via
719  // get_session_it_unsafe(...))
720 
721  const auto session_id = session_it->second->get_session_id();
722  std::exception_ptr leaf_exception = nullptr;
723  try {
724  if (leaf_aggregator_.leafCount() > 0) {
725  leaf_aggregator_.disconnect(session_id);
726  }
727  } catch (...) {
728  leaf_exception = std::current_exception();
729  }
730 
731  {
732  std::lock_guard<std::mutex> lock(render_group_assignment_mutex_);
733  render_group_assignment_map_.erase(session_id);
734  }
735 
736  sessions_.erase(session_it);
737  write_lock.unlock();
738 
739  if (render_handler_) {
740  render_handler_->disconnect(session_id);
741  }
742 }
std::mutex render_group_assignment_mutex_
Definition: DBHandler.h:1071
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:588
void disconnect(const TSessionId session)
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:607
size_t leafCount() const
RenderGroupAnalyzerSessionMap render_group_assignment_map_
Definition: DBHandler.h:1070
mapd_unique_lock< mapd_shared_mutex > write_lock
SessionMap sessions_
Definition: DBHandler.h:923

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::emergency_shutdown ( )

Definition at line 7232 of file DBHandler.cpp.

References calcite_.

Referenced by shutdown().

7232  {
7233  if (calcite_) {
7234  calcite_->close_calcite_server(false);
7235  }
7236 }
std::shared_ptr< Calcite > calcite_
Definition: DBHandler.h:610

+ Here is the caller graph for this function:

void DBHandler::execute_distributed_copy_statement ( Parser::CopyTableStmt copy_stmt,
const Catalog_Namespace::SessionInfo session_info 
)
private

Definition at line 6662 of file DBHandler.cpp.

References Parser::CopyTableStmt::execute(), and leaf_aggregator_.

Referenced by sql_execute_impl().

6664  {
6665  auto importer_factory = [&session_info, this](
6666  const Catalog& catalog,
6667  const TableDescriptor* td,
6668  const std::string& file_path,
6669  const import_export::CopyParams& copy_params)
6670  -> std::unique_ptr<import_export::AbstractImporter> {
6671  return std::make_unique<import_export::Importer>(
6672  new DistributedLoader(session_info, td, &leaf_aggregator_),
6673  file_path,
6674  copy_params);
6675  };
6676  copy_stmt->execute(session_info, importer_factory);
6677 }
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:114
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:588
void execute(const Catalog_Namespace::SessionInfo &session) override

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::execute_next_render_step ( TRenderStepResult &  _return,
const TPendingRenderQuery &  pending_render,
const TRenderAggDataMap &  merged_data 
)
override

Definition at line 7040 of file DBHandler.cpp.

References measure< TimeT >::execution(), logger::INFO, LOG, render_handler_, and THROW_MAPD_EXCEPTION.

7042  {
7043  if (!render_handler_) {
7044  THROW_MAPD_EXCEPTION("Backend rendering is disabled.");
7045  }
7046 
7047  LOG(INFO) << "execute_next_render_step: id:" << pending_render.id;
7048  auto time_ms = measure<>::execution([&]() {
7049  try {
7050  render_handler_->execute_next_render_step(_return, pending_render, merged_data);
7051  } catch (std::exception& e) {
7052  THROW_MAPD_EXCEPTION(e.what());
7053  }
7054  });
7055  LOG(INFO) << "execute_next_render_step-COMPLETED id: " << pending_render.id
7056  << ", time: " << time_ms << "ms ";
7057 }
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:205
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:607
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:128

+ Here is the call graph for this function:

void DBHandler::execute_query_step ( TStepResult &  _return,
const TPendingQuery &  pending_query,
const TSubqueryId  subquery_id,
const std::string &  start_time_str 
)
override

Definition at line 6863 of file DBHandler.cpp.

References measure< TimeT >::execution(), logger::INFO, leaf_handler_, LOG, and THROW_MAPD_EXCEPTION.

6866  {
6867  if (!leaf_handler_) {
6868  THROW_MAPD_EXCEPTION("Distributed support is disabled.");
6869  }
6870  LOG(INFO) << "execute_query_step : id:" << pending_query.id;
6871  auto time_ms = measure<>::execution([&]() {
6872  try {
6873  leaf_handler_->execute_query_step(
6874  _return, pending_query, subquery_id, start_time_str);
6875  } catch (std::exception& e) {
6876  THROW_MAPD_EXCEPTION(e.what());
6877  }
6878  });
6879  LOG(INFO) << "execute_query_step-COMPLETED " << time_ms << "ms";
6880 }
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:205
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
std::unique_ptr< MapDLeafHandler > leaf_handler_
Definition: DBHandler.h:609

+ Here is the call graph for this function:

std::vector< PushedDownFilterInfo > DBHandler::execute_rel_alg ( ExecutionResult _return,
QueryStateProxy  query_state_proxy,
const std::string &  query_ra,
const bool  column_format,
const ExecutorDeviceType  executor_device_type,
const int32_t  first_n,
const int32_t  at_most_n,
const bool  just_validate,
const bool  find_push_down_candidates,
const ExplainInfo explain_info,
const std::optional< size_t >  executor_index = std::nullopt 
) const
private

Definition at line 5861 of file DBHandler.cpp.

References allow_loop_joins_, allow_multifrag_, cat(), query_state::QueryStateProxy::createTimer(), Default, measure< TimeT >::execution(), ExplainInfo::explain_optimized, ExplainInfo::explain_plan, ExecutionResult::Explaination, g_dynamic_watchdog_time_limit, g_enable_columnar_output, g_enable_dynamic_watchdog, g_enable_runtime_query_interrupt, g_enable_watchdog, g_pending_query_interrupt_freq, g_running_query_interrupt_freq, query_state::QueryState::getConstSessionInfo(), ExecutionResult::getExecutionTime(), Executor::getExecutor(), ExecutionResult::getPushedDownFilterInfo(), query_state::QueryStateProxy::getQueryState(), ExecutionResult::getRows(), SystemParameters::gpu_input_mem_limit, lockmgr::TableSchemaLockMgr::instance(), lockmgr::TableDataLockMgr::instance(), intel_jit_profile_, jit_debug_, ExplainInfo::justCalciteExplain(), ExplainInfo::justExplain(), Optimized, ExecutionResult::QueryResult, ExecutionResult::setExecutionTime(), ExecutionResult::setResultType(), system_parameters_, Executor::UNITARY_EXECUTOR_ID, and VLOG.

Referenced by execute_rel_alg_with_filter_push_down(), get_tables_meta_impl(), sql_execute_impl(), and validate_rel_alg().

5872  {
5873  query_state::Timer timer = query_state_proxy.createTimer(__func__);
5874 
5875  VLOG(1) << "Table Schema Locks:\n" << lockmgr::TableSchemaLockMgr::instance();
5876  VLOG(1) << "Table Data Locks:\n" << lockmgr::TableDataLockMgr::instance();
5877 
5878  const auto& cat = query_state_proxy.getQueryState().getConstSessionInfo()->getCatalog();
5879  auto executor = Executor::getExecutor(
5880  executor_index ? *executor_index : Executor::UNITARY_EXECUTOR_ID,
5881  jit_debug_ ? "/tmp" : "",
5882  jit_debug_ ? "mapdquery" : "",
5884  RelAlgExecutor ra_executor(executor.get(),
5885  cat,
5886  query_ra,
5887  query_state_proxy.getQueryState().shared_from_this());
5888  CompilationOptions co = {executor_device_type,
5889  /*hoist_literals=*/true,
5892  /*allow_lazy_fetch=*/true,
5893  /*filter_on_deleted_column=*/true,
5897  auto validate_or_explain_query =
5898  explain_info.justExplain() || explain_info.justCalciteExplain() || just_validate;
5901  explain_info.justExplain(),
5902  allow_loop_joins_ || just_validate,
5904  jit_debug_,
5905  just_validate,
5908  find_push_down_candidates,
5909  explain_info.justCalciteExplain(),
5911  g_enable_runtime_query_interrupt && !validate_or_explain_query &&
5912  !query_state_proxy.getQueryState()
5914  ->get_session_id()
5915  .empty(),
5918  auto execution_time_ms = _return.getExecutionTime() + measure<>::execution([&]() {
5919  _return = ra_executor.executeRelAlgQuery(
5920  co, eo, explain_info.explain_plan, nullptr);
5921  });
5922  // reduce execution time by the time spent during queue waiting
5923  const auto rs = _return.getRows();
5924  if (rs) {
5925  execution_time_ms -= rs->getQueueTime();
5926  }
5927  _return.setExecutionTime(execution_time_ms);
5928  VLOG(1) << cat.getDataMgr().getSystemMemoryUsage();
5929  const auto& filter_push_down_info = _return.getPushedDownFilterInfo();
5930  if (!filter_push_down_info.empty()) {
5931  return filter_push_down_info;
5932  }
5933  if (explain_info.justExplain()) {
5935  } else if (!explain_info.justCalciteExplain()) {
5937  }
5938  return {};
5939 }
double g_running_query_interrupt_freq
Definition: Execute.cpp:123
static TableSchemaLockMgr & instance()
Definition: LockMgr.h:40
bool g_enable_watchdog
std::string cat(Ts &&...args)
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
unsigned g_pending_query_interrupt_freq
Definition: Execute.cpp:122
const bool jit_debug_
Definition: DBHandler.h:596
bool g_enable_dynamic_watchdog
Definition: Execute.cpp:77
Timer createTimer(char const *event_name)
Definition: QueryState.cpp:138
static TableDataLockMgr & instance()
Definition: LockMgr.h:76
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:177
bool g_enable_columnar_output
Definition: Execute.cpp:95
bool justExplain() const
Definition: ParserWrapper.h:39
bool explain_plan
Definition: ParserWrapper.h:34
QueryState & getQueryState()
Definition: QueryState.h:181
const std::shared_ptr< ResultSet > & getRows() const
SystemParameters & system_parameters_
Definition: DBHandler.h:606
void setResultType(RType type)
const bool allow_loop_joins_
Definition: DBHandler.h:600
const bool intel_jit_profile_
Definition: DBHandler.h:597
const std::vector< PushedDownFilterInfo > & getPushedDownFilterInfo() const
bool explain_optimized
Definition: ParserWrapper.h:33
void setExecutionTime(int64_t execution_time_ms)
bool allow_multifrag_
Definition: DBHandler.h:598
unsigned g_dynamic_watchdog_time_limit
Definition: Execute.cpp:81
bool justCalciteExplain() const
Definition: ParserWrapper.h:41
std::shared_ptr< Catalog_Namespace::SessionInfo const > getConstSessionInfo() const
Definition: QueryState.cpp:88
int64_t getExecutionTime() const
bool g_enable_runtime_query_interrupt
Definition: Execute.cpp:119
#define VLOG(n)
Definition: Logger.h:305
static const ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:376

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::execute_rel_alg_df ( TDataFrame &  _return,
const std::string &  query_ra,
QueryStateProxy  query_state_proxy,
const Catalog_Namespace::SessionInfo session_info,
const ExecutorDeviceType  executor_device_type,
const ExecutorDeviceType  results_device_type,
const size_t  device_id,
const int32_t  first_n,
const TArrowTransport::type  transport_method 
) const
private

Definition at line 5941 of file DBHandler.cpp.

References allow_loop_joins_, allow_multifrag_, cat(), CHECK, CPU, data_mgr_, Default, measure< TimeT >::execution(), g_dynamic_watchdog_time_limit, g_enable_columnar_output, g_enable_dynamic_watchdog, g_enable_runtime_query_interrupt, g_enable_watchdog, g_pending_query_interrupt_freq, g_running_query_interrupt_freq, Catalog_Namespace::SessionInfo::getCatalog(), query_state::QueryState::getConstSessionInfo(), Executor::getExecutor(), query_state::QueryStateProxy::getQueryState(), getTargetNames(), GPU, SystemParameters::gpu_input_mem_limit, handle_to_dev_ptr_mutex_, intel_jit_profile_, ipc_handle_to_dev_ptr_, jit_debug_, system_parameters_, and Executor::UNITARY_EXECUTOR_ID.

Referenced by sql_execute_df().

5949  {
5950  const auto& cat = session_info.getCatalog();
5952  jit_debug_ ? "/tmp" : "",
5953  jit_debug_ ? "mapdquery" : "",
5955  RelAlgExecutor ra_executor(executor.get(),
5956  cat,
5957  query_ra,
5958  query_state_proxy.getQueryState().shared_from_this());
5959  CompilationOptions co = {executor_device_type,
5960  /*hoist_literals=*/true,
5963  /*allow_lazy_fetch=*/true,
5964  /*filter_on_deleted_column=*/true,
5967  ExecutionOptions eo = {
5970  false,
5973  jit_debug_,
5974  false,
5977  false,
5978  false,
5980  g_enable_runtime_query_interrupt && !query_state_proxy.getQueryState()
5982  ->get_session_id()
5983  .empty(),
5986  ExecutionResult result{std::make_shared<ResultSet>(std::vector<TargetInfo>{},
5989  nullptr,
5990  nullptr,
5991  0,
5992  0),
5993  {}};
5994  _return.execution_time_ms += measure<>::execution(
5995  [&]() { result = ra_executor.executeRelAlgQuery(co, eo, false, nullptr); });
5996  _return.execution_time_ms -= result.getRows()->getQueueTime();
5997  const auto rs = result.getRows();
5998  const auto converter =
5999  std::make_unique<ArrowResultSetConverter>(rs,
6000  data_mgr_,
6001  results_device_type,
6002  device_id,
6003  getTargetNames(result.getTargetsMeta()),
6004  first_n,
6005  ArrowTransport(transport_method));
6006  ArrowResult arrow_result;
6007  _return.arrow_conversion_time_ms +=
6008  measure<>::execution([&] { arrow_result = converter->getArrowResult(); });
6009  _return.sm_handle =
6010  std::string(arrow_result.sm_handle.begin(), arrow_result.sm_handle.end());
6011  _return.sm_size = arrow_result.sm_size;
6012  _return.df_handle =
6013  std::string(arrow_result.df_handle.begin(), arrow_result.df_handle.end());
6014  _return.df_buffer =
6015  std::string(arrow_result.df_buffer.begin(), arrow_result.df_buffer.end());
6016  if (results_device_type == ExecutorDeviceType::GPU) {
6017  std::lock_guard<std::mutex> map_lock(handle_to_dev_ptr_mutex_);
6018  CHECK(!ipc_handle_to_dev_ptr_.count(_return.df_handle));
6019  ipc_handle_to_dev_ptr_.insert(
6020  std::make_pair(_return.df_handle, arrow_result.serialized_cuda_handle));
6021  }
6022  _return.df_size = arrow_result.df_size;
6023 }
double g_running_query_interrupt_freq
Definition: Execute.cpp:123
bool g_enable_watchdog
std::string cat(Ts &&...args)
std::mutex handle_to_dev_ptr_mutex_
Definition: DBHandler.h:986
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
unsigned g_pending_query_interrupt_freq
Definition: Execute.cpp:122
ArrowTransport
std::vector< std::string > getTargetNames(const std::vector< TargetMetaInfo > &targets) const
Definition: DBHandler.cpp:6047
const bool jit_debug_
Definition: DBHandler.h:596
bool g_enable_dynamic_watchdog
Definition: Execute.cpp:77
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:177
bool g_enable_columnar_output
Definition: Execute.cpp:95
QueryState & getQueryState()
Definition: QueryState.h:181
SystemParameters & system_parameters_
Definition: DBHandler.h:606
const bool allow_loop_joins_
Definition: DBHandler.h:600
const bool intel_jit_profile_
Definition: DBHandler.h:597
Catalog & getCatalog() const
Definition: SessionInfo.h:67
std::shared_ptr< Data_Namespace::DataMgr > data_mgr_
Definition: DBHandler.h:586
#define CHECK(condition)
Definition: Logger.h:211
std::unordered_map< std::string, std::string > ipc_handle_to_dev_ptr_
Definition: DBHandler.h:987
bool allow_multifrag_
Definition: DBHandler.h:598
unsigned g_dynamic_watchdog_time_limit
Definition: Execute.cpp:81
std::shared_ptr< Catalog_Namespace::SessionInfo const > getConstSessionInfo() const
Definition: QueryState.cpp:88
bool g_enable_runtime_query_interrupt
Definition: Execute.cpp:119
static const ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:376

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::execute_rel_alg_with_filter_push_down ( ExecutionResult _return,
QueryStateProxy  query_state_proxy,
std::string &  query_ra,
const bool  column_format,
const ExecutorDeviceType  executor_device_type,
const int32_t  first_n,
const int32_t  at_most_n,
const bool  just_explain,
const bool  just_calcite_explain,
const std::vector< PushedDownFilterInfo > &  filter_push_down_requests 
)
private

Definition at line 6611 of file DBHandler.cpp.

References ExecutionResult::addExecutionTime(), ExplainInfo::defaults(), execute_rel_alg(), ExecutionResult::Explaination, query_state::QueryStateProxy::getQueryState(), query_state::QueryState::getQueryStr(), parse_to_ra(), system_parameters_, and ExecutionResult::updateResultSet().

Referenced by sql_execute_impl().

6621  {
6622  // collecting the selected filters' info to be sent to Calcite:
6623  std::vector<TFilterPushDownInfo> filter_push_down_info;
6624  for (const auto& req : filter_push_down_requests) {
6625  TFilterPushDownInfo filter_push_down_info_for_request;
6626  filter_push_down_info_for_request.input_prev = req.input_prev;
6627  filter_push_down_info_for_request.input_start = req.input_start;
6628  filter_push_down_info_for_request.input_next = req.input_next;
6629  filter_push_down_info.push_back(filter_push_down_info_for_request);
6630  }
6631  // deriving the new relational algebra plan with respect to the pushed down filters
6632  _return.addExecutionTime(measure<>::execution([&]() {
6633  query_ra = parse_to_ra(query_state_proxy,
6634  query_state_proxy.getQueryState().getQueryStr(),
6635  filter_push_down_info,
6636  false,
6638  .first.plan_result;
6639  }));
6640 
6641  if (just_calcite_explain) {
6642  // return the new ra as the result
6644  return;
6645  }
6646 
6647  // execute the new relational algebra plan:
6648  auto explain_info = ExplainInfo::defaults();
6649  explain_info.explain = just_explain;
6650  execute_rel_alg(_return,
6651  query_state_proxy,
6652  query_ra,
6653  column_format,
6654  executor_device_type,
6655  first_n,
6656  at_most_n,
6657  /*just_validate=*/false,
6658  /*find_push_down_candidates=*/false,
6659  explain_info);
6660 }
static ExplainInfo defaults()
Definition: ParserWrapper.h:37
std::vector< PushedDownFilterInfo > execute_rel_alg(ExecutionResult &_return, QueryStateProxy, const std::string &query_ra, const bool column_format, const ExecutorDeviceType executor_device_type, const int32_t first_n, const int32_t at_most_n, const bool just_validate, const bool find_push_down_candidates, const ExplainInfo &explain_info, const std::optional< size_t > executor_index=std::nullopt) const
Definition: DBHandler.cpp:5861
std::string const & getQueryStr() const
Definition: QueryState.h:157
void addExecutionTime(int64_t execution_time_ms)
QueryState & getQueryState()
Definition: QueryState.h:181
void updateResultSet(const std::string &query_ra, RType type, bool success=true)
SystemParameters & system_parameters_
Definition: DBHandler.h:606
std::pair< TPlanResult, lockmgr::LockedTableDescriptors > parse_to_ra(QueryStateProxy, const std::string &query_str, const std::vector< TFilterPushDownInfo > &filter_push_down_info, const bool acquire_locks, const SystemParameters &system_parameters, bool check_privileges=true)
Definition: DBHandler.cpp:6679

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::executeDdl ( TQueryResult &  _return,
const std::string &  query_ra,
std::shared_ptr< Catalog_Namespace::SessionInfo const session_ptr 
)
private

Definition at line 7652 of file DBHandler.cpp.

References clear_cpu_memory(), clear_gpu_memory(), clearRenderMemory(), DdlCommandExecutor::commandStr(), convertResult(), convertResultSet(), ExecutionResult::empty(), DdlCommandExecutor::execute(), measure< TimeT >::execution(), ExecutionResult::getDataPtr(), getQueries(), ExecutionResult::getRows(), DdlCommandExecutor::getTargetQuerySessionToKill(), getUserSessions(), interruptQuery(), DdlCommandExecutor::isAlterSystemClear(), DdlCommandExecutor::isKillQuery(), DdlCommandExecutor::isShowCreateTable(), DdlCommandExecutor::isShowQueries(), DdlCommandExecutor::isShowUserSessions(), run_benchmark_import::result, and DdlCommandExecutor::returnCacheType().

Referenced by sql_execute_impl().

7655  {
7656  DdlCommandExecutor executor = DdlCommandExecutor(query_ra, session_ptr);
7657  std::string commandStr = executor.commandStr();
7658 
7659  if (executor.isKillQuery()) {
7660  interruptQuery(*session_ptr, executor.getTargetQuerySessionToKill());
7661  } else {
7663 
7664  if (executor.isShowQueries()) {
7665  // getQueries still requires Thrift cannot be nested into DdlCommandExecutor
7666  _return.execution_time_ms +=
7667  measure<>::execution([&]() { result = getQueries(session_ptr); });
7668  } else if (executor.isShowUserSessions()) {
7669  // getUserSessions still requires Thrift cannot be nested into DdlCommandExecutor
7670  _return.execution_time_ms +=
7671  measure<>::execution([&]() { result = getUserSessions(session_ptr); });
7672  } else if (executor.isAlterSystemClear()) {
7673  result = ExecutionResult();
7674  if (executor.returnCacheType() == "CPU") {
7675  _return.execution_time_ms += measure<>::execution(
7676  [&]() { clear_cpu_memory(session_ptr->get_session_id()); });
7677  } else if (executor.returnCacheType() == "GPU") {
7678  _return.execution_time_ms += measure<>::execution(
7679  [&]() { clear_gpu_memory(session_ptr->get_session_id()); });
7680  } else if (executor.returnCacheType() == "RENDER") {
7681  _return.execution_time_ms += measure<>::execution(
7682  [&]() { clearRenderMemory(session_ptr->get_session_id()); });
7683  } else {
7684  throw std::runtime_error("Unknwon cache type. Cannot clear");
7685  }
7686  } else {
7687  _return.execution_time_ms +=
7688  measure<>::execution([&]() { result = executor.execute(); });
7689  }
7690 
7691  if (!result.empty()) {
7692  // reduce execution time by the time spent during queue waiting
7693  _return.execution_time_ms -= result.getRows()->getQueueTime();
7694 
7695  if (executor.isShowCreateTable()) {
7696  convertResult(_return, *result.getDataPtr(), true);
7697  } else {
7698  convertResultSet(result, *session_ptr, commandStr, _return);
7699  }
7700  }
7701  }
7702 }
std::string returnCacheType()
void clearRenderMemory(const TSessionId &session)
Definition: DBHandler.cpp:2707
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
static void convertResult(TQueryResult &_return, const ResultSet &results, const bool column_format)
Definition: DBHandler.cpp:6170
const ResultSetPtr & getDataPtr() const
void convertResultSet(ExecutionResult &result, const Catalog_Namespace::SessionInfo &session_info, const std::string &query_state_str, TQueryResult &_return)
Definition: DBHandler.cpp:7344
const std::string getTargetQuerySessionToKill()
const std::string commandStr()
const std::shared_ptr< ResultSet > & getRows() const
ExecutionResult execute()
void interruptQuery(const Catalog_Namespace::SessionInfo &session_info, const std::string &target_session)
Definition: DBHandler.cpp:7552
void clear_gpu_memory(const TSessionId &session) override
Definition: DBHandler.cpp:2665
ExecutionResult getQueries(std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
Definition: DBHandler.cpp:7418
ExecutionResult getUserSessions(std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
Definition: DBHandler.cpp:7374
void clear_cpu_memory(const TSessionId &session) override
Definition: DBHandler.cpp:2686

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::executeDdl ( ExecutionResult _return,
const std::string &  query_ra,
std::shared_ptr< Catalog_Namespace::SessionInfo const session_ptr 
)
private

Definition at line 7704 of file DBHandler.cpp.

References ExecutionResult::CalciteDdl, clear_cpu_memory(), clear_gpu_memory(), clearRenderMemory(), DdlCommandExecutor::commandStr(), DdlCommandExecutor::execute(), measure< TimeT >::execution(), getQueries(), DdlCommandExecutor::getTargetQuerySessionToKill(), getUserSessions(), interruptQuery(), DdlCommandExecutor::isAlterSystemClear(), DdlCommandExecutor::isKillQuery(), DdlCommandExecutor::isShowCreateTable(), DdlCommandExecutor::isShowQueries(), DdlCommandExecutor::isShowUserSessions(), DdlCommandExecutor::returnCacheType(), ExecutionResult::setExecutionTime(), ExecutionResult::setResultType(), and ExecutionResult::SimpleResult.

7707  {
7708  DdlCommandExecutor executor = DdlCommandExecutor(query_ra, session_ptr);
7709  std::string commandStr = executor.commandStr();
7710 
7711  if (executor.isKillQuery()) {
7712  interruptQuery(*session_ptr, executor.getTargetQuerySessionToKill());
7713  } else {
7714  int64_t execution_time_ms;
7715  if (executor.isShowQueries()) {
7716  // getQueries still requires Thrift cannot be nested into DdlCommandExecutor
7717  execution_time_ms =
7718  measure<>::execution([&]() { _return = getQueries(session_ptr); });
7719  } else if (executor.isShowUserSessions()) {
7720  // getUserSessions still requires Thrift cannot be nested into DdlCommandExecutor
7721  execution_time_ms =
7722  measure<>::execution([&]() { _return = getUserSessions(session_ptr); });
7723  } else if (executor.isAlterSystemClear()) {
7724  _return = ExecutionResult();
7725  if (executor.returnCacheType() == "CPU") {
7726  execution_time_ms = measure<>::execution(
7727  [&]() { clear_cpu_memory(session_ptr->get_session_id()); });
7728  } else if (executor.returnCacheType() == "GPU") {
7729  execution_time_ms = measure<>::execution(
7730  [&]() { clear_gpu_memory(session_ptr->get_session_id()); });
7731  } else if (executor.returnCacheType() == "RENDER") {
7732  execution_time_ms = measure<>::execution(
7733  [&]() { clearRenderMemory(session_ptr->get_session_id()); });
7734  } else {
7735  throw std::runtime_error("Unknwon cache type. Cannot clear");
7736  }
7737  } else {
7738  execution_time_ms = measure<>::execution([&]() { _return = executor.execute(); });
7739  }
7740  _return.setExecutionTime(execution_time_ms);
7741  }
7742  if (executor.isShowCreateTable()) {
7744  } else {
7746  }
7747 }
std::string returnCacheType()
void clearRenderMemory(const TSessionId &session)
Definition: DBHandler.cpp:2707
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
const std::string getTargetQuerySessionToKill()
const std::string commandStr()
void setResultType(RType type)
ExecutionResult execute()
void setExecutionTime(int64_t execution_time_ms)
void interruptQuery(const Catalog_Namespace::SessionInfo &session_info, const std::string &target_session)
Definition: DBHandler.cpp:7552
void clear_gpu_memory(const TSessionId &session) override
Definition: DBHandler.cpp:2665
ExecutionResult getQueries(std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
Definition: DBHandler.cpp:7418
ExecutionResult getUserSessions(std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr)
Definition: DBHandler.cpp:7374
void clear_cpu_memory(const TSessionId &session) override
Definition: DBHandler.cpp:2686

+ Here is the call graph for this function:

template<>
void DBHandler::expire_idle_sessions_unsafe ( mapd_unique_lock< mapd_shared_mutex > &  write_lock)

Definition at line 188 of file DBHandler.cpp.

References check_session_exp_unsafe(), LeafAggregator::disconnect(), anonymous_namespace{DBHandler.cpp}::get_session_from_map(), logger::INFO, leaf_aggregator_, LeafAggregator::leafCount(), LOG, render_handler_, and sessions_.

189  {
190  std::vector<std::string> expired_sessions;
191  for (auto session_pair : sessions_) {
192  auto session_it = get_session_from_map(session_pair.first, sessions_);
193  try {
194  check_session_exp_unsafe(session_it);
195  } catch (const ForceDisconnect& e) {
196  expired_sessions.emplace_back(session_it->second->get_session_id());
197  }
198  }
199 
200  for (auto session_id : expired_sessions) {
201  if (leaf_aggregator_.leafCount() > 0) {
202  try {
203  leaf_aggregator_.disconnect(session_id);
204  } catch (TOmniSciException& toe) {
205  LOG(INFO) << " Problem disconnecting from leaves : " << toe.what();
206  } catch (std::exception& e) {
207  LOG(INFO)
208  << " Problem disconnecting from leaves, check leaf logs for additonal info";
209  }
210  }
211  sessions_.erase(session_id);
212  }
213  if (render_handler_) {
214  write_lock.unlock();
215  for (auto session_id : expired_sessions) {
216  // NOTE: the render disconnect is done after the session lock is released to
217  // avoid a deadlock. See: https://omnisci.atlassian.net/browse/BE-3324
218  // This out-of-scope solution is a compromise for now until a better session
219  // handling/locking mechanism is developed for the renderer. Note as well that the
220  // session_id cannot be immediately reused. If a render request were to slip in
221  // after the lock is released and before the render disconnect could cause a
222  // problem.
223  render_handler_->disconnect(session_id);
224  }
225  }
226 }
#define LOG(tag)
Definition: Logger.h:205
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:588
void disconnect(const TSessionId session)
SessionMap::iterator get_session_from_map(const TSessionId &session, SessionMap &session_map)
Definition: DBHandler.cpp:141
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:607
size_t leafCount() const
void check_session_exp_unsafe(const SessionMap::iterator &session_it)
Definition: DBHandler.cpp:5751
mapd_unique_lock< mapd_shared_mutex > write_lock
SessionMap sessions_
Definition: DBHandler.h:923

+ Here is the call graph for this function:

template<typename SESSION_MAP_LOCK >
void DBHandler::expire_idle_sessions_unsafe ( SESSION_MAP_LOCK &  lock)
private

Referenced by connect_impl().

+ Here is the caller graph for this function:

std::unordered_map< std::string, std::unordered_set< std::string > > DBHandler::fill_column_names_by_table ( std::vector< std::string > &  table_names,
query_state::StdLog stdlog 
)
private

Definition at line 1727 of file DBHandler.cpp.

References get_table_details_impl().

Referenced by get_token_based_completions().

1728  {
1729  std::unordered_map<std::string, std::unordered_set<std::string>> column_names_by_table;
1730  for (auto it = table_names.begin(); it != table_names.end();) {
1731  TTableDetails table_details;
1732  try {
1733  get_table_details_impl(table_details, stdlog, *it, false, false);
1734  } catch (const TOmniSciException& e) {
1735  // Remove the corrupted Table/View name from the list for further processing.
1736  it = table_names.erase(it);
1737  continue;
1738  }
1739  for (const auto& column_type : table_details.row_desc) {
1740  column_names_by_table[*it].emplace(column_type.col_name);
1741  }
1742  ++it;
1743  }
1744  return column_names_by_table;
1745 }
void get_table_details_impl(TTableDetails &_return, query_state::StdLog &stdlog, const std::string &table_name, const bool get_system, const bool get_physical, const std::string &database_name={})
Definition: DBHandler.cpp:2354

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::fillGeoColumns ( const TSessionId &  session,
const Catalog_Namespace::Catalog catalog,
std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &  import_buffers,
const ColumnDescriptor cd,
size_t &  col_idx,
size_t  num_rows,
const std::string &  table_name,
bool  assign_render_groups 
)
private

Definition at line 2906 of file DBHandler.cpp.

References CHECK, CHECK_EQ, ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::get_type(), Geospatial::GeoTypesFactory::getGeoColumns(), logger::INFO, IS_GEO_POLY, LOG, LOG_IF, render_group_assignment_map_, render_group_assignment_mutex_, import_export::Importer::set_geo_physical_import_buffer_columnar(), and THROW_MAPD_EXCEPTION.

Referenced by fillMissingBuffers(), load_table(), and load_table_binary_columnar_internal().

2914  {
2915  auto geo_col_idx = col_idx - 1;
2916  const auto wkt_or_wkb_hex_column = import_buffers[geo_col_idx]->getGeoStringBuffer();
2917  std::vector<std::vector<double>> coords_column, bounds_column;
2918  std::vector<std::vector<int>> ring_sizes_column, poly_rings_column;
2919  std::vector<int> render_groups_column;
2920  SQLTypeInfo ti = cd->columnType;
2921  if (num_rows != wkt_or_wkb_hex_column->size() ||
2922  !Geospatial::GeoTypesFactory::getGeoColumns(wkt_or_wkb_hex_column,
2923  ti,
2924  coords_column,
2925  bounds_column,
2926  ring_sizes_column,
2927  poly_rings_column,
2928  false)) {
2929  std::ostringstream oss;
2930  oss << "Invalid geometry in column " << cd->columnName;
2931  THROW_MAPD_EXCEPTION(oss.str());
2932  }
2933 
2934  // start or continue assigning render groups for poly columns?
2935  if (IS_GEO_POLY(cd->columnType.get_type()) && assign_render_groups) {
2936  // get RGA to use
2937  import_export::RenderGroupAnalyzer* render_group_analyzer{};
2938  {
2939  // mutex the map access
2940  std::lock_guard<std::mutex> lock(render_group_assignment_mutex_);
2941 
2942  // emplace new RGA or fetch existing RGA from map
2943  auto [itr_table, emplaced_table] = render_group_assignment_map_.try_emplace(
2944  session, RenderGroupAssignmentTableMap());
2945  LOG_IF(INFO, emplaced_table)
2946  << "load_table_binary_columnar_polys: Creating Render Group Assignment "
2947  "Persistent Data for Session '"
2948  << session << "'";
2949  auto [itr_column, emplaced_column] =
2950  itr_table->second.try_emplace(table_name, RenderGroupAssignmentColumnMap());
2951  LOG_IF(INFO, emplaced_column)
2952  << "load_table_binary_columnar_polys: Creating Render Group Assignment "
2953  "Persistent Data for Table '"
2954  << table_name << "'";
2955  auto [itr_analyzer, emplaced_analyzer] = itr_column->second.try_emplace(
2956  cd->columnName, std::make_unique<import_export::RenderGroupAnalyzer>());
2957  LOG_IF(INFO, emplaced_analyzer)
2958  << "load_table_binary_columnar_polys: Creating Render Group Assignment "
2959  "Persistent Data for Column '"
2960  << cd->columnName << "'";
2961  render_group_analyzer = itr_analyzer->second.get();
2962  CHECK(render_group_analyzer);
2963 
2964  // seed new RGA from existing table/column, to handle appends
2965  if (emplaced_analyzer) {
2966  LOG(INFO) << "load_table_binary_columnar_polys: Seeding Render Groups from "
2967  "existing table...";
2968  render_group_analyzer->seedFromExistingTableContents(
2969  catalog, table_name, cd->columnName);
2970  LOG(INFO) << "load_table_binary_columnar_polys: Done";
2971  }
2972  }
2973 
2974  // assign render groups for this set of bounds
2975  LOG(INFO) << "load_table_binary_columnar_polys: Assigning Render Groups...";
2976  render_groups_column.reserve(bounds_column.size());
2977  for (auto const& bounds : bounds_column) {
2978  CHECK_EQ(bounds.size(), 4u);
2979  int rg = render_group_analyzer->insertBoundsAndReturnRenderGroup(bounds);
2980  render_groups_column.push_back(rg);
2981  }
2982  LOG(INFO) << "load_table_binary_columnar_polys: Done";
2983  } else {
2984  // render groups all zero
2985  render_groups_column.resize(bounds_column.size(), 0);
2986  }
2987 
2988  // Populate physical columns, advance col_idx
2990  cd,
2991  import_buffers,
2992  col_idx,
2993  coords_column,
2994  bounds_column,
2995  ring_sizes_column,
2996  poly_rings_column,
2997  render_groups_column);
2998 }
#define CHECK_EQ(x, y)
Definition: Logger.h:219
std::unordered_map< std::string, RenderGroupAssignmentColumnMap > RenderGroupAssignmentTableMap
Definition: DBHandler.h:1067
std::mutex render_group_assignment_mutex_
Definition: DBHandler.h:1071
#define LOG(tag)
Definition: Logger.h:205
static void set_geo_physical_import_buffer_columnar(const Catalog_Namespace::Catalog &catalog, const ColumnDescriptor *cd, std::vector< std::unique_ptr< TypedImportBuffer >> &import_buffers, size_t &col_idx, std::vector< std::vector< double >> &coords_column, std::vector< std::vector< double >> &bounds_column, std::vector< std::vector< int >> &ring_sizes_column, std::vector< std::vector< int >> &poly_rings_column, std::vector< int > &render_groups_column)
Definition: Importer.cpp:1735
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:329
#define LOG_IF(severity, condition)
Definition: Logger.h:301
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
static bool getGeoColumns(const std::string &wkt_or_wkb_hex, SQLTypeInfo &ti, std::vector< double > &coords, std::vector< double > &bounds, std::vector< int > &ring_sizes, std::vector< int > &poly_rings, const bool promote_poly_to_mpoly=false)
Definition: Types.cpp:937
#define CHECK(condition)
Definition: Logger.h:211
RenderGroupAnalyzerSessionMap render_group_assignment_map_
Definition: DBHandler.h:1070
SQLTypeInfo columnType
std::unordered_map< std::string, std::unique_ptr< import_export::RenderGroupAnalyzer >> RenderGroupAssignmentColumnMap
Definition: DBHandler.h:1065
std::string columnName
#define IS_GEO_POLY(T)
Definition: sqltypes.h:255

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::fillMissingBuffers ( const TSessionId &  session,
const Catalog_Namespace::Catalog catalog,
std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &  import_buffers,
const std::list< const ColumnDescriptor * > &  cds,
const std::vector< int > &  desc_id_to_column_id,
size_t  num_rows,
const std::string &  table_name,
bool  assign_render_groups 
)
private

Definition at line 3000 of file DBHandler.cpp.

References CHECK, and fillGeoColumns().

Referenced by load_table(), load_table_binary(), load_table_binary_arrow(), and load_table_binary_columnar_internal().

3008  {
3009  size_t skip_physical_cols = 0;
3010  size_t col_idx = 0, import_idx = 0;
3011  for (const auto& cd : cds) {
3012  if (skip_physical_cols > 0) {
3013  CHECK(cd->isGeoPhyCol);
3014  skip_physical_cols--;
3015  continue;
3016  } else if (cd->columnType.is_geometry()) {
3017  skip_physical_cols = cd->columnType.get_physical_cols();
3018  }
3019  if (desc_id_to_column_id[import_idx] == -1) {
3020  import_buffers[col_idx]->addDefaultValues(cd, num_rows);
3021  col_idx++;
3022  if (cd->columnType.is_geometry()) {
3023  fillGeoColumns(session,
3024  catalog,
3025  import_buffers,
3026  cd,
3027  col_idx,
3028  num_rows,
3029  table_name,
3030  assign_render_groups);
3031  }
3032  } else {
3033  col_idx++;
3034  col_idx += skip_physical_cols;
3035  }
3036  import_idx++;
3037  }
3038 }
void fillGeoColumns(const TSessionId &session, const Catalog_Namespace::Catalog &catalog, std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &import_buffers, const ColumnDescriptor *cd, size_t &col_idx, size_t num_rows, const std::string &table_name, bool assign_render_groups)
Definition: DBHandler.cpp:2906
#define CHECK(condition)
Definition: Logger.h:211

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

TRowDescriptor DBHandler::fixup_row_descriptor ( const TRowDescriptor &  row_desc,
const Catalog_Namespace::Catalog cat 
)
private

Definition at line 6110 of file DBHandler.cpp.

References DictDescriptor::dictNBits, and Catalog_Namespace::Catalog::getMetadataForDict().

Referenced by get_table_details_impl(), and sql_validate().

6111  {
6112  TRowDescriptor fixedup_row_desc;
6113  for (const TColumnType& col_desc : row_desc) {
6114  auto fixedup_col_desc = col_desc;
6115  if (col_desc.col_type.encoding == TEncodingType::DICT &&
6116  col_desc.col_type.comp_param > 0) {
6117  const auto dd = cat.getMetadataForDict(col_desc.col_type.comp_param, false);
6118  fixedup_col_desc.col_type.comp_param = dd->dictNBits;
6119  }
6120  fixedup_row_desc.push_back(fixedup_col_desc);
6121  }
6122 
6123  return fixedup_row_desc;
6124 }
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1554

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void DBHandler::get_all_effective_roles_for_user ( std::vector< std::string > &  _return,
const TSessionId &  session,
const std::string &  granteeName 
)
override

Definition at line 2206 of file DBHandler.cpp.

References get_session_ptr(), getAllRolesForUserImpl(), and STDLOG.

2208  {
2209  auto stdlog = STDLOG(get_session_ptr(sessionId));
2210  auto session_ptr = stdlog.getConstSessionInfo();
2211  getAllRolesForUserImpl(session_ptr, roles, sessionId, granteeName, /*effective=*/true);
2212 }
void getAllRolesForUserImpl(std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr, std::vector< std::string > &roles, const TSessionId &sessionId, const std::string &granteeName, bool effective)
Definition: DBHandler.cpp:2162
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5801
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

void DBHandler::get_all_files_in_archive ( std::vector< std::string > &  _return,
const TSessionId &  session,
const std::string &  archive_path,
const TCopyParams &  copy_params 
)
override

Definition at line 5604 of file DBHandler.cpp.

References anonymous_namespace{DBHandler.cpp}::add_vsi_archive_prefix(), anonymous_namespace{DBHandler.cpp}::add_vsi_network_prefix(), import_export::Importer::gdalFileExists(), import_export::Importer::gdalGetAllFilesInArchive(), get_session_ptr(), getConnectionInfo(), import_path_, anonymous_namespace{DBHandler.cpp}::is_a_supported_archive_file(), anonymous_namespace{DBHandler.cpp}::path_is_relative(), STDLOG, thrift_to_copyparams(), THROW_MAPD_EXCEPTION, toString(), and anonymous_namespace{DBHandler.cpp}::validate_import_file_path_if_local().

5607  {
5608  auto stdlog =
5609  STDLOG(get_session_ptr(session), "get_all_files_in_archive", archive_path_in);
5610  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
5611 
5612  std::string archive_path(archive_path_in);
5613  if (path_is_relative(archive_path)) {
5614  // assume relative paths are relative to data_path / mapd_import / <session>
5615  auto file_path = import_path_ / picosha2::hash256_hex_string(session) /
5616  boost::filesystem::path(archive_path).filename();
5617  archive_path = file_path.string();
5618  }
5619  validate_import_file_path_if_local(archive_path);
5620 
5621  if (is_a_supported_archive_file(archive_path)) {
5622  // find the archive file
5623  add_vsi_network_prefix(archive_path);
5624  if (!import_export::Importer::gdalFileExists(archive_path,
5625  thrift_to_copyparams(copy_params))) {
5626  THROW_MAPD_EXCEPTION("Archive does not exist: " + archive_path_in);
5627  }
5628  // find all files in archive
5629  add_vsi_archive_prefix(archive_path);
5631  archive_path, thrift_to_copyparams(copy_params));
5632  // prepend them all with original path
5633  for (auto& s : _return) {
5634  s = archive_path_in + '/' + s;
5635  }
5636  }
5637 }
boost::filesystem::path import_path_
Definition: DBHandler.h:592
void add_vsi_archive_prefix(std::string &path)
Definition: DBHandler.cpp:3892
bool is_a_supported_archive_file(const std::string &path)
Definition: DBHandler.cpp:3958
std::string toString(const ExtArgumentType &sig_type)
bool path_is_relative(const std::string &path)
Definition: DBHandler.cpp:3926
static bool gdalFileExists(const std::string &path, const CopyParams &copy_params)
Definition: Importer.cpp:4982
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5801
void validate_import_file_path_if_local(const std::string &file_path)
Definition: DBHandler.cpp:4012
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
void add_vsi_network_prefix(std::string &path)
Definition: DBHandler.cpp:3863
static std::vector< std::string > gdalGetAllFilesInArchive(const std::string &archive_path, const CopyParams &copy_params)
Definition: Importer.cpp:5059
import_export::CopyParams thrift_to_copyparams(const TCopyParams &cp)
Definition: DBHandler.cpp:3564
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1747
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

void DBHandler::get_all_roles_for_user ( std::vector< std::string > &  _return,
const TSessionId &  session,
const std::string &  granteeName 
)
override

Definition at line 2196 of file DBHandler.cpp.

References get_session_ptr(), getAllRolesForUserImpl(), and STDLOG.

2198  {
2199  // WARNING: This function only returns directly granted roles.
2200  // See also: get_all_effective_roles_for_user() for all of a user's roles.
2201  auto stdlog = STDLOG(get_session_ptr(sessionId));
2202  auto session_ptr = stdlog.getConstSessionInfo();
2203  getAllRolesForUserImpl(session_ptr, roles, sessionId, granteeName, /*effective=*/false);
2204 }
void getAllRolesForUserImpl(std::shared_ptr< Catalog_Namespace::SessionInfo const > session_ptr, std::vector< std::string > &roles, const TSessionId &sessionId, const std::string &granteeName, bool effective)
Definition: DBHandler.cpp:2162
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5801
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

void DBHandler::get_completion_hints ( std::vector< TCompletionHint > &  hints,
const TSessionId &  session,
const std::string &  sql,
const int  cursor 
)
override

Definition at line 1620 of file DBHandler.cpp.

References anonymous_namespace{DBHandler.cpp}::extract_projection_tokens_for_completion(), get_completion_hints_unsorted(), get_session_ptr(), get_uc_compatible_table_names_by_column(), gpu_enabled::sort(), STDLOG, and to_upper().

1623  {
1624  auto stdlog = STDLOG(get_session_ptr(session));
1625  std::vector<std::string> visible_tables; // Tables allowed for the given session.
1626  get_completion_hints_unsorted(hints, visible_tables, stdlog, sql, cursor);
1627  const auto proj_tokens = extract_projection_tokens_for_completion(sql);
1628  auto compatible_table_names = get_uc_compatible_table_names_by_column(
1629  proj_tokens.uc_column_names, visible_tables, stdlog);
1630  // Add the table qualifiers explicitly specified by the user.
1631  compatible_table_names.insert(proj_tokens.uc_column_table_qualifiers.begin(),
1632  proj_tokens.uc_column_table_qualifiers.end());
1633  // Sort the hints by category, from COLUMN (most specific) to KEYWORD.
1634  std::sort(
1635  hints.begin(),
1636  hints.end(),
1637  [&compatible_table_names](const TCompletionHint& lhs, const TCompletionHint& rhs) {
1638  if (lhs.type == TCompletionHintType::TABLE &&
1639  rhs.type == TCompletionHintType::TABLE) {
1640  // Between two tables, one which is compatible with the specified
1641  // projections and one which isn't, pick the one which is compatible.
1642  if (compatible_table_names.find(to_upper(lhs.hints.back())) !=
1643  compatible_table_names.end() &&
1644  compatible_table_names.find(to_upper(rhs.hints.back())) ==
1645  compatible_table_names.end()) {
1646  return true;
1647  }
1648  }
1649  return lhs.type < rhs.type;
1650  });
1651 }
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5801
ProjectionTokensForCompletion extract_projection_tokens_for_completion(const std::string &sql)
Definition: DBHandler.cpp:1596
void get_completion_hints_unsorted(std::vector< TCompletionHint > &hints, std::vector< std::string > &visible_tables, query_state::StdLog &stdlog, const std::string &sql, const int cursor)
Definition: DBHandler.cpp:1653
std::string to_upper(const std::string &str)
std::unordered_set< std::string > get_uc_compatible_table_names_by_column(const std::unordered_set< std::string > &uc_column_names, std::vector< std::string > &table_names, query_state::StdLog &stdlog)
Definition: DBHandler.cpp:1752
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

void DBHandler::get_completion_hints_unsorted ( std::vector< TCompletionHint > &  hints,
std::vector< std::string > &  visible_tables,
query_state::StdLog stdlog,
const std::string &  sql,
const int  cursor 
)
private

Definition at line 1653 of file DBHandler.cpp.

References calcite_, logger::ERROR, GET_PHYSICAL_TABLES_AND_VIEWS, get_tables_impl(), get_token_based_completions(), query_state::StdLog::getConstSessionInfo(), just_whitelisted_keyword_hints(), and LOG.

Referenced by get_completion_hints().

1657  {
1658  const auto& session_info = *stdlog.getConstSessionInfo();
1659  try {
1660  get_tables_impl(visible_tables, session_info, GET_PHYSICAL_TABLES_AND_VIEWS);
1661 
1662  // Filter out keywords suggested by Calcite which we don't support.
1664  calcite_->getCompletionHints(session_info, visible_tables, sql, cursor));
1665  } catch (const std::exception& e) {
1666  TOmniSciException ex;
1667  ex.error_msg = std::string(e.what());
1668  LOG(ERROR) << ex.error_msg;
1669  throw ex;
1670  }
1671  boost::regex from_expr{R"(\s+from\s+)", boost::regex::extended | boost::regex::icase};
1672  const size_t length_to_cursor =
1673  cursor < 0 ? sql.size() : std::min(sql.size(), static_cast<size_t>(cursor));
1674  // Trust hints from Calcite after the FROM keyword.
1675  if (boost::regex_search(sql.cbegin(), sql.cbegin() + length_to_cursor, from_expr)) {
1676  return;
1677  }
1678  // Before FROM, the query is too incomplete for context-sensitive completions.
1679  get_token_based_completions(hints, stdlog, visible_tables, sql, cursor);
1680 }
#define LOG(tag)
Definition: Logger.h:205
std::vector< TCompletionHint > just_whitelisted_keyword_hints(const std::vector< TCompletionHint > &hints)
void get_tables_impl(std::vector< std::string > &table_names, const Catalog_Namespace::SessionInfo &, const GetTablesType get_tables_type, const std::string &database_name={})
Definition: DBHandler.cpp:2483
void get_token_based_completions(std::vector< TCompletionHint > &hints, query_state::StdLog &stdlog, std::vector< std::string > &visible_tables, const std::string &sql, const int cursor)
Definition: DBHandler.cpp:1682
std::shared_ptr< Calcite > calcite_
Definition: DBHandler.h:610
std::shared_ptr< Catalog_Namespace::SessionInfo const > getConstSessionInfo() const
Definition: QueryState.cpp:159

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::shared_ptr< const Catalog_Namespace::SessionInfo > DBHandler::get_const_session_ptr ( const TSessionId &  session)
private

Definition at line 5778 of file DBHandler.cpp.

References get_session_ptr().

5779  {
5780  return get_session_ptr(session);
5781 }
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5801

+ Here is the call graph for this function:

void DBHandler::get_custom_expressions ( std::vector< TCustomExpression > &  _return,
const TSessionId &  session 
)
override

Definition at line 4296 of file DBHandler.cpp.

References anonymous_namespace{DBHandler.cpp}::create_thrift_obj_from_custom_expr(), custom_expressions_mutex_, get_session_ptr(), getConnectionInfo(), STDLOG, and toString().

4297  {
4298  auto stdlog = STDLOG(get_session_ptr(session));
4299  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4300 
4301  auto session_ptr = stdlog.getConstSessionInfo();
4302  auto& catalog = session_ptr->getCatalog();
4303  mapd_shared_lock<mapd_shared_mutex> read_lock(custom_expressions_mutex_);
4304  auto custom_expressions =
4305  catalog.getCustomExpressionsForUser(session_ptr->get_currentUser());
4306  for (const auto& custom_expression : custom_expressions) {
4307  _return.emplace_back(create_thrift_obj_from_custom_expr(*custom_expression, catalog));
4308  }
4309 }
std::string toString(const ExtArgumentType &sig_type)
mapd_shared_mutex custom_expressions_mutex_
Definition: DBHandler.h:1072
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5801
TCustomExpression create_thrift_obj_from_custom_expr(const CustomExpression &custom_expr, const Catalog &catalog)
Definition: DBHandler.cpp:4256
mapd_shared_lock< mapd_shared_mutex > read_lock
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1747
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

void DBHandler::get_dashboard ( TDashboard &  _return,
const TSessionId &  session,
const int32_t  dashboard_id 
)
override

Definition at line 4345 of file DBHandler.cpp.

References cat(), get_dashboard_impl(), get_session_ptr(), getConnectionInfo(), is_allowed_on_dashboard(), STDLOG, THROW_MAPD_EXCEPTION, to_string(), toString(), Catalog_Namespace::UserMetadata::userName, and AccessPrivileges::VIEW_DASHBOARD.

4347  {
4348  auto stdlog = STDLOG(get_session_ptr(session));
4349  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4350  auto session_ptr = stdlog.getConstSessionInfo();
4351  auto const& cat = session_ptr->getCatalog();
4353  auto dash = cat.getMetadataForDashboard(dashboard_id);
4354  if (!dash) {
4355  THROW_MAPD_EXCEPTION("Dashboard with dashboard id " + std::to_string(dashboard_id) +
4356  " doesn't exist");
4357  }
4359  *session_ptr, dash->dashboardId, AccessPrivileges::VIEW_DASHBOARD)) {
4360  THROW_MAPD_EXCEPTION("User has no view privileges for the dashboard with id " +
4361  std::to_string(dashboard_id));
4362  }
4363  user_meta.userName = "";
4364  SysCatalog::instance().getMetadataForUserById(dash->userId, user_meta);
4365  dashboard = get_dashboard_impl(session_ptr, user_meta, dash);
4366 }
static const AccessPrivileges VIEW_DASHBOARD
Definition: DBObject.h:173
std::string cat(Ts &&...args)
std::string toString(const ExtArgumentType &sig_type)
std::string to_string(char const *&&v)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5801
TDashboard get_dashboard_impl(const std::shared_ptr< Catalog_Namespace::SessionInfo const > &session_ptr, Catalog_Namespace::UserMetadata &user_meta, const DashboardDescriptor *dash, const bool populate_state=true)
Definition: DBHandler.cpp:4388
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
static bool is_allowed_on_dashboard(const Catalog_Namespace::SessionInfo &session_info, int32_t dashboard_id, AccessPrivileges requestedPermissions)
Definition: DBHandler.cpp:4220
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1747
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

void DBHandler::get_dashboard_grantees ( std::vector< TDashboardGrantees > &  _return,
const TSessionId &  session,
const int32_t  dashboard_id 
)
override

Definition at line 4713 of file DBHandler.cpp.

References cat(), DashboardPrivileges::CREATE_DASHBOARD, DashboardDBObjectType, DashboardPrivileges::DELETE_DASHBOARD, DashboardPrivileges::EDIT_DASHBOARD, get_session_ptr(), getConnectionInfo(), STDLOG, THROW_MAPD_EXCEPTION, to_string(), toString(), Catalog_Namespace::UserMetadata::userId, Catalog_Namespace::UserMetadata::userName, and DashboardPrivileges::VIEW_DASHBOARD.

4716  {
4717  auto stdlog = STDLOG(get_session_ptr(session));
4718  stdlog.appendNameValuePairs("client", getConnectionInfo().toString());
4719  auto session_ptr = stdlog.getConstSessionInfo();
4720  auto const& cat = session_ptr->getCatalog();
4722  auto dash = cat.getMetadataForDashboard(dashboard_id);
4723  if (!dash) {
4724  THROW_MAPD_EXCEPTION("Dashboard id " + std::to_string(dashboard_id) +
4725  " does not exist");
4726  } else if (session_ptr->get_currentUser().userId != dash->userId &&
4727  !session_ptr->get_currentUser().isSuper) {
4729  "User should be either owner of dashboard or super user to access grantees");
4730  }
4731  std::vector<ObjectRoleDescriptor*> objectsList;
4732  objectsList = SysCatalog::instance().getMetadataForObject(
4733  cat.getCurrentDB().dbId,
4734  static_cast<int>(DBObjectType::DashboardDBObjectType),
4735  dashboard_id); // By default objecttypecan be only dashabaords
4736  user_meta.userId = -1;
4737  user_meta.userName = "";
4738  SysCatalog::instance().getMetadataForUserById(dash->userId, user_meta);
4739  for (auto object : objectsList) {
4740  if (user_meta.userName == object->roleName) {
4741  // Mask owner
4742  continue;
4743  }
4744  TDashboardGrantees grantee;
4745  TDashboardPermissions perm;
4746  grantee.name = object->roleName;
4747  grantee.is_user = object->roleType;
4748  perm.create_ = object->privs.hasPermission(DashboardPrivileges::CREATE_DASHBOARD);
4749  perm.delete_ = object->privs.hasPermission(DashboardPrivileges::DELETE_DASHBOARD);
4750  perm.edit_ = object->privs.hasPermission(DashboardPrivileges::EDIT_DASHBOARD);
4751  perm.view_ = object->privs.hasPermission(DashboardPrivileges::VIEW_DASHBOARD);
4752  grantee.permissions = perm;
4753  dashboard_grantees.push_back(grantee);
4754  }
4755 }
std::string cat(Ts &&...args)
std::string toString(const ExtArgumentType &sig_type)
std::string to_string(char const *&&v)
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
Definition: DBHandler.cpp:5801
static const int32_t EDIT_DASHBOARD
Definition: DBObject.h:106
static const int32_t DELETE_DASHBOARD
Definition: DBObject.h:104
#define THROW_MAPD_EXCEPTION(errstr)
Definition: DBHandler.cpp:128
static const int32_t VIEW_DASHBOARD
Definition: DBObject.h:105
static const int32_t CREATE_DASHBOARD
Definition: DBObject.h:103
ConnectionInfo getConnectionInfo() const
Definition: DBHandler.cpp:1747
#define STDLOG(...)
Definition: QueryState.h:235

+ Here is the call graph for this function:

TDashboard DBHandler::get_dashboard_impl ( const std::shared_ptr< Catalog_Namespace::SessionInfo const > &  session_ptr,
Catalog_Namespace::UserMetadata user_meta,
const DashboardDescriptor dash,
const bool  populate_state = true 
)
private

Definition at line 4388 of file DBHandler.cpp.

References cat(), DashboardPrivileges::CREATE_DASHBOARD, DashboardDBObjectType, DashboardDescriptor::dashboardId, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, DashboardPrivileges::DELETE_DASHBOARD, DashboardPrivileges::EDIT_DASHBOARD, DBObject::getPrivileges(), AccessPrivileges::hasPermission(), DashboardDescriptor::imageHash, DashboardDescriptor::updateTime, DashboardDescriptor::user, DashboardDescriptor::userId, Catalog_Namespace::UserMetadata::userName, and DashboardPrivileges::VIEW_DASHBOARD.

Referenced by get_dashboard(), and get_dashboards().

4392  {
4393  auto const& cat = session_ptr->getCatalog();
4394  SysCatalog::instance().getMetadataForUserById(dash->userId, user_meta);
4395  auto objects_list = SysCatalog::instance().getMetadataForObject(
4396  cat.getCurrentDB().dbId,
4397  static_cast<int>(DBObjectType::DashboardDBObjectType),
4398  dash->dashboardId);
4399  TDashboard dashboard;
4400  dashboard.dashboard_name = dash->dashboardName;
4401  if (populate_state) {
4402  dashboard.dashboard_state = dash->dashboardState;
4403  }
4404  dashboard.image_hash = dash->imageHash;
4405  dashboard.update_time = dash->updateTime;
4406  dashboard.dashboard_metadata = dash->dashboardMetadata;
4407  dashboard.dashboard_id = dash->dashboardId;
4408  dashboard.dashboard_owner = dash->user;
4409  TDashboardPermissions perms;
4410  // Super user has all permissions.
4411  if (session_ptr->get_currentUser().isSuper) {
4412  perms.create_ = true;
4413  perms.delete_ = true;
4414  perms.edit_ = true;
4415  perms.view_ = true;
4416  } else {
4417  // Collect all grants on current user
4418  // add them to the permissions.
4419  auto obj_to_find =
4420  DBObject(dashboard.dashboard_id, DBObjectType::DashboardDBObjectType);
4421  obj_to_find.loadKey(session_ptr->getCatalog());
4422  std::vector<std::string> grantees =
4423  SysCatalog::instance().getRoles(true,
4424  session_ptr->get_currentUser().isSuper,
4425  session_ptr->get_currentUser().userName);
4426  for (const auto& grantee : grantees) {
4427  DBObject* object_found;
4428  auto* gr = SysCatalog::instance().getGrantee(grantee);
4429  if (gr && (object_found = gr->findDbObject(obj_to_find.getObjectKey(), true))) {
4430  const auto obj_privs = object_found->getPrivileges();
4431  perms.create_ |= obj_privs.hasPermission(DashboardPrivileges::CREATE_DASHBOARD);
4432  perms.delete_ |= obj_privs.hasPermission(DashboardPrivileges::DELETE_DASHBOARD);
4433  perms.edit_ |= obj_privs.hasPermission(DashboardPrivileges::EDIT_DASHBOARD);
4434  perms.view_ |= obj_privs.hasPermission(DashboardPrivileges::VIEW_DASHBOARD);
4435  }
4436  }
4437  }
4438  dashboard.dashboard_permissions = perms;
4439  if (objects_list.empty() ||
4440  (objects_list.size() == 1 && objects_list[0]->roleName == user_meta.userName)) {
4441  dashboard.is_dash_shared = false;
4442  } else {
4443  dashboard.is_dash_shared = true;
4444  }
4445  return dashboard;
4446 }
std::string cat(Ts &&...args)
static const int32_t EDIT_DASHBOARD
Definition: DBObject.h:106