OmniSciDB  d2f719934e
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
DBHandler.h
Go to the documentation of this file.
1 /*
2  * Copyright 2020 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 /*
18  * File: DBHandler.h
19  * Author: michael
20  *
21  * Created on Jan 1, 2017, 12:40 PM
22  */
23 
24 #pragma once
25 
26 #include "LeafAggregator.h"
27 
28 #ifdef HAVE_PROFILER
29 #include <gperftools/heap-profiler.h>
30 #endif // HAVE_PROFILER
31 
32 #include "Calcite/Calcite.h"
33 #include "Catalog/Catalog.h"
35 #include "Geospatial/Transforms.h"
36 #include "ImportExport/Importer.h"
38 #include "LockMgr/LockMgr.h"
39 #include "Logger/Logger.h"
40 #include "Parser/ParserWrapper.h"
42 #include "Parser/parser.h"
45 #include "QueryEngine/Execute.h"
51 #include "Shared/StringTransform.h"
54 #include "Shared/measure.h"
55 #include "Shared/scope.h"
61 
62 #include <sys/types.h>
63 #include <thrift/server/TServer.h>
64 #include <thrift/transport/THttpTransport.h>
65 #include <thrift/transport/TSocket.h>
66 #include <thrift/transport/TTransport.h>
67 #include <atomic>
68 #include <boost/algorithm/string.hpp>
69 #include <boost/algorithm/string/replace.hpp>
70 #include <boost/algorithm/string/trim.hpp>
71 #include <boost/filesystem.hpp>
72 #include <boost/make_shared.hpp>
73 #include <boost/noncopyable.hpp>
74 #include <boost/none_t.hpp>
75 #include <boost/optional.hpp>
76 #include <boost/program_options.hpp>
77 #include <boost/regex.hpp>
78 #include <boost/tokenizer.hpp>
79 #include <cmath>
80 #include <csignal>
81 #include <fstream>
82 #include <list>
83 #include <map>
84 #include <memory>
85 #include <mutex>
86 #include <random>
87 #include <regex>
88 #include <string>
89 #include <thread>
90 #include <typeinfo>
91 #include <unordered_map>
92 
93 #include "gen-cpp/OmniSci.h"
94 #include "gen-cpp/extension_functions_types.h"
95 
96 using namespace std::string_literals;
97 
98 class MapDAggHandler;
99 class MapDLeafHandler;
100 
101 // Multiple concurrent requests for the same session can occur. For that reason, each
102 // request briefly takes a lock to make a copy of the appropriate SessionInfo object. Then
103 // it releases the lock and uses the copy for the remainder of the request.
104 using SessionMap = std::map<TSessionId, std::shared_ptr<Catalog_Namespace::SessionInfo>>;
105 using PermissionFuncPtr = bool (*)(const AccessPrivileges&, const TDBObjectPermissions&);
107 
108 class TrackingProcessor : public OmniSciProcessor {
109  public:
110  TrackingProcessor(std::shared_ptr<OmniSciIf> handler, const bool check_origin)
111  : OmniSciProcessor(handler), check_origin_(check_origin) {}
112 
113  bool process(std::shared_ptr<::apache::thrift::protocol::TProtocol> in,
114  std::shared_ptr<::apache::thrift::protocol::TProtocol> out,
115  void* connectionContext) override {
116  using namespace ::apache::thrift;
117 
118  auto transport = in->getTransport();
119  if (transport && check_origin_) {
120  static std::mutex processor_mutex;
121  std::lock_guard lock(processor_mutex);
122  const auto origin_str = transport->getOrigin();
123  std::vector<std::string> origins;
124  boost::split(origins, origin_str, boost::is_any_of(","));
125  if (origins.empty()) {
127  } else {
128  // Take the first origin, which should be the client IP before any intermediate
129  // servers (e.g. the web server)
130  auto trimmed_origin = origins.front();
131  boost::algorithm::trim(trimmed_origin);
132  TrackingProcessor::client_address = trimmed_origin;
133  }
134  if (dynamic_cast<transport::THttpTransport*>(transport.get())) {
136  } else if (dynamic_cast<transport::TBufferedTransport*>(transport.get())) {
138  } else {
140  }
141  } else {
143  }
144 
145  return OmniSciProcessor::process(in, out, connectionContext);
146  }
147 
148  static thread_local std::string client_address;
149  static thread_local ClientProtocol client_protocol;
150 
151  private:
152  const bool check_origin_;
153 };
154 
155 namespace File_Namespace {
156 struct DiskCacheConfig;
157 }
158 
159 class DBHandler : public OmniSciIf {
160  public:
161  DBHandler(const std::vector<LeafHostInfo>& db_leaves,
162  const std::vector<LeafHostInfo>& string_leaves,
163  const std::string& base_data_path,
164  const bool allow_multifrag,
165  const bool jit_debug,
166  const bool intel_jit_profile,
167  const bool read_only,
168  const bool allow_loop_joins,
169  const bool enable_rendering,
170  const bool renderer_use_vulkan_driver,
171  const bool renderer_prefer_igpu,
172  const unsigned renderer_vulkan_timeout_ms,
173  const bool enable_auto_clear_render_mem,
174  const int render_oom_retry_threshold,
175  const size_t render_mem_bytes,
176  const size_t max_concurrent_render_sessions,
177  const size_t reserved_gpu_mem,
178  const bool render_compositor_use_last_gpu,
179  const size_t num_reader_threads,
180  const AuthMetadata& authMetadata,
181  SystemParameters& system_parameters,
182  const bool legacy_syntax,
183  const int idle_session_duration,
184  const int max_session_duration,
185  const bool enable_runtime_udf_registration,
186  const std::string& udf_filename,
187  const std::string& clang_path,
188  const std::vector<std::string>& clang_options,
189 #ifdef ENABLE_GEOS
190  const std::string& libgeos_so_filename,
191 #endif
192  const File_Namespace::DiskCacheConfig& disk_cache_config,
193  const bool is_new_db);
194  void initialize(const bool is_new_db);
195  ~DBHandler() override;
196 
197  static inline size_t max_bytes_for_thrift() { return 2 * 1000 * 1000 * 1000LL; }
198 
199  // Important ****
200  // This block must be keep in sync with mapd.thrift and HAHandler.h
201  // Please keep in same order for easy check and cut and paste
202  // Important ****
203  static void parser_with_error_handler(
204  const std::string& query_str,
205  std::list<std::unique_ptr<Parser::Stmt>>& parse_trees);
206 
207  void krb5_connect(TKrb5Session& session,
208  const std::string& token,
209  const std::string& dbname) override;
210  // connection, admin
211  void connect(TSessionId& session,
212  const std::string& username,
213  const std::string& passwd,
214  const std::string& dbname) override;
215  void disconnect(const TSessionId& session) override;
216  void switch_database(const TSessionId& session, const std::string& dbname) override;
217  void clone_session(TSessionId& session2, const TSessionId& session1) override;
218  void get_server_status(TServerStatus& _return, const TSessionId& session) override;
219  void get_status(std::vector<TServerStatus>& _return,
220  const TSessionId& session) override;
221  void get_hardware_info(TClusterHardwareInfo& _return,
222  const TSessionId& session) override;
223 
224  bool hasTableAccessPrivileges(const TableDescriptor* td,
225  const Catalog_Namespace::SessionInfo& session_info);
226  void get_tables(std::vector<std::string>& _return, const TSessionId& session) override;
227  void get_tables_for_database(std::vector<std::string>& _return,
228  const TSessionId& session,
229  const std::string& database_name) override;
230  void get_physical_tables(std::vector<std::string>& _return,
231  const TSessionId& session) override;
232  void get_views(std::vector<std::string>& _return, const TSessionId& session) override;
233  void get_tables_meta(std::vector<TTableMeta>& _return,
234  const TSessionId& session) override;
235  void get_table_details(TTableDetails& _return,
236  const TSessionId& session,
237  const std::string& table_name) override;
238  void get_table_details_for_database(TTableDetails& _return,
239  const TSessionId& session,
240  const std::string& table_name,
241  const std::string& database_name) override;
242  void get_internal_table_details(TTableDetails& _return,
243  const TSessionId& session,
244  const std::string& table_name) override;
245  void get_internal_table_details_for_database(TTableDetails& _return,
246  const TSessionId& session,
247  const std::string& table_name,
248  const std::string& database_name) override;
249  void get_users(std::vector<std::string>& _return, const TSessionId& session) override;
250  void get_databases(std::vector<TDBInfo>& _return, const TSessionId& session) override;
251 
252  void get_version(std::string& _return) override;
253  void start_heap_profile(const TSessionId& session) override;
254  void stop_heap_profile(const TSessionId& session) override;
255  void get_heap_profile(std::string& _return, const TSessionId& session) override;
256  void get_memory(std::vector<TNodeMemoryInfo>& _return,
257  const TSessionId& session,
258  const std::string& memory_level) override;
259  void clear_cpu_memory(const TSessionId& session) override;
260  void clear_gpu_memory(const TSessionId& session) override;
261  void clearRenderMemory(const TSessionId& session); // it's not declared on thrifth
262  // and on persisten leaf client
263  void set_cur_session(const TSessionId& parent_session,
264  const TSessionId& leaf_session,
265  const std::string& start_time_str,
266  const std::string& label,
267  bool for_running_query_kernel) override;
268  void invalidate_cur_session(const TSessionId& parent_session,
269  const TSessionId& leaf_session,
270  const std::string& start_time_str,
271  const std::string& label,
272  bool for_running_query_kernel) override;
273  void set_table_epoch(const TSessionId& session,
274  const int db_id,
275  const int table_id,
276  const int new_epoch) override;
277  void set_table_epoch_by_name(const TSessionId& session,
278  const std::string& table_name,
279  const int new_epoch) override;
280  int32_t get_table_epoch(const TSessionId& session,
281  const int32_t db_id,
282  const int32_t table_id) override;
283  int32_t get_table_epoch_by_name(const TSessionId& session,
284  const std::string& table_name) override;
285  void get_table_epochs(std::vector<TTableEpochInfo>& _return,
286  const TSessionId& session,
287  const int32_t db_id,
288  const int32_t table_id) override;
289  void set_table_epochs(const TSessionId& session,
290  const int32_t db_id,
291  const std::vector<TTableEpochInfo>& table_epochs) override;
292 
293  void get_session_info(TSessionInfo& _return, const TSessionId& session) override;
294 
295  void sql_execute(ExecutionResult& _return,
296  const TSessionId& session,
297  const std::string& query,
298  const bool column_format,
299  const int32_t first_n,
300  const int32_t at_most_n);
301  // query, render
302  void sql_execute(TQueryResult& _return,
303  const TSessionId& session,
304  const std::string& query,
305  const bool column_format,
306  const std::string& nonce,
307  const int32_t first_n,
308  const int32_t at_most_n) override;
309  void get_completion_hints(std::vector<TCompletionHint>& hints,
310  const TSessionId& session,
311  const std::string& sql,
312  const int cursor) override;
313  // TODO(miyu): merge the following two data frame APIs.
314  void sql_execute_df(TDataFrame& _return,
315  const TSessionId& session,
316  const std::string& query,
317  const TDeviceType::type device_type,
318  const int32_t device_id,
319  const int32_t first_n,
320  const TArrowTransport::type transport_method) override;
321  void sql_execute_gdf(TDataFrame& _return,
322  const TSessionId& session,
323  const std::string& query,
324  const int32_t device_id,
325  const int32_t first_n) override;
326  void deallocate_df(const TSessionId& session,
327  const TDataFrame& df,
328  const TDeviceType::type device_type,
329  const int32_t device_id) override;
330  void interrupt(const TSessionId& query_session,
331  const TSessionId& interrupt_session) override;
332  void sql_validate(TRowDescriptor& _return,
333  const TSessionId& session,
334  const std::string& query) override;
335 
336  void set_execution_mode(const TSessionId& session,
337  const TExecuteMode::type mode) override;
338  void render_vega(TRenderResult& _return,
339  const TSessionId& session,
340  const int64_t widget_id,
341  const std::string& vega_json,
342  const int32_t compression_level,
343  const std::string& nonce) override;
344  void get_result_row_for_pixel(
345  TPixelTableRowResult& _return,
346  const TSessionId& session,
347  const int64_t widget_id,
348  const TPixel& pixel,
349  const std::map<std::string, std::vector<std::string>>& table_col_names,
350  const bool column_format,
351  const int32_t pixel_radius,
352  const std::string& nonce) override;
353 
354  // custom expressions
355  int32_t create_custom_expression(const TSessionId& session,
356  const TCustomExpression& custom_expression) override;
357  void get_custom_expressions(std::vector<TCustomExpression>& _return,
358  const TSessionId& session) override;
359  void update_custom_expression(const TSessionId& session,
360  const int32_t id,
361  const std::string& expression_json) override;
362  void delete_custom_expressions(const TSessionId& session,
363  const std::vector<int32_t>& custom_expression_ids,
364  const bool do_soft_delete) override;
365 
366  // dashboards
367  void get_dashboard(TDashboard& _return,
368  const TSessionId& session,
369  const int32_t dashboard_id) override;
370  void get_dashboards(std::vector<TDashboard>& _return,
371  const TSessionId& session) override;
372  int32_t create_dashboard(const TSessionId& session,
373  const std::string& dashboard_name,
374  const std::string& dashboard_state,
375  const std::string& image_hash,
376  const std::string& dashboard_metadata) override;
377  void replace_dashboard(const TSessionId& session,
378  const int32_t dashboard_id,
379  const std::string& dashboard_name,
380  const std::string& dashboard_owner,
381  const std::string& dashboard_state,
382  const std::string& image_hash,
383  const std::string& dashboard_metadata) override;
384  void delete_dashboard(const TSessionId& session, const int32_t dashboard_id) override;
385  void share_dashboards(const TSessionId& session,
386  const std::vector<int32_t>& dashboard_ids,
387  const std::vector<std::string>& groups,
388  const TDashboardPermissions& permissions) override;
389  void delete_dashboards(const TSessionId& session,
390  const std::vector<int32_t>& dashboard_ids) override;
391  void share_dashboard(const TSessionId& session,
392  const int32_t dashboard_id,
393  const std::vector<std::string>& groups,
394  const std::vector<std::string>& objects,
395  const TDashboardPermissions& permissions,
396  const bool grant_role) override;
397  void unshare_dashboards(const TSessionId& session,
398  const std::vector<int32_t>& dashboard_ids,
399  const std::vector<std::string>& groups,
400  const TDashboardPermissions& permissions) override;
401  void unshare_dashboard(const TSessionId& session,
402  const int32_t dashboard_id,
403  const std::vector<std::string>& groups,
404  const std::vector<std::string>& objects,
405  const TDashboardPermissions& permissions) override;
406  void get_dashboard_grantees(std::vector<TDashboardGrantees>& _return,
407  const TSessionId& session,
408  const int32_t dashboard_id) override;
409 
410  void get_link_view(TFrontendView& _return,
411  const TSessionId& session,
412  const std::string& link) override;
413  void create_link(std::string& _return,
414  const TSessionId& session,
415  const std::string& view_state,
416  const std::string& view_metadata) override;
417  // import
418  void load_table_binary(const TSessionId& session,
419  const std::string& table_name,
420  const std::vector<TRow>& rows,
421  const std::vector<std::string>& column_names) override;
422 
423  std::unique_ptr<lockmgr::AbstractLockContainer<const TableDescriptor*>>
424  prepare_columnar_loader(
425  const Catalog_Namespace::SessionInfo& session_info,
426  const std::string& table_name,
427  size_t num_cols,
428  std::unique_ptr<import_export::Loader>* loader,
429  std::vector<std::unique_ptr<import_export::TypedImportBuffer>>* import_buffers,
430  const std::vector<std::string>& column_names);
431 
432  void load_table_binary_columnar(const TSessionId& session,
433  const std::string& table_name,
434  const std::vector<TColumn>& cols,
435  const std::vector<std::string>& column_names) override;
436  void load_table_binary_columnar_polys(const TSessionId& session,
437  const std::string& table_name,
438  const std::vector<TColumn>& cols,
439  const std::vector<std::string>& column_names,
440  const bool assign_render_groups) override;
441  void load_table_binary_arrow(const TSessionId& session,
442  const std::string& table_name,
443  const std::string& arrow_stream,
444  const bool use_column_names) override;
445 
446  void load_table(const TSessionId& session,
447  const std::string& table_name,
448  const std::vector<TStringRow>& rows,
449  const std::vector<std::string>& column_names) override;
450  void detect_column_types(TDetectResult& _return,
451  const TSessionId& session,
452  const std::string& file_name,
453  const TCopyParams& copy_params) override;
454  void create_table(const TSessionId& session,
455  const std::string& table_name,
456  const TRowDescriptor& row_desc,
457  const TFileType::type file_type,
458  const TCreateParams& create_params) override;
459  void import_table(const TSessionId& session,
460  const std::string& table_name,
461  const std::string& file_name,
462  const TCopyParams& copy_params) override;
463  void import_geo_table(const TSessionId& session,
464  const std::string& table_name,
465  const std::string& file_name,
466  const TCopyParams& copy_params,
467  const TRowDescriptor& row_desc,
468  const TCreateParams& create_params) override;
469  void import_table_status(TImportStatus& _return,
470  const TSessionId& session,
471  const std::string& import_id) override;
472  void get_first_geo_file_in_archive(std::string& _return,
473  const TSessionId& session,
474  const std::string& archive_path,
475  const TCopyParams& copy_params) override;
476  void get_all_files_in_archive(std::vector<std::string>& _return,
477  const TSessionId& session,
478  const std::string& archive_path,
479  const TCopyParams& copy_params) override;
480  void get_layers_in_geo_file(std::vector<TGeoFileLayerInfo>& _return,
481  const TSessionId& session,
482  const std::string& file_name,
483  const TCopyParams& copy_params) override;
484  // distributed
485  int64_t query_get_outer_fragment_count(const TSessionId& session,
486  const std::string& select_query) override;
487 
488  void check_table_consistency(TTableMeta& _return,
489  const TSessionId& session,
490  const int32_t table_id) override;
491  void start_query(TPendingQuery& _return,
492  const TSessionId& leaf_session,
493  const TSessionId& parent_session,
494  const std::string& query_ra,
495  const std::string& start_time_str,
496  const bool just_explain,
497  const std::vector<int64_t>& outer_fragment_indices) override;
498  void execute_query_step(TStepResult& _return,
499  const TPendingQuery& pending_query,
500  const TSubqueryId subquery_id,
501  const std::string& start_time_str) override;
502  void broadcast_serialized_rows(const TSerializedRows& serialized_rows,
503  const TRowDescriptor& row_desc,
504  const TQueryId query_id,
505  const TSubqueryId subquery_id,
506  const bool is_final_subquery_result) override;
507 
508  void start_render_query(TPendingRenderQuery& _return,
509  const TSessionId& session,
510  const int64_t widget_id,
511  const int16_t node_idx,
512  const std::string& vega_json) override;
513  void execute_next_render_step(TRenderStepResult& _return,
514  const TPendingRenderQuery& pending_render,
515  const TRenderAggDataMap& merged_data) override;
516 
517  void insert_data(const TSessionId& session, const TInsertData& insert_data) override;
518  void checkpoint(const TSessionId& session, const int32_t table_id) override;
519  // DB Object Privileges
520  void get_roles(std::vector<std::string>& _return, const TSessionId& session) override;
521  bool has_role(const TSessionId& sessionId,
522  const std::string& granteeName,
523  const std::string& roleName) override;
524  bool has_object_privilege(const TSessionId& sessionId,
525  const std::string& granteeName,
526  const std::string& objectName,
527  const TDBObjectType::type object_type,
528  const TDBObjectPermissions& permissions) override;
529  void get_db_objects_for_grantee(std::vector<TDBObject>& _return,
530  const TSessionId& session,
531  const std::string& roleName) override;
532  void get_db_object_privs(std::vector<TDBObject>& _return,
533  const TSessionId& session,
534  const std::string& objectName,
535  const TDBObjectType::type type) override;
536  void get_all_roles_for_user(std::vector<std::string>& _return,
537  const TSessionId& session,
538  const std::string& granteeName) override;
539  void get_all_effective_roles_for_user(std::vector<std::string>& _return,
540  const TSessionId& session,
541  const std::string& granteeName) override;
542  std::vector<std::string> get_valid_groups(const TSessionId& session,
543  int32_t dashboard_id,
544  std::vector<std::string> groups);
545  // licensing
546  void set_license_key(TLicenseInfo& _return,
547  const TSessionId& session,
548  const std::string& key,
549  const std::string& nonce) override;
550  void get_license_claims(TLicenseInfo& _return,
551  const TSessionId& session,
552  const std::string& nonce) override;
553  // user-defined functions
554  /*
555  Returns a mapping of device (CPU, GPU) parameters (name, LLVM IR
556  triplet, features, etc)
557  */
558  void get_device_parameters(std::map<std::string, std::string>& _return,
559  const TSessionId& session) override;
560 
561  /*
562  Register Runtime Extension Functions (UDFs, UDTFs) with given
563  signatures. The extension functions implementations are given in a
564  mapping of a device and the corresponding LLVM/NVVM IR string.
565  */
566 
567  void register_runtime_extension_functions(
568  const TSessionId& session,
569  const std::vector<TUserDefinedFunction>& udfs,
570  const std::vector<TUserDefinedTableFunction>& udtfs,
571  const std::map<std::string, std::string>& device_ir_map) override;
572 
573  // end of sync block for HAHandler and mapd.thrift
574 
575  void shutdown();
576  void emergency_shutdown();
577 
578  TSessionId getInvalidSessionId() const;
579 
580  void internal_connect(TSessionId& session,
581  const std::string& username,
582  const std::string& dbname);
583 
584  bool isAggregator() const;
585 
586  std::shared_ptr<Data_Namespace::DataMgr> data_mgr_;
587 
589  std::vector<LeafHostInfo> db_leaves_;
590  std::vector<LeafHostInfo> string_leaves_;
591  const std::string base_data_path_;
592  boost::filesystem::path import_path_;
594  std::default_random_engine random_gen_;
595  std::uniform_int_distribution<int64_t> session_id_dist_;
596  const bool jit_debug_;
597  const bool intel_jit_profile_;
599  const bool read_only_;
600  const bool allow_loop_joins_;
603  std::mutex render_mutex_;
604  int64_t start_time_;
607  std::unique_ptr<RenderHandler> render_handler_;
608  std::unique_ptr<MapDAggHandler> agg_handler_;
609  std::unique_ptr<MapDLeafHandler> leaf_handler_;
610  std::shared_ptr<Calcite> calcite_;
611  const bool legacy_syntax_;
612 
613  std::unique_ptr<QueryDispatchQueue> dispatch_queue_;
614 
615  template <typename... ARGS>
616  std::shared_ptr<query_state::QueryState> create_query_state(ARGS&&... args) {
617  return query_states_.create(std::forward<ARGS>(args)...);
618  }
619 
620  // Exactly one immutable SessionInfo copy should be taken by a typical request.
621  Catalog_Namespace::SessionInfo get_session_copy(const TSessionId& session);
622  std::shared_ptr<Catalog_Namespace::SessionInfo> get_session_copy_ptr(
623  const TSessionId& session);
624 
625  void get_tables_meta_impl(std::vector<TTableMeta>& _return,
626  QueryStateProxy query_state_proxy,
627  const Catalog_Namespace::SessionInfo& session_info,
628  const bool with_table_locks = true);
629 
630  // Visible for use in tests.
631  void resizeDispatchQueue(size_t queue_size);
632 
633  protected:
634  // Returns empty std::shared_ptr if !check_license && session.empty().
635  std::shared_ptr<Catalog_Namespace::SessionInfo> get_session_ptr(
636  const TSessionId& session_id);
637 
638  ConnectionInfo getConnectionInfo() const;
639 
640  private:
641  std::atomic<bool> initialized_{false};
642  std::shared_ptr<Catalog_Namespace::SessionInfo> create_new_session(
643  TSessionId& session,
644  const std::string& dbname,
645  const Catalog_Namespace::UserMetadata& user_meta,
646  std::shared_ptr<Catalog_Namespace::Catalog> cat);
647  void connect_impl(TSessionId& session,
648  const std::string& passwd,
649  const std::string& dbname,
650  const Catalog_Namespace::UserMetadata& user_meta,
651  std::shared_ptr<Catalog_Namespace::Catalog> cat,
652  query_state::StdLog& stdlog);
653  void disconnect_impl(const SessionMap::iterator& session_it,
654  mapd_unique_lock<mapd_shared_mutex>& write_lock);
655  void check_table_load_privileges(const TSessionId& session,
656  const std::string& table_name);
657  void check_table_load_privileges(const Catalog_Namespace::SessionInfo& session_info,
658  const std::string& table_name);
659  void get_tables_impl(std::vector<std::string>& table_names,
661  const GetTablesType get_tables_type,
662  const std::string& database_name = {});
663  void get_table_details_impl(TTableDetails& _return,
664  query_state::StdLog& stdlog,
665  const std::string& table_name,
666  const bool get_system,
667  const bool get_physical,
668  const std::string& database_name = {});
669  void getAllRolesForUserImpl(
670  std::shared_ptr<Catalog_Namespace::SessionInfo const> session_ptr,
671  std::vector<std::string>& roles,
672  const TSessionId& sessionId,
673  const std::string& granteeName,
674  bool effective);
675  void check_read_only(const std::string& str);
676  void check_session_exp_unsafe(const SessionMap::iterator& session_it);
677  void validateGroups(const std::vector<std::string>& groups);
678  void validateDashboardIdsForSharing(const Catalog_Namespace::SessionInfo& session_info,
679  const std::vector<int32_t>& dashboard_ids);
680  void shareOrUnshareDashboards(const TSessionId& session,
681  const std::vector<int32_t>& dashboard_ids,
682  const std::vector<std::string>& groups,
683  const TDashboardPermissions& permissions,
684  const bool do_share);
685 
686  // Use get_session_copy() or get_session_copy_ptr() instead of get_const_session_ptr()
687  // unless you know what you are doing. If you need to save a SessionInfo beyond the
688  // present Thrift call, then the safe way to do this is by saving the return value of
689  // get_const_session_ptr() as a std::weak_ptr.
690  // Returns empty std::shared_ptr if session.empty().
691  std::shared_ptr<const Catalog_Namespace::SessionInfo> get_const_session_ptr(
692  const TSessionId& session);
693 
694  template <typename SESSION_MAP_LOCK>
695  SessionMap::iterator get_session_it_unsafe(const TSessionId& session,
696  SESSION_MAP_LOCK& lock);
697  template <typename SESSION_MAP_LOCK>
698  void expire_idle_sessions_unsafe(SESSION_MAP_LOCK& lock);
699  static void value_to_thrift_column(const TargetValue& tv,
700  const SQLTypeInfo& ti,
701  TColumn& column);
702  static TDatum value_to_thrift(const TargetValue& tv, const SQLTypeInfo& ti);
703 
704  std::pair<TPlanResult, lockmgr::LockedTableDescriptors> parse_to_ra(
706  const std::string& query_str,
707  const std::vector<TFilterPushDownInfo>& filter_push_down_info,
708  const bool acquire_locks,
709  const SystemParameters& system_parameters,
710  bool check_privileges = true);
711 
712  void sql_execute_local(
713  TQueryResult& _return,
714  const QueryStateProxy& query_state_proxy,
715  const std::shared_ptr<Catalog_Namespace::SessionInfo> session_ptr,
716  const std::string& query_str,
717  const bool column_format,
718  const std::string& nonce,
719  const int32_t first_n,
720  const int32_t at_most_n,
721  const bool use_calcite);
722 
723  int64_t process_deferred_copy_from(const TSessionId& session_id);
724 
725  static void convertData(TQueryResult& _return,
727  const QueryStateProxy& query_state_proxy,
728  const std::string& query_str,
729  const bool column_format,
730  const int32_t first_n,
731  const int32_t at_most_n);
732 
733  void sql_execute_impl(ExecutionResult& _return,
735  const bool column_format,
736  const ExecutorDeviceType executor_device_type,
737  const int32_t first_n,
738  const int32_t at_most_n,
739  const bool use_calcite);
740 
742  const TableDescriptor* td,
743  const AccessPrivileges acess_priv);
744 
745  void execute_distributed_copy_statement(
747  const Catalog_Namespace::SessionInfo& session_info);
748 
749  TQueryResult validate_rel_alg(const std::string& query_ra, QueryStateProxy);
750 
751  std::vector<PushedDownFilterInfo> execute_rel_alg(
752  ExecutionResult& _return,
754  const std::string& query_ra,
755  const bool column_format,
756  const ExecutorDeviceType executor_device_type,
757  const int32_t first_n,
758  const int32_t at_most_n,
759  const bool just_validate,
760  const bool find_push_down_candidates,
761  const ExplainInfo& explain_info,
762  const std::optional<size_t> executor_index = std::nullopt) const;
763 
764  void execute_rel_alg_with_filter_push_down(
765  ExecutionResult& _return,
767  std::string& query_ra,
768  const bool column_format,
769  const ExecutorDeviceType executor_device_type,
770  const int32_t first_n,
771  const int32_t at_most_n,
772  const bool just_explain,
773  const bool just_calcite_explain,
774  const std::vector<PushedDownFilterInfo>& filter_push_down_requests);
775 
776  void execute_rel_alg_df(TDataFrame& _return,
777  const std::string& query_ra,
778  QueryStateProxy query_state_proxy,
779  const Catalog_Namespace::SessionInfo& session_info,
780  const ExecutorDeviceType executor_device_type,
781  const ExecutorDeviceType results_device_type,
782  const size_t device_id,
783  const int32_t first_n,
784  const TArrowTransport::type transport_method) const;
785 
786  void executeDdl(TQueryResult& _return,
787  const std::string& query_ra,
788  std::shared_ptr<Catalog_Namespace::SessionInfo const> session_ptr);
789 
790  void executeDdl(ExecutionResult& _return,
791  const std::string& query_ra,
792  std::shared_ptr<Catalog_Namespace::SessionInfo const> session_ptr);
793 
794  TColumnType populateThriftColumnType(const Catalog_Namespace::Catalog* cat,
795  const ColumnDescriptor* cd);
796  TRowDescriptor fixup_row_descriptor(const TRowDescriptor& row_desc,
798  void set_execution_mode_nolock(Catalog_Namespace::SessionInfo* session_ptr,
799  const TExecuteMode::type mode);
800  char unescape_char(std::string str);
801  import_export::CopyParams thrift_to_copyparams(const TCopyParams& cp);
802  TCopyParams copyparams_to_thrift(const import_export::CopyParams& cp);
803  void check_geospatial_files(const boost::filesystem::path file_path,
804  const import_export::CopyParams& copy_params);
805  void render_rel_alg(TRenderResult& _return,
806  const std::string& query_ra,
807  const std::string& query_str,
808  const Catalog_Namespace::SessionInfo& session_info,
809  const std::string& render_type,
810  const bool is_projection_query);
811 
812  TColumnType create_geo_column(const TDatumType::type type,
813  const std::string& name,
814  const bool is_array);
815 
816  static void convertExplain(TQueryResult& _return,
817  const ResultSet& results,
818  const bool column_format);
819  static void convertResult(TQueryResult& _return,
820  const ResultSet& results,
821  const bool column_format);
822 
823  static void convertRows(TQueryResult& _return,
824  QueryStateProxy query_state_proxy,
825  const std::vector<TargetMetaInfo>& targets,
826  const ResultSet& results,
827  const bool column_format,
828  const int32_t first_n,
829  const int32_t at_most_n);
830 
831  // Use ExecutionResult to populate a TQueryResult
832  // calls convertRows, but after some setup using session_info
833  void convertResultSet(ExecutionResult& result,
834  const Catalog_Namespace::SessionInfo& session_info,
835  const std::string& query_state_str,
836  TQueryResult& _return);
837 
838  static void createSimpleResult(TQueryResult& _return,
839  const ResultSet& results,
840  const bool column_format,
841  const std::string label);
842 
843  std::vector<TargetMetaInfo> getTargetMetaInfo(
844  const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& targets) const;
845 
846  std::vector<std::string> getTargetNames(
847  const std::vector<TargetMetaInfo>& targets) const;
848 
849  std::vector<std::string> getTargetNames(
850  const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& targets) const;
851 
852  void get_completion_hints_unsorted(std::vector<TCompletionHint>& hints,
853  std::vector<std::string>& visible_tables,
854  query_state::StdLog& stdlog,
855  const std::string& sql,
856  const int cursor);
857  void get_token_based_completions(std::vector<TCompletionHint>& hints,
858  query_state::StdLog& stdlog,
859  std::vector<std::string>& visible_tables,
860  const std::string& sql,
861  const int cursor);
862 
863  std::unordered_map<std::string, std::unordered_set<std::string>>
864  fill_column_names_by_table(std::vector<std::string>& table_names,
865  query_state::StdLog& stdlog);
866 
867  TDashboard get_dashboard_impl(
868  const std::shared_ptr<Catalog_Namespace::SessionInfo const>& session_ptr,
870  const DashboardDescriptor* dash,
871  const bool populate_state = true);
872 
873  static bool has_database_permission(const AccessPrivileges& privs,
874  const TDBObjectPermissions& permissions);
875  static bool has_table_permission(const AccessPrivileges& privs,
876  const TDBObjectPermissions& permission);
877  static bool has_dashboard_permission(const AccessPrivileges& privs,
878  const TDBObjectPermissions& permissions);
879  static bool has_view_permission(const AccessPrivileges& privs,
880  const TDBObjectPermissions& permissions);
881  static bool has_server_permission(const AccessPrivileges& privs,
882  const TDBObjectPermissions& permissions);
883  // For the provided upper case column names `uc_column_names`, return
884  // the tables from `table_names` which contain at least one of them.
885  // Used to rank the TABLE auto-completion hints by the columns
886  // specified in the projection.
887  std::unordered_set<std::string> get_uc_compatible_table_names_by_column(
888  const std::unordered_set<std::string>& uc_column_names,
889  std::vector<std::string>& table_names,
890  query_state::StdLog& stdlog);
891 
892  std::unique_ptr<lockmgr::AbstractLockContainer<const TableDescriptor*>>
893  prepare_loader_generic(
894  const Catalog_Namespace::SessionInfo& session_info,
895  const std::string& table_name,
896  size_t num_cols,
897  std::unique_ptr<import_export::Loader>* loader,
898  std::vector<std::unique_ptr<import_export::TypedImportBuffer>>* import_buffers,
899  const std::vector<std::string>& column_names,
900  std::string load_type);
901 
902  void fillGeoColumns(
903  const TSessionId& session,
904  const Catalog_Namespace::Catalog& catalog,
905  std::vector<std::unique_ptr<import_export::TypedImportBuffer>>& import_buffers,
906  const ColumnDescriptor* cd,
907  size_t& col_idx,
908  size_t num_rows,
909  const std::string& table_name,
910  bool assign_render_groups);
911 
912  void fillMissingBuffers(
913  const TSessionId& session,
914  const Catalog_Namespace::Catalog& catalog,
915  std::vector<std::unique_ptr<import_export::TypedImportBuffer>>& import_buffers,
916  const std::list<const ColumnDescriptor*>& cds,
917  const std::vector<int>& desc_id_to_column_id,
918  size_t num_rows,
919  const std::string& table_name,
920  bool assign_render_groups);
921 
924 
925  bool super_user_rights_; // default is "false"; setting to "true"
926  // ignores passwd checks in "connect(..)"
927  // method
928  const int idle_session_duration_; // max duration of idle session
929  const int max_session_duration_; // max duration of session
930 
932 
933  const bool enable_rendering_;
936  const unsigned renderer_vulkan_timeout_;
940  const size_t reserved_gpu_mem_;
942  const size_t render_mem_bytes_;
943  const size_t num_reader_threads_;
944 #ifdef ENABLE_GEOS
945  const std::string& libgeos_so_filename_;
946 #endif
948  const std::string& udf_filename_;
949  const std::string& clang_path_;
950  const std::vector<std::string>& clang_options_;
951 
953  std::string table;
954  std::string file_name;
956  std::string partitions;
957  };
958 
960  std::unordered_map<std::string, DeferredCopyFromState> was_deferred_copy_from;
962 
963  std::optional<DeferredCopyFromState> operator()(const std::string& session_id) {
964  std::lock_guard<std::mutex> map_lock(deferred_copy_from_mutex);
965  auto itr = was_deferred_copy_from.find(session_id);
966  if (itr == was_deferred_copy_from.end()) {
967  return std::nullopt;
968  }
969  return itr->second;
970  }
971 
972  void add(const std::string& session_id, const DeferredCopyFromState& state) {
973  std::lock_guard<std::mutex> map_lock(deferred_copy_from_mutex);
974  const auto ret = was_deferred_copy_from.insert(std::make_pair(session_id, state));
975  CHECK(ret.second);
976  }
977 
978  void remove(const std::string& session_id) {
979  std::lock_guard<std::mutex> map_lock(deferred_copy_from_mutex);
980  was_deferred_copy_from.erase(session_id);
981  }
982  };
984 
985  // Only for IPC device memory deallocation
986  mutable std::mutex handle_to_dev_ptr_mutex_;
987  mutable std::unordered_map<std::string, std::string> ipc_handle_to_dev_ptr_;
988 
989  friend void run_warmup_queries(std::shared_ptr<DBHandler> handler,
990  std::string base_path,
991  std::string query_file_path);
992 
993  friend class RenderHandler::Impl;
994  friend class MapDAggHandler;
995  friend class MapDLeafHandler;
996 
997  std::map<const std::string, const PermissionFuncPtr> permissionFuncMap_ = {
998  {"database"s, has_database_permission},
999  {"dashboard"s, has_dashboard_permission},
1000  {"table"s, has_table_permission},
1001  {"view"s, has_view_permission},
1002  {"server"s, has_server_permission}};
1003 
1004  void check_and_invalidate_sessions(Parser::DDLStmt* ddl);
1005 
1006  template <typename STMT_TYPE>
1007  void invalidate_sessions(std::string& name, STMT_TYPE* stmt) {
1008  using namespace Parser;
1009  auto is_match = [&](auto session_it) {
1011  return boost::iequals(name,
1012  session_it->second->getCatalog().getCurrentDB().dbName);
1014  return boost::iequals(name, session_it->second->get_currentUser().userName);
1015  }
1016  return false;
1017  };
1018  auto check_and_remove_sessions = [&]() {
1019  for (auto it = sessions_.begin(); it != sessions_.end();) {
1020  if (is_match(it)) {
1021  it = sessions_.erase(it);
1022  } else {
1023  ++it;
1024  }
1025  }
1026  };
1027  check_and_remove_sessions();
1028  }
1029 
1030  std::string const createInMemoryCalciteSession(
1031  const std::shared_ptr<Catalog_Namespace::Catalog>& catalog_ptr);
1032  bool isInMemoryCalciteSession(const Catalog_Namespace::UserMetadata user_meta);
1033  void removeInMemoryCalciteSession(const std::string& session_id);
1034 
1035  ExecutionResult getUserSessions(
1036  std::shared_ptr<Catalog_Namespace::SessionInfo const> session_ptr);
1037 
1038  // getQueries returns a set of queries queued in the DB
1039  // that belongs to the same DB in the caller's session
1040 
1041  ExecutionResult getQueries(
1042  std::shared_ptr<Catalog_Namespace::SessionInfo const> session_ptr);
1043 
1044  void get_queries_info(std::vector<TQueryInfo>& _return, const TSessionId& session);
1045 
1046  // this function passes the interrupt request to the DB executor
1047  void interruptQuery(const Catalog_Namespace::SessionInfo& session_info,
1048  const std::string& target_session);
1049 
1050  // render group assignment
1051 
1052  enum class AssignRenderGroupsMode { kNone, kAssign, kCleanUp };
1053 
1054  void load_table_binary_columnar_internal(
1055  const TSessionId& session,
1056  const std::string& table_name,
1057  const std::vector<TColumn>& cols,
1058  const std::vector<std::string>& column_names,
1059  const AssignRenderGroupsMode assign_render_groups_mode);
1060 
1061  TRole::type getServerRole() const;
1062 
1063  using RenderGroupAssignmentColumnMap =
1064  std::unordered_map<std::string,
1065  std::unique_ptr<import_export::RenderGroupAnalyzer>>;
1067  std::unordered_map<std::string, RenderGroupAssignmentColumnMap>;
1069  std::unordered_map<TSessionId, RenderGroupAssignmentTableMap>;
1073 
1074  void import_geo_table_internal(const TSessionId& session,
1075  const std::string& table_name,
1076  const std::string& file_name,
1077  const TCopyParams& copy_params,
1078  const TRowDescriptor& row_desc,
1079  const TCreateParams& create_params);
1080 };
Classes used to wrap parser calls for calcite redirection.
AssignRenderGroupsMode
Definition: DBHandler.h:1052
std::vector< LeafHostInfo > string_leaves_
Definition: DBHandler.h:590
auto get_users(SysCatalog &syscat, std::unique_ptr< SqliteConnector > &sqliteConnector, const int32_t dbId=-1)
const std::vector< std::string > & clang_options_
Definition: DBHandler.h:950
boost::filesystem::path import_path_
Definition: DBHandler.h:592
std::unique_ptr< QueryDispatchQueue > dispatch_queue_
Definition: DBHandler.h:613
std::unordered_map< std::string, RenderGroupAssignmentColumnMap > RenderGroupAssignmentTableMap
Definition: DBHandler.h:1067
const bool runtime_udf_registration_enabled_
Definition: DBHandler.h:931
ClientProtocol
const std::string & udf_filename_
Definition: DBHandler.h:948
std::string cat(Ts &&...args)
const int render_oom_retry_threshold_
Definition: DBHandler.h:938
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:114
std::mutex handle_to_dev_ptr_mutex_
Definition: DBHandler.h:986
std::shared_ptr< query_state::QueryState > create_query_state(ARGS &&...args)
Definition: DBHandler.h:616
static thread_local std::string client_address
Definition: DBHandler.h:148
ExecutorDeviceType
std::mutex render_group_assignment_mutex_
Definition: DBHandler.h:1071
void add(const std::string &session_id, const DeferredCopyFromState &state)
Definition: DBHandler.h:972
bool user_can_access_table(const Catalog_Namespace::SessionInfo &session_info, const TableDescriptor *td, const AccessPrivileges access_priv)
bool(*)(const AccessPrivileges &, const TDBObjectPermissions &) PermissionFuncPtr
Definition: DBHandler.h:105
static thread_local ClientProtocol client_protocol
Definition: DBHandler.h:149
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:588
string name
Definition: setup.in.py:72
mapd_shared_mutex custom_expressions_mutex_
Definition: DBHandler.h:1072
std::unordered_map< TSessionId, RenderGroupAssignmentTableMap > RenderGroupAnalyzerSessionMap
Definition: DBHandler.h:1069
const unsigned renderer_vulkan_timeout_
Definition: DBHandler.h:936
const std::string base_data_path_
Definition: DBHandler.h:591
const bool jit_debug_
Definition: DBHandler.h:596
const bool check_origin_
Definition: DBHandler.h:152
const size_t render_mem_bytes_
Definition: DBHandler.h:942
const bool renderer_use_vulkan_driver_
Definition: DBHandler.h:934
std::map< TSessionId, std::shared_ptr< Catalog_Namespace::SessionInfo >> SessionMap
Definition: DBHandler.h:104
DeferredCopyFromSessions deferred_copy_from_sessions
Definition: DBHandler.h:983
std::vector< std::string > split(std::string_view str, std::string_view delim, std::optional< size_t > maxsplit)
split apart a string into a vector of substrings
int64_t start_time_
Definition: DBHandler.h:604
import_export::CopyParams copy_params
Definition: DBHandler.h:955
mapd_shared_mutex sessions_mutex_
Definition: DBHandler.h:602
This file contains the class specification and related data structures for Catalog.
std::mutex render_mutex_
Definition: DBHandler.h:603
static size_t max_bytes_for_thrift()
Definition: DBHandler.h:197
query_state::QueryStates query_states_
Definition: DBHandler.h:922
const size_t reserved_gpu_mem_
Definition: DBHandler.h:940
void run_warmup_queries(std::shared_ptr< DBHandler > handler, std::string base_path, std::string query_file_path)
Definition: MapDServer.cpp:200
std::optional< DeferredCopyFromState > operator()(const std::string &session_id)
Definition: DBHandler.h:963
const bool render_compositor_use_last_gpu_
Definition: DBHandler.h:941
GetTablesType
Definition: Catalog.h:63
std::shared_timed_mutex mapd_shared_mutex
const int max_session_duration_
Definition: DBHandler.h:929
ExecutorDeviceType executor_device_type_
Definition: DBHandler.h:593
std::vector< LeafHostInfo > db_leaves_
Definition: DBHandler.h:589
const File_Namespace::DiskCacheConfig & disk_cache_config_
Definition: DBHandler.h:947
const std::string & clang_path_
Definition: DBHandler.h:949
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:607
SystemParameters & system_parameters_
Definition: DBHandler.h:606
const size_t num_reader_threads_
Definition: DBHandler.h:943
specifies the content in-memory of a row in the column metadata table
std::unique_ptr< MapDLeafHandler > leaf_handler_
Definition: DBHandler.h:609
const bool enable_auto_clear_render_mem_
Definition: DBHandler.h:937
const bool renderer_prefer_igpu_
Definition: DBHandler.h:935
std::map< std::string, std::string > get_device_parameters(bool cpu_only)
const bool allow_loop_joins_
Definition: DBHandler.h:600
const bool enable_rendering_
Definition: DBHandler.h:933
std::unordered_map< std::string, DeferredCopyFromState > was_deferred_copy_from
Definition: DBHandler.h:960
const bool intel_jit_profile_
Definition: DBHandler.h:597
bool super_user_rights_
Definition: DBHandler.h:925
TrackingProcessor(std::shared_ptr< OmniSciIf > handler, const bool check_origin)
Definition: DBHandler.h:110
std::shared_ptr< Calcite > calcite_
Definition: DBHandler.h:610
void shutdown()
Definition: Logger.cpp:340
std::shared_ptr< Data_Namespace::DataMgr > data_mgr_
Definition: DBHandler.h:586
const bool read_only_
Definition: DBHandler.h:599
bool process(std::shared_ptr<::apache::thrift::protocol::TProtocol > in, std::shared_ptr<::apache::thrift::protocol::TProtocol > out, void *connectionContext) override
Definition: DBHandler.h:113
const bool legacy_syntax_
Definition: DBHandler.h:611
#define CHECK(condition)
Definition: Logger.h:211
QueryId query_id()
Definition: Logger.cpp:469
const int idle_session_duration_
Definition: DBHandler.h:928
std::unique_ptr< MapDAggHandler > agg_handler_
Definition: DBHandler.h:608
std::unordered_map< std::string, std::string > ipc_handle_to_dev_ptr_
Definition: DBHandler.h:987
RenderGroupAnalyzerSessionMap render_group_assignment_map_
Definition: DBHandler.h:1070
const size_t max_concurrent_render_sessions_
Definition: DBHandler.h:939
boost::variant< ScalarTargetValue, ArrayTargetValue, GeoTargetValue, GeoTargetValuePtr > TargetValue
Definition: TargetValue.h:167
bool allow_multifrag_
Definition: DBHandler.h:598
mapd_unique_lock< mapd_shared_mutex > write_lock
const AuthMetadata & authMetadata_
Definition: DBHandler.h:605
void invalidate_sessions(std::string &name, STMT_TYPE *stmt)
Definition: DBHandler.h:1007
bool cpu_mode_only_
Definition: DBHandler.h:601
std::default_random_engine random_gen_
Definition: DBHandler.h:594
SessionMap sessions_
Definition: DBHandler.h:923
std::uniform_int_distribution< int64_t > session_id_dist_
Definition: DBHandler.h:595