OmniSciDB  8fa3bf436f
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
DBHandler.h
Go to the documentation of this file.
1 /*
2  * Copyright 2020 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 /*
18  * File: DBHandler.h
19  * Author: michael
20  *
21  * Created on Jan 1, 2017, 12:40 PM
22  */
23 
24 #pragma once
25 
26 #include "LeafAggregator.h"
27 
28 #ifdef HAVE_PROFILER
29 #include <gperftools/heap-profiler.h>
30 #endif // HAVE_PROFILER
31 
32 #include "Calcite/Calcite.h"
33 #include "Catalog/Catalog.h"
35 #include "Geospatial/Transforms.h"
36 #include "ImportExport/Importer.h"
37 #include "LockMgr/LockMgr.h"
38 #include "Logger/Logger.h"
39 #include "Parser/ParserWrapper.h"
41 #include "Parser/parser.h"
44 #include "QueryEngine/Execute.h"
50 #include "Shared/StringTransform.h"
53 #include "Shared/mapd_shared_ptr.h"
54 #include "Shared/measure.h"
55 #include "Shared/scope.h"
61 
62 #include <sys/types.h>
63 #include <thrift/server/TServer.h>
64 #include <thrift/transport/THttpTransport.h>
65 #include <thrift/transport/TSocket.h>
66 #include <thrift/transport/TTransport.h>
67 #include <atomic>
68 #include <boost/algorithm/string.hpp>
69 #include <boost/algorithm/string/replace.hpp>
70 #include <boost/algorithm/string/trim.hpp>
71 #include <boost/filesystem.hpp>
72 #include <boost/make_shared.hpp>
73 #include <boost/noncopyable.hpp>
74 #include <boost/none_t.hpp>
75 #include <boost/optional.hpp>
76 #include <boost/program_options.hpp>
77 #include <boost/regex.hpp>
78 #include <boost/tokenizer.hpp>
79 #include <cmath>
80 #include <csignal>
81 #include <fstream>
82 #include <list>
83 #include <map>
84 #include <memory>
85 #include <mutex>
86 #include <random>
87 #include <regex>
88 #include <string>
89 #include <thread>
90 #include <typeinfo>
91 #include <unordered_map>
92 
93 #include "gen-cpp/OmniSci.h"
94 #include "gen-cpp/extension_functions_types.h"
95 
96 using namespace std::string_literals;
97 
98 class MapDAggHandler;
99 class MapDLeafHandler;
100 
101 // Multiple concurrent requests for the same session can occur. For that reason, each
102 // request briefly takes a lock to make a copy of the appropriate SessionInfo object. Then
103 // it releases the lock and uses the copy for the remainder of the request.
104 using SessionMap = std::map<TSessionId, std::shared_ptr<Catalog_Namespace::SessionInfo>>;
105 using PermissionFuncPtr = bool (*)(const AccessPrivileges&, const TDBObjectPermissions&);
107 
108 class TrackingProcessor : public OmniSciProcessor {
109  public:
110  TrackingProcessor(mapd::shared_ptr<OmniSciIf> handler, const bool check_origin)
111  : OmniSciProcessor(handler), check_origin_(check_origin) {}
112 
113  bool process(mapd::shared_ptr<::apache::thrift::protocol::TProtocol> in,
114  mapd::shared_ptr<::apache::thrift::protocol::TProtocol> out,
115  void* connectionContext) {
116  using namespace ::apache::thrift;
117 
118  auto transport = in->getTransport();
119  if (transport && check_origin_) {
120  static std::mutex processor_mutex;
121  std::lock_guard lock(processor_mutex);
122  const auto origin_str = transport->getOrigin();
123  std::vector<std::string> origins;
124  boost::split(origins, origin_str, boost::is_any_of(","));
125  if (origins.empty()) {
127  } else {
128  // Take the first origin, which should be the client IP before any intermediate
129  // servers (e.g. the web server)
130  auto trimmed_origin = origins.front();
131  boost::algorithm::trim(trimmed_origin);
132  TrackingProcessor::client_address = trimmed_origin;
133  }
134  if (dynamic_cast<transport::THttpTransport*>(transport.get())) {
136  } else if (dynamic_cast<transport::TBufferedTransport*>(transport.get())) {
138  } else {
140  }
141  } else {
143  }
144 
145  return OmniSciProcessor::process(in, out, connectionContext);
146  }
147 
148  static thread_local std::string client_address;
149  static thread_local ClientProtocol client_protocol;
150 
151  private:
152  const bool check_origin_;
153 };
154 
155 struct DiskCacheConfig;
156 
157 class DBHandler : public OmniSciIf {
158  public:
159  DBHandler(const std::vector<LeafHostInfo>& db_leaves,
160  const std::vector<LeafHostInfo>& string_leaves,
161  const std::string& base_data_path,
162  const bool allow_multifrag,
163  const bool jit_debug,
164  const bool intel_jit_profile,
165  const bool read_only,
166  const bool allow_loop_joins,
167  const bool enable_rendering,
168  const bool renderer_use_vulkan_driver,
169  const bool enable_auto_clear_render_mem,
170  const int render_oom_retry_threshold,
171  const size_t render_mem_bytes,
172  const size_t max_concurrent_render_sessions,
173  const size_t reserved_gpu_mem,
174  const bool render_compositor_use_last_gpu,
175  const size_t num_reader_threads,
176  const AuthMetadata& authMetadata,
177  SystemParameters& system_parameters,
178  const bool legacy_syntax,
179  const int idle_session_duration,
180  const int max_session_duration,
181  const bool enable_runtime_udf_registration,
182  const std::string& udf_filename,
183  const std::string& clang_path,
184  const std::vector<std::string>& clang_options,
185 #ifdef ENABLE_GEOS
186  const std::string& libgeos_so_filename,
187 #endif
188  const DiskCacheConfig& disk_cache_config,
189  const bool is_new_db);
190  void initialize(const bool is_new_db);
191  ~DBHandler() override;
192 
193  static inline size_t max_bytes_for_thrift() { return 2 * 1000 * 1000 * 1000LL; }
194 
195  // Important ****
196  // This block must be keep in sync with mapd.thrift and HAHandler.h
197  // Please keep in same order for easy check and cut and paste
198  // Important ****
199  static void parser_with_error_handler(
200  const std::string& query_str,
201  std::list<std::unique_ptr<Parser::Stmt>>& parse_trees);
202 
203  void krb5_connect(TKrb5Session& session,
204  const std::string& token,
205  const std::string& dbname) override;
206  // connection, admin
207  void connect(TSessionId& session,
208  const std::string& username,
209  const std::string& passwd,
210  const std::string& dbname) override;
211  void disconnect(const TSessionId& session) override;
212  void switch_database(const TSessionId& session, const std::string& dbname) override;
213  void clone_session(TSessionId& session2, const TSessionId& session1) override;
214  void get_server_status(TServerStatus& _return, const TSessionId& session) override;
215  void get_status(std::vector<TServerStatus>& _return,
216  const TSessionId& session) override;
217  void get_hardware_info(TClusterHardwareInfo& _return,
218  const TSessionId& session) override;
219 
220  bool hasTableAccessPrivileges(const TableDescriptor* td,
221  const Catalog_Namespace::SessionInfo& session_info);
222  void get_tables(std::vector<std::string>& _return, const TSessionId& session) override;
223  void get_physical_tables(std::vector<std::string>& _return,
224  const TSessionId& session) override;
225  void get_views(std::vector<std::string>& _return, const TSessionId& session) override;
226  void get_tables_meta(std::vector<TTableMeta>& _return,
227  const TSessionId& session) override;
228  void get_table_details(TTableDetails& _return,
229  const TSessionId& session,
230  const std::string& table_name) override;
231  void get_internal_table_details(TTableDetails& _return,
232  const TSessionId& session,
233  const std::string& table_name) override;
234  void get_users(std::vector<std::string>& _return, const TSessionId& session) override;
235  void get_databases(std::vector<TDBInfo>& _return, const TSessionId& session) override;
236 
237  void get_version(std::string& _return) override;
238  void start_heap_profile(const TSessionId& session) override;
239  void stop_heap_profile(const TSessionId& session) override;
240  void get_heap_profile(std::string& _return, const TSessionId& session) override;
241  void get_memory(std::vector<TNodeMemoryInfo>& _return,
242  const TSessionId& session,
243  const std::string& memory_level) override;
244  void clear_cpu_memory(const TSessionId& session) override;
245  void clear_gpu_memory(const TSessionId& session) override;
246  void set_cur_session(const TSessionId& parent_session,
247  const TSessionId& leaf_session,
248  const std::string& start_time_str,
249  const std::string& label) override;
250  void invalidate_cur_session(const TSessionId& parent_session,
251  const TSessionId& leaf_session,
252  const std::string& start_time_str,
253  const std::string& label) override;
254  void set_table_epoch(const TSessionId& session,
255  const int db_id,
256  const int table_id,
257  const int new_epoch) override;
258  void set_table_epoch_by_name(const TSessionId& session,
259  const std::string& table_name,
260  const int new_epoch) override;
261  int32_t get_table_epoch(const TSessionId& session,
262  const int32_t db_id,
263  const int32_t table_id) override;
264  int32_t get_table_epoch_by_name(const TSessionId& session,
265  const std::string& table_name) override;
266  void get_table_epochs(std::vector<TTableEpochInfo>& _return,
267  const TSessionId& session,
268  const int32_t db_id,
269  const int32_t table_id) override;
270  void set_table_epochs(const TSessionId& session,
271  const int32_t db_id,
272  const std::vector<TTableEpochInfo>& table_epochs) override;
273 
274  void get_session_info(TSessionInfo& _return, const TSessionId& session) override;
275 
276  void sql_execute(ExecutionResult& _return,
277  const TSessionId& session,
278  const std::string& query,
279  const bool column_format,
280  const int32_t first_n,
281  const int32_t at_most_n);
282  // query, render
283  void sql_execute(TQueryResult& _return,
284  const TSessionId& session,
285  const std::string& query,
286  const bool column_format,
287  const std::string& nonce,
288  const int32_t first_n,
289  const int32_t at_most_n) override;
290  void get_completion_hints(std::vector<TCompletionHint>& hints,
291  const TSessionId& session,
292  const std::string& sql,
293  const int cursor) override;
294  // TODO(miyu): merge the following two data frame APIs.
295  void sql_execute_df(TDataFrame& _return,
296  const TSessionId& session,
297  const std::string& query,
298  const TDeviceType::type device_type,
299  const int32_t device_id,
300  const int32_t first_n,
301  const TArrowTransport::type transport_method) override;
302  void sql_execute_gdf(TDataFrame& _return,
303  const TSessionId& session,
304  const std::string& query,
305  const int32_t device_id,
306  const int32_t first_n) override;
307  void deallocate_df(const TSessionId& session,
308  const TDataFrame& df,
309  const TDeviceType::type device_type,
310  const int32_t device_id) override;
311  void interrupt(const TSessionId& query_session,
312  const TSessionId& interrupt_session) override;
313  void sql_validate(TRowDescriptor& _return,
314  const TSessionId& session,
315  const std::string& query) override;
316 
317  void set_execution_mode(const TSessionId& session,
318  const TExecuteMode::type mode) override;
319  void render_vega(TRenderResult& _return,
320  const TSessionId& session,
321  const int64_t widget_id,
322  const std::string& vega_json,
323  const int32_t compression_level,
324  const std::string& nonce) override;
325  void get_result_row_for_pixel(
326  TPixelTableRowResult& _return,
327  const TSessionId& session,
328  const int64_t widget_id,
329  const TPixel& pixel,
330  const std::map<std::string, std::vector<std::string>>& table_col_names,
331  const bool column_format,
332  const int32_t pixel_radius,
333  const std::string& nonce) override;
334 
335  // custom expressions
336  int32_t create_custom_expression(const TSessionId& session,
337  const TCustomExpression& custom_expression) override;
338  void get_custom_expressions(std::vector<TCustomExpression>& _return,
339  const TSessionId& session) override;
340  void update_custom_expression(const TSessionId& session,
341  const int32_t id,
342  const std::string& expression_json) override;
343  void delete_custom_expressions(const TSessionId& session,
344  const std::vector<int32_t>& custom_expression_ids,
345  const bool do_soft_delete) override;
346 
347  // dashboards
348  void get_dashboard(TDashboard& _return,
349  const TSessionId& session,
350  const int32_t dashboard_id) override;
351  void get_dashboards(std::vector<TDashboard>& _return,
352  const TSessionId& session) override;
353  int32_t create_dashboard(const TSessionId& session,
354  const std::string& dashboard_name,
355  const std::string& dashboard_state,
356  const std::string& image_hash,
357  const std::string& dashboard_metadata) override;
358  void replace_dashboard(const TSessionId& session,
359  const int32_t dashboard_id,
360  const std::string& dashboard_name,
361  const std::string& dashboard_owner,
362  const std::string& dashboard_state,
363  const std::string& image_hash,
364  const std::string& dashboard_metadata) override;
365  void delete_dashboard(const TSessionId& session, const int32_t dashboard_id) override;
366  void share_dashboards(const TSessionId& session,
367  const std::vector<int32_t>& dashboard_ids,
368  const std::vector<std::string>& groups,
369  const TDashboardPermissions& permissions) override;
370  void delete_dashboards(const TSessionId& session,
371  const std::vector<int32_t>& dashboard_ids) override;
372  void share_dashboard(const TSessionId& session,
373  const int32_t dashboard_id,
374  const std::vector<std::string>& groups,
375  const std::vector<std::string>& objects,
376  const TDashboardPermissions& permissions,
377  const bool grant_role) override;
378  void unshare_dashboards(const TSessionId& session,
379  const std::vector<int32_t>& dashboard_ids,
380  const std::vector<std::string>& groups,
381  const TDashboardPermissions& permissions) override;
382  void unshare_dashboard(const TSessionId& session,
383  const int32_t dashboard_id,
384  const std::vector<std::string>& groups,
385  const std::vector<std::string>& objects,
386  const TDashboardPermissions& permissions) override;
387  void get_dashboard_grantees(std::vector<TDashboardGrantees>& _return,
388  const TSessionId& session,
389  const int32_t dashboard_id) override;
390 
391  void get_link_view(TFrontendView& _return,
392  const TSessionId& session,
393  const std::string& link) override;
394  void create_link(std::string& _return,
395  const TSessionId& session,
396  const std::string& view_state,
397  const std::string& view_metadata) override;
398  // import
399  void load_table_binary(const TSessionId& session,
400  const std::string& table_name,
401  const std::vector<TRow>& rows,
402  const std::vector<std::string>& column_names) override;
403 
404  std::unique_ptr<lockmgr::AbstractLockContainer<const TableDescriptor*>>
405  prepare_columnar_loader(
406  const Catalog_Namespace::SessionInfo& session_info,
407  const std::string& table_name,
408  size_t num_cols,
409  std::unique_ptr<import_export::Loader>* loader,
410  std::vector<std::unique_ptr<import_export::TypedImportBuffer>>* import_buffers,
411  const std::vector<std::string>& column_names);
412 
413  void load_table_binary_columnar(const TSessionId& session,
414  const std::string& table_name,
415  const std::vector<TColumn>& cols,
416  const std::vector<std::string>& column_names) override;
417  void load_table_binary_columnar_polys(const TSessionId& session,
418  const std::string& table_name,
419  const std::vector<TColumn>& cols,
420  const std::vector<std::string>& column_names,
421  const bool assign_render_groups) override;
422  void load_table_binary_arrow(const TSessionId& session,
423  const std::string& table_name,
424  const std::string& arrow_stream,
425  const bool use_column_names) override;
426 
427  void load_table(const TSessionId& session,
428  const std::string& table_name,
429  const std::vector<TStringRow>& rows,
430  const std::vector<std::string>& column_names) override;
431  void detect_column_types(TDetectResult& _return,
432  const TSessionId& session,
433  const std::string& file_name,
434  const TCopyParams& copy_params) override;
435  void create_table(const TSessionId& session,
436  const std::string& table_name,
437  const TRowDescriptor& row_desc,
438  const TFileType::type file_type,
439  const TCreateParams& create_params) override;
440  void import_table(const TSessionId& session,
441  const std::string& table_name,
442  const std::string& file_name,
443  const TCopyParams& copy_params) override;
444  void import_geo_table(const TSessionId& session,
445  const std::string& table_name,
446  const std::string& file_name,
447  const TCopyParams& copy_params,
448  const TRowDescriptor& row_desc,
449  const TCreateParams& create_params) override;
450  void import_table_status(TImportStatus& _return,
451  const TSessionId& session,
452  const std::string& import_id) override;
453  void get_first_geo_file_in_archive(std::string& _return,
454  const TSessionId& session,
455  const std::string& archive_path,
456  const TCopyParams& copy_params) override;
457  void get_all_files_in_archive(std::vector<std::string>& _return,
458  const TSessionId& session,
459  const std::string& archive_path,
460  const TCopyParams& copy_params) override;
461  void get_layers_in_geo_file(std::vector<TGeoFileLayerInfo>& _return,
462  const TSessionId& session,
463  const std::string& file_name,
464  const TCopyParams& copy_params) override;
465  // distributed
466  int64_t query_get_outer_fragment_count(const TSessionId& session,
467  const std::string& select_query) override;
468 
469  void check_table_consistency(TTableMeta& _return,
470  const TSessionId& session,
471  const int32_t table_id) override;
472  void start_query(TPendingQuery& _return,
473  const TSessionId& leaf_session,
474  const TSessionId& parent_session,
475  const std::string& query_ra,
476  const std::string& start_time_str,
477  const bool just_explain,
478  const std::vector<int64_t>& outer_fragment_indices) override;
479  void execute_query_step(TStepResult& _return,
480  const TPendingQuery& pending_query,
481  const TSubqueryId subquery_id,
482  const std::string& start_time_str) override;
483  void broadcast_serialized_rows(const TSerializedRows& serialized_rows,
484  const TRowDescriptor& row_desc,
485  const TQueryId query_id,
486  const TSubqueryId subquery_id,
487  const bool is_final_subquery_result) override;
488 
489  void start_render_query(TPendingRenderQuery& _return,
490  const TSessionId& session,
491  const int64_t widget_id,
492  const int16_t node_idx,
493  const std::string& vega_json) override;
494  void execute_next_render_step(TRenderStepResult& _return,
495  const TPendingRenderQuery& pending_render,
496  const TRenderAggDataMap& merged_data) override;
497 
498  void insert_data(const TSessionId& session, const TInsertData& insert_data) override;
499  void checkpoint(const TSessionId& session, const int32_t table_id) override;
500  // DB Object Privileges
501  void get_roles(std::vector<std::string>& _return, const TSessionId& session) override;
502  bool has_role(const TSessionId& sessionId,
503  const std::string& granteeName,
504  const std::string& roleName) override;
505  bool has_object_privilege(const TSessionId& sessionId,
506  const std::string& granteeName,
507  const std::string& objectName,
508  const TDBObjectType::type object_type,
509  const TDBObjectPermissions& permissions) override;
510  void get_db_objects_for_grantee(std::vector<TDBObject>& _return,
511  const TSessionId& session,
512  const std::string& roleName) override;
513  void get_db_object_privs(std::vector<TDBObject>& _return,
514  const TSessionId& session,
515  const std::string& objectName,
516  const TDBObjectType::type type) override;
517  void get_all_roles_for_user(std::vector<std::string>& _return,
518  const TSessionId& session,
519  const std::string& granteeName) override;
520  std::vector<std::string> get_valid_groups(const TSessionId& session,
521  int32_t dashboard_id,
522  std::vector<std::string> groups);
523  // licensing
524  void set_license_key(TLicenseInfo& _return,
525  const TSessionId& session,
526  const std::string& key,
527  const std::string& nonce) override;
528  void get_license_claims(TLicenseInfo& _return,
529  const TSessionId& session,
530  const std::string& nonce) override;
531  // user-defined functions
532  /*
533  Returns a mapping of device (CPU, GPU) parameters (name, LLVM IR
534  triplet, features, etc)
535  */
536  void get_device_parameters(std::map<std::string, std::string>& _return,
537  const TSessionId& session) override;
538 
539  /*
540  Register Runtime Extension Functions (UDFs, UDTFs) with given
541  signatures. The extension functions implementations are given in a
542  mapping of a device and the corresponding LLVM/NVVM IR string.
543  */
544 
545  void register_runtime_extension_functions(
546  const TSessionId& session,
547  const std::vector<TUserDefinedFunction>& udfs,
548  const std::vector<TUserDefinedTableFunction>& udtfs,
549  const std::map<std::string, std::string>& device_ir_map) override;
550 
551  // end of sync block for HAHandler and mapd.thrift
552 
553  void shutdown();
554  void emergency_shutdown();
555 
556  TSessionId getInvalidSessionId() const;
557 
558  void internal_connect(TSessionId& session,
559  const std::string& username,
560  const std::string& dbname);
561 
562  bool isAggregator() const;
563 
564  std::shared_ptr<Data_Namespace::DataMgr> data_mgr_;
565 
567  std::vector<LeafHostInfo> db_leaves_;
568  std::vector<LeafHostInfo> string_leaves_;
569  const std::string base_data_path_;
570  boost::filesystem::path import_path_;
572  std::default_random_engine random_gen_;
573  std::uniform_int_distribution<int64_t> session_id_dist_;
574  const bool jit_debug_;
575  const bool intel_jit_profile_;
577  const bool read_only_;
578  const bool allow_loop_joins_;
581  std::mutex render_mutex_;
582  int64_t start_time_;
585  std::unique_ptr<RenderHandler> render_handler_;
586  std::unique_ptr<MapDAggHandler> agg_handler_;
587  std::unique_ptr<MapDLeafHandler> leaf_handler_;
588  std::shared_ptr<Calcite> calcite_;
589  const bool legacy_syntax_;
590 
591  std::unique_ptr<QueryDispatchQueue> dispatch_queue_;
592 
593  template <typename... ARGS>
594  std::shared_ptr<query_state::QueryState> create_query_state(ARGS&&... args) {
595  return query_states_.create(std::forward<ARGS>(args)...);
596  }
597 
598  // Exactly one immutable SessionInfo copy should be taken by a typical request.
599  Catalog_Namespace::SessionInfo get_session_copy(const TSessionId& session);
600  std::shared_ptr<Catalog_Namespace::SessionInfo> get_session_copy_ptr(
601  const TSessionId& session);
602 
603  void get_tables_meta_impl(std::vector<TTableMeta>& _return,
604  QueryStateProxy query_state_proxy,
605  const Catalog_Namespace::SessionInfo& session_info,
606  const bool with_table_locks = true);
607 
608  private:
609  std::atomic<bool> initialized_{false};
610  std::shared_ptr<Catalog_Namespace::SessionInfo> create_new_session(
611  TSessionId& session,
612  const std::string& dbname,
613  const Catalog_Namespace::UserMetadata& user_meta,
614  std::shared_ptr<Catalog_Namespace::Catalog> cat);
615  void connect_impl(TSessionId& session,
616  const std::string& passwd,
617  const std::string& dbname,
618  const Catalog_Namespace::UserMetadata& user_meta,
619  std::shared_ptr<Catalog_Namespace::Catalog> cat,
620  query_state::StdLog& stdlog);
621  void disconnect_impl(const SessionMap::iterator& session_it,
622  mapd_unique_lock<mapd_shared_mutex>& write_lock);
623  void check_table_load_privileges(const TSessionId& session,
624  const std::string& table_name);
625  void check_table_load_privileges(const Catalog_Namespace::SessionInfo& session_info,
626  const std::string& table_name);
627  void get_tables_impl(std::vector<std::string>& table_names,
629  const GetTablesType get_tables_type);
630  void get_table_details_impl(TTableDetails& _return,
631  query_state::StdLog& stdlog,
632  const std::string& table_name,
633  const bool get_system,
634  const bool get_physical);
635  void check_read_only(const std::string& str);
636  void check_session_exp_unsafe(const SessionMap::iterator& session_it);
637  void validateGroups(const std::vector<std::string>& groups);
638  void validateDashboardIdsForSharing(const Catalog_Namespace::SessionInfo& session_info,
639  const std::vector<int32_t>& dashboard_ids);
640  void shareOrUnshareDashboards(const TSessionId& session,
641  const std::vector<int32_t>& dashboard_ids,
642  const std::vector<std::string>& groups,
643  const TDashboardPermissions& permissions,
644  const bool do_share);
645 
646  // Use get_session_copy() or get_session_copy_ptr() instead of get_const_session_ptr()
647  // unless you know what you are doing. If you need to save a SessionInfo beyond the
648  // present Thrift call, then the safe way to do this is by saving the return value of
649  // get_const_session_ptr() as a std::weak_ptr.
650  // Returns empty std::shared_ptr if session.empty().
651  std::shared_ptr<const Catalog_Namespace::SessionInfo> get_const_session_ptr(
652  const TSessionId& session);
653  std::shared_ptr<Catalog_Namespace::SessionInfo> get_session_ptr(
654  const TSessionId& session_id);
655  template <typename SESSION_MAP_LOCK>
656  SessionMap::iterator get_session_it_unsafe(const TSessionId& session,
657  SESSION_MAP_LOCK& lock);
658  template <typename SESSION_MAP_LOCK>
659  void expire_idle_sessions_unsafe(SESSION_MAP_LOCK& lock);
660  static void value_to_thrift_column(const TargetValue& tv,
661  const SQLTypeInfo& ti,
662  TColumn& column);
663  static TDatum value_to_thrift(const TargetValue& tv, const SQLTypeInfo& ti);
664  static std::string apply_copy_to_shim(const std::string& query_str);
665 
666  std::pair<TPlanResult, lockmgr::LockedTableDescriptors> parse_to_ra(
668  const std::string& query_str,
669  const std::vector<TFilterPushDownInfo>& filter_push_down_info,
670  const bool acquire_locks,
671  const SystemParameters& system_parameters,
672  bool check_privileges = true);
673 
674  void sql_execute_local(
675  TQueryResult& _return,
676  const QueryStateProxy& query_state_proxy,
677  const std::shared_ptr<Catalog_Namespace::SessionInfo> session_ptr,
678  const std::string& query_str,
679  const bool column_format,
680  const std::string& nonce,
681  const int32_t first_n,
682  const int32_t at_most_n,
683  const bool use_calcite);
684 
685  int64_t process_geo_copy_from(const TSessionId& session_id);
686 
687  static void convertData(TQueryResult& _return,
689  const QueryStateProxy& query_state_proxy,
690  const std::string& query_str,
691  const bool column_format,
692  const int32_t first_n,
693  const int32_t at_most_n);
694 
695  void sql_execute_impl(ExecutionResult& _return,
697  const bool column_format,
698  const ExecutorDeviceType executor_device_type,
699  const int32_t first_n,
700  const int32_t at_most_n,
701  const bool use_calcite);
702 
703  bool user_can_access_table(const Catalog_Namespace::SessionInfo&,
704  const TableDescriptor* td,
705  const AccessPrivileges acess_priv);
706 
707  void execute_distributed_copy_statement(
709  const Catalog_Namespace::SessionInfo& session_info);
710 
711  TQueryResult validate_rel_alg(const std::string& query_ra, QueryStateProxy);
712 
713  std::vector<PushedDownFilterInfo> execute_rel_alg(
714  ExecutionResult& _return,
716  const std::string& query_ra,
717  const bool column_format,
718  const ExecutorDeviceType executor_device_type,
719  const int32_t first_n,
720  const int32_t at_most_n,
721  const bool just_validate,
722  const bool find_push_down_candidates,
723  const ExplainInfo& explain_info,
724  const std::optional<size_t> executor_index = std::nullopt) const;
725 
726  void execute_rel_alg_with_filter_push_down(
727  ExecutionResult& _return,
729  std::string& query_ra,
730  const bool column_format,
731  const ExecutorDeviceType executor_device_type,
732  const int32_t first_n,
733  const int32_t at_most_n,
734  const bool just_explain,
735  const bool just_calcite_explain,
736  const std::vector<PushedDownFilterInfo>& filter_push_down_requests);
737 
738  void execute_rel_alg_df(TDataFrame& _return,
739  const std::string& query_ra,
740  QueryStateProxy query_state_proxy,
741  const Catalog_Namespace::SessionInfo& session_info,
742  const ExecutorDeviceType device_type,
743  const size_t device_id,
744  const int32_t first_n,
745  const TArrowTransport::type transport_method) const;
746 
747  void executeDdl(TQueryResult& _return,
748  const std::string& query_ra,
749  std::shared_ptr<Catalog_Namespace::SessionInfo const> session_ptr);
750 
751  void executeDdl(ExecutionResult& _return,
752  const std::string& query_ra,
753  std::shared_ptr<Catalog_Namespace::SessionInfo const> session_ptr);
754 
755  TColumnType populateThriftColumnType(const Catalog_Namespace::Catalog* cat,
756  const ColumnDescriptor* cd);
757  TRowDescriptor fixup_row_descriptor(const TRowDescriptor& row_desc,
759  void set_execution_mode_nolock(Catalog_Namespace::SessionInfo* session_ptr,
760  const TExecuteMode::type mode);
761  char unescape_char(std::string str);
762  import_export::CopyParams thrift_to_copyparams(const TCopyParams& cp);
763  TCopyParams copyparams_to_thrift(const import_export::CopyParams& cp);
764  void check_geospatial_files(const boost::filesystem::path file_path,
765  const import_export::CopyParams& copy_params);
766  void render_rel_alg(TRenderResult& _return,
767  const std::string& query_ra,
768  const std::string& query_str,
769  const Catalog_Namespace::SessionInfo& session_info,
770  const std::string& render_type,
771  const bool is_projection_query);
772 
773  TColumnType create_geo_column(const TDatumType::type type,
774  const std::string& name,
775  const bool is_array);
776 
777  static void convertExplain(TQueryResult& _return,
778  const ResultSet& results,
779  const bool column_format);
780  static void convertResult(TQueryResult& _return,
781  const ResultSet& results,
782  const bool column_format);
783 
784  static void convertRows(TQueryResult& _return,
785  QueryStateProxy query_state_proxy,
786  const std::vector<TargetMetaInfo>& targets,
787  const ResultSet& results,
788  const bool column_format,
789  const int32_t first_n,
790  const int32_t at_most_n);
791 
792  // Use ExecutionResult to populate a TQueryResult
793  // calls convertRows, but after some setup using session_info
794  void convertResultSet(ExecutionResult& result,
795  const Catalog_Namespace::SessionInfo& session_info,
796  const std::string& query_state_str,
797  TQueryResult& _return);
798 
799  static void createSimpleResult(TQueryResult& _return,
800  const ResultSet& results,
801  const bool column_format,
802  const std::string label);
803 
804  std::vector<TargetMetaInfo> getTargetMetaInfo(
805  const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& targets) const;
806 
807  std::vector<std::string> getTargetNames(
808  const std::vector<TargetMetaInfo>& targets) const;
809 
810  std::vector<std::string> getTargetNames(
811  const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& targets) const;
812 
813  void get_completion_hints_unsorted(std::vector<TCompletionHint>& hints,
814  std::vector<std::string>& visible_tables,
815  query_state::StdLog& stdlog,
816  const std::string& sql,
817  const int cursor);
818  void get_token_based_completions(std::vector<TCompletionHint>& hints,
819  query_state::StdLog& stdlog,
820  std::vector<std::string>& visible_tables,
821  const std::string& sql,
822  const int cursor);
823 
824  std::unordered_map<std::string, std::unordered_set<std::string>>
825  fill_column_names_by_table(std::vector<std::string>& table_names,
826  query_state::StdLog& stdlog);
827 
828  ConnectionInfo getConnectionInfo() const;
829 
830  TDashboard get_dashboard_impl(
831  const std::shared_ptr<Catalog_Namespace::SessionInfo const>& session_ptr,
833  const DashboardDescriptor* dash,
834  const bool populate_state = true);
835 
836  static bool has_database_permission(const AccessPrivileges& privs,
837  const TDBObjectPermissions& permissions);
838  static bool has_table_permission(const AccessPrivileges& privs,
839  const TDBObjectPermissions& permission);
840  static bool has_dashboard_permission(const AccessPrivileges& privs,
841  const TDBObjectPermissions& permissions);
842  static bool has_view_permission(const AccessPrivileges& privs,
843  const TDBObjectPermissions& permissions);
844  static bool has_server_permission(const AccessPrivileges& privs,
845  const TDBObjectPermissions& permissions);
846  // For the provided upper case column names `uc_column_names`, return
847  // the tables from `table_names` which contain at least one of them.
848  // Used to rank the TABLE auto-completion hints by the columns
849  // specified in the projection.
850  std::unordered_set<std::string> get_uc_compatible_table_names_by_column(
851  const std::unordered_set<std::string>& uc_column_names,
852  std::vector<std::string>& table_names,
853  query_state::StdLog& stdlog);
854 
855  std::unique_ptr<lockmgr::AbstractLockContainer<const TableDescriptor*>>
856  prepare_loader_generic(
857  const Catalog_Namespace::SessionInfo& session_info,
858  const std::string& table_name,
859  size_t num_cols,
860  std::unique_ptr<import_export::Loader>* loader,
861  std::vector<std::unique_ptr<import_export::TypedImportBuffer>>* import_buffers,
862  const std::vector<std::string>& column_names,
863  std::string load_type);
864 
867 
868  bool super_user_rights_; // default is "false"; setting to "true"
869  // ignores passwd checks in "connect(..)"
870  // method
871  const int idle_session_duration_; // max duration of idle session
872  const int max_session_duration_; // max duration of session
873 
875 
876  const bool enable_rendering_;
881  const size_t reserved_gpu_mem_;
883  const size_t render_mem_bytes_;
884  const size_t num_reader_threads_;
885 #ifdef ENABLE_GEOS
886  const std::string& libgeos_so_filename_;
887 #endif
889  const std::string& udf_filename_;
890  const std::string& clang_path_;
891  const std::vector<std::string>& clang_options_;
892 
894  std::string geo_copy_from_table;
898  };
899 
901  std::unordered_map<std::string, GeoCopyFromState> was_geo_copy_from;
903 
904  std::optional<GeoCopyFromState> operator()(const std::string& session_id) {
905  std::lock_guard<std::mutex> map_lock(geo_copy_from_mutex);
906  auto itr = was_geo_copy_from.find(session_id);
907  if (itr == was_geo_copy_from.end()) {
908  return std::nullopt;
909  }
910  return itr->second;
911  }
912 
913  void add(const std::string& session_id, const GeoCopyFromState& state) {
914  std::lock_guard<std::mutex> map_lock(geo_copy_from_mutex);
915  const auto ret = was_geo_copy_from.insert(std::make_pair(session_id, state));
916  CHECK(ret.second);
917  }
918 
919  void remove(const std::string& session_id) {
920  std::lock_guard<std::mutex> map_lock(geo_copy_from_mutex);
921  was_geo_copy_from.erase(session_id);
922  }
923  };
925 
926  // Only for IPC device memory deallocation
927  mutable std::mutex handle_to_dev_ptr_mutex_;
928  mutable std::unordered_map<std::string, std::string> ipc_handle_to_dev_ptr_;
929 
930  friend void run_warmup_queries(mapd::shared_ptr<DBHandler> handler,
931  std::string base_path,
932  std::string query_file_path);
933 
934  friend class RenderHandler::Impl;
935  friend class MapDAggHandler;
936  friend class MapDLeafHandler;
937 
938  std::map<const std::string, const PermissionFuncPtr> permissionFuncMap_ = {
939  {"database"s, has_database_permission},
940  {"dashboard"s, has_dashboard_permission},
941  {"table"s, has_table_permission},
942  {"view"s, has_view_permission},
943  {"server"s, has_server_permission}};
944 
945  void check_and_invalidate_sessions(Parser::DDLStmt* ddl);
946 
947  template <typename STMT_TYPE>
948  void invalidate_sessions(std::string& name, STMT_TYPE* stmt) {
949  using namespace Parser;
950  auto is_match = [&](auto session_it) {
952  return boost::iequals(name,
953  session_it->second->getCatalog().getCurrentDB().dbName);
955  return boost::iequals(name, session_it->second->get_currentUser().userName);
956  }
957  return false;
958  };
959  auto check_and_remove_sessions = [&]() {
960  for (auto it = sessions_.begin(); it != sessions_.end();) {
961  if (is_match(it)) {
962  it = sessions_.erase(it);
963  } else {
964  ++it;
965  }
966  }
967  };
968  check_and_remove_sessions();
969  }
970 
971  std::string const createInMemoryCalciteSession(
972  const std::shared_ptr<Catalog_Namespace::Catalog>& catalog_ptr);
973  bool isInMemoryCalciteSession(const Catalog_Namespace::UserMetadata user_meta);
974  void removeInMemoryCalciteSession(const std::string& session_id);
975 
976  ExecutionResult getUserSessions(
977  std::shared_ptr<Catalog_Namespace::SessionInfo const> session_ptr);
978 
979  // getQueries returns a set of queries queued in the DB
980  // that belongs to the same DB in the caller's session
981 
982  ExecutionResult getQueries(
983  std::shared_ptr<Catalog_Namespace::SessionInfo const> session_ptr);
984 
985  // this function passes the interrupt request to the DB executor
986  void interruptQuery(const Catalog_Namespace::SessionInfo& session_info,
987  const std::string& target_session);
988 
989  // render group assignment
990 
991  enum class AssignRenderGroupsMode { kNone, kAssign, kCleanUp };
992 
993  void load_table_binary_columnar_internal(
994  const TSessionId& session,
995  const std::string& table_name,
996  const std::vector<TColumn>& cols,
997  const std::vector<std::string>& column_names,
998  const AssignRenderGroupsMode assign_render_groups_mode);
999 
1000  using RenderGroupAssignmentColumnMap =
1001  std::unordered_map<std::string,
1002  std::unique_ptr<import_export::RenderGroupAnalyzer>>;
1004  std::unordered_map<std::string, RenderGroupAssignmentColumnMap>;
1006  std::unordered_map<TSessionId, RenderGroupAssignmentTableMap>;
1010 };
Classes used to wrap parser calls for calcite redirection.
AssignRenderGroupsMode
Definition: DBHandler.h:991
std::vector< LeafHostInfo > string_leaves_
Definition: DBHandler.h:568
const std::vector< std::string > & clang_options_
Definition: DBHandler.h:891
boost::filesystem::path import_path_
Definition: DBHandler.h:570
std::unique_ptr< QueryDispatchQueue > dispatch_queue_
Definition: DBHandler.h:591
bool process(mapd::shared_ptr<::apache::thrift::protocol::TProtocol > in, mapd::shared_ptr<::apache::thrift::protocol::TProtocol > out, void *connectionContext)
Definition: DBHandler.h:113
std::unordered_map< std::string, RenderGroupAssignmentColumnMap > RenderGroupAssignmentTableMap
Definition: DBHandler.h:1004
const bool runtime_udf_registration_enabled_
Definition: DBHandler.h:874
ClientProtocol
const std::string & udf_filename_
Definition: DBHandler.h:889
std::unordered_map< std::string, GeoCopyFromState > was_geo_copy_from
Definition: DBHandler.h:901
std::string cat(Ts &&...args)
const int render_oom_retry_threshold_
Definition: DBHandler.h:879
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:102
std::mutex handle_to_dev_ptr_mutex_
Definition: DBHandler.h:927
std::shared_ptr< query_state::QueryState > create_query_state(ARGS &&...args)
Definition: DBHandler.h:594
static thread_local std::string client_address
Definition: DBHandler.h:148
ExecutorDeviceType
std::mutex render_group_assignment_mutex_
Definition: DBHandler.h:1008
bool(*)(const AccessPrivileges &, const TDBObjectPermissions &) PermissionFuncPtr
Definition: DBHandler.h:105
static thread_local ClientProtocol client_protocol
Definition: DBHandler.h:149
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:566
string name
Definition: setup.in.py:72
mapd_shared_mutex custom_expressions_mutex_
Definition: DBHandler.h:1009
std::unordered_map< TSessionId, RenderGroupAssignmentTableMap > RenderGroupAnalyzerSessionMap
Definition: DBHandler.h:1006
const std::string base_data_path_
Definition: DBHandler.h:569
const bool jit_debug_
Definition: DBHandler.h:574
const bool check_origin_
Definition: DBHandler.h:152
const size_t render_mem_bytes_
Definition: DBHandler.h:883
std::string geo_copy_from_table
Definition: DBHandler.h:894
const bool renderer_use_vulkan_driver_
Definition: DBHandler.h:877
std::map< TSessionId, std::shared_ptr< Catalog_Namespace::SessionInfo >> SessionMap
Definition: DBHandler.h:104
std::vector< std::string > split(std::string_view str, std::string_view delim, std::optional< size_t > maxsplit)
split apart a string into a vector of substrings
int64_t start_time_
Definition: DBHandler.h:582
void run_warmup_queries(mapd::shared_ptr< DBHandler > handler, std::string base_path, std::string query_file_path)
Definition: MapDServer.cpp:203
mapd_shared_mutex sessions_mutex_
Definition: DBHandler.h:580
This file contains the class specification and related data structures for Catalog.
std::mutex render_mutex_
Definition: DBHandler.h:581
static size_t max_bytes_for_thrift()
Definition: DBHandler.h:193
query_state::QueryStates query_states_
Definition: DBHandler.h:865
const size_t reserved_gpu_mem_
Definition: DBHandler.h:881
const bool render_compositor_use_last_gpu_
Definition: DBHandler.h:882
GetTablesType
Definition: Catalog.h:62
std::shared_timed_mutex mapd_shared_mutex
const int max_session_duration_
Definition: DBHandler.h:872
ExecutorDeviceType executor_device_type_
Definition: DBHandler.h:571
std::vector< LeafHostInfo > db_leaves_
Definition: DBHandler.h:567
std::string geo_copy_from_file_name
Definition: DBHandler.h:895
const std::string & clang_path_
Definition: DBHandler.h:890
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:585
SystemParameters & system_parameters_
Definition: DBHandler.h:584
const size_t num_reader_threads_
Definition: DBHandler.h:884
specifies the content in-memory of a row in the column metadata table
TrackingProcessor(mapd::shared_ptr< OmniSciIf > handler, const bool check_origin)
Definition: DBHandler.h:110
const DiskCacheConfig & disk_cache_config_
Definition: DBHandler.h:888
std::unique_ptr< MapDLeafHandler > leaf_handler_
Definition: DBHandler.h:587
const bool enable_auto_clear_render_mem_
Definition: DBHandler.h:878
std::map< std::string, std::string > get_device_parameters(bool cpu_only)
const bool allow_loop_joins_
Definition: DBHandler.h:578
const bool enable_rendering_
Definition: DBHandler.h:876
GeoCopyFromSessions geo_copy_from_sessions
Definition: DBHandler.h:924
const bool intel_jit_profile_
Definition: DBHandler.h:575
bool super_user_rights_
Definition: DBHandler.h:868
void add(const std::string &session_id, const GeoCopyFromState &state)
Definition: DBHandler.h:913
std::shared_ptr< Calcite > calcite_
Definition: DBHandler.h:588
void shutdown()
Definition: Logger.cpp:314
std::shared_ptr< Data_Namespace::DataMgr > data_mgr_
Definition: DBHandler.h:564
const bool read_only_
Definition: DBHandler.h:577
const bool legacy_syntax_
Definition: DBHandler.h:589
auto get_users(std::unique_ptr< SqliteConnector > &sqliteConnector, const int32_t dbId=-1)
#define CHECK(condition)
Definition: Logger.h:203
const int idle_session_duration_
Definition: DBHandler.h:871
std::unique_ptr< MapDAggHandler > agg_handler_
Definition: DBHandler.h:586
std::unordered_map< std::string, std::string > ipc_handle_to_dev_ptr_
Definition: DBHandler.h:928
RenderGroupAnalyzerSessionMap render_group_assignment_map_
Definition: DBHandler.h:1007
const size_t max_concurrent_render_sessions_
Definition: DBHandler.h:880
boost::variant< ScalarTargetValue, ArrayTargetValue, GeoTargetValue, GeoTargetValuePtr > TargetValue
Definition: TargetValue.h:167
bool allow_multifrag_
Definition: DBHandler.h:576
mapd_unique_lock< mapd_shared_mutex > write_lock
const AuthMetadata & authMetadata_
Definition: DBHandler.h:583
std::optional< GeoCopyFromState > operator()(const std::string &session_id)
Definition: DBHandler.h:904
std::string geo_copy_from_partitions
Definition: DBHandler.h:897
void invalidate_sessions(std::string &name, STMT_TYPE *stmt)
Definition: DBHandler.h:948
bool cpu_mode_only_
Definition: DBHandler.h:579
std::default_random_engine random_gen_
Definition: DBHandler.h:572
SessionMap sessions_
Definition: DBHandler.h:866
import_export::CopyParams geo_copy_from_copy_params
Definition: DBHandler.h:896
std::uniform_int_distribution< int64_t > session_id_dist_
Definition: DBHandler.h:573
std::string apply_copy_to_shim(const std::string &query_str)