OmniSciDB  2e3a973ef4
DBHandler.h
Go to the documentation of this file.
1 /*
2  * Copyright 2020 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 /*
18  * File: DBHandler.h
19  * Author: michael
20  *
21  * Created on Jan 1, 2017, 12:40 PM
22  */
23 
24 #pragma once
25 
26 #include "LeafAggregator.h"
27 
28 #ifdef HAVE_PROFILER
29 #include <gperftools/heap-profiler.h>
30 #endif // HAVE_PROFILER
31 
32 #include "Calcite/Calcite.h"
33 #include "Catalog/Catalog.h"
35 #include "Geospatial/Transforms.h"
36 #include "ImportExport/Importer.h"
37 #include "LockMgr/LockMgr.h"
38 #include "Logger/Logger.h"
39 #include "Parser/ParserWrapper.h"
41 #include "Parser/parser.h"
44 #include "QueryEngine/Execute.h"
50 #include "Shared/StringTransform.h"
53 #include "Shared/mapd_shared_ptr.h"
54 #include "Shared/measure.h"
55 #include "Shared/scope.h"
61 
62 #include <sys/time.h>
63 #include <sys/types.h>
64 #include <sys/wait.h>
65 #include <thrift/server/TServer.h>
66 #include <thrift/transport/THttpTransport.h>
67 #include <thrift/transport/TSocket.h>
68 #include <thrift/transport/TTransport.h>
69 #include <atomic>
70 #include <boost/algorithm/string.hpp>
71 #include <boost/algorithm/string/replace.hpp>
72 #include <boost/algorithm/string/trim.hpp>
73 #include <boost/filesystem.hpp>
74 #include <boost/make_shared.hpp>
75 #include <boost/noncopyable.hpp>
76 #include <boost/none_t.hpp>
77 #include <boost/optional.hpp>
78 #include <boost/program_options.hpp>
79 #include <boost/regex.hpp>
80 #include <boost/tokenizer.hpp>
81 #include <cmath>
82 #include <csignal>
83 #include <fstream>
84 #include <list>
85 #include <map>
86 #include <memory>
87 #include <mutex>
88 #include <random>
89 #include <regex>
90 #include <string>
91 #include <thread>
92 #include <typeinfo>
93 #include <unordered_map>
94 
95 #include "gen-cpp/OmniSci.h"
96 #include "gen-cpp/extension_functions_types.h"
97 
98 using namespace std::string_literals;
99 
100 class MapDAggHandler;
101 class MapDLeafHandler;
102 
103 // Multiple concurrent requests for the same session can occur. For that reason, each
104 // request briefly takes a lock to make a copy of the appropriate SessionInfo object. Then
105 // it releases the lock and uses the copy for the remainder of the request.
106 using SessionMap = std::map<TSessionId, std::shared_ptr<Catalog_Namespace::SessionInfo>>;
107 using PermissionFuncPtr = bool (*)(const AccessPrivileges&, const TDBObjectPermissions&);
109 
110 class TrackingProcessor : public OmniSciProcessor {
111  public:
112  TrackingProcessor(mapd::shared_ptr<OmniSciIf> handler, const bool check_origin)
113  : OmniSciProcessor(handler), check_origin_(check_origin) {}
114 
115  bool process(mapd::shared_ptr<::apache::thrift::protocol::TProtocol> in,
116  mapd::shared_ptr<::apache::thrift::protocol::TProtocol> out,
117  void* connectionContext) {
118  using namespace ::apache::thrift;
119 
120  auto transport = in->getTransport();
121  if (transport && check_origin_) {
122  static std::mutex processor_mutex;
123  std::lock_guard lock(processor_mutex);
124  const auto origin_str = transport->getOrigin();
125  std::vector<std::string> origins;
126  boost::split(origins, origin_str, boost::is_any_of(","));
127  if (origins.empty()) {
129  } else {
130  // Take the first origin, which should be the client IP before any intermediate
131  // servers (e.g. the web server)
132  auto trimmed_origin = origins.front();
133  boost::algorithm::trim(trimmed_origin);
134  TrackingProcessor::client_address = trimmed_origin;
135  }
136  if (dynamic_cast<transport::THttpTransport*>(transport.get())) {
138  } else if (dynamic_cast<transport::TBufferedTransport*>(transport.get())) {
140  } else {
142  }
143  } else {
145  }
146 
147  return OmniSciProcessor::process(in, out, connectionContext);
148  }
149 
150  static thread_local std::string client_address;
151  static thread_local ClientProtocol client_protocol;
152 
153  private:
154  const bool check_origin_;
155 };
156 
157 struct DiskCacheConfig;
158 
159 class DBHandler : public OmniSciIf {
160  public:
161  DBHandler(const std::vector<LeafHostInfo>& db_leaves,
162  const std::vector<LeafHostInfo>& string_leaves,
163  const std::string& base_data_path,
164  const bool cpu_only,
165  const bool allow_multifrag,
166  const bool jit_debug,
167  const bool intel_jit_profile,
168  const bool read_only,
169  const bool allow_loop_joins,
170  const bool enable_rendering,
171  const bool renderer_use_vulkan_driver,
172  const bool enable_auto_clear_render_mem,
173  const int render_oom_retry_threshold,
174  const size_t render_mem_bytes,
175  const size_t max_concurrent_render_sessions,
176  const int num_gpus,
177  const int start_gpu,
178  const size_t reserved_gpu_mem,
179  const bool render_compositor_use_last_gpu,
180  const size_t num_reader_threads,
181  const AuthMetadata& authMetadata,
182  const SystemParameters& system_parameters,
183  const bool legacy_syntax,
184  const int idle_session_duration,
185  const int max_session_duration,
186  const bool enable_runtime_udf_registration,
187  const std::string& udf_filename,
188  const std::string& clang_path,
189  const std::vector<std::string>& clang_options,
190 #ifdef ENABLE_GEOS
191  const std::string& libgeos_so_filename,
192 #endif
193  const DiskCacheConfig& disk_cache_config);
194 
195  ~DBHandler() override;
196 
197  static inline size_t max_bytes_for_thrift() { return 2 * 1000 * 1000 * 1000L; }
198 
199  // Important ****
200  // This block must be keep in sync with mapd.thrift and HAHandler.h
201  // Please keep in same order for easy check and cut and paste
202  // Important ****
203  static void parser_with_error_handler(
204  const std::string& query_str,
205  std::list<std::unique_ptr<Parser::Stmt>>& parse_trees);
206 
207  void krb5_connect(TKrb5Session& session,
208  const std::string& token,
209  const std::string& dbname) override;
210  // connection, admin
211  void connect(TSessionId& session,
212  const std::string& username,
213  const std::string& passwd,
214  const std::string& dbname) override;
215  void disconnect(const TSessionId& session) override;
216  void switch_database(const TSessionId& session, const std::string& dbname) override;
217  void clone_session(TSessionId& session2, const TSessionId& session1) override;
218  void get_server_status(TServerStatus& _return, const TSessionId& session) override;
219  void get_status(std::vector<TServerStatus>& _return,
220  const TSessionId& session) override;
221  void get_hardware_info(TClusterHardwareInfo& _return,
222  const TSessionId& session) override;
223 
224  bool hasTableAccessPrivileges(const TableDescriptor* td,
225  const Catalog_Namespace::SessionInfo& session_info);
226  void get_tables(std::vector<std::string>& _return, const TSessionId& session) override;
227  void get_physical_tables(std::vector<std::string>& _return,
228  const TSessionId& session) override;
229  void get_views(std::vector<std::string>& _return, const TSessionId& session) override;
230  void get_tables_meta(std::vector<TTableMeta>& _return,
231  const TSessionId& session) override;
232  void get_table_details(TTableDetails& _return,
233  const TSessionId& session,
234  const std::string& table_name) override;
235  void get_internal_table_details(TTableDetails& _return,
236  const TSessionId& session,
237  const std::string& table_name) override;
238  void get_users(std::vector<std::string>& _return, const TSessionId& session) override;
239  void get_databases(std::vector<TDBInfo>& _return, const TSessionId& session) override;
240 
241  void get_version(std::string& _return) override;
242  void start_heap_profile(const TSessionId& session) override;
243  void stop_heap_profile(const TSessionId& session) override;
244  void get_heap_profile(std::string& _return, const TSessionId& session) override;
245  void get_memory(std::vector<TNodeMemoryInfo>& _return,
246  const TSessionId& session,
247  const std::string& memory_level) override;
248  void clear_cpu_memory(const TSessionId& session) override;
249  void clear_gpu_memory(const TSessionId& session) override;
250  void set_table_epoch(const TSessionId& session,
251  const int db_id,
252  const int table_id,
253  const int new_epoch) override;
254  void set_table_epoch_by_name(const TSessionId& session,
255  const std::string& table_name,
256  const int new_epoch) override;
257  int32_t get_table_epoch(const TSessionId& session,
258  const int32_t db_id,
259  const int32_t table_id) override;
260  int32_t get_table_epoch_by_name(const TSessionId& session,
261  const std::string& table_name) override;
262  void get_table_epochs(std::vector<TTableEpochInfo>& _return,
263  const TSessionId& session,
264  const int32_t db_id,
265  const int32_t table_id) override;
266  void set_table_epochs(const TSessionId& session,
267  const int32_t db_id,
268  const std::vector<TTableEpochInfo>& table_epochs) override;
269 
270  void get_session_info(TSessionInfo& _return, const TSessionId& session) override;
271  // query, render
272  void sql_execute(TQueryResult& _return,
273  const TSessionId& session,
274  const std::string& query,
275  const bool column_format,
276  const std::string& nonce,
277  const int32_t first_n,
278  const int32_t at_most_n) override;
279  void get_completion_hints(std::vector<TCompletionHint>& hints,
280  const TSessionId& session,
281  const std::string& sql,
282  const int cursor) override;
283  // TODO(miyu): merge the following two data frame APIs.
284  void sql_execute_df(TDataFrame& _return,
285  const TSessionId& session,
286  const std::string& query,
287  const TDeviceType::type device_type,
288  const int32_t device_id,
289  const int32_t first_n,
290  const TArrowTransport::type transport_method) override;
291  void sql_execute_gdf(TDataFrame& _return,
292  const TSessionId& session,
293  const std::string& query,
294  const int32_t device_id,
295  const int32_t first_n) override;
296  void deallocate_df(const TSessionId& session,
297  const TDataFrame& df,
298  const TDeviceType::type device_type,
299  const int32_t device_id) override;
300  void interrupt(const TSessionId& query_session,
301  const TSessionId& interrupt_session) override;
302  void sql_validate(TRowDescriptor& _return,
303  const TSessionId& session,
304  const std::string& query) override;
305 
306  void set_execution_mode(const TSessionId& session,
307  const TExecuteMode::type mode) override;
308  void render_vega(TRenderResult& _return,
309  const TSessionId& session,
310  const int64_t widget_id,
311  const std::string& vega_json,
312  const int32_t compression_level,
313  const std::string& nonce) override;
314  void get_result_row_for_pixel(
315  TPixelTableRowResult& _return,
316  const TSessionId& session,
317  const int64_t widget_id,
318  const TPixel& pixel,
319  const std::map<std::string, std::vector<std::string>>& table_col_names,
320  const bool column_format,
321  const int32_t pixel_radius,
322  const std::string& nonce) override;
323 
324  // dashboards
325  void get_dashboard(TDashboard& _return,
326  const TSessionId& session,
327  int32_t dashboard_id) override;
328  void get_dashboards(std::vector<TDashboard>& _return,
329  const TSessionId& session) override;
330  int32_t create_dashboard(const TSessionId& session,
331  const std::string& dashboard_name,
332  const std::string& dashboard_state,
333  const std::string& image_hash,
334  const std::string& dashboard_metadata) override;
335  void replace_dashboard(const TSessionId& session,
336  const int32_t dashboard_id,
337  const std::string& dashboard_name,
338  const std::string& dashboard_owner,
339  const std::string& dashboard_state,
340  const std::string& image_hash,
341  const std::string& dashboard_metadata) override;
342  void delete_dashboard(const TSessionId& session, const int32_t dashboard_id) override;
343  void share_dashboards(const TSessionId& session,
344  const std::vector<int32_t>& dashboard_ids,
345  const std::vector<std::string>& groups,
346  const TDashboardPermissions& permissions) override;
347  void delete_dashboards(const TSessionId& session,
348  const std::vector<int32_t>& dashboard_ids) override;
349  void share_dashboard(const TSessionId& session,
350  const int32_t dashboard_id,
351  const std::vector<std::string>& groups,
352  const std::vector<std::string>& objects,
353  const TDashboardPermissions& permissions,
354  const bool grant_role) override;
355  void unshare_dashboards(const TSessionId& session,
356  const std::vector<int32_t>& dashboard_ids,
357  const std::vector<std::string>& groups,
358  const TDashboardPermissions& permissions) override;
359  void unshare_dashboard(const TSessionId& session,
360  const int32_t dashboard_id,
361  const std::vector<std::string>& groups,
362  const std::vector<std::string>& objects,
363  const TDashboardPermissions& permissions) override;
364  void get_dashboard_grantees(std::vector<TDashboardGrantees>& _return,
365  const TSessionId& session,
366  int32_t dashboard_id) override;
367 
368  void get_link_view(TFrontendView& _return,
369  const TSessionId& session,
370  const std::string& link) override;
371  void create_link(std::string& _return,
372  const TSessionId& session,
373  const std::string& view_state,
374  const std::string& view_metadata) override;
375  // import
376  void load_table_binary(const TSessionId& session,
377  const std::string& table_name,
378  const std::vector<TRow>& rows) override;
379 
380  std::unique_ptr<lockmgr::AbstractLockContainer<const TableDescriptor*>>
381  prepare_columnar_loader(
382  const Catalog_Namespace::SessionInfo& session_info,
383  const std::string& table_name,
384  size_t num_cols,
385  std::unique_ptr<import_export::Loader>* loader,
386  std::vector<std::unique_ptr<import_export::TypedImportBuffer>>* import_buffers);
387 
388  void load_table_binary_columnar(const TSessionId& session,
389  const std::string& table_name,
390  const std::vector<TColumn>& cols) override;
391  void load_table_binary_arrow(const TSessionId& session,
392  const std::string& table_name,
393  const std::string& arrow_stream) override;
394 
395  void load_table(const TSessionId& session,
396  const std::string& table_name,
397  const std::vector<TStringRow>& rows) override;
398  void detect_column_types(TDetectResult& _return,
399  const TSessionId& session,
400  const std::string& file_name,
401  const TCopyParams& copy_params) override;
402  void create_table(const TSessionId& session,
403  const std::string& table_name,
404  const TRowDescriptor& row_desc,
405  const TFileType::type file_type,
406  const TCreateParams& create_params) override;
407  void import_table(const TSessionId& session,
408  const std::string& table_name,
409  const std::string& file_name,
410  const TCopyParams& copy_params) override;
411  void import_geo_table(const TSessionId& session,
412  const std::string& table_name,
413  const std::string& file_name,
414  const TCopyParams& copy_params,
415  const TRowDescriptor& row_desc,
416  const TCreateParams& create_params) override;
417  void import_table_status(TImportStatus& _return,
418  const TSessionId& session,
419  const std::string& import_id) override;
420  void get_first_geo_file_in_archive(std::string& _return,
421  const TSessionId& session,
422  const std::string& archive_path,
423  const TCopyParams& copy_params) override;
424  void get_all_files_in_archive(std::vector<std::string>& _return,
425  const TSessionId& session,
426  const std::string& archive_path,
427  const TCopyParams& copy_params) override;
428  void get_layers_in_geo_file(std::vector<TGeoFileLayerInfo>& _return,
429  const TSessionId& session,
430  const std::string& file_name,
431  const TCopyParams& copy_params) override;
432  // distributed
433  int64_t query_get_outer_fragment_count(const TSessionId& session,
434  const std::string& select_query) override;
435 
436  void check_table_consistency(TTableMeta& _return,
437  const TSessionId& session,
438  const int32_t table_id) override;
439  void start_query(TPendingQuery& _return,
440  const TSessionId& leaf_session,
441  const TSessionId& parent_session,
442  const std::string& query_ra,
443  const bool just_explain,
444  const std::vector<int64_t>& outer_fragment_indices) override;
445  void execute_query_step(TStepResult& _return,
446  const TPendingQuery& pending_query) override;
447  void broadcast_serialized_rows(const TSerializedRows& serialized_rows,
448  const TRowDescriptor& row_desc,
449  const TQueryId query_id) override;
450 
451  void start_render_query(TPendingRenderQuery& _return,
452  const TSessionId& session,
453  const int64_t widget_id,
454  const int16_t node_idx,
455  const std::string& vega_json) override;
456  void execute_next_render_step(TRenderStepResult& _return,
457  const TPendingRenderQuery& pending_render,
458  const TRenderAggDataMap& merged_data) override;
459 
460  void insert_data(const TSessionId& session, const TInsertData& insert_data) override;
461  void checkpoint(const TSessionId& session,
462  const int32_t db_id,
463  const int32_t table_id) override;
464  // DB Object Privileges
465  void get_roles(std::vector<std::string>& _return, const TSessionId& session) override;
466  bool has_role(const TSessionId& sessionId,
467  const std::string& granteeName,
468  const std::string& roleName) override;
469  bool has_object_privilege(const TSessionId& sessionId,
470  const std::string& granteeName,
471  const std::string& objectName,
472  const TDBObjectType::type object_type,
473  const TDBObjectPermissions& permissions) override;
474  void get_db_objects_for_grantee(std::vector<TDBObject>& _return,
475  const TSessionId& session,
476  const std::string& roleName) override;
477  void get_db_object_privs(std::vector<TDBObject>& _return,
478  const TSessionId& session,
479  const std::string& objectName,
480  const TDBObjectType::type type) override;
481  void get_all_roles_for_user(std::vector<std::string>& _return,
482  const TSessionId& session,
483  const std::string& granteeName) override;
484  std::vector<std::string> get_valid_groups(const TSessionId& session,
485  int32_t dashboard_id,
486  std::vector<std::string> groups);
487  // licensing
488  void set_license_key(TLicenseInfo& _return,
489  const TSessionId& session,
490  const std::string& key,
491  const std::string& nonce) override;
492  void get_license_claims(TLicenseInfo& _return,
493  const TSessionId& session,
494  const std::string& nonce) override;
495  // user-defined functions
496  /*
497  Returns a mapping of device (CPU, GPU) parameters (name, LLVM IR
498  triplet, features, etc)
499  */
500  void get_device_parameters(std::map<std::string, std::string>& _return,
501  const TSessionId& session) override;
502 
503  /*
504  Register Runtime Extension Functions (UDFs, UDTFs) with given
505  signatures. The extension functions implementations are given in a
506  mapping of a device and the corresponding LLVM/NVVM IR string.
507  */
508 
509  void register_runtime_extension_functions(
510  const TSessionId& session,
511  const std::vector<TUserDefinedFunction>& udfs,
512  const std::vector<TUserDefinedTableFunction>& udtfs,
513  const std::map<std::string, std::string>& device_ir_map) override;
514 
515  // end of sync block for HAHandler and mapd.thrift
516 
517  void shutdown();
518  void emergency_shutdown();
519 
520  TSessionId getInvalidSessionId() const;
521 
522  void internal_connect(TSessionId& session,
523  const std::string& username,
524  const std::string& dbname);
525 
526  std::shared_ptr<Data_Namespace::DataMgr> data_mgr_;
527 
529  const std::vector<LeafHostInfo> string_leaves_;
530  const std::string base_data_path_;
531  boost::filesystem::path import_path_;
533  std::default_random_engine random_gen_;
534  std::uniform_int_distribution<int64_t> session_id_dist_;
535  const bool jit_debug_;
536  const bool intel_jit_profile_;
538  const bool read_only_;
539  const bool allow_loop_joins_;
542  std::mutex render_mutex_;
543  int64_t start_time_;
546  std::unique_ptr<RenderHandler> render_handler_;
547  std::unique_ptr<MapDAggHandler> agg_handler_;
548  std::unique_ptr<MapDLeafHandler> leaf_handler_;
549  std::shared_ptr<Calcite> calcite_;
550  const bool legacy_syntax_;
551 
552  std::unique_ptr<QueryDispatchQueue> dispatch_queue_;
553 
554  template <typename... ARGS>
555  std::shared_ptr<query_state::QueryState> create_query_state(ARGS&&... args) {
556  return query_states_.create(std::forward<ARGS>(args)...);
557  }
558 
559  // Exactly one immutable SessionInfo copy should be taken by a typical request.
560  Catalog_Namespace::SessionInfo get_session_copy(const TSessionId& session);
561  std::shared_ptr<Catalog_Namespace::SessionInfo> get_session_copy_ptr(
562  const TSessionId& session);
563 
564  void get_tables_meta_impl(std::vector<TTableMeta>& _return,
565  QueryStateProxy query_state_proxy,
566  const Catalog_Namespace::SessionInfo& session_info,
567  const bool with_table_locks = true);
568 
569  private:
570  std::shared_ptr<Catalog_Namespace::SessionInfo> create_new_session(
571  TSessionId& session,
572  const std::string& dbname,
573  const Catalog_Namespace::UserMetadata& user_meta,
574  std::shared_ptr<Catalog_Namespace::Catalog> cat);
575  void connect_impl(TSessionId& session,
576  const std::string& passwd,
577  const std::string& dbname,
578  const Catalog_Namespace::UserMetadata& user_meta,
579  std::shared_ptr<Catalog_Namespace::Catalog> cat,
580  query_state::StdLog& stdlog);
581  void disconnect_impl(const SessionMap::iterator& session_it,
582  mapd_unique_lock<mapd_shared_mutex>& write_lock);
583  void check_table_load_privileges(const TSessionId& session,
584  const std::string& table_name);
585  void check_table_load_privileges(const Catalog_Namespace::SessionInfo& session_info,
586  const std::string& table_name);
587  void get_tables_impl(std::vector<std::string>& table_names,
589  const GetTablesType get_tables_type);
590  void get_table_details_impl(TTableDetails& _return,
591  query_state::StdLog& stdlog,
592  const std::string& table_name,
593  const bool get_system,
594  const bool get_physical);
595  void check_read_only(const std::string& str);
596  void check_session_exp_unsafe(const SessionMap::iterator& session_it);
597  void validateGroups(const std::vector<std::string>& groups);
598  void validateDashboardIdsForSharing(const Catalog_Namespace::SessionInfo& session_info,
599  const std::vector<int32_t>& dashboard_ids);
600  void shareOrUnshareDashboards(const TSessionId& session,
601  const std::vector<int32_t>& dashboard_ids,
602  const std::vector<std::string>& groups,
603  const TDashboardPermissions& permissions,
604  const bool do_share);
605 
606  // Use get_session_copy() or get_session_copy_ptr() instead of get_const_session_ptr()
607  // unless you know what you are doing. If you need to save a SessionInfo beyond the
608  // present Thrift call, then the safe way to do this is by saving the return value of
609  // get_const_session_ptr() as a std::weak_ptr.
610  // Returns empty std::shared_ptr if session.empty().
611  std::shared_ptr<const Catalog_Namespace::SessionInfo> get_const_session_ptr(
612  const TSessionId& session);
613  std::shared_ptr<Catalog_Namespace::SessionInfo> get_session_ptr(
614  const TSessionId& session_id);
615  template <typename SESSION_MAP_LOCK>
616  SessionMap::iterator get_session_it_unsafe(const TSessionId& session,
617  SESSION_MAP_LOCK& lock);
618  static void value_to_thrift_column(const TargetValue& tv,
619  const SQLTypeInfo& ti,
620  TColumn& column);
621  static TDatum value_to_thrift(const TargetValue& tv, const SQLTypeInfo& ti);
622  static std::string apply_copy_to_shim(const std::string& query_str);
623 
624  std::pair<TPlanResult, lockmgr::LockedTableDescriptors> parse_to_ra(
626  const std::string& query_str,
627  const std::vector<TFilterPushDownInfo>& filter_push_down_info,
628  const bool acquire_locks,
629  const SystemParameters system_parameters,
630  bool check_privileges = true);
631 
632  void sql_execute_impl(TQueryResult& _return,
634  const bool column_format,
635  const std::string& nonce,
636  const ExecutorDeviceType executor_device_type,
637  const int32_t first_n,
638  const int32_t at_most_n);
639 
640  bool user_can_access_table(const Catalog_Namespace::SessionInfo&,
641  const TableDescriptor* td,
642  const AccessPrivileges acess_priv);
643 
644  void execute_distributed_copy_statement(
646  const Catalog_Namespace::SessionInfo& session_info);
647 
648  TQueryResult validate_rel_alg(const std::string& query_ra, QueryStateProxy);
649 
650  std::vector<PushedDownFilterInfo> execute_rel_alg(
651  TQueryResult& _return,
653  const std::string& query_ra,
654  const bool column_format,
655  const ExecutorDeviceType executor_device_type,
656  const int32_t first_n,
657  const int32_t at_most_n,
658  const bool just_validate,
659  const bool find_push_down_candidates,
660  const ExplainInfo& explain_info,
661  const std::optional<size_t> executor_index = std::nullopt) const;
662 
663  void execute_rel_alg_with_filter_push_down(
664  TQueryResult& _return,
666  std::string& query_ra,
667  const bool column_format,
668  const ExecutorDeviceType executor_device_type,
669  const int32_t first_n,
670  const int32_t at_most_n,
671  const bool just_explain,
672  const bool just_calcite_explain,
673  const std::vector<PushedDownFilterInfo>& filter_push_down_requests);
674 
675  void execute_rel_alg_df(TDataFrame& _return,
676  const std::string& query_ra,
677  QueryStateProxy query_state_proxy,
678  const Catalog_Namespace::SessionInfo& session_info,
679  const ExecutorDeviceType device_type,
680  const size_t device_id,
681  const int32_t first_n,
682  const TArrowTransport::type transport_method) const;
683 
684  void executeDdl(TQueryResult& _return,
685  const std::string& query_ra,
686  std::shared_ptr<Catalog_Namespace::SessionInfo const> session_ptr);
687 
688  TColumnType populateThriftColumnType(const Catalog_Namespace::Catalog* cat,
689  const ColumnDescriptor* cd);
690  TRowDescriptor fixup_row_descriptor(const TRowDescriptor& row_desc,
691  const Catalog_Namespace::Catalog& cat);
692  void set_execution_mode_nolock(Catalog_Namespace::SessionInfo* session_ptr,
693  const TExecuteMode::type mode);
694  char unescape_char(std::string str);
695  import_export::CopyParams thrift_to_copyparams(const TCopyParams& cp);
696  TCopyParams copyparams_to_thrift(const import_export::CopyParams& cp);
697  void check_geospatial_files(const boost::filesystem::path file_path,
698  const import_export::CopyParams& copy_params);
699  void render_rel_alg(TRenderResult& _return,
700  const std::string& query_ra,
701  const std::string& query_str,
702  const Catalog_Namespace::SessionInfo& session_info,
703  const std::string& render_type,
704  const bool is_projection_query);
705 
706  TColumnType create_geo_column(const TDatumType::type type,
707  const std::string& name,
708  const bool is_array);
709 
710  void convert_explain(TQueryResult& _return,
711  const ResultSet& results,
712  const bool column_format) const;
713  void convert_result(TQueryResult& _return,
714  const ResultSet& results,
715  const bool column_format) const;
716 
717  template <class R>
718  void convert_rows(TQueryResult& _return,
719  QueryStateProxy query_state_proxy,
720  const std::vector<TargetMetaInfo>& targets,
721  const R& results,
722  const bool column_format,
723  const int32_t first_n,
724  const int32_t at_most_n) const;
725 
726  void create_simple_result(TQueryResult& _return,
727  const ResultSet& results,
728  const bool column_format,
729  const std::string label) const;
730 
731  std::vector<TargetMetaInfo> getTargetMetaInfo(
732  const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& targets) const;
733 
734  std::vector<std::string> getTargetNames(
735  const std::vector<TargetMetaInfo>& targets) const;
736 
737  std::vector<std::string> getTargetNames(
738  const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& targets) const;
739 
740  TColumnType convert_target_metainfo(const TargetMetaInfo& target,
741  const size_t idx) const;
742 
743  TRowDescriptor convert_target_metainfo(
744  const std::vector<TargetMetaInfo>& targets) const;
745 
746  void get_completion_hints_unsorted(std::vector<TCompletionHint>& hints,
747  std::vector<std::string>& visible_tables,
748  query_state::StdLog& stdlog,
749  const std::string& sql,
750  const int cursor);
751  void get_token_based_completions(std::vector<TCompletionHint>& hints,
752  query_state::StdLog& stdlog,
753  std::vector<std::string>& visible_tables,
754  const std::string& sql,
755  const int cursor);
756 
757  std::unordered_map<std::string, std::unordered_set<std::string>>
758  fill_column_names_by_table(std::vector<std::string>& table_names,
759  query_state::StdLog& stdlog);
760 
761  ConnectionInfo getConnectionInfo() const;
762 
763  static bool has_database_permission(const AccessPrivileges& privs,
764  const TDBObjectPermissions& permissions);
765  static bool has_table_permission(const AccessPrivileges& privs,
766  const TDBObjectPermissions& permission);
767  static bool has_dashboard_permission(const AccessPrivileges& privs,
768  const TDBObjectPermissions& permissions);
769  static bool has_view_permission(const AccessPrivileges& privs,
770  const TDBObjectPermissions& permissions);
771 
772  // For the provided upper case column names `uc_column_names`, return
773  // the tables from `table_names` which contain at least one of them.
774  // Used to rank the TABLE auto-completion hints by the columns
775  // specified in the projection.
776  std::unordered_set<std::string> get_uc_compatible_table_names_by_column(
777  const std::unordered_set<std::string>& uc_column_names,
778  std::vector<std::string>& table_names,
779  query_state::StdLog& stdlog);
780 
783 
784  bool super_user_rights_; // default is "false"; setting to "true"
785  // ignores passwd checks in "connect(..)"
786  // method
787  const int idle_session_duration_; // max duration of idle session
788  const int max_session_duration_; // max duration of session
789 
791 
793  std::string geo_copy_from_table;
797  };
798 
800  std::unordered_map<std::string, GeoCopyFromState> was_geo_copy_from;
802 
803  std::optional<GeoCopyFromState> operator()(const std::string& session_id) {
804  std::lock_guard<std::mutex> map_lock(geo_copy_from_mutex);
805  auto itr = was_geo_copy_from.find(session_id);
806  if (itr == was_geo_copy_from.end()) {
807  return std::nullopt;
808  }
809  return itr->second;
810  }
811 
812  void add(const std::string& session_id, const GeoCopyFromState& state) {
813  std::lock_guard<std::mutex> map_lock(geo_copy_from_mutex);
814  const auto ret = was_geo_copy_from.insert(std::make_pair(session_id, state));
815  CHECK(ret.second);
816  }
817 
818  void remove(const std::string& session_id) {
819  std::lock_guard<std::mutex> map_lock(geo_copy_from_mutex);
820  was_geo_copy_from.erase(session_id);
821  }
822  };
824 
825  // Only for IPC device memory deallocation
826  mutable std::mutex handle_to_dev_ptr_mutex_;
827  mutable std::unordered_map<std::string, std::string> ipc_handle_to_dev_ptr_;
828 
829  friend void run_warmup_queries(mapd::shared_ptr<DBHandler> handler,
830  std::string base_path,
831  std::string query_file_path);
832 
833  friend class RenderHandler::Impl;
834  friend class MapDAggHandler;
835  friend class MapDLeafHandler;
836 
837  std::map<const std::string, const PermissionFuncPtr> permissionFuncMap_ = {
838  {"database"s, has_database_permission},
839  {"dashboard"s, has_dashboard_permission},
840  {"table"s, has_table_permission},
841  {"view"s, has_view_permission}};
842 
843  void check_and_invalidate_sessions(Parser::DDLStmt* ddl);
844 
845  template <typename STMT_TYPE>
846  void invalidate_sessions(std::string& name, STMT_TYPE* stmt) {
847  using namespace Parser;
848  auto is_match = [&](auto session_it) {
850  return boost::iequals(name,
851  session_it->second->getCatalog().getCurrentDB().dbName);
853  return boost::iequals(name, session_it->second->get_currentUser().userName);
854  }
855  return false;
856  };
857  auto check_and_remove_sessions = [&]() {
858  for (auto it = sessions_.begin(); it != sessions_.end();) {
859  if (is_match(it)) {
860  it = sessions_.erase(it);
861  } else {
862  ++it;
863  }
864  }
865  };
866  check_and_remove_sessions();
867  }
868 
869  std::string const createInMemoryCalciteSession(
870  const std::shared_ptr<Catalog_Namespace::Catalog>& catalog_ptr);
871  bool isInMemoryCalciteSession(const Catalog_Namespace::UserMetadata user_meta);
872  void removeInMemoryCalciteSession(const std::string& session_id);
873 
874  void getUserSessions(const Catalog_Namespace::SessionInfo& session_info,
875  TQueryResult& _return);
876 
877  // this function returns a set of queries queued in the DB
878  // that belongs to the same DB in the caller's session
879  void getQueries(const Catalog_Namespace::SessionInfo& session_info,
880  TQueryResult& _return);
881 
882  // this function passes the interrupt request to the DB executor
883  void interruptQuery(const Catalog_Namespace::SessionInfo& session_info,
884  const std::string& target_session);
885 };
Classes used to wrap parser calls for calcite redirection.
boost::filesystem::path import_path_
Definition: DBHandler.h:531
std::unique_ptr< QueryDispatchQueue > dispatch_queue_
Definition: DBHandler.h:552
bool process(mapd::shared_ptr<::apache::thrift::protocol::TProtocol > in, mapd::shared_ptr<::apache::thrift::protocol::TProtocol > out, void *connectionContext)
Definition: DBHandler.h:115
const bool runtime_udf_registration_enabled_
Definition: DBHandler.h:790
ClientProtocol
std::unordered_map< std::string, GeoCopyFromState > was_geo_copy_from
Definition: DBHandler.h:800
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:97
std::mutex handle_to_dev_ptr_mutex_
Definition: DBHandler.h:826
static thread_local std::string client_address
Definition: DBHandler.h:150
ExecutorDeviceType
bool(*)(const AccessPrivileges &, const TDBObjectPermissions &) PermissionFuncPtr
Definition: DBHandler.h:107
const std::vector< LeafHostInfo > string_leaves_
Definition: DBHandler.h:529
static thread_local ClientProtocol client_protocol
Definition: DBHandler.h:151
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:528
std::shared_ptr< query_state::QueryState > create_query_state(ARGS &&... args)
Definition: DBHandler.h:555
const std::string base_data_path_
Definition: DBHandler.h:530
const bool jit_debug_
Definition: DBHandler.h:535
const bool check_origin_
Definition: DBHandler.h:154
std::string geo_copy_from_table
Definition: DBHandler.h:793
std::vector< std::string > split(std::string_view str, std::string_view delim, std::optional< size_t > maxsplit)
split apart a string into a vector of substrings
int64_t start_time_
Definition: DBHandler.h:543
void run_warmup_queries(mapd::shared_ptr< DBHandler > handler, std::string base_path, std::string query_file_path)
Definition: MapDServer.cpp:181
mapd_shared_mutex sessions_mutex_
Definition: DBHandler.h:541
std::map< TSessionId, std::shared_ptr< Catalog_Namespace::SessionInfo > > SessionMap
Definition: DBHandler.h:106
This file contains the class specification and related data structures for Catalog.
std::mutex render_mutex_
Definition: DBHandler.h:542
static size_t max_bytes_for_thrift()
Definition: DBHandler.h:197
query_state::QueryStates query_states_
Definition: DBHandler.h:781
std::string cat(Ts &&... args)
GetTablesType
Definition: Catalog.h:60
std::shared_timed_mutex mapd_shared_mutex
const int max_session_duration_
Definition: DBHandler.h:788
ExecutorDeviceType executor_device_type_
Definition: DBHandler.h:532
std::string geo_copy_from_file_name
Definition: DBHandler.h:794
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:546
specifies the content in-memory of a row in the column metadata table
TrackingProcessor(mapd::shared_ptr< OmniSciIf > handler, const bool check_origin)
Definition: DBHandler.h:112
std::unique_ptr< MapDLeafHandler > leaf_handler_
Definition: DBHandler.h:548
std::map< std::string, std::string > get_device_parameters(bool cpu_only)
const bool allow_loop_joins_
Definition: DBHandler.h:539
GeoCopyFromSessions geo_copy_from_sessions
Definition: DBHandler.h:823
const bool intel_jit_profile_
Definition: DBHandler.h:536
bool super_user_rights_
Definition: DBHandler.h:784
void add(const std::string &session_id, const GeoCopyFromState &state)
Definition: DBHandler.h:812
std::shared_ptr< Calcite > calcite_
Definition: DBHandler.h:549
void shutdown()
Definition: Logger.cpp:314
const SystemParameters & system_parameters_
Definition: DBHandler.h:545
std::shared_ptr< Data_Namespace::DataMgr > data_mgr_
Definition: DBHandler.h:526
const bool read_only_
Definition: DBHandler.h:538
Definition: Catalog.h:62
const bool legacy_syntax_
Definition: DBHandler.h:550
auto get_users(std::unique_ptr< SqliteConnector > &sqliteConnector, const int32_t dbId=-1)
#define CHECK(condition)
Definition: Logger.h:197
const int idle_session_duration_
Definition: DBHandler.h:787
std::unique_ptr< MapDAggHandler > agg_handler_
Definition: DBHandler.h:547
std::unordered_map< std::string, std::string > ipc_handle_to_dev_ptr_
Definition: DBHandler.h:827
boost::variant< ScalarTargetValue, ArrayTargetValue, GeoTargetValue, GeoTargetValuePtr > TargetValue
Definition: TargetValue.h:167
bool allow_multifrag_
Definition: DBHandler.h:537
mapd_unique_lock< mapd_shared_mutex > write_lock
specifies the content in-memory of a row in the table metadata table
const int8_t const int64_t const uint64_t const int32_t const int64_t int64_t ** out
const AuthMetadata & authMetadata_
Definition: DBHandler.h:544
std::optional< GeoCopyFromState > operator()(const std::string &session_id)
Definition: DBHandler.h:803
std::string geo_copy_from_partitions
Definition: DBHandler.h:796
void invalidate_sessions(std::string &name, STMT_TYPE *stmt)
Definition: DBHandler.h:846
bool cpu_mode_only_
Definition: DBHandler.h:540
std::default_random_engine random_gen_
Definition: DBHandler.h:533
SessionMap sessions_
Definition: DBHandler.h:782
import_export::CopyParams geo_copy_from_copy_params
Definition: DBHandler.h:795
std::uniform_int_distribution< int64_t > session_id_dist_
Definition: DBHandler.h:534
std::string apply_copy_to_shim(const std::string &query_str)