OmniSciDB  3a86f6ec37
DBHandler.h
Go to the documentation of this file.
1 /*
2  * Copyright 2020 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 /*
18  * File: DBHandler.h
19  * Author: michael
20  *
21  * Created on Jan 1, 2017, 12:40 PM
22  */
23 
24 #pragma once
25 
26 #include "LeafAggregator.h"
27 
28 #ifdef HAVE_PROFILER
29 #include <gperftools/heap-profiler.h>
30 #endif // HAVE_PROFILER
31 
32 #include "Calcite/Calcite.h"
33 #include "Catalog/Catalog.h"
35 #include "Geospatial/Transforms.h"
36 #include "ImportExport/Importer.h"
37 #include "LockMgr/LockMgr.h"
38 #include "Logger/Logger.h"
39 #include "Parser/ParserWrapper.h"
41 #include "Parser/parser.h"
44 #include "QueryEngine/Execute.h"
50 #include "Shared/StringTransform.h"
53 #include "Shared/mapd_shared_ptr.h"
54 #include "Shared/measure.h"
55 #include "Shared/scope.h"
61 
62 #include <sys/types.h>
63 #include <thrift/server/TServer.h>
64 #include <thrift/transport/THttpTransport.h>
65 #include <thrift/transport/TSocket.h>
66 #include <thrift/transport/TTransport.h>
67 #include <atomic>
68 #include <boost/algorithm/string.hpp>
69 #include <boost/algorithm/string/replace.hpp>
70 #include <boost/algorithm/string/trim.hpp>
71 #include <boost/filesystem.hpp>
72 #include <boost/make_shared.hpp>
73 #include <boost/noncopyable.hpp>
74 #include <boost/none_t.hpp>
75 #include <boost/optional.hpp>
76 #include <boost/program_options.hpp>
77 #include <boost/regex.hpp>
78 #include <boost/tokenizer.hpp>
79 #include <cmath>
80 #include <csignal>
81 #include <fstream>
82 #include <list>
83 #include <map>
84 #include <memory>
85 #include <mutex>
86 #include <random>
87 #include <regex>
88 #include <string>
89 #include <thread>
90 #include <typeinfo>
91 #include <unordered_map>
92 
93 #include "gen-cpp/OmniSci.h"
94 #include "gen-cpp/extension_functions_types.h"
95 
96 using namespace std::string_literals;
97 
98 class MapDAggHandler;
99 class MapDLeafHandler;
100 
101 // Multiple concurrent requests for the same session can occur. For that reason, each
102 // request briefly takes a lock to make a copy of the appropriate SessionInfo object. Then
103 // it releases the lock and uses the copy for the remainder of the request.
104 using SessionMap = std::map<TSessionId, std::shared_ptr<Catalog_Namespace::SessionInfo>>;
105 using PermissionFuncPtr = bool (*)(const AccessPrivileges&, const TDBObjectPermissions&);
107 
108 class TrackingProcessor : public OmniSciProcessor {
109  public:
110  TrackingProcessor(mapd::shared_ptr<OmniSciIf> handler, const bool check_origin)
111  : OmniSciProcessor(handler), check_origin_(check_origin) {}
112 
113  bool process(mapd::shared_ptr<::apache::thrift::protocol::TProtocol> in,
114  mapd::shared_ptr<::apache::thrift::protocol::TProtocol> out,
115  void* connectionContext) {
116  using namespace ::apache::thrift;
117 
118  auto transport = in->getTransport();
119  if (transport && check_origin_) {
120  static std::mutex processor_mutex;
121  std::lock_guard lock(processor_mutex);
122  const auto origin_str = transport->getOrigin();
123  std::vector<std::string> origins;
124  boost::split(origins, origin_str, boost::is_any_of(","));
125  if (origins.empty()) {
127  } else {
128  // Take the first origin, which should be the client IP before any intermediate
129  // servers (e.g. the web server)
130  auto trimmed_origin = origins.front();
131  boost::algorithm::trim(trimmed_origin);
132  TrackingProcessor::client_address = trimmed_origin;
133  }
134  if (dynamic_cast<transport::THttpTransport*>(transport.get())) {
136  } else if (dynamic_cast<transport::TBufferedTransport*>(transport.get())) {
138  } else {
140  }
141  } else {
143  }
144 
145  return OmniSciProcessor::process(in, out, connectionContext);
146  }
147 
148  static thread_local std::string client_address;
149  static thread_local ClientProtocol client_protocol;
150 
151  private:
152  const bool check_origin_;
153 };
154 
155 struct DiskCacheConfig;
156 
157 class DBHandler : public OmniSciIf {
158  public:
159  DBHandler(const std::vector<LeafHostInfo>& db_leaves,
160  const std::vector<LeafHostInfo>& string_leaves,
161  const std::string& base_data_path,
162  const bool cpu_only,
163  const bool allow_multifrag,
164  const bool jit_debug,
165  const bool intel_jit_profile,
166  const bool read_only,
167  const bool allow_loop_joins,
168  const bool enable_rendering,
169  const bool renderer_use_vulkan_driver,
170  const bool enable_auto_clear_render_mem,
171  const int render_oom_retry_threshold,
172  const size_t render_mem_bytes,
173  const size_t max_concurrent_render_sessions,
174  const int num_gpus,
175  const int start_gpu,
176  const size_t reserved_gpu_mem,
177  const bool render_compositor_use_last_gpu,
178  const size_t num_reader_threads,
179  const AuthMetadata& authMetadata,
180  const SystemParameters& system_parameters,
181  const bool legacy_syntax,
182  const int idle_session_duration,
183  const int max_session_duration,
184  const bool enable_runtime_udf_registration,
185  const std::string& udf_filename,
186  const std::string& clang_path,
187  const std::vector<std::string>& clang_options,
188 #ifdef ENABLE_GEOS
189  const std::string& libgeos_so_filename,
190 #endif
191  const DiskCacheConfig& disk_cache_config);
192 
193  ~DBHandler() override;
194 
195  static inline size_t max_bytes_for_thrift() { return 2 * 1000 * 1000 * 1000L; }
196 
197  // Important ****
198  // This block must be keep in sync with mapd.thrift and HAHandler.h
199  // Please keep in same order for easy check and cut and paste
200  // Important ****
201  static void parser_with_error_handler(
202  const std::string& query_str,
203  std::list<std::unique_ptr<Parser::Stmt>>& parse_trees);
204 
205  void krb5_connect(TKrb5Session& session,
206  const std::string& token,
207  const std::string& dbname) override;
208  // connection, admin
209  void connect(TSessionId& session,
210  const std::string& username,
211  const std::string& passwd,
212  const std::string& dbname) override;
213  void disconnect(const TSessionId& session) override;
214  void switch_database(const TSessionId& session, const std::string& dbname) override;
215  void clone_session(TSessionId& session2, const TSessionId& session1) override;
216  void get_server_status(TServerStatus& _return, const TSessionId& session) override;
217  void get_status(std::vector<TServerStatus>& _return,
218  const TSessionId& session) override;
219  void get_hardware_info(TClusterHardwareInfo& _return,
220  const TSessionId& session) override;
221 
222  bool hasTableAccessPrivileges(const TableDescriptor* td,
223  const Catalog_Namespace::SessionInfo& session_info);
224  void get_tables(std::vector<std::string>& _return, const TSessionId& session) override;
225  void get_physical_tables(std::vector<std::string>& _return,
226  const TSessionId& session) override;
227  void get_views(std::vector<std::string>& _return, const TSessionId& session) override;
228  void get_tables_meta(std::vector<TTableMeta>& _return,
229  const TSessionId& session) override;
230  void get_table_details(TTableDetails& _return,
231  const TSessionId& session,
232  const std::string& table_name) override;
233  void get_internal_table_details(TTableDetails& _return,
234  const TSessionId& session,
235  const std::string& table_name) override;
236  void get_users(std::vector<std::string>& _return, const TSessionId& session) override;
237  void get_databases(std::vector<TDBInfo>& _return, const TSessionId& session) override;
238 
239  void get_version(std::string& _return) override;
240  void start_heap_profile(const TSessionId& session) override;
241  void stop_heap_profile(const TSessionId& session) override;
242  void get_heap_profile(std::string& _return, const TSessionId& session) override;
243  void get_memory(std::vector<TNodeMemoryInfo>& _return,
244  const TSessionId& session,
245  const std::string& memory_level) override;
246  void clear_cpu_memory(const TSessionId& session) override;
247  void clear_gpu_memory(const TSessionId& session) override;
248  void set_table_epoch(const TSessionId& session,
249  const int db_id,
250  const int table_id,
251  const int new_epoch) override;
252  void set_table_epoch_by_name(const TSessionId& session,
253  const std::string& table_name,
254  const int new_epoch) override;
255  int32_t get_table_epoch(const TSessionId& session,
256  const int32_t db_id,
257  const int32_t table_id) override;
258  int32_t get_table_epoch_by_name(const TSessionId& session,
259  const std::string& table_name) override;
260  void get_table_epochs(std::vector<TTableEpochInfo>& _return,
261  const TSessionId& session,
262  const int32_t db_id,
263  const int32_t table_id) override;
264  void set_table_epochs(const TSessionId& session,
265  const int32_t db_id,
266  const std::vector<TTableEpochInfo>& table_epochs) override;
267 
268  void get_session_info(TSessionInfo& _return, const TSessionId& session) override;
269  // query, render
270  void sql_execute(TQueryResult& _return,
271  const TSessionId& session,
272  const std::string& query,
273  const bool column_format,
274  const std::string& nonce,
275  const int32_t first_n,
276  const int32_t at_most_n) override;
277  void get_completion_hints(std::vector<TCompletionHint>& hints,
278  const TSessionId& session,
279  const std::string& sql,
280  const int cursor) override;
281  // TODO(miyu): merge the following two data frame APIs.
282  void sql_execute_df(TDataFrame& _return,
283  const TSessionId& session,
284  const std::string& query,
285  const TDeviceType::type device_type,
286  const int32_t device_id,
287  const int32_t first_n,
288  const TArrowTransport::type transport_method) override;
289  void sql_execute_gdf(TDataFrame& _return,
290  const TSessionId& session,
291  const std::string& query,
292  const int32_t device_id,
293  const int32_t first_n) override;
294  void deallocate_df(const TSessionId& session,
295  const TDataFrame& df,
296  const TDeviceType::type device_type,
297  const int32_t device_id) override;
298  void interrupt(const TSessionId& query_session,
299  const TSessionId& interrupt_session) override;
300  void sql_validate(TRowDescriptor& _return,
301  const TSessionId& session,
302  const std::string& query) override;
303 
304  void set_execution_mode(const TSessionId& session,
305  const TExecuteMode::type mode) override;
306  void render_vega(TRenderResult& _return,
307  const TSessionId& session,
308  const int64_t widget_id,
309  const std::string& vega_json,
310  const int32_t compression_level,
311  const std::string& nonce) override;
312  void get_result_row_for_pixel(
313  TPixelTableRowResult& _return,
314  const TSessionId& session,
315  const int64_t widget_id,
316  const TPixel& pixel,
317  const std::map<std::string, std::vector<std::string>>& table_col_names,
318  const bool column_format,
319  const int32_t pixel_radius,
320  const std::string& nonce) override;
321 
322  // dashboards
323  void get_dashboard(TDashboard& _return,
324  const TSessionId& session,
325  int32_t dashboard_id) override;
326  void get_dashboards(std::vector<TDashboard>& _return,
327  const TSessionId& session) override;
328  int32_t create_dashboard(const TSessionId& session,
329  const std::string& dashboard_name,
330  const std::string& dashboard_state,
331  const std::string& image_hash,
332  const std::string& dashboard_metadata) override;
333  void replace_dashboard(const TSessionId& session,
334  const int32_t dashboard_id,
335  const std::string& dashboard_name,
336  const std::string& dashboard_owner,
337  const std::string& dashboard_state,
338  const std::string& image_hash,
339  const std::string& dashboard_metadata) override;
340  void delete_dashboard(const TSessionId& session, const int32_t dashboard_id) override;
341  void share_dashboards(const TSessionId& session,
342  const std::vector<int32_t>& dashboard_ids,
343  const std::vector<std::string>& groups,
344  const TDashboardPermissions& permissions) override;
345  void delete_dashboards(const TSessionId& session,
346  const std::vector<int32_t>& dashboard_ids) override;
347  void share_dashboard(const TSessionId& session,
348  const int32_t dashboard_id,
349  const std::vector<std::string>& groups,
350  const std::vector<std::string>& objects,
351  const TDashboardPermissions& permissions,
352  const bool grant_role) override;
353  void unshare_dashboards(const TSessionId& session,
354  const std::vector<int32_t>& dashboard_ids,
355  const std::vector<std::string>& groups,
356  const TDashboardPermissions& permissions) override;
357  void unshare_dashboard(const TSessionId& session,
358  const int32_t dashboard_id,
359  const std::vector<std::string>& groups,
360  const std::vector<std::string>& objects,
361  const TDashboardPermissions& permissions) override;
362  void get_dashboard_grantees(std::vector<TDashboardGrantees>& _return,
363  const TSessionId& session,
364  int32_t dashboard_id) override;
365 
366  void get_link_view(TFrontendView& _return,
367  const TSessionId& session,
368  const std::string& link) override;
369  void create_link(std::string& _return,
370  const TSessionId& session,
371  const std::string& view_state,
372  const std::string& view_metadata) override;
373  // import
374  void load_table_binary(const TSessionId& session,
375  const std::string& table_name,
376  const std::vector<TRow>& rows,
377  const std::vector<std::string>& column_names) override;
378 
379  std::unique_ptr<lockmgr::AbstractLockContainer<const TableDescriptor*>>
380  prepare_columnar_loader(
381  const Catalog_Namespace::SessionInfo& session_info,
382  const std::string& table_name,
383  size_t num_cols,
384  std::unique_ptr<import_export::Loader>* loader,
385  std::vector<std::unique_ptr<import_export::TypedImportBuffer>>* import_buffers,
386  const std::vector<std::string>& column_names);
387 
388  void load_table_binary_columnar(const TSessionId& session,
389  const std::string& table_name,
390  const std::vector<TColumn>& cols,
391  const std::vector<std::string>& column_names) override;
392  void load_table_binary_arrow(const TSessionId& session,
393  const std::string& table_name,
394  const std::string& arrow_stream,
395  const bool use_column_names) override;
396 
397  void load_table(const TSessionId& session,
398  const std::string& table_name,
399  const std::vector<TStringRow>& rows,
400  const std::vector<std::string>& column_names) override;
401  void detect_column_types(TDetectResult& _return,
402  const TSessionId& session,
403  const std::string& file_name,
404  const TCopyParams& copy_params) override;
405  void create_table(const TSessionId& session,
406  const std::string& table_name,
407  const TRowDescriptor& row_desc,
408  const TFileType::type file_type,
409  const TCreateParams& create_params) override;
410  void import_table(const TSessionId& session,
411  const std::string& table_name,
412  const std::string& file_name,
413  const TCopyParams& copy_params) override;
414  void import_geo_table(const TSessionId& session,
415  const std::string& table_name,
416  const std::string& file_name,
417  const TCopyParams& copy_params,
418  const TRowDescriptor& row_desc,
419  const TCreateParams& create_params) override;
420  void import_table_status(TImportStatus& _return,
421  const TSessionId& session,
422  const std::string& import_id) override;
423  void get_first_geo_file_in_archive(std::string& _return,
424  const TSessionId& session,
425  const std::string& archive_path,
426  const TCopyParams& copy_params) override;
427  void get_all_files_in_archive(std::vector<std::string>& _return,
428  const TSessionId& session,
429  const std::string& archive_path,
430  const TCopyParams& copy_params) override;
431  void get_layers_in_geo_file(std::vector<TGeoFileLayerInfo>& _return,
432  const TSessionId& session,
433  const std::string& file_name,
434  const TCopyParams& copy_params) override;
435  // distributed
436  int64_t query_get_outer_fragment_count(const TSessionId& session,
437  const std::string& select_query) override;
438 
439  void check_table_consistency(TTableMeta& _return,
440  const TSessionId& session,
441  const int32_t table_id) override;
442  void start_query(TPendingQuery& _return,
443  const TSessionId& leaf_session,
444  const TSessionId& parent_session,
445  const std::string& query_ra,
446  const bool just_explain,
447  const std::vector<int64_t>& outer_fragment_indices) override;
448  void execute_query_step(TStepResult& _return,
449  const TPendingQuery& pending_query,
450  const TSubqueryId subquery_id) override;
451  void broadcast_serialized_rows(const TSerializedRows& serialized_rows,
452  const TRowDescriptor& row_desc,
453  const TQueryId query_id,
454  const TSubqueryId subquery_id,
455  const bool is_final_subquery_result) override;
456 
457  void start_render_query(TPendingRenderQuery& _return,
458  const TSessionId& session,
459  const int64_t widget_id,
460  const int16_t node_idx,
461  const std::string& vega_json) override;
462  void execute_next_render_step(TRenderStepResult& _return,
463  const TPendingRenderQuery& pending_render,
464  const TRenderAggDataMap& merged_data) override;
465 
466  void insert_data(const TSessionId& session, const TInsertData& insert_data) override;
467  void checkpoint(const TSessionId& session, const int32_t table_id) override;
468  // DB Object Privileges
469  void get_roles(std::vector<std::string>& _return, const TSessionId& session) override;
470  bool has_role(const TSessionId& sessionId,
471  const std::string& granteeName,
472  const std::string& roleName) override;
473  bool has_object_privilege(const TSessionId& sessionId,
474  const std::string& granteeName,
475  const std::string& objectName,
476  const TDBObjectType::type object_type,
477  const TDBObjectPermissions& permissions) override;
478  void get_db_objects_for_grantee(std::vector<TDBObject>& _return,
479  const TSessionId& session,
480  const std::string& roleName) override;
481  void get_db_object_privs(std::vector<TDBObject>& _return,
482  const TSessionId& session,
483  const std::string& objectName,
484  const TDBObjectType::type type) override;
485  void get_all_roles_for_user(std::vector<std::string>& _return,
486  const TSessionId& session,
487  const std::string& granteeName) override;
488  std::vector<std::string> get_valid_groups(const TSessionId& session,
489  int32_t dashboard_id,
490  std::vector<std::string> groups);
491  // licensing
492  void set_license_key(TLicenseInfo& _return,
493  const TSessionId& session,
494  const std::string& key,
495  const std::string& nonce) override;
496  void get_license_claims(TLicenseInfo& _return,
497  const TSessionId& session,
498  const std::string& nonce) override;
499  // user-defined functions
500  /*
501  Returns a mapping of device (CPU, GPU) parameters (name, LLVM IR
502  triplet, features, etc)
503  */
504  void get_device_parameters(std::map<std::string, std::string>& _return,
505  const TSessionId& session) override;
506 
507  /*
508  Register Runtime Extension Functions (UDFs, UDTFs) with given
509  signatures. The extension functions implementations are given in a
510  mapping of a device and the corresponding LLVM/NVVM IR string.
511  */
512 
513  void register_runtime_extension_functions(
514  const TSessionId& session,
515  const std::vector<TUserDefinedFunction>& udfs,
516  const std::vector<TUserDefinedTableFunction>& udtfs,
517  const std::map<std::string, std::string>& device_ir_map) override;
518 
519  // end of sync block for HAHandler and mapd.thrift
520 
521  void shutdown();
522  void emergency_shutdown();
523 
524  TSessionId getInvalidSessionId() const;
525 
526  void internal_connect(TSessionId& session,
527  const std::string& username,
528  const std::string& dbname);
529 
530  std::shared_ptr<Data_Namespace::DataMgr> data_mgr_;
531 
533  const std::vector<LeafHostInfo> string_leaves_;
534  const std::string base_data_path_;
535  boost::filesystem::path import_path_;
537  std::default_random_engine random_gen_;
538  std::uniform_int_distribution<int64_t> session_id_dist_;
539  const bool jit_debug_;
540  const bool intel_jit_profile_;
542  const bool read_only_;
543  const bool allow_loop_joins_;
546  std::mutex render_mutex_;
547  int64_t start_time_;
550  std::unique_ptr<RenderHandler> render_handler_;
551  std::unique_ptr<MapDAggHandler> agg_handler_;
552  std::unique_ptr<MapDLeafHandler> leaf_handler_;
553  std::shared_ptr<Calcite> calcite_;
554  const bool legacy_syntax_;
555 
556  std::unique_ptr<QueryDispatchQueue> dispatch_queue_;
557 
558  template <typename... ARGS>
559  std::shared_ptr<query_state::QueryState> create_query_state(ARGS&&... args) {
560  return query_states_.create(std::forward<ARGS>(args)...);
561  }
562 
563  // Exactly one immutable SessionInfo copy should be taken by a typical request.
564  Catalog_Namespace::SessionInfo get_session_copy(const TSessionId& session);
565  std::shared_ptr<Catalog_Namespace::SessionInfo> get_session_copy_ptr(
566  const TSessionId& session);
567 
568  void get_tables_meta_impl(std::vector<TTableMeta>& _return,
569  QueryStateProxy query_state_proxy,
570  const Catalog_Namespace::SessionInfo& session_info,
571  const bool with_table_locks = true);
572 
573  private:
574  std::shared_ptr<Catalog_Namespace::SessionInfo> create_new_session(
575  TSessionId& session,
576  const std::string& dbname,
577  const Catalog_Namespace::UserMetadata& user_meta,
578  std::shared_ptr<Catalog_Namespace::Catalog> cat);
579  void connect_impl(TSessionId& session,
580  const std::string& passwd,
581  const std::string& dbname,
582  const Catalog_Namespace::UserMetadata& user_meta,
583  std::shared_ptr<Catalog_Namespace::Catalog> cat,
584  query_state::StdLog& stdlog);
585  void disconnect_impl(const SessionMap::iterator& session_it,
586  mapd_unique_lock<mapd_shared_mutex>& write_lock);
587  void check_table_load_privileges(const TSessionId& session,
588  const std::string& table_name);
589  void check_table_load_privileges(const Catalog_Namespace::SessionInfo& session_info,
590  const std::string& table_name);
591  void get_tables_impl(std::vector<std::string>& table_names,
593  const GetTablesType get_tables_type);
594  void get_table_details_impl(TTableDetails& _return,
595  query_state::StdLog& stdlog,
596  const std::string& table_name,
597  const bool get_system,
598  const bool get_physical);
599  void check_read_only(const std::string& str);
600  void check_session_exp_unsafe(const SessionMap::iterator& session_it);
601  void validateGroups(const std::vector<std::string>& groups);
602  void validateDashboardIdsForSharing(const Catalog_Namespace::SessionInfo& session_info,
603  const std::vector<int32_t>& dashboard_ids);
604  void shareOrUnshareDashboards(const TSessionId& session,
605  const std::vector<int32_t>& dashboard_ids,
606  const std::vector<std::string>& groups,
607  const TDashboardPermissions& permissions,
608  const bool do_share);
609 
610  // Use get_session_copy() or get_session_copy_ptr() instead of get_const_session_ptr()
611  // unless you know what you are doing. If you need to save a SessionInfo beyond the
612  // present Thrift call, then the safe way to do this is by saving the return value of
613  // get_const_session_ptr() as a std::weak_ptr.
614  // Returns empty std::shared_ptr if session.empty().
615  std::shared_ptr<const Catalog_Namespace::SessionInfo> get_const_session_ptr(
616  const TSessionId& session);
617  std::shared_ptr<Catalog_Namespace::SessionInfo> get_session_ptr(
618  const TSessionId& session_id);
619  template <typename SESSION_MAP_LOCK>
620  SessionMap::iterator get_session_it_unsafe(const TSessionId& session,
621  SESSION_MAP_LOCK& lock);
622  static void value_to_thrift_column(const TargetValue& tv,
623  const SQLTypeInfo& ti,
624  TColumn& column);
625  static TDatum value_to_thrift(const TargetValue& tv, const SQLTypeInfo& ti);
626  static std::string apply_copy_to_shim(const std::string& query_str);
627 
628  std::pair<TPlanResult, lockmgr::LockedTableDescriptors> parse_to_ra(
630  const std::string& query_str,
631  const std::vector<TFilterPushDownInfo>& filter_push_down_info,
632  const bool acquire_locks,
633  const SystemParameters system_parameters,
634  bool check_privileges = true);
635 
636  void sql_execute_impl(TQueryResult& _return,
638  const bool column_format,
639  const std::string& nonce,
640  const ExecutorDeviceType executor_device_type,
641  const int32_t first_n,
642  const int32_t at_most_n);
643 
644  bool user_can_access_table(const Catalog_Namespace::SessionInfo&,
645  const TableDescriptor* td,
646  const AccessPrivileges acess_priv);
647 
648  void execute_distributed_copy_statement(
650  const Catalog_Namespace::SessionInfo& session_info);
651 
652  TQueryResult validate_rel_alg(const std::string& query_ra, QueryStateProxy);
653 
654  std::vector<PushedDownFilterInfo> execute_rel_alg(
655  TQueryResult& _return,
657  const std::string& query_ra,
658  const bool column_format,
659  const ExecutorDeviceType executor_device_type,
660  const int32_t first_n,
661  const int32_t at_most_n,
662  const bool just_validate,
663  const bool find_push_down_candidates,
664  const ExplainInfo& explain_info,
665  const std::optional<size_t> executor_index = std::nullopt) const;
666 
667  void execute_rel_alg_with_filter_push_down(
668  TQueryResult& _return,
670  std::string& query_ra,
671  const bool column_format,
672  const ExecutorDeviceType executor_device_type,
673  const int32_t first_n,
674  const int32_t at_most_n,
675  const bool just_explain,
676  const bool just_calcite_explain,
677  const std::vector<PushedDownFilterInfo>& filter_push_down_requests);
678 
679  void execute_rel_alg_df(TDataFrame& _return,
680  const std::string& query_ra,
681  QueryStateProxy query_state_proxy,
682  const Catalog_Namespace::SessionInfo& session_info,
683  const ExecutorDeviceType device_type,
684  const size_t device_id,
685  const int32_t first_n,
686  const TArrowTransport::type transport_method) const;
687 
688  void executeDdl(TQueryResult& _return,
689  const std::string& query_ra,
690  std::shared_ptr<Catalog_Namespace::SessionInfo const> session_ptr);
691 
692  TColumnType populateThriftColumnType(const Catalog_Namespace::Catalog* cat,
693  const ColumnDescriptor* cd);
694  TRowDescriptor fixup_row_descriptor(const TRowDescriptor& row_desc,
695  const Catalog_Namespace::Catalog& cat);
696  void set_execution_mode_nolock(Catalog_Namespace::SessionInfo* session_ptr,
697  const TExecuteMode::type mode);
698  char unescape_char(std::string str);
699  import_export::CopyParams thrift_to_copyparams(const TCopyParams& cp);
700  TCopyParams copyparams_to_thrift(const import_export::CopyParams& cp);
701  void check_geospatial_files(const boost::filesystem::path file_path,
702  const import_export::CopyParams& copy_params);
703  void render_rel_alg(TRenderResult& _return,
704  const std::string& query_ra,
705  const std::string& query_str,
706  const Catalog_Namespace::SessionInfo& session_info,
707  const std::string& render_type,
708  const bool is_projection_query);
709 
710  TColumnType create_geo_column(const TDatumType::type type,
711  const std::string& name,
712  const bool is_array);
713 
714  void convert_explain(TQueryResult& _return,
715  const ResultSet& results,
716  const bool column_format) const;
717  void convert_result(TQueryResult& _return,
718  const ResultSet& results,
719  const bool column_format) const;
720 
721  void convert_rows(TQueryResult& _return,
722  QueryStateProxy query_state_proxy,
723  const std::vector<TargetMetaInfo>& targets,
724  const ResultSet& results,
725  const bool column_format,
726  const int32_t first_n,
727  const int32_t at_most_n) const;
728 
729  // Use ExecutionResult to populate a TQueryResult
730  // calls convert_rows, but after some setup using session_info
731  void convert_result_set(ExecutionResult& result,
732  const Catalog_Namespace::SessionInfo& session_info,
733  const std::string& query_state_str,
734  TQueryResult& _return);
735 
736  void create_simple_result(TQueryResult& _return,
737  const ResultSet& results,
738  const bool column_format,
739  const std::string label) const;
740 
741  std::vector<TargetMetaInfo> getTargetMetaInfo(
742  const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& targets) const;
743 
744  std::vector<std::string> getTargetNames(
745  const std::vector<TargetMetaInfo>& targets) const;
746 
747  std::vector<std::string> getTargetNames(
748  const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& targets) const;
749 
750  void get_completion_hints_unsorted(std::vector<TCompletionHint>& hints,
751  std::vector<std::string>& visible_tables,
752  query_state::StdLog& stdlog,
753  const std::string& sql,
754  const int cursor);
755  void get_token_based_completions(std::vector<TCompletionHint>& hints,
756  query_state::StdLog& stdlog,
757  std::vector<std::string>& visible_tables,
758  const std::string& sql,
759  const int cursor);
760 
761  std::unordered_map<std::string, std::unordered_set<std::string>>
762  fill_column_names_by_table(std::vector<std::string>& table_names,
763  query_state::StdLog& stdlog);
764 
765  ConnectionInfo getConnectionInfo() const;
766 
767  TDashboard get_dashboard_impl(
768  const std::shared_ptr<Catalog_Namespace::SessionInfo const>& session_ptr,
770  const DashboardDescriptor* dash,
771  const bool populate_state = true);
772 
773  static bool has_database_permission(const AccessPrivileges& privs,
774  const TDBObjectPermissions& permissions);
775  static bool has_table_permission(const AccessPrivileges& privs,
776  const TDBObjectPermissions& permission);
777  static bool has_dashboard_permission(const AccessPrivileges& privs,
778  const TDBObjectPermissions& permissions);
779  static bool has_view_permission(const AccessPrivileges& privs,
780  const TDBObjectPermissions& permissions);
781  static bool has_server_permission(const AccessPrivileges& privs,
782  const TDBObjectPermissions& permissions);
783  // For the provided upper case column names `uc_column_names`, return
784  // the tables from `table_names` which contain at least one of them.
785  // Used to rank the TABLE auto-completion hints by the columns
786  // specified in the projection.
787  std::unordered_set<std::string> get_uc_compatible_table_names_by_column(
788  const std::unordered_set<std::string>& uc_column_names,
789  std::vector<std::string>& table_names,
790  query_state::StdLog& stdlog);
791 
794 
795  bool super_user_rights_; // default is "false"; setting to "true"
796  // ignores passwd checks in "connect(..)"
797  // method
798  const int idle_session_duration_; // max duration of idle session
799  const int max_session_duration_; // max duration of session
800 
802 
804  std::string geo_copy_from_table;
808  };
809 
811  std::unordered_map<std::string, GeoCopyFromState> was_geo_copy_from;
813 
814  std::optional<GeoCopyFromState> operator()(const std::string& session_id) {
815  std::lock_guard<std::mutex> map_lock(geo_copy_from_mutex);
816  auto itr = was_geo_copy_from.find(session_id);
817  if (itr == was_geo_copy_from.end()) {
818  return std::nullopt;
819  }
820  return itr->second;
821  }
822 
823  void add(const std::string& session_id, const GeoCopyFromState& state) {
824  std::lock_guard<std::mutex> map_lock(geo_copy_from_mutex);
825  const auto ret = was_geo_copy_from.insert(std::make_pair(session_id, state));
826  CHECK(ret.second);
827  }
828 
829  void remove(const std::string& session_id) {
830  std::lock_guard<std::mutex> map_lock(geo_copy_from_mutex);
831  was_geo_copy_from.erase(session_id);
832  }
833  };
835 
836  // Only for IPC device memory deallocation
837  mutable std::mutex handle_to_dev_ptr_mutex_;
838  mutable std::unordered_map<std::string, std::string> ipc_handle_to_dev_ptr_;
839 
840  friend void run_warmup_queries(mapd::shared_ptr<DBHandler> handler,
841  std::string base_path,
842  std::string query_file_path);
843 
844  friend class RenderHandler::Impl;
845  friend class MapDAggHandler;
846  friend class MapDLeafHandler;
847 
848  std::map<const std::string, const PermissionFuncPtr> permissionFuncMap_ = {
849  {"database"s, has_database_permission},
850  {"dashboard"s, has_dashboard_permission},
851  {"table"s, has_table_permission},
852  {"view"s, has_view_permission},
853  {"server"s, has_server_permission}};
854 
855  void check_and_invalidate_sessions(Parser::DDLStmt* ddl);
856 
857  template <typename STMT_TYPE>
858  void invalidate_sessions(std::string& name, STMT_TYPE* stmt) {
859  using namespace Parser;
860  auto is_match = [&](auto session_it) {
862  return boost::iequals(name,
863  session_it->second->getCatalog().getCurrentDB().dbName);
865  return boost::iequals(name, session_it->second->get_currentUser().userName);
866  }
867  return false;
868  };
869  auto check_and_remove_sessions = [&]() {
870  for (auto it = sessions_.begin(); it != sessions_.end();) {
871  if (is_match(it)) {
872  it = sessions_.erase(it);
873  } else {
874  ++it;
875  }
876  }
877  };
878  check_and_remove_sessions();
879  }
880 
881  std::string const createInMemoryCalciteSession(
882  const std::shared_ptr<Catalog_Namespace::Catalog>& catalog_ptr);
883  bool isInMemoryCalciteSession(const Catalog_Namespace::UserMetadata user_meta);
884  void removeInMemoryCalciteSession(const std::string& session_id);
885 
886  ExecutionResult getUserSessions(
887  std::shared_ptr<Catalog_Namespace::SessionInfo const> session_ptr);
888 
889  // getQueries returns a set of queries queued in the DB
890  // that belongs to the same DB in the caller's session
891 
892  ExecutionResult getQueries(
893  std::shared_ptr<Catalog_Namespace::SessionInfo const> session_ptr);
894 
895  // this function passes the interrupt request to the DB executor
896  void interruptQuery(const Catalog_Namespace::SessionInfo& session_info,
897  const std::string& target_session);
898 };
Classes used to wrap parser calls for calcite redirection.
boost::filesystem::path import_path_
Definition: DBHandler.h:535
std::unique_ptr< QueryDispatchQueue > dispatch_queue_
Definition: DBHandler.h:556
bool process(mapd::shared_ptr<::apache::thrift::protocol::TProtocol > in, mapd::shared_ptr<::apache::thrift::protocol::TProtocol > out, void *connectionContext)
Definition: DBHandler.h:113
const bool runtime_udf_registration_enabled_
Definition: DBHandler.h:801
ClientProtocol
std::unordered_map< std::string, GeoCopyFromState > was_geo_copy_from
Definition: DBHandler.h:811
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:101
std::mutex handle_to_dev_ptr_mutex_
Definition: DBHandler.h:837
static thread_local std::string client_address
Definition: DBHandler.h:148
ExecutorDeviceType
bool(*)(const AccessPrivileges &, const TDBObjectPermissions &) PermissionFuncPtr
Definition: DBHandler.h:105
const std::vector< LeafHostInfo > string_leaves_
Definition: DBHandler.h:533
static thread_local ClientProtocol client_protocol
Definition: DBHandler.h:149
LeafAggregator leaf_aggregator_
Definition: DBHandler.h:532
std::shared_ptr< query_state::QueryState > create_query_state(ARGS &&... args)
Definition: DBHandler.h:559
const std::string base_data_path_
Definition: DBHandler.h:534
const bool jit_debug_
Definition: DBHandler.h:539
const bool check_origin_
Definition: DBHandler.h:152
std::string geo_copy_from_table
Definition: DBHandler.h:804
name
Definition: setup.py:35
std::vector< std::string > split(std::string_view str, std::string_view delim, std::optional< size_t > maxsplit)
split apart a string into a vector of substrings
int64_t start_time_
Definition: DBHandler.h:547
void run_warmup_queries(mapd::shared_ptr< DBHandler > handler, std::string base_path, std::string query_file_path)
Definition: MapDServer.cpp:182
mapd_shared_mutex sessions_mutex_
Definition: DBHandler.h:545
std::map< TSessionId, std::shared_ptr< Catalog_Namespace::SessionInfo > > SessionMap
Definition: DBHandler.h:104
This file contains the class specification and related data structures for Catalog.
std::mutex render_mutex_
Definition: DBHandler.h:546
static size_t max_bytes_for_thrift()
Definition: DBHandler.h:195
query_state::QueryStates query_states_
Definition: DBHandler.h:792
std::string cat(Ts &&... args)
GetTablesType
Definition: Catalog.h:61
std::shared_timed_mutex mapd_shared_mutex
const int max_session_duration_
Definition: DBHandler.h:799
ExecutorDeviceType executor_device_type_
Definition: DBHandler.h:536
std::string geo_copy_from_file_name
Definition: DBHandler.h:805
def connect(uri=None, user=None, password=None, host=None, port=6274, dbname=None, protocol='binary', sessionid=None, bin_cert_validate=None, bin_ca_certs=None, idpurl=None, idpformusernamefield='username', idpformpasswordfield='password', idpsslverify=True)
Definition: connection.py:54
std::unique_ptr< RenderHandler > render_handler_
Definition: DBHandler.h:550
specifies the content in-memory of a row in the column metadata table
TrackingProcessor(mapd::shared_ptr< OmniSciIf > handler, const bool check_origin)
Definition: DBHandler.h:110
std::unique_ptr< MapDLeafHandler > leaf_handler_
Definition: DBHandler.h:552
std::map< std::string, std::string > get_device_parameters(bool cpu_only)
const bool allow_loop_joins_
Definition: DBHandler.h:543
GeoCopyFromSessions geo_copy_from_sessions
Definition: DBHandler.h:834
const bool intel_jit_profile_
Definition: DBHandler.h:540
bool super_user_rights_
Definition: DBHandler.h:795
void add(const std::string &session_id, const GeoCopyFromState &state)
Definition: DBHandler.h:823
std::shared_ptr< Calcite > calcite_
Definition: DBHandler.h:553
void shutdown()
Definition: Logger.cpp:314
const SystemParameters & system_parameters_
Definition: DBHandler.h:549
std::shared_ptr< Data_Namespace::DataMgr > data_mgr_
Definition: DBHandler.h:530
const bool read_only_
Definition: DBHandler.h:542
Definition: Catalog.h:63
const bool legacy_syntax_
Definition: DBHandler.h:554
auto get_users(std::unique_ptr< SqliteConnector > &sqliteConnector, const int32_t dbId=-1)
#define CHECK(condition)
Definition: Logger.h:197
const int idle_session_duration_
Definition: DBHandler.h:798
std::unique_ptr< MapDAggHandler > agg_handler_
Definition: DBHandler.h:551
std::unordered_map< std::string, std::string > ipc_handle_to_dev_ptr_
Definition: DBHandler.h:838
boost::variant< ScalarTargetValue, ArrayTargetValue, GeoTargetValue, GeoTargetValuePtr > TargetValue
Definition: TargetValue.h:167
bool allow_multifrag_
Definition: DBHandler.h:541
mapd_unique_lock< mapd_shared_mutex > write_lock
const AuthMetadata & authMetadata_
Definition: DBHandler.h:548
std::optional< GeoCopyFromState > operator()(const std::string &session_id)
Definition: DBHandler.h:814
std::string geo_copy_from_partitions
Definition: DBHandler.h:807
void invalidate_sessions(std::string &name, STMT_TYPE *stmt)
Definition: DBHandler.h:858
bool cpu_mode_only_
Definition: DBHandler.h:544
def create_table(omni_con, table_name, is_temporary=False, max_rollback_epochs=-1)
std::default_random_engine random_gen_
Definition: DBHandler.h:537
SessionMap sessions_
Definition: DBHandler.h:793
import_export::CopyParams geo_copy_from_copy_params
Definition: DBHandler.h:806
std::uniform_int_distribution< int64_t > session_id_dist_
Definition: DBHandler.h:538
std::string apply_copy_to_shim(const std::string &query_str)