OmniSciDB  b24e664e58
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
MapDHandler.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 /*
18  * File: MapDHandler.h
19  * Author: michael
20  *
21  * Created on Jan 1, 2017, 12:40 PM
22  */
23 
24 #ifndef MAPDHANDLER_H
25 #define MAPDHANDLER_H
26 
27 #include "LeafAggregator.h"
28 
29 #ifdef HAVE_PROFILER
30 #include <gperftools/heap-profiler.h>
31 #endif // HAVE_PROFILER
32 
33 #include "MapDRelease.h"
34 
35 #include "Calcite/Calcite.h"
36 #include "Catalog/Catalog.h"
38 #include "Import/Importer.h"
39 #include "LockMgr/LockMgr.h"
40 #include "Parser/ParserWrapper.h"
42 #include "Parser/parser.h"
43 #include "Planner/Planner.h"
46 #include "QueryEngine/Execute.h"
51 #include "QueryState.h"
52 #include "Shared/ConfigResolve.h"
54 #include "Shared/Logger.h"
55 #include "Shared/MapDParameters.h"
56 #include "Shared/StringTransform.h"
57 #include "Shared/geosupport.h"
59 #include "Shared/mapd_shared_ptr.h"
60 #include "Shared/measure.h"
61 #include "Shared/scope.h"
64 
65 #include <fcntl.h>
66 #include <sys/time.h>
67 #include <sys/types.h>
68 #include <sys/wait.h>
69 #include <unistd.h>
70 #include <atomic>
71 #include <boost/algorithm/string.hpp>
72 #include <boost/algorithm/string/replace.hpp>
73 #include <boost/algorithm/string/trim.hpp>
74 #include <boost/filesystem.hpp>
75 #include <boost/make_shared.hpp>
76 #include <boost/noncopyable.hpp>
77 #include <boost/none_t.hpp>
78 #include <boost/optional.hpp>
79 #include <boost/program_options.hpp>
80 #include <boost/regex.hpp>
81 #include <boost/tokenizer.hpp>
82 #include <cmath>
83 #include <csignal>
84 #include <fstream>
85 #include <list>
86 #include <map>
87 #include <memory>
88 #include <mutex>
89 #include <random>
90 #include <regex>
91 #include <string>
92 #include <thread>
93 #include <typeinfo>
94 #include <unordered_map>
95 #include "gen-cpp/MapD.h"
96 #include "gen-cpp/extension_functions_types.h"
97 
98 using namespace std::string_literals;
99 
100 class MapDRenderHandler;
101 class MapDAggHandler;
102 class MapDLeafHandler;
103 
105 
106 // Multiple concurrent requests for the same session can occur. For that reason, each
107 // request briefly takes a lock to make a copy of the appropriate SessionInfo object. Then
108 // it releases the lock and uses the copy for the remainder of the request.
109 using SessionMap = std::map<TSessionId, std::shared_ptr<Catalog_Namespace::SessionInfo>>;
110 using permissionFuncPtr = bool (*)(const AccessPrivileges&, const TDBObjectPermissions&);
111 using TableMap = std::map<std::string, bool>;
112 using OptionalTableMap = boost::optional<TableMap&>;
114 
115 class MapDHandler : public MapDIf {
116  public:
117  MapDHandler(const std::vector<LeafHostInfo>& db_leaves,
118  const std::vector<LeafHostInfo>& string_leaves,
119  const std::string& base_data_path,
120  const bool cpu_only,
121  const bool allow_multifrag,
122  const bool jit_debug,
123  const bool intel_jit_profile,
124  const bool read_only,
125  const bool allow_loop_joins,
126  const bool enable_rendering,
127  const bool enable_auto_clear_render_mem,
128  const int render_oom_retry_threshold,
129  const size_t render_mem_bytes,
130  const int num_gpus,
131  const int start_gpu,
132  const size_t reserved_gpu_mem,
133  const size_t num_reader_threads,
134  const AuthMetadata& authMetadata,
135  const MapDParameters& mapd_parameters,
136  const bool legacy_syntax,
137  const int idle_session_duration,
138  const int max_session_duration,
139  const bool enable_runtime_udf_registration,
140  const std::string& udf_filename);
141 
142  ~MapDHandler() override;
143 
144  static inline size_t max_bytes_for_thrift() { return 2 * 1000 * 1000 * 1000L; }
145 
146  // Important ****
147  // This block must be keep in sync with mapd.thrift and HAHandler.h
148  // Please keep in same order for easy check and cut and paste
149  // Important ****
150 
151  void krb5_connect(TKrb5Session& session,
152  const std::string& token,
153  const std::string& dbname) override;
154  // connection, admin
155  void connect(TSessionId& session,
156  const std::string& username,
157  const std::string& passwd,
158  const std::string& dbname) override;
159  void disconnect(const TSessionId& session) override;
160  void switch_database(const TSessionId& session, const std::string& dbname) override;
161  void get_server_status(TServerStatus& _return, const TSessionId& session) override;
162  void get_status(std::vector<TServerStatus>& _return,
163  const TSessionId& session) override;
164  void get_hardware_info(TClusterHardwareInfo& _return,
165  const TSessionId& session) override;
166 
167  bool hasTableAccessPrivileges(const TableDescriptor* td, const TSessionId& session);
168  void get_tables(std::vector<std::string>& _return, const TSessionId& session) override;
169  void get_physical_tables(std::vector<std::string>& _return,
170  const TSessionId& session) override;
171  void get_views(std::vector<std::string>& _return, const TSessionId& session) override;
172  void get_tables_meta(std::vector<TTableMeta>& _return,
173  const TSessionId& session) override;
174  void get_table_details(TTableDetails& _return,
175  const TSessionId& session,
176  const std::string& table_name) override;
177  void get_internal_table_details(TTableDetails& _return,
178  const TSessionId& session,
179  const std::string& table_name) override;
180  void get_users(std::vector<std::string>& _return, const TSessionId& session) override;
181  void get_databases(std::vector<TDBInfo>& _return, const TSessionId& session) override;
182 
183  void get_version(std::string& _return) override;
184  void start_heap_profile(const TSessionId& session) override;
185  void stop_heap_profile(const TSessionId& session) override;
186  void get_heap_profile(std::string& _return, const TSessionId& session) override;
187  void get_memory(std::vector<TNodeMemoryInfo>& _return,
188  const TSessionId& session,
189  const std::string& memory_level) override;
190  void clear_cpu_memory(const TSessionId& session) override;
191  void clear_gpu_memory(const TSessionId& session) override;
192  void set_table_epoch(const TSessionId& session,
193  const int db_id,
194  const int table_id,
195  const int new_epoch) override;
196  void set_table_epoch_by_name(const TSessionId& session,
197  const std::string& table_name,
198  const int new_epoch) override;
199  int32_t get_table_epoch(const TSessionId& session,
200  const int32_t db_id,
201  const int32_t table_id) override;
202  int32_t get_table_epoch_by_name(const TSessionId& session,
203  const std::string& table_name) override;
204  void get_session_info(TSessionInfo& _return, const TSessionId& session) override;
205  // query, render
206  void sql_execute(TQueryResult& _return,
207  const TSessionId& session,
208  const std::string& query,
209  const bool column_format,
210  const std::string& nonce,
211  const int32_t first_n,
212  const int32_t at_most_n) override;
213  void get_completion_hints(std::vector<TCompletionHint>& hints,
214  const TSessionId& session,
215  const std::string& sql,
216  const int cursor) override;
217  // TODO(miyu): merge the following two data frame APIs.
218  void sql_execute_df(TDataFrame& _return,
219  const TSessionId& session,
220  const std::string& query,
221  const TDeviceType::type device_type,
222  const int32_t device_id,
223  const int32_t first_n) override;
224  void sql_execute_gdf(TDataFrame& _return,
225  const TSessionId& session,
226  const std::string& query,
227  const int32_t device_id,
228  const int32_t first_n) override;
229  void deallocate_df(const TSessionId& session,
230  const TDataFrame& df,
231  const TDeviceType::type device_type,
232  const int32_t device_id) override;
233  void interrupt(const TSessionId& session) override;
234  void sql_validate(TTableDescriptor& _return,
235  const TSessionId& session,
236  const std::string& query) override;
237  void set_execution_mode(const TSessionId& session,
238  const TExecuteMode::type mode) override;
239  void render_vega(TRenderResult& _return,
240  const TSessionId& session,
241  const int64_t widget_id,
242  const std::string& vega_json,
243  const int32_t compression_level,
244  const std::string& nonce) override;
245  void get_result_row_for_pixel(
246  TPixelTableRowResult& _return,
247  const TSessionId& session,
248  const int64_t widget_id,
249  const TPixel& pixel,
250  const std::map<std::string, std::vector<std::string>>& table_col_names,
251  const bool column_format,
252  const int32_t pixel_radius,
253  const std::string& nonce) override;
254 
255  // dashboards
256  void get_dashboard(TDashboard& _return,
257  const TSessionId& session,
258  int32_t dashboard_id) override;
259  void get_dashboards(std::vector<TDashboard>& _return,
260  const TSessionId& session) override;
261  int32_t create_dashboard(const TSessionId& session,
262  const std::string& dashboard_name,
263  const std::string& dashboard_state,
264  const std::string& image_hash,
265  const std::string& dashboard_metadata) override;
266  void replace_dashboard(const TSessionId& session,
267  const int32_t dashboard_id,
268  const std::string& dashboard_name,
269  const std::string& dashboard_owner,
270  const std::string& dashboard_state,
271  const std::string& image_hash,
272  const std::string& dashboard_metadata) override;
273  void delete_dashboard(const TSessionId& session, const int32_t dashboard_id) override;
274  void share_dashboard(const TSessionId& session,
275  const int32_t dashboard_id,
276  const std::vector<std::string>& groups,
277  const std::vector<std::string>& objects,
278  const TDashboardPermissions& permissions,
279  const bool grant_role) override;
280  void unshare_dashboard(const TSessionId& session,
281  const int32_t dashboard_id,
282  const std::vector<std::string>& groups,
283  const std::vector<std::string>& objects,
284  const TDashboardPermissions& permissions) override;
285  void get_dashboard_grantees(std::vector<TDashboardGrantees>& _return,
286  const TSessionId& session,
287  int32_t dashboard_id) override;
288 
289  void get_link_view(TFrontendView& _return,
290  const TSessionId& session,
291  const std::string& link) override;
292  void create_link(std::string& _return,
293  const TSessionId& session,
294  const std::string& view_state,
295  const std::string& view_metadata) override;
296  // import
297  void load_table_binary(const TSessionId& session,
298  const std::string& table_name,
299  const std::vector<TRow>& rows) override;
300 
301  void prepare_columnar_loader(
302  const Catalog_Namespace::SessionInfo& session_info,
303  const std::string& table_name,
304  size_t num_cols,
305  std::unique_ptr<Importer_NS::Loader>* loader,
306  std::vector<std::unique_ptr<Importer_NS::TypedImportBuffer>>* import_buffers);
307 
308  void load_table_binary_columnar(const TSessionId& session,
309  const std::string& table_name,
310  const std::vector<TColumn>& cols) override;
311  void load_table_binary_arrow(const TSessionId& session,
312  const std::string& table_name,
313  const std::string& arrow_stream) override;
314 
315  void load_table(const TSessionId& session,
316  const std::string& table_name,
317  const std::vector<TStringRow>& rows) override;
318  void detect_column_types(TDetectResult& _return,
319  const TSessionId& session,
320  const std::string& file_name,
321  const TCopyParams& copy_params) override;
322  void create_table(const TSessionId& session,
323  const std::string& table_name,
324  const TRowDescriptor& row_desc,
325  const TFileType::type file_type,
326  const TCreateParams& create_params) override;
327  void import_table(const TSessionId& session,
328  const std::string& table_name,
329  const std::string& file_name,
330  const TCopyParams& copy_params) override;
331  void import_geo_table(const TSessionId& session,
332  const std::string& table_name,
333  const std::string& file_name,
334  const TCopyParams& copy_params,
335  const TRowDescriptor& row_desc,
336  const TCreateParams& create_params) override;
337  void import_table_status(TImportStatus& _return,
338  const TSessionId& session,
339  const std::string& import_id) override;
340  void get_first_geo_file_in_archive(std::string& _return,
341  const TSessionId& session,
342  const std::string& archive_path,
343  const TCopyParams& copy_params) override;
344  void get_all_files_in_archive(std::vector<std::string>& _return,
345  const TSessionId& session,
346  const std::string& archive_path,
347  const TCopyParams& copy_params) override;
348  void get_layers_in_geo_file(std::vector<TGeoFileLayerInfo>& _return,
349  const TSessionId& session,
350  const std::string& file_name,
351  const TCopyParams& copy_params) override;
352  // distributed
353  void check_table_consistency(TTableMeta& _return,
354  const TSessionId& session,
355  const int32_t table_id) override;
356  void start_query(TPendingQuery& _return,
357  const TSessionId& session,
358  const std::string& query_ra,
359  const bool just_explain) override;
360  void execute_query_step(TStepResult& _return,
361  const TPendingQuery& pending_query) override;
362  void broadcast_serialized_rows(const TSerializedRows& serialized_rows,
363  const TRowDescriptor& row_desc,
364  const TQueryId query_id) override;
365 
366  void start_render_query(TPendingRenderQuery& _return,
367  const TSessionId& session,
368  const int64_t widget_id,
369  const int16_t node_idx,
370  const std::string& vega_json) override;
371  void execute_next_render_step(TRenderStepResult& _return,
372  const TPendingRenderQuery& pending_render,
373  const TRenderAggDataMap& merged_data) override;
374 
375  void insert_data(const TSessionId& session, const TInsertData& insert_data) override;
376  void checkpoint(const TSessionId& session,
377  const int32_t db_id,
378  const int32_t table_id) override;
379  // DB Object Privileges
380  void get_roles(std::vector<std::string>& _return, const TSessionId& session) override;
381  bool has_role(const TSessionId& sessionId,
382  const std::string& granteeName,
383  const std::string& roleName) override;
384  bool has_object_privilege(const TSessionId& sessionId,
385  const std::string& granteeName,
386  const std::string& objectName,
387  const TDBObjectType::type object_type,
388  const TDBObjectPermissions& permissions) override;
389  void get_db_objects_for_grantee(std::vector<TDBObject>& _return,
390  const TSessionId& session,
391  const std::string& roleName) override;
392  void get_db_object_privs(std::vector<TDBObject>& _return,
393  const TSessionId& session,
394  const std::string& objectName,
395  const TDBObjectType::type type) override;
396  void get_all_roles_for_user(std::vector<std::string>& _return,
397  const TSessionId& session,
398  const std::string& granteeName) override;
399  std::vector<std::string> get_valid_groups(const TSessionId& session,
400  int32_t dashboard_id,
401  std::vector<std::string> groups);
402  // licensing
403  void set_license_key(TLicenseInfo& _return,
404  const TSessionId& session,
405  const std::string& key,
406  const std::string& nonce) override;
407  void get_license_claims(TLicenseInfo& _return,
408  const TSessionId& session,
409  const std::string& nonce) override;
410  // user-defined functions
411  /*
412  Returns a mapping of device (CPU, GPU) parameters (name, LLVM IR
413  triplet, features, etc)
414  */
415  void get_device_parameters(std::map<std::string, std::string>& _return,
416  const TSessionId& session) override;
417 
418  /*
419  Register Runtime Extension Functions (UDFs, UDTFs) with given
420  signatures. The extension functions implementations are given in a
421  mapping of a device and the corresponding LLVM/NVVM IR string.
422  */
423 
424  void register_runtime_extension_functions(
425  const TSessionId& session,
426  const std::vector<TUserDefinedFunction>& udfs,
427  const std::vector<TUserDefinedTableFunction>& udtfs,
428  const std::map<std::string, std::string>& device_ir_map) override;
429 
430  // end of sync block for HAHandler and mapd.thrift
431 
432  void shutdown();
433  void emergency_shutdown();
434 
435  TSessionId getInvalidSessionId() const;
436 
437  void internal_connect(TSessionId& session,
438  const std::string& username,
439  const std::string& dbname);
440 
441  std::shared_ptr<Data_Namespace::DataMgr> data_mgr_;
442 
444  const std::vector<LeafHostInfo> string_leaves_;
445  const std::string base_data_path_;
446  boost::filesystem::path import_path_;
448  std::default_random_engine random_gen_;
449  std::uniform_int_distribution<int64_t> session_id_dist_;
450  const bool jit_debug_;
451  const bool intel_jit_profile_;
453  const bool read_only_;
454  const bool allow_loop_joins_;
457  std::mutex render_mutex_;
458  int64_t start_time_;
461  std::unique_ptr<MapDRenderHandler> render_handler_;
462  std::unique_ptr<MapDAggHandler> agg_handler_;
463  std::unique_ptr<MapDLeafHandler> leaf_handler_;
464  std::shared_ptr<Calcite> calcite_;
465  const bool legacy_syntax_;
466 
467  template <typename... ARGS>
468  std::shared_ptr<query_state::QueryState> create_query_state(ARGS&&... args) {
469  return query_states_.create(std::forward<ARGS>(args)...);
470  }
471 
472  // Exactly one immutable SessionInfo copy should be taken by a typical request.
473  Catalog_Namespace::SessionInfo get_session_copy(const TSessionId& session);
474  std::shared_ptr<Catalog_Namespace::SessionInfo> get_session_copy_ptr(
475  const TSessionId& session);
476 
477  private:
478  void connect_impl(TSessionId& session,
479  const std::string& passwd,
480  const std::string& dbname,
482  std::shared_ptr<Catalog_Namespace::Catalog> cat,
484  void disconnect_impl(const SessionMap::iterator& session_it);
485  void check_table_load_privileges(const TSessionId& session,
486  const std::string& table_name);
487  void check_table_load_privileges(const Catalog_Namespace::SessionInfo& session_info,
488  const std::string& table_name);
489  void get_tables_impl(std::vector<std::string>& table_names,
490  query_state::StdLog& stdlog,
491  const GetTablesType get_tables_type);
492  void get_table_details_impl(TTableDetails& _return,
494  const TSessionId& session,
495  const std::string& table_name,
496  const bool get_system,
497  const bool get_physical);
498  void check_read_only(const std::string& str);
499  void check_session_exp_unsafe(const SessionMap::iterator& session_it);
500 
501  // Use get_session_copy() or get_session_copy_ptr() instead of get_const_session_ptr()
502  // unless you know what you are doing. If you need to save a SessionInfo beyond the
503  // present Thrift call, then the safe way to do this is by saving the return value of
504  // get_const_session_ptr() as a std::weak_ptr.
505  // Returns empty std::shared_ptr if session.empty().
506  std::shared_ptr<const Catalog_Namespace::SessionInfo> get_const_session_ptr(
507  const TSessionId& session);
508  std::shared_ptr<Catalog_Namespace::SessionInfo> get_session_ptr(
509  const TSessionId& session_id);
510  template <typename SESSION_MAP_LOCK>
511  SessionMap::iterator get_session_it_unsafe(const TSessionId& session,
512  SESSION_MAP_LOCK& lock);
513  static void value_to_thrift_column(const TargetValue& tv,
514  const SQLTypeInfo& ti,
515  TColumn& column);
516  static TDatum value_to_thrift(const TargetValue& tv, const SQLTypeInfo& ti);
517  static std::string apply_copy_to_shim(const std::string& query_str);
518 
519  std::string parse_to_ra(QueryStateProxy,
520  const std::string& query_str,
521  const std::vector<TFilterPushDownInfo>& filter_push_down_info,
522  OptionalTableMap tableNames,
523  const MapDParameters mapd_parameters,
524  RenderInfo* render_info = nullptr);
525 
526  void sql_execute_impl(TQueryResult& _return,
528  const bool column_format,
529  const std::string& nonce,
530  const ExecutorDeviceType executor_device_type,
531  const int32_t first_n,
532  const int32_t at_most_n);
533 
534  bool user_can_access_table(const Catalog_Namespace::SessionInfo&,
535  const TableDescriptor* td,
536  const AccessPrivileges acess_priv);
537 
538  void execute_distributed_copy_statement(
540  const Catalog_Namespace::SessionInfo& session_info);
541 
542  void validate_rel_alg(TTableDescriptor& _return, QueryStateProxy);
543 
544  std::vector<PushedDownFilterInfo> execute_rel_alg(
545  TQueryResult& _return,
547  const std::string& query_ra,
548  const bool column_format,
549  const ExecutorDeviceType executor_device_type,
550  const int32_t first_n,
551  const int32_t at_most_n,
552  const bool just_explain,
553  const bool just_validate,
554  const bool find_push_down_candidates,
555  const bool just_calcite_explain,
556  const bool explain_optimized_ir) const;
557 
558  void execute_rel_alg_with_filter_push_down(
559  TQueryResult& _return,
561  std::string& query_ra,
562  const bool column_format,
563  const ExecutorDeviceType executor_device_type,
564  const int32_t first_n,
565  const int32_t at_most_n,
566  const bool just_explain,
567  const bool just_calcite_explain,
568  const std::vector<PushedDownFilterInfo> filter_push_down_requests);
569 
570  void execute_rel_alg_df(TDataFrame& _return,
571  const std::string& query_ra,
572  const Catalog_Namespace::SessionInfo& session_info,
573  const ExecutorDeviceType device_type,
574  const size_t device_id,
575  const int32_t first_n) const;
576  TColumnType populateThriftColumnType(const Catalog_Namespace::Catalog* cat,
577  const ColumnDescriptor* cd);
578  TRowDescriptor fixup_row_descriptor(const TRowDescriptor& row_desc,
579  const Catalog_Namespace::Catalog& cat);
580  void set_execution_mode_nolock(Catalog_Namespace::SessionInfo* session_ptr,
581  const TExecuteMode::type mode);
582  char unescape_char(std::string str);
583  Importer_NS::CopyParams thrift_to_copyparams(const TCopyParams& cp);
584  TCopyParams copyparams_to_thrift(const Importer_NS::CopyParams& cp);
585  void check_geospatial_files(const boost::filesystem::path file_path,
586  const Importer_NS::CopyParams& copy_params);
587  void render_rel_alg(TRenderResult& _return,
588  const std::string& query_ra,
589  const std::string& query_str,
590  const Catalog_Namespace::SessionInfo& session_info,
591  const std::string& render_type,
592  const bool is_projection_query);
593 
594  TColumnType create_geo_column(const TDatumType::type type,
595  const std::string& name,
596  const bool is_array);
597 
598  void convert_explain(TQueryResult& _return,
599  const ResultSet& results,
600  const bool column_format) const;
601  void convert_result(TQueryResult& _return,
602  const ResultSet& results,
603  const bool column_format) const;
604 
605  template <class R>
606  void convert_rows(TQueryResult& _return,
607  QueryStateProxy query_state_proxy,
608  const std::vector<TargetMetaInfo>& targets,
609  const R& results,
610  const bool column_format,
611  const int32_t first_n,
612  const int32_t at_most_n) const;
613 
614  void create_simple_result(TQueryResult& _return,
615  const ResultSet& results,
616  const bool column_format,
617  const std::string label) const;
618 
619  void execute_root_plan(TQueryResult& _return,
620  QueryStateProxy query_state_proxy,
621  const Planner::RootPlan* root_plan,
622  const bool column_format,
623  const Catalog_Namespace::SessionInfo& session_info,
624  const ExecutorDeviceType executor_device_type,
625  const int32_t first_n) const;
626 
627  std::vector<TargetMetaInfo> getTargetMetaInfo(
628  const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& targets) const;
629 
630  std::vector<std::string> getTargetNames(
631  const std::vector<TargetMetaInfo>& targets) const;
632 
633  std::vector<std::string> getTargetNames(
634  const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& targets) const;
635 
636  TColumnType convert_target_metainfo(const TargetMetaInfo& target,
637  const size_t idx) const;
638 
639  TRowDescriptor convert_target_metainfo(
640  const std::vector<TargetMetaInfo>& targets) const;
641 
642  void get_completion_hints_unsorted(std::vector<TCompletionHint>& hints,
643  std::vector<std::string>& visible_tables,
644  const TSessionId& session,
645  const std::string& sql,
646  const int cursor);
647  void get_token_based_completions(std::vector<TCompletionHint>& hints,
648  const TSessionId& session,
649  const std::vector<std::string>& visible_tables,
650  const std::string& sql,
651  const int cursor);
652  Planner::RootPlan* parse_to_plan_legacy(
653  const std::string& query_str,
654  const Catalog_Namespace::SessionInfo& session_info,
655  const std::string& action /* render or validate */);
656 
657  std::unordered_map<std::string, std::unordered_set<std::string>>
658  fill_column_names_by_table(const std::vector<std::string>& table_names,
659  const TSessionId& session);
660 
661  static bool has_database_permission(const AccessPrivileges& privs,
662  const TDBObjectPermissions& permissions);
663  static bool has_table_permission(const AccessPrivileges& privs,
664  const TDBObjectPermissions& permission);
665  static bool has_dashboard_permission(const AccessPrivileges& privs,
666  const TDBObjectPermissions& permissions);
667  static bool has_view_permission(const AccessPrivileges& privs,
668  const TDBObjectPermissions& permissions);
669 
670  // For the provided upper case column names `uc_column_names`, return
671  // the tables from `table_names` which contain at least one of them.
672  // Used to rank the TABLE auto-completion hints by the columns
673  // specified in the projection.
674  std::unordered_set<std::string> get_uc_compatible_table_names_by_column(
675  const std::unordered_set<std::string>& uc_column_names,
676  const std::vector<std::string>& table_names,
677  const TSessionId& session);
678 
681 
682  bool super_user_rights_; // default is "false"; setting to "true"
683  // ignores passwd checks in "connect(..)"
684  // method
685  const int idle_session_duration_; // max duration of idle session
686  const int max_session_duration_; // max duration of session
687 
689 
691  std::string _geo_copy_from_table;
695 
696  // Only for IPC device memory deallocation
697  mutable std::mutex handle_to_dev_ptr_mutex_;
698  mutable std::unordered_map<std::string, int8_t*> ipc_handle_to_dev_ptr_;
699 
700  friend void run_warmup_queries(mapd::shared_ptr<MapDHandler> handler,
701  std::string base_path,
702  std::string query_file_path);
703 
704  friend class MapDRenderHandler;
705  friend class MapDAggHandler;
706  friend class MapDLeafHandler;
707 
708  std::map<const std::string, const permissionFuncPtr> permissionFuncMap_ = {
709  {"database"s, has_database_permission},
710  {"dashboard"s, has_dashboard_permission},
711  {"table"s, has_table_permission},
712  {"view"s, has_view_permission}};
713 
714  void check_and_invalidate_sessions(Parser::DDLStmt* ddl);
715 
716  template <typename STMT_TYPE>
717  void invalidate_sessions(std::string& name, STMT_TYPE* stmt) {
718  using namespace Parser;
719  auto is_match = [&](auto session_it) {
721  return boost::iequals(name,
722  session_it->second->getCatalog().getCurrentDB().dbName);
724  return boost::iequals(name, session_it->second->get_currentUser().userName);
725  }
726  return false;
727  };
728  auto check_and_remove_sessions = [&]() {
729  for (auto it = sessions_.begin(); it != sessions_.end();) {
730  if (is_match(it)) {
731  it = sessions_.erase(it);
732  } else {
733  ++it;
734  }
735  }
736  };
737  check_and_remove_sessions();
738  }
739 
740  std::string const createInMemoryCalciteSession(
741  const std::shared_ptr<Catalog_Namespace::Catalog>& catalog_ptr);
742  bool isInMemoryCalciteSession(const Catalog_Namespace::UserMetadata user_meta);
743  void removeInMemoryCalciteSession(const std::string& session_id);
744 };
745 
746 #endif /* MAPDHANDLER_H */
std::map< std::string, std::string > get_device_parameters()
SessionMap sessions_
Definition: MapDHandler.h:680
const bool allow_loop_joins_
Definition: MapDHandler.h:454
Classes used to wrap parser calls for calcite redirection.
std::unique_ptr< MapDRenderHandler > render_handler_
Definition: MapDHandler.h:461
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:81
boost::filesystem::path import_path_
Definition: MapDHandler.h:446
ExecutorDeviceType
std::string parse_to_ra(query_state::QueryStateProxy query_state_proxy, const std::string &query_str)
Definition: LockMgr.cpp:77
std::mutex render_mutex_
Definition: MapDHandler.h:457
void run_warmup_queries(mapd::shared_ptr< MapDHandler > handler, std::string base_path, std::string query_file_path)
Definition: MapDServer.cpp:192
std::unique_ptr< MapDAggHandler > agg_handler_
Definition: MapDHandler.h:462
bool cpu_mode_only_
Definition: MapDHandler.h:455
const bool runtime_udf_registration_enabled_
Definition: MapDHandler.h:688
const bool intel_jit_profile_
Definition: MapDHandler.h:451
int64_t start_time_
Definition: MapDHandler.h:458
bool(*)(const AccessPrivileges &, const TDBObjectPermissions &) permissionFuncPtr
Definition: MapDHandler.h:110
mapd_shared_mutex sessions_mutex_
Definition: MapDHandler.h:456
const int idle_session_duration_
Definition: MapDHandler.h:685
This file contains the class specification and related data structures for Catalog.
const MapDParameters & mapd_parameters_
Definition: MapDHandler.h:460
std::map< TSessionId, std::shared_ptr< Catalog_Namespace::SessionInfo >> SessionMap
Definition: MapDHandler.h:109
bool allow_multifrag_
Definition: MapDHandler.h:452
const bool read_only_
Definition: MapDHandler.h:453
std::shared_timed_mutex mapd_shared_mutex
const std::vector< LeafHostInfo > string_leaves_
Definition: MapDHandler.h:444
static size_t max_bytes_for_thrift()
Definition: MapDHandler.h:144
std::map< std::string, bool > TableMap
Definition: MapDHandler.h:111
LeafAggregator leaf_aggregator_
Definition: MapDHandler.h:443
GetTablesType
Definition: MapDHandler.h:104
std::shared_ptr< query_state::QueryState > create_query_state(ARGS &&...args)
Definition: MapDHandler.h:468
std::string _geo_copy_from_table
Definition: MapDHandler.h:691
specifies the content in-memory of a row in the column metadata table
Defines data structures for query plan nodes.
std::shared_ptr< Calcite > calcite_
Definition: MapDHandler.h:464
Importer_NS::CopyParams _geo_copy_from_copy_params
Definition: MapDHandler.h:693
std::string _geo_copy_from_file_name
Definition: MapDHandler.h:692
query_state::QueryStates query_states_
Definition: MapDHandler.h:679
bool _was_geo_copy_from
Definition: MapDHandler.h:690
std::shared_ptr< Data_Namespace::DataMgr > data_mgr_
Definition: MapDHandler.h:441
void shutdown()
Definition: Logger.cpp:299
boost::optional< TableMap & > OptionalTableMap
Definition: MapDHandler.h:112
ExecutorDeviceType executor_device_type_
Definition: MapDHandler.h:447
std::uniform_int_distribution< int64_t > session_id_dist_
Definition: MapDHandler.h:449
auto get_users(std::unique_ptr< SqliteConnector > &sqliteConnector, const int32_t dbId=-1)
void invalidate_sessions(std::string &name, STMT_TYPE *stmt)
Definition: MapDHandler.h:717
const AuthMetadata & authMetadata_
Definition: MapDHandler.h:459
std::unique_ptr< MapDLeafHandler > leaf_handler_
Definition: MapDHandler.h:463
boost::variant< ScalarTargetValue, ArrayTargetValue, GeoTargetValue, GeoTargetValuePtr > TargetValue
Definition: TargetValue.h:167
std::mutex handle_to_dev_ptr_mutex_
Definition: MapDHandler.h:697
specifies the content in-memory of a row in the table metadata table
const int max_session_duration_
Definition: MapDHandler.h:686
const bool jit_debug_
Definition: MapDHandler.h:450
bool super_user_rights_
Definition: MapDHandler.h:682
std::unordered_map< std::string, int8_t * > ipc_handle_to_dev_ptr_
Definition: MapDHandler.h:698
std::default_random_engine random_gen_
Definition: MapDHandler.h:448
std::string _geo_copy_from_partitions
Definition: MapDHandler.h:694
const bool legacy_syntax_
Definition: MapDHandler.h:465
const std::string base_data_path_
Definition: MapDHandler.h:445