OmniSciDB  04ee39c94c
MapDHandler.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 /*
18  * File: MapDHandler.h
19  * Author: michael
20  *
21  * Created on Jan 1, 2017, 12:40 PM
22  */
23 
24 #ifndef MAPDHANDLER_H
25 #define MAPDHANDLER_H
26 
27 #include "LeafAggregator.h"
28 #ifdef HAVE_PROFILER
29 #include <gperftools/heap-profiler.h>
30 #endif // HAVE_PROFILER
31 #include <thrift/concurrency/PlatformThreadFactory.h>
32 #include <thrift/concurrency/ThreadManager.h>
33 #include <thrift/protocol/TBinaryProtocol.h>
34 #include <thrift/protocol/TJSONProtocol.h>
35 #include <thrift/server/TThreadedServer.h>
36 #include <thrift/transport/TBufferTransports.h>
37 #include <thrift/transport/THttpServer.h>
38 #include <thrift/transport/TServerSocket.h>
39 
40 #include "MapDRelease.h"
41 
42 #include "Calcite/Calcite.h"
43 #include "Catalog/Catalog.h"
45 #include "Import/Importer.h"
46 #include "LockMgr/LockMgr.h"
47 #include "Parser/ParserWrapper.h"
49 #include "Parser/parser.h"
50 #include "Planner/Planner.h"
53 #include "QueryEngine/Execute.h"
58 #include "Shared/ConfigResolve.h"
60 #include "Shared/Logger.h"
61 #include "Shared/MapDParameters.h"
62 #include "Shared/StringTransform.h"
63 #include "Shared/geosupport.h"
65 #include "Shared/mapd_shared_ptr.h"
66 #include "Shared/measure.h"
67 #include "Shared/scope.h"
70 
71 #include <fcntl.h>
72 #include <sys/time.h>
73 #include <sys/types.h>
74 #include <sys/wait.h>
75 #include <unistd.h>
76 #include <atomic>
77 #include <boost/algorithm/string.hpp>
78 #include <boost/algorithm/string/replace.hpp>
79 #include <boost/algorithm/string/trim.hpp>
80 #include <boost/filesystem.hpp>
81 #include <boost/make_shared.hpp>
82 #include <boost/noncopyable.hpp>
83 #include <boost/none_t.hpp>
84 #include <boost/optional.hpp>
85 #include <boost/program_options.hpp>
86 #include <boost/regex.hpp>
87 #include <boost/tokenizer.hpp>
88 #include <cmath>
89 #include <csignal>
90 #include <fstream>
91 #include <list>
92 #include <map>
93 #include <memory>
94 #include <mutex>
95 #include <random>
96 #include <regex>
97 #include <string>
98 #include <thread>
99 #include <typeinfo>
100 #include <unordered_map>
101 #include "gen-cpp/MapD.h"
102 
103 using namespace std::string_literals;
104 
105 class LogSession;
106 class MapDRenderHandler;
107 class MapDAggHandler;
108 class MapDLeafHandler;
109 
111 
112 // Multiple concurrent requests for the same session can occur. For that reason, each
113 // request briefly takes a lock to make a copy of the appropriate SessionInfo object. Then
114 // it releases the lock and uses the copy for the remainder of the request.
115 using SessionMap = std::map<TSessionId, std::shared_ptr<Catalog_Namespace::SessionInfo>>;
116 using permissionFuncPtr = bool (*)(const AccessPrivileges&, const TDBObjectPermissions&);
117 using TableMap = std::map<std::string, bool>;
118 using OptionalTableMap = boost::optional<TableMap&>;
119 
120 class MapDHandler : public MapDIf {
121  public:
122  MapDHandler(const std::vector<LeafHostInfo>& db_leaves,
123  const std::vector<LeafHostInfo>& string_leaves,
124  const std::string& base_data_path,
125  const bool cpu_only,
126  const bool allow_multifrag,
127  const bool jit_debug,
128  const bool intel_jit_profile,
129  const bool read_only,
130  const bool allow_loop_joins,
131  const bool enable_rendering,
132  const bool enable_spirv,
133  const bool enable_auto_clear_render_mem,
134  const int render_oom_retry_threshold,
135  const size_t render_mem_bytes,
136  const int num_gpus,
137  const int start_gpu,
138  const size_t reserved_gpu_mem,
139  const size_t num_reader_threads,
140  const AuthMetadata& authMetadata,
141  const MapDParameters& mapd_parameters,
142  const bool legacy_syntax,
143  const int idle_session_duration,
144  const int max_session_duration,
145  const bool enable_runtime_udf_registration,
146  const std::string& udf_filename);
147 
148  ~MapDHandler() override;
149 
150  static inline size_t max_bytes_for_thrift() { return 2 * 1000 * 1000 * 1000L; }
151 
152  // Important ****
153  // This block must be keep in sync with mapd.thrift and HAHandler.h
154  // Please keep in same order for easy check and cut and paste
155  // Important ****
156 
157  // connection, admin
158  void connect(TSessionId& session,
159  const std::string& username,
160  const std::string& passwd,
161  const std::string& dbname) override;
162  void disconnect(const TSessionId& session) override;
163  void switch_database(const TSessionId& session, const std::string& dbname) override;
164  void get_server_status(TServerStatus& _return, const TSessionId& session) override;
165  void get_status(std::vector<TServerStatus>& _return,
166  const TSessionId& session) override;
167  void get_hardware_info(TClusterHardwareInfo& _return,
168  const TSessionId& session) override;
169 
170  bool hasTableAccessPrivileges(const TableDescriptor* td, const TSessionId& session);
171  void get_tables(std::vector<std::string>& _return, const TSessionId& session) override;
172  void get_physical_tables(std::vector<std::string>& _return,
173  const TSessionId& session) override;
174  void get_views(std::vector<std::string>& _return, const TSessionId& session) override;
175  void get_tables_meta(std::vector<TTableMeta>& _return,
176  const TSessionId& session) override;
177  void get_table_details(TTableDetails& _return,
178  const TSessionId& session,
179  const std::string& table_name) override;
180  void get_internal_table_details(TTableDetails& _return,
181  const TSessionId& session,
182  const std::string& table_name) override;
183  void get_users(std::vector<std::string>& _return, const TSessionId& session) override;
184  void get_databases(std::vector<TDBInfo>& _return, const TSessionId& session) override;
185 
186  void get_version(std::string& _return) override;
187  void start_heap_profile(const TSessionId& session) override;
188  void stop_heap_profile(const TSessionId& session) override;
189  void get_heap_profile(std::string& _return, const TSessionId& session) override;
190  void get_memory(std::vector<TNodeMemoryInfo>& _return,
191  const TSessionId& session,
192  const std::string& memory_level) override;
193  void clear_cpu_memory(const TSessionId& session) override;
194  void clear_gpu_memory(const TSessionId& session) override;
195  void set_table_epoch(const TSessionId& session,
196  const int db_id,
197  const int table_id,
198  const int new_epoch) override;
199  void set_table_epoch_by_name(const TSessionId& session,
200  const std::string& table_name,
201  const int new_epoch) override;
202  int32_t get_table_epoch(const TSessionId& session,
203  const int32_t db_id,
204  const int32_t table_id) override;
205  int32_t get_table_epoch_by_name(const TSessionId& session,
206  const std::string& table_name) override;
207  void get_session_info(TSessionInfo& _return, const TSessionId& session) override;
208  // query, render
209  void sql_execute(TQueryResult& _return,
210  const TSessionId& session,
211  const std::string& query,
212  const bool column_format,
213  const std::string& nonce,
214  const int32_t first_n,
215  const int32_t at_most_n) override;
216  void get_completion_hints(std::vector<TCompletionHint>& hints,
217  const TSessionId& session,
218  const std::string& sql,
219  const int cursor) override;
220  // TODO(miyu): merge the following two data frame APIs.
221  void sql_execute_df(TDataFrame& _return,
222  const TSessionId& session,
223  const std::string& query,
224  const TDeviceType::type device_type,
225  const int32_t device_id,
226  const int32_t first_n) override;
227  void sql_execute_gdf(TDataFrame& _return,
228  const TSessionId& session,
229  const std::string& query,
230  const int32_t device_id,
231  const int32_t first_n) override;
232  void deallocate_df(const TSessionId& session,
233  const TDataFrame& df,
234  const TDeviceType::type device_type,
235  const int32_t device_id) override;
236  void interrupt(const TSessionId& session) override;
237  void sql_validate(TTableDescriptor& _return,
238  const TSessionId& session,
239  const std::string& query) override;
240  void set_execution_mode(const TSessionId& session,
241  const TExecuteMode::type mode) override;
242  void render_vega(TRenderResult& _return,
243  const TSessionId& session,
244  const int64_t widget_id,
245  const std::string& vega_json,
246  const int32_t compression_level,
247  const std::string& nonce) override;
248  void get_result_row_for_pixel(
249  TPixelTableRowResult& _return,
250  const TSessionId& session,
251  const int64_t widget_id,
252  const TPixel& pixel,
253  const std::map<std::string, std::vector<std::string>>& table_col_names,
254  const bool column_format,
255  const int32_t pixel_radius,
256  const std::string& nonce) override;
257 
258  // dashboards
259  void get_dashboard(TDashboard& _return,
260  const TSessionId& session,
261  int32_t dashboard_id) override;
262  void get_dashboards(std::vector<TDashboard>& _return,
263  const TSessionId& session) override;
264  int32_t create_dashboard(const TSessionId& session,
265  const std::string& dashboard_name,
266  const std::string& dashboard_state,
267  const std::string& image_hash,
268  const std::string& dashboard_metadata) override;
269  void replace_dashboard(const TSessionId& session,
270  const int32_t dashboard_id,
271  const std::string& dashboard_name,
272  const std::string& dashboard_owner,
273  const std::string& dashboard_state,
274  const std::string& image_hash,
275  const std::string& dashboard_metadata) override;
276  void delete_dashboard(const TSessionId& session, const int32_t dashboard_id) override;
277  void share_dashboard(const TSessionId& session,
278  const int32_t dashboard_id,
279  const std::vector<std::string>& groups,
280  const std::vector<std::string>& objects,
281  const TDashboardPermissions& permissions,
282  const bool grant_role) override;
283  void unshare_dashboard(const TSessionId& session,
284  const int32_t dashboard_id,
285  const std::vector<std::string>& groups,
286  const std::vector<std::string>& objects,
287  const TDashboardPermissions& permissions) override;
288  void get_dashboard_grantees(std::vector<TDashboardGrantees>& _return,
289  const TSessionId& session,
290  int32_t dashboard_id) override;
291 
292  void get_link_view(TFrontendView& _return,
293  const TSessionId& session,
294  const std::string& link) override;
295  void create_link(std::string& _return,
296  const TSessionId& session,
297  const std::string& view_state,
298  const std::string& view_metadata) override;
299  // import
300  void load_table_binary(const TSessionId& session,
301  const std::string& table_name,
302  const std::vector<TRow>& rows) override;
303 
304  void prepare_columnar_loader(
305  const Catalog_Namespace::SessionInfo& session_info,
306  const std::string& table_name,
307  size_t num_cols,
308  std::unique_ptr<Importer_NS::Loader>* loader,
309  std::vector<std::unique_ptr<Importer_NS::TypedImportBuffer>>* import_buffers);
310 
311  void load_table_binary_columnar(const TSessionId& session,
312  const std::string& table_name,
313  const std::vector<TColumn>& cols) override;
314  void load_table_binary_arrow(const TSessionId& session,
315  const std::string& table_name,
316  const std::string& arrow_stream) override;
317 
318  void load_table(const TSessionId& session,
319  const std::string& table_name,
320  const std::vector<TStringRow>& rows) override;
321  void detect_column_types(TDetectResult& _return,
322  const TSessionId& session,
323  const std::string& file_name,
324  const TCopyParams& copy_params) override;
325  void create_table(const TSessionId& session,
326  const std::string& table_name,
327  const TRowDescriptor& row_desc,
328  const TFileType::type file_type,
329  const TCreateParams& create_params) override;
330  void import_table(const TSessionId& session,
331  const std::string& table_name,
332  const std::string& file_name,
333  const TCopyParams& copy_params) override;
334  void import_geo_table(const TSessionId& session,
335  const std::string& table_name,
336  const std::string& file_name,
337  const TCopyParams& copy_params,
338  const TRowDescriptor& row_desc,
339  const TCreateParams& create_params) override;
340  void import_table_status(TImportStatus& _return,
341  const TSessionId& session,
342  const std::string& import_id) override;
343  void get_first_geo_file_in_archive(std::string& _return,
344  const TSessionId& session,
345  const std::string& archive_path,
346  const TCopyParams& copy_params) override;
347  void get_all_files_in_archive(std::vector<std::string>& _return,
348  const TSessionId& session,
349  const std::string& archive_path,
350  const TCopyParams& copy_params) override;
351  void get_layers_in_geo_file(std::vector<TGeoFileLayerInfo>& _return,
352  const TSessionId& session,
353  const std::string& file_name,
354  const TCopyParams& copy_params) override;
355  // distributed
356  void check_table_consistency(TTableMeta& _return,
357  const TSessionId& session,
358  const int32_t table_id) override;
359  void start_query(TPendingQuery& _return,
360  const TSessionId& session,
361  const std::string& query_ra,
362  const bool just_explain) override;
363  void execute_first_step(TStepResult& _return,
364  const TPendingQuery& pending_query) override;
365  void broadcast_serialized_rows(const TSerializedRows& serialized_rows,
366  const TRowDescriptor& row_desc,
367  const TQueryId query_id) override;
368 
369  void start_render_query(TPendingRenderQuery& _return,
370  const TSessionId& session,
371  const int64_t widget_id,
372  const int16_t node_idx,
373  const std::string& vega_json) override;
374  void execute_next_render_step(TRenderStepResult& _return,
375  const TPendingRenderQuery& pending_render,
376  const TRenderAggDataMap& merged_data) override;
377 
378  void insert_data(const TSessionId& session, const TInsertData& insert_data) override;
379  void checkpoint(const TSessionId& session,
380  const int32_t db_id,
381  const int32_t table_id) override;
382  // DB Object Privileges
383  void get_roles(std::vector<std::string>& _return, const TSessionId& session) override;
384  bool has_object_privilege(const TSessionId& sessionId,
385  const std::string& granteeName,
386  const std::string& objectName,
387  const TDBObjectType::type object_type,
388  const TDBObjectPermissions& permissions) override;
389  void get_db_objects_for_grantee(std::vector<TDBObject>& _return,
390  const TSessionId& session,
391  const std::string& roleName) override;
392  void get_db_object_privs(std::vector<TDBObject>& _return,
393  const TSessionId& session,
394  const std::string& objectName,
395  const TDBObjectType::type type) override;
396  void get_all_roles_for_user(std::vector<std::string>& _return,
397  const TSessionId& session,
398  const std::string& granteeName) override;
399  std::vector<std::string> get_valid_groups(const TSessionId& session,
400  int32_t dashboard_id,
401  std::vector<std::string> groups);
402  // licensing
403  void set_license_key(TLicenseInfo& _return,
404  const TSessionId& session,
405  const std::string& key,
406  const std::string& nonce) override;
407  void get_license_claims(TLicenseInfo& _return,
408  const TSessionId& session,
409  const std::string& nonce) override;
410  // user-defined functions
411  /*
412  Returns a mapping of device (CPU, GPU) parameters (name, LLVM IR
413  triplet, features, etc)
414  */
415  void get_device_parameters(std::map<std::string, std::string>& _return) override;
416  /*
417  Register UDFs with given signatures. The UDF implementations are
418  given in a mapping of a device and the corresponding LLVM IR
419  string.
420  */
421  void register_runtime_udf(
422  const TSessionId& session,
423  const std::string& signatures,
424  const std::map<std::string, std::string>& device_ir_map) override;
425 
426  // end of sync block for HAHandler and mapd.thrift
427 
428  void shutdown();
429  void emergency_shutdown();
430 
431  TSessionId getInvalidSessionId() const;
432 
433  void internal_connect(TSessionId& session,
434  const std::string& username,
435  const std::string& dbname);
436 
437  std::shared_ptr<Data_Namespace::DataMgr> data_mgr_;
438 
440  const std::vector<LeafHostInfo> string_leaves_;
441  const std::string base_data_path_;
442  boost::filesystem::path import_path_;
444  std::default_random_engine random_gen_;
445  std::uniform_int_distribution<int64_t> session_id_dist_;
446  const bool jit_debug_;
447  const bool intel_jit_profile_;
449  const bool read_only_;
450  const bool allow_loop_joins_;
453  std::mutex render_mutex_;
454  int64_t start_time_;
457  std::unique_ptr<MapDRenderHandler> render_handler_;
458  std::unique_ptr<MapDAggHandler> agg_handler_;
459  std::unique_ptr<MapDLeafHandler> leaf_handler_;
460  std::shared_ptr<Calcite> calcite_;
461  const bool legacy_syntax_;
462 
463  // Exactly one immutable SessionInfo copy should be taken by a typical request.
464  Catalog_Namespace::SessionInfo get_session_copy(const TSessionId& session);
465  std::shared_ptr<Catalog_Namespace::SessionInfo> get_session_copy_ptr(
466  const TSessionId& session);
467 
468  private:
469  void connect_impl(TSessionId& session,
470  const std::string& passwd,
471  const std::string& dbname,
473  std::shared_ptr<Catalog_Namespace::Catalog> cat,
474  LogSession&);
475  void disconnect_impl(const SessionMap::iterator& session_it);
476  void check_table_load_privileges(const TSessionId& session,
477  const std::string& table_name);
478  void check_table_load_privileges(const Catalog_Namespace::SessionInfo& session_info,
479  const std::string& table_name);
480  void get_tables_impl(std::vector<std::string>& table_names,
481  const TSessionId& session,
482  const GetTablesType get_tables_type);
483  void get_table_details_impl(TTableDetails& _return,
484  const TSessionId& session,
485  const std::string& table_name,
486  const bool get_system,
487  const bool get_physical);
488  void check_read_only(const std::string& str);
489  void check_session_exp_unsafe(const SessionMap::iterator& session_it);
490 
491  // Use get_session_copy() or get_session_copy_ptr() instead of get_const_session_ptr()
492  // unless you know what you are doing. If you need to save a SessionInfo beyond the
493  // present Thrift call, then the safe way to do this is by saving the return value of
494  // get_const_session_ptr() as a std::weak_ptr.
495  // Returns empty std::shared_ptr if session.empty().
496  std::shared_ptr<const Catalog_Namespace::SessionInfo> get_const_session_ptr(
497  const TSessionId& session);
498  std::shared_ptr<Catalog_Namespace::SessionInfo> get_session_ptr(
499  const TSessionId& session_id);
500  SessionMap::iterator get_session_it_unsafe(const TSessionId& session);
501  static void value_to_thrift_column(const TargetValue& tv,
502  const SQLTypeInfo& ti,
503  TColumn& column);
504  static TDatum value_to_thrift(const TargetValue& tv, const SQLTypeInfo& ti);
505  static std::string apply_copy_to_shim(const std::string& query_str);
506 
507  std::string parse_to_ra(const std::string& query_str,
508  const std::vector<TFilterPushDownInfo>& filter_push_down_info,
509  const Catalog_Namespace::SessionInfo& session_info,
510  OptionalTableMap tableNames,
511  const MapDParameters mapd_parameters,
512  RenderInfo* render_info = nullptr);
513 
514  void sql_execute_impl(TQueryResult& _return,
515  const Catalog_Namespace::SessionInfo& session_info,
516  const std::string& query_str,
517  const bool column_format,
518  const std::string& nonce,
519  const ExecutorDeviceType executor_device_type,
520  const int32_t first_n,
521  const int32_t at_most_n);
522 
523  bool user_can_access_table(const Catalog_Namespace::SessionInfo&,
524  const TableDescriptor* td,
525  const AccessPrivileges acess_priv);
526 
527  void execute_distributed_copy_statement(
529  const Catalog_Namespace::SessionInfo& session_info);
530 
531  void validate_rel_alg(TTableDescriptor& _return,
532  const std::string& query_str,
533  const Catalog_Namespace::SessionInfo& session_info);
534  std::vector<PushedDownFilterInfo> execute_rel_alg(
535  TQueryResult& _return,
536  const std::string& query_ra,
537  const bool column_format,
538  const Catalog_Namespace::SessionInfo& session_info,
539  const ExecutorDeviceType executor_device_type,
540  const int32_t first_n,
541  const int32_t at_most_n,
542  const bool just_explain,
543  const bool just_validate,
544  const bool find_push_down_candidates,
545  const bool just_calcite_explain,
546  const bool explain_optimized_ir) const;
547 
548  void execute_rel_alg_with_filter_push_down(
549  TQueryResult& _return,
550  std::string& query_ra,
551  const bool column_format,
552  const Catalog_Namespace::SessionInfo& session_info,
553  const ExecutorDeviceType executor_device_type,
554  const int32_t first_n,
555  const int32_t at_most_n,
556  const bool just_explain,
557  const bool just_calcite_explain,
558  const std::string& query_str,
559  const std::vector<PushedDownFilterInfo> filter_push_down_requests);
560 
561  void execute_rel_alg_df(TDataFrame& _return,
562  const std::string& query_ra,
563  const Catalog_Namespace::SessionInfo& session_info,
564  const ExecutorDeviceType device_type,
565  const size_t device_id,
566  const int32_t first_n) const;
567  TColumnType populateThriftColumnType(const Catalog_Namespace::Catalog* cat,
568  const ColumnDescriptor* cd);
569  TRowDescriptor fixup_row_descriptor(const TRowDescriptor& row_desc,
570  const Catalog_Namespace::Catalog& cat);
571  void set_execution_mode_nolock(Catalog_Namespace::SessionInfo* session_ptr,
572  const TExecuteMode::type mode);
573  char unescape_char(std::string str);
574  Importer_NS::CopyParams thrift_to_copyparams(const TCopyParams& cp);
575  TCopyParams copyparams_to_thrift(const Importer_NS::CopyParams& cp);
576  void check_geospatial_files(const boost::filesystem::path file_path,
577  const Importer_NS::CopyParams& copy_params);
578  void render_rel_alg(TRenderResult& _return,
579  const std::string& query_ra,
580  const std::string& query_str,
581  const Catalog_Namespace::SessionInfo& session_info,
582  const std::string& render_type,
583  const bool is_projection_query);
584 
585  TColumnType create_geo_column(const TDatumType::type type,
586  const std::string& name,
587  const bool is_array);
588 
589  void convert_explain(TQueryResult& _return,
590  const ResultSet& results,
591  const bool column_format) const;
592  void convert_result(TQueryResult& _return,
593  const ResultSet& results,
594  const bool column_format) const;
595 
596  template <class R>
597  void convert_rows(TQueryResult& _return,
598  const std::vector<TargetMetaInfo>& targets,
599  const R& results,
600  const bool column_format,
601  const int32_t first_n,
602  const int32_t at_most_n) const;
603 
604  void create_simple_result(TQueryResult& _return,
605  const ResultSet& results,
606  const bool column_format,
607  const std::string label) const;
608 
609  void execute_root_plan(TQueryResult& _return,
610  const Planner::RootPlan* root_plan,
611  const bool column_format,
612  const Catalog_Namespace::SessionInfo& session_info,
613  const ExecutorDeviceType executor_device_type,
614  const int32_t first_n) const;
615 
616  std::vector<TargetMetaInfo> getTargetMetaInfo(
617  const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& targets) const;
618 
619  std::vector<std::string> getTargetNames(
620  const std::vector<TargetMetaInfo>& targets) const;
621 
622  std::vector<std::string> getTargetNames(
623  const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& targets) const;
624 
625  TColumnType convert_target_metainfo(const TargetMetaInfo& target,
626  const size_t idx) const;
627 
628  TRowDescriptor convert_target_metainfo(
629  const std::vector<TargetMetaInfo>& targets) const;
630 
631  void get_completion_hints_unsorted(std::vector<TCompletionHint>& hints,
632  std::vector<std::string>& visible_tables,
633  const TSessionId& session,
634  const std::string& sql,
635  const int cursor);
636  void get_token_based_completions(std::vector<TCompletionHint>& hints,
637  const TSessionId& session,
638  const std::vector<std::string>& visible_tables,
639  const std::string& sql,
640  const int cursor);
641  Planner::RootPlan* parse_to_plan(const std::string& query_str,
642  const Catalog_Namespace::SessionInfo& session_info);
643  Planner::RootPlan* parse_to_plan_legacy(
644  const std::string& query_str,
645  const Catalog_Namespace::SessionInfo& session_info,
646  const std::string& action /* render or validate */);
647 
648  std::unordered_map<std::string, std::unordered_set<std::string>>
649  fill_column_names_by_table(const std::vector<std::string>& table_names,
650  const TSessionId& session);
651 
652  static bool has_database_permission(const AccessPrivileges& privs,
653  const TDBObjectPermissions& permissions);
654  static bool has_table_permission(const AccessPrivileges& privs,
655  const TDBObjectPermissions& permission);
656  static bool has_dashboard_permission(const AccessPrivileges& privs,
657  const TDBObjectPermissions& permissions);
658  static bool has_view_permission(const AccessPrivileges& privs,
659  const TDBObjectPermissions& permissions);
660 
661  // For the provided upper case column names `uc_column_names`, return the tables
662  // from `table_names` which contain at least one of them. Used to rank the TABLE
663  // auto-completion hints by the columns specified in the projection.
664  std::unordered_set<std::string> get_uc_compatible_table_names_by_column(
665  const std::unordered_set<std::string>& uc_column_names,
666  const std::vector<std::string>& table_names,
667  const TSessionId& session);
668 
670 
671  bool super_user_rights_; // default is "false"; setting to "true" ignores passwd checks
672  // in "connect(..)" method
673  const int idle_session_duration_; // max duration of idle session
674  const int max_session_duration_; // max duration of session
675 
677 
679  std::string _geo_copy_from_table;
683 
684  // Only for IPC device memory deallocation
685  mutable std::mutex handle_to_dev_ptr_mutex_;
686  mutable std::unordered_map<std::string, int8_t*> ipc_handle_to_dev_ptr_;
687 
688  friend void run_warmup_queries(mapd::shared_ptr<MapDHandler> handler,
689  std::string base_path,
690  std::string query_file_path);
691 
692  friend class LogSession;
693  friend class MapDRenderHandler;
694  friend class MapDAggHandler;
695  friend class MapDLeafHandler;
696 
697  std::map<const std::string, const permissionFuncPtr> permissionFuncMap_ = {
698  {"database"s, has_database_permission},
699  {"dashboard"s, has_dashboard_permission},
700  {"table"s, has_table_permission},
701  {"view"s, has_view_permission}};
702 
703  void check_and_invalidate_sessions(Parser::DDLStmt* ddl);
704 
705  template <typename STMT_TYPE>
706  void invalidate_sessions(std::string& name, STMT_TYPE* stmt) {
707  using namespace Parser;
708 
709  auto is_match = [&name](const std::string& session_name) {
710  return boost::iequals(name, session_name);
711  };
713  for (auto it = sessions_.begin(); it != sessions_.end();) {
714  if (is_match(it->second.get()->getCatalog().getCurrentDB().dbName)) {
715  it = sessions_.erase(it);
716  } else {
717  ++it;
718  }
719  }
721  for (auto it = sessions_.begin(); it != sessions_.end();) {
722  if (is_match(it->second.get()->get_currentUser().userName)) {
723  it = sessions_.erase(it);
724  } else {
725  ++it;
726  }
727  }
728  }
729  }
730 };
731 
732 // Log Format:
733 // YYYY-MM-DDTHH:MM::SS.FFFFFF [pid] [file]:[line] [label] [func] [match] [dur_ms]
734 // [dbname] [user] [pubsessid] {[names]} {[values]}
735 // Call at both beginning(label="stdlog_begin") and end(label="stdlog") of Thrift call,
736 // with dur_ms = current age of LogSession object in milliseconds.
737 // stdlog_begin is logged at DEBUG1 level, stdlog is logged at INFO level.
738 // The only required parameter is session, which can be either:
739 // * std::shared_ptr<const Catalog_Namespace::SessionInfo> - No locking is done.
740 // * TSessionId string - will call get_const_session_ptr() to get shared_ptr.
741 // All remaining optional parameters are name,value pairs that will be included in log.
742 #define LOG_SESSION(session, ...) \
743  LogSession log_session(*this, session, __FILE__, __LINE__, __func__, ##__VA_ARGS__)
744 
745 class LogSession : boost::noncopyable {
746  std::string const file_;
747  size_t const line_;
748  char const* const func_;
749  std::list<std::string> name_value_pairs_;
750  std::chrono::steady_clock::time_point const start_;
751  std::shared_ptr<Catalog_Namespace::SessionInfo> session_ptr_;
752  static std::atomic<int64_t> s_match;
753  int64_t const match_; // Unique to each begin/end pair to match them together.
754  template <typename... Pairs>
755  LogSession(char const* file, size_t line, char const* func, Pairs&&... pairs)
756  : file_(boost::filesystem::path(file).filename().string())
757  , line_(line)
758  , func_(func)
759  , name_value_pairs_{to_string(std::forward<Pairs>(pairs))...}
760  , start_(std::chrono::steady_clock::now())
761  , match_(s_match++) {
762  static_assert(sizeof...(Pairs) % 2 == 0,
763  "LogSession() requires an even number of name/value parameters.");
764  }
765  void stdlog(logger::Severity, char const* label);
766  void update_session_last_used_duration();
767 
768  public:
769  template <typename... Pairs>
771  SessionMap::mapped_type& session_ptr,
772  char const* file,
773  size_t line,
774  char const* func,
775  Pairs&&... pairs)
776  : LogSession(file, line, func, std::forward<Pairs>(pairs)...) {
777  session_ptr_ = session_ptr;
778  stdlog(logger::Severity::DEBUG1, "stdlog_begin");
779  }
780  template <typename... Pairs>
782  TSessionId const& session_id,
783  char const* file,
784  size_t line,
785  char const* func,
786  Pairs&&... pairs)
787  : LogSession(file, line, func, std::forward<Pairs>(pairs)...) {
788  session_ptr_ = mh.get_session_ptr(session_id);
789  stdlog(logger::Severity::DEBUG1, "stdlog_begin");
790  }
791  ~LogSession();
792  template <typename... Pairs>
793  void append_name_value_pairs(Pairs&&... pairs) {
794  static_assert(sizeof...(Pairs) % 2 == 0,
795  "append_name_value_pairs() requires an even number of parameters.");
796  name_value_pairs_.splice(name_value_pairs_.cend(),
797  {to_string(std::forward<Pairs>(pairs))...});
798  }
799  template <typename Units = std::chrono::milliseconds>
800  typename Units::rep duration() const {
801  using namespace std::chrono;
802  return duration_cast<Units>(steady_clock::now() - start_).count();
803  }
804  void set_session(SessionMap::mapped_type& session_ptr);
805 };
806 
807 #endif /* MAPDHANDLER_H */
std::map< std::string, std::string > get_device_parameters()
SessionMap sessions_
Definition: MapDHandler.h:669
const bool allow_loop_joins_
Definition: MapDHandler.h:450
Classes used to wrap parser calls for calcite redirection.
std::unique_ptr< MapDRenderHandler > render_handler_
Definition: MapDHandler.h:457
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:81
void get_license_claims(ClientContext &context)
Definition: omnisql.cpp:961
boost::filesystem::path import_path_
Definition: MapDHandler.h:442
ExecutorDeviceType
static std::atomic< int64_t > s_match
Definition: MapDHandler.h:752
std::mutex render_mutex_
Definition: MapDHandler.h:453
auto sql(const std::string &sql_stmts)
void get_table_epoch(ClientContext &context, const std::string &table_specifier)
Definition: omnisql.cpp:220
void get_db_objects_for_grantee(ClientContext &context)
Definition: omnisql.cpp:881
void run_warmup_queries(mapd::shared_ptr< MapDHandler > handler, std::string base_path, std::string query_file_path)
Definition: MapDServer.cpp:176
LogSession(char const *file, size_t line, char const *func, Pairs &&... pairs)
Definition: MapDHandler.h:755
std::string const file_
Definition: MapDHandler.h:746
std::unique_ptr< MapDAggHandler > agg_handler_
Definition: MapDHandler.h:458
void get_db_object_privs(ClientContext &context)
Definition: omnisql.cpp:923
bool cpu_mode_only_
Definition: MapDHandler.h:451
const bool runtime_udf_registration_enabled_
Definition: MapDHandler.h:676
std::shared_ptr< Catalog_Namespace::SessionInfo > get_session_ptr(const TSessionId &session_id)
const bool intel_jit_profile_
Definition: MapDHandler.h:447
int64_t start_time_
Definition: MapDHandler.h:454
bool(*)(const AccessPrivileges &, const TDBObjectPermissions &) permissionFuncPtr
Definition: MapDHandler.h:116
mapd_shared_mutex sessions_mutex_
Definition: MapDHandler.h:452
std::string to_string(char const *&&v)
const int idle_session_duration_
Definition: MapDHandler.h:673
This file contains the class specification and related data structures for Catalog.
void set_license_key(ClientContext &context, const std::string &token)
Definition: omnisql.cpp:948
const MapDParameters & mapd_parameters_
Definition: MapDHandler.h:456
bool allow_multifrag_
Definition: MapDHandler.h:448
std::list< std::string > name_value_pairs_
Definition: MapDHandler.h:749
const bool read_only_
Definition: MapDHandler.h:449
std::shared_timed_mutex mapd_shared_mutex
const std::vector< LeafHostInfo > string_leaves_
Definition: MapDHandler.h:440
static size_t max_bytes_for_thrift()
Definition: MapDHandler.h:150
std::map< std::string, bool > TableMap
Definition: MapDHandler.h:117
LeafAggregator leaf_aggregator_
Definition: MapDHandler.h:439
GetTablesType
Definition: MapDHandler.h:110
std::string _geo_copy_from_table
Definition: MapDHandler.h:679
specifies the content in-memory of a row in the column metadata table
Defines data structures for query plan nodes.
std::shared_ptr< Calcite > calcite_
Definition: MapDHandler.h:460
LogSession(MapDHandler &mh, SessionMap::mapped_type &session_ptr, char const *file, size_t line, char const *func, Pairs &&... pairs)
Definition: MapDHandler.h:770
Importer_NS::CopyParams _geo_copy_from_copy_params
Definition: MapDHandler.h:681
std::string _geo_copy_from_file_name
Definition: MapDHandler.h:680
char const *const func_
Definition: MapDHandler.h:748
std::map< TSessionId, std::shared_ptr< Catalog_Namespace::SessionInfo > > SessionMap
Definition: MapDHandler.h:115
bool _was_geo_copy_from
Definition: MapDHandler.h:678
void append_name_value_pairs(Pairs &&... pairs)
Definition: MapDHandler.h:793
std::shared_ptr< Data_Namespace::DataMgr > data_mgr_
Definition: MapDHandler.h:437
Severity
Definition: Logger.h:70
void shutdown()
Definition: Logger.cpp:294
boost::optional< TableMap & > OptionalTableMap
Definition: MapDHandler.h:118
Definition: Catalog.h:60
ExecutorDeviceType executor_device_type_
Definition: MapDHandler.h:443
LogSession(MapDHandler &mh, TSessionId const &session_id, char const *file, size_t line, char const *func, Pairs &&... pairs)
Definition: MapDHandler.h:781
std::shared_ptr< Catalog_Namespace::SessionInfo > session_ptr_
Definition: MapDHandler.h:751
void set_table_epoch(ClientContext &context, const std::string &table_details)
Definition: omnisql.cpp:282
std::uniform_int_distribution< int64_t > session_id_dist_
Definition: MapDHandler.h:445
Units::rep duration() const
Definition: MapDHandler.h:800
std::string parse_to_ra(const Catalog_Namespace::Catalog &cat, const std::string &query_str, const Catalog_Namespace::SessionInfo &session_info)
Definition: LockMgr.cpp:76
auto get_users(std::unique_ptr< SqliteConnector > &sqliteConnector, const int32_t dbId=-1)
size_t const line_
Definition: MapDHandler.h:747
void invalidate_sessions(std::string &name, STMT_TYPE *stmt)
Definition: MapDHandler.h:706
const AuthMetadata & authMetadata_
Definition: MapDHandler.h:455
std::unique_ptr< MapDLeafHandler > leaf_handler_
Definition: MapDHandler.h:459
boost::variant< ScalarTargetValue, ArrayTargetValue, GeoTargetValue, GeoTargetValuePtr > TargetValue
Definition: TargetValue.h:167
std::mutex handle_to_dev_ptr_mutex_
Definition: MapDHandler.h:685
int64_t const match_
Definition: MapDHandler.h:753
specifies the content in-memory of a row in the table metadata table
const int max_session_duration_
Definition: MapDHandler.h:674
const bool jit_debug_
Definition: MapDHandler.h:446
TSessionId session
std::chrono::steady_clock::time_point const start_
Definition: MapDHandler.h:750
bool super_user_rights_
Definition: MapDHandler.h:671
std::unordered_map< std::string, int8_t * > ipc_handle_to_dev_ptr_
Definition: MapDHandler.h:686
std::default_random_engine random_gen_
Definition: MapDHandler.h:444
std::string _geo_copy_from_partitions
Definition: MapDHandler.h:682
const bool legacy_syntax_
Definition: MapDHandler.h:461
const std::string base_data_path_
Definition: MapDHandler.h:441