28 #include <gperftools/heap-profiler.h>
29 #endif // HAVE_PROFILER
63 #include <sys/types.h>
64 #include <thrift/server/TServer.h>
65 #include <thrift/transport/THttpClient.h>
66 #include <thrift/transport/THttpTransport.h>
67 #include <thrift/transport/TSocket.h>
68 #include <thrift/transport/TTransport.h>
70 #include <boost/algorithm/string.hpp>
71 #include <boost/algorithm/string/replace.hpp>
72 #include <boost/algorithm/string/trim.hpp>
73 #include <boost/filesystem.hpp>
74 #include <boost/make_shared.hpp>
75 #include <boost/noncopyable.hpp>
76 #include <boost/none_t.hpp>
77 #include <boost/optional.hpp>
78 #include <boost/program_options.hpp>
79 #include <boost/tokenizer.hpp>
91 #include <unordered_map>
93 #include "gen-cpp/Heavy.h"
94 #include "gen-cpp/extension_functions_types.h"
96 using namespace std::string_literals;
104 using SessionMap = std::map<TSessionId, std::shared_ptr<Catalog_Namespace::SessionInfo>>;
108 namespace dbhandler {
112 bool throw_db_exception =
false);
118 : HeavyProcessor(handler), check_origin_(check_origin) {}
120 bool process(std::shared_ptr<::apache::thrift::protocol::TProtocol> in,
121 std::shared_ptr<::apache::thrift::protocol::TProtocol> out,
122 void* connectionContext)
override {
123 using namespace ::apache::thrift;
125 auto transport = in->getTransport();
126 if (transport && check_origin_) {
127 static std::mutex processor_mutex;
129 const auto origin_str = transport->getOrigin();
130 std::vector<std::string> origins;
131 boost::split(origins, origin_str, boost::is_any_of(
","));
132 if (origins.empty()) {
137 auto trimmed_origin = origins.front();
138 boost::algorithm::trim(trimmed_origin);
141 if (dynamic_cast<transport::THttpTransport*>(transport.get())) {
143 }
else if (dynamic_cast<transport::TBufferedTransport*>(transport.get())) {
152 return HeavyProcessor::process(in, out, connectionContext);
162 namespace File_Namespace {
163 struct DiskCacheConfig;
168 DBHandler(
const std::vector<LeafHostInfo>& db_leaves,
169 const std::vector<LeafHostInfo>& string_leaves,
170 const std::string& base_data_path,
171 const bool allow_multifrag,
172 const bool jit_debug,
173 const bool intel_jit_profile,
174 const bool read_only,
175 const bool allow_loop_joins,
176 const bool enable_rendering,
177 const bool renderer_use_ppll_polys,
178 const bool renderer_prefer_igpu,
179 const unsigned renderer_vulkan_timeout_ms,
180 const bool enable_auto_clear_render_mem,
181 const int render_oom_retry_threshold,
182 const size_t render_mem_bytes,
183 const size_t max_concurrent_render_sessions,
184 const size_t reserved_gpu_mem,
185 const bool render_compositor_use_last_gpu,
186 const size_t num_reader_threads,
189 const bool legacy_syntax,
190 const int idle_session_duration,
191 const int max_session_duration,
192 const std::string& udf_filename,
193 const std::string& clang_path,
194 const std::vector<std::string>& clang_options,
196 const std::string& libgeos_so_filename,
199 const bool is_new_db);
200 void initialize(
const bool is_new_db);
210 void krb5_connect(TKrb5Session& session,
211 const std::string& token,
212 const std::string& dbname)
override;
214 void connect(TSessionId& session,
215 const std::string& username,
216 const std::string& passwd,
217 const std::string& dbname)
override;
218 void disconnect(
const TSessionId& session)
override;
219 void switch_database(
const TSessionId& session,
const std::string& dbname)
override;
220 void clone_session(TSessionId& session2,
const TSessionId& session1)
override;
221 void get_server_status(TServerStatus& _return,
const TSessionId& session)
override;
222 void get_status(std::vector<TServerStatus>& _return,
223 const TSessionId& session)
override;
224 void get_hardware_info(TClusterHardwareInfo& _return,
225 const TSessionId& session)
override;
229 void get_tables(std::vector<std::string>& _return,
const TSessionId& session)
override;
230 void get_tables_for_database(std::vector<std::string>& _return,
231 const TSessionId& session,
232 const std::string& database_name)
override;
233 void get_physical_tables(std::vector<std::string>& _return,
234 const TSessionId& session)
override;
235 void get_views(std::vector<std::string>& _return,
const TSessionId& session)
override;
236 void get_tables_meta(std::vector<TTableMeta>& _return,
237 const TSessionId& session)
override;
238 void get_table_details(TTableDetails& _return,
239 const TSessionId& session,
240 const std::string& table_name)
override;
241 void get_table_details_for_database(TTableDetails& _return,
242 const TSessionId& session,
243 const std::string& table_name,
244 const std::string& database_name)
override;
245 void get_internal_table_details(TTableDetails& _return,
246 const TSessionId& session,
247 const std::string& table_name,
248 const bool include_system_columns)
override;
249 void get_internal_table_details_for_database(TTableDetails& _return,
250 const TSessionId& session,
251 const std::string& table_name,
252 const std::string& database_name)
override;
253 void get_users(std::vector<std::string>& _return,
const TSessionId& session)
override;
254 void get_databases(std::vector<TDBInfo>& _return,
const TSessionId& session)
override;
256 void get_version(std::string& _return)
override;
257 void start_heap_profile(
const TSessionId& session)
override;
258 void stop_heap_profile(
const TSessionId& session)
override;
259 void get_heap_profile(std::string& _return,
const TSessionId& session)
override;
260 void get_memory(std::vector<TNodeMemoryInfo>& _return,
261 const TSessionId& session,
262 const std::string& memory_level)
override;
263 void clear_cpu_memory(
const TSessionId& session)
override;
264 void clear_gpu_memory(
const TSessionId& session)
override;
265 void clearRenderMemory(
const TSessionId& session);
267 void set_cur_session(
const TSessionId& parent_session,
268 const TSessionId& leaf_session,
269 const std::string& start_time_str,
270 const std::string&
label,
271 bool for_running_query_kernel)
override;
272 void invalidate_cur_session(
const TSessionId& parent_session,
273 const TSessionId& leaf_session,
274 const std::string& start_time_str,
275 const std::string&
label,
276 bool for_running_query_kernel)
override;
277 void set_table_epoch(
const TSessionId& session,
280 const int new_epoch)
override;
281 void set_table_epoch_by_name(
const TSessionId& session,
282 const std::string& table_name,
283 const int new_epoch)
override;
284 int32_t get_table_epoch(
const TSessionId& session,
286 const int32_t table_id)
override;
287 int32_t get_table_epoch_by_name(
const TSessionId& session,
288 const std::string& table_name)
override;
289 void get_table_epochs(std::vector<TTableEpochInfo>& _return,
290 const TSessionId& session,
292 const int32_t table_id)
override;
293 void set_table_epochs(
const TSessionId& session,
295 const std::vector<TTableEpochInfo>& table_epochs)
override;
297 void get_session_info(TSessionInfo& _return,
const TSessionId& session)
override;
299 void set_leaf_info(
const TSessionId& session,
const TLeafInfo& info)
override;
302 const TSessionId& session,
303 const std::string& query,
304 const bool column_format,
305 const int32_t first_n,
306 const int32_t at_most_n,
309 void sql_execute(TQueryResult& _return,
310 const TSessionId& session,
311 const std::string& query,
312 const bool column_format,
313 const std::string& nonce,
314 const int32_t first_n,
315 const int32_t at_most_n)
override;
316 void get_completion_hints(std::vector<TCompletionHint>& hints,
317 const TSessionId& session,
318 const std::string& sql,
319 const int cursor)
override;
321 void sql_execute_df(TDataFrame& _return,
322 const TSessionId& session,
323 const std::string& query,
325 const int32_t device_id,
326 const int32_t first_n,
328 void sql_execute_gdf(TDataFrame& _return,
329 const TSessionId& session,
330 const std::string& query,
331 const int32_t device_id,
332 const int32_t first_n)
override;
333 void deallocate_df(
const TSessionId& session,
334 const TDataFrame& df,
336 const int32_t device_id)
override;
337 void interrupt(
const TSessionId& query_session,
338 const TSessionId& interrupt_session)
override;
339 void sql_validate(TRowDescriptor& _return,
340 const TSessionId& session,
341 const std::string& query)
override;
343 void set_execution_mode(
const TSessionId& session,
345 void render_vega(TRenderResult& _return,
346 const TSessionId& session,
347 const int64_t widget_id,
348 const std::string& vega_json,
349 const int32_t compression_level,
350 const std::string& nonce)
override;
351 void get_result_row_for_pixel(
352 TPixelTableRowResult& _return,
353 const TSessionId& session,
354 const int64_t widget_id,
356 const std::map<std::string, std::vector<std::string>>& table_col_names,
357 const bool column_format,
358 const int32_t pixel_radius,
359 const std::string& nonce)
override;
362 int32_t create_custom_expression(
const TSessionId& session,
363 const TCustomExpression& custom_expression)
override;
364 void get_custom_expressions(std::vector<TCustomExpression>& _return,
365 const TSessionId& session)
override;
366 void update_custom_expression(
const TSessionId& session,
368 const std::string& expression_json)
override;
369 void delete_custom_expressions(
const TSessionId& session,
370 const std::vector<int32_t>& custom_expression_ids,
371 const bool do_soft_delete)
override;
374 void get_dashboard(TDashboard& _return,
375 const TSessionId& session,
376 const int32_t dashboard_id)
override;
377 void get_dashboards(std::vector<TDashboard>& _return,
378 const TSessionId& session)
override;
380 const std::string& dashboard_name,
381 const std::string& dashboard_state,
382 const std::string& image_hash,
383 const std::string& dashboard_metadata)
override;
384 void replace_dashboard(
const TSessionId& session,
385 const int32_t dashboard_id,
386 const std::string& dashboard_name,
387 const std::string& dashboard_owner,
388 const std::string& dashboard_state,
389 const std::string& image_hash,
390 const std::string& dashboard_metadata)
override;
391 void delete_dashboard(
const TSessionId& session,
const int32_t dashboard_id)
override;
392 void share_dashboards(
const TSessionId& session,
393 const std::vector<int32_t>& dashboard_ids,
394 const std::vector<std::string>& groups,
395 const TDashboardPermissions& permissions)
override;
396 void delete_dashboards(
const TSessionId& session,
397 const std::vector<int32_t>& dashboard_ids)
override;
398 void share_dashboard(
const TSessionId& session,
399 const int32_t dashboard_id,
400 const std::vector<std::string>& groups,
401 const std::vector<std::string>& objects,
402 const TDashboardPermissions& permissions,
403 const bool grant_role)
override;
404 void unshare_dashboards(
const TSessionId& session,
405 const std::vector<int32_t>& dashboard_ids,
406 const std::vector<std::string>& groups,
407 const TDashboardPermissions& permissions)
override;
408 void unshare_dashboard(
const TSessionId& session,
409 const int32_t dashboard_id,
410 const std::vector<std::string>& groups,
411 const std::vector<std::string>& objects,
412 const TDashboardPermissions& permissions)
override;
413 void get_dashboard_grantees(std::vector<TDashboardGrantees>& _return,
414 const TSessionId& session,
415 const int32_t dashboard_id)
override;
417 void get_link_view(TFrontendView& _return,
418 const TSessionId& session,
419 const std::string& link)
override;
420 void create_link(std::string& _return,
421 const TSessionId& session,
422 const std::string& view_state,
423 const std::string& view_metadata)
override;
425 void load_table_binary(
const TSessionId& session,
426 const std::string& table_name,
427 const std::vector<TRow>& rows,
428 const std::vector<std::string>& column_names)
override;
430 void load_table_binary_columnar(
const TSessionId& session,
431 const std::string& table_name,
432 const std::vector<TColumn>& cols,
433 const std::vector<std::string>& column_names)
override;
434 void load_table_binary_columnar_polys(
const TSessionId& session,
435 const std::string& table_name,
436 const std::vector<TColumn>& cols,
437 const std::vector<std::string>& column_names,
438 const bool assign_render_groups)
override;
439 void load_table_binary_arrow(
const TSessionId& session,
440 const std::string& table_name,
441 const std::string& arrow_stream,
442 const bool use_column_names)
override;
444 void load_table(
const TSessionId& session,
445 const std::string& table_name,
446 const std::vector<TStringRow>& rows,
447 const std::vector<std::string>& column_names)
override;
448 void detect_column_types(TDetectResult& _return,
449 const TSessionId& session,
450 const std::string& file_name,
451 const TCopyParams& copy_params)
override;
453 const std::string& table_name,
454 const TRowDescriptor& row_desc,
455 const TCreateParams& create_params)
override;
456 void import_table(
const TSessionId& session,
457 const std::string& table_name,
458 const std::string& file_name,
459 const TCopyParams& copy_params)
override;
460 void import_geo_table(
const TSessionId& session,
461 const std::string& table_name,
462 const std::string& file_name,
463 const TCopyParams& copy_params,
464 const TRowDescriptor& row_desc,
465 const TCreateParams& create_params)
override;
466 void import_table_status(TImportStatus& _return,
467 const TSessionId& session,
468 const std::string& import_id)
override;
469 void get_first_geo_file_in_archive(std::string& _return,
470 const TSessionId& session,
471 const std::string& archive_path,
472 const TCopyParams& copy_params)
override;
473 void get_all_files_in_archive(std::vector<std::string>& _return,
474 const TSessionId& session,
475 const std::string& archive_path,
476 const TCopyParams& copy_params)
override;
477 void get_layers_in_geo_file(std::vector<TGeoFileLayerInfo>& _return,
478 const TSessionId& session,
479 const std::string& file_name,
480 const TCopyParams& copy_params)
override;
482 int64_t query_get_outer_fragment_count(
const TSessionId& session,
483 const std::string& select_query)
override;
485 void check_table_consistency(TTableMeta& _return,
486 const TSessionId& session,
487 const int32_t table_id)
override;
488 void start_query(TPendingQuery& _return,
489 const TSessionId& leaf_session,
490 const TSessionId& parent_session,
491 const std::string& query_ra,
492 const std::string& start_time_str,
493 const bool just_explain,
494 const std::vector<int64_t>& outer_fragment_indices)
override;
495 void execute_query_step(TStepResult& _return,
496 const TPendingQuery& pending_query,
497 const TSubqueryId subquery_id,
498 const std::string& start_time_str)
override;
499 void broadcast_serialized_rows(
const TSerializedRows& serialized_rows,
500 const TRowDescriptor& row_desc,
502 const TSubqueryId subquery_id,
503 const bool is_final_subquery_result)
override;
505 void start_render_query(TPendingRenderQuery& _return,
506 const TSessionId& session,
507 const int64_t widget_id,
508 const int16_t node_idx,
509 const std::string& vega_json)
override;
510 void execute_next_render_step(TRenderStepResult& _return,
511 const TPendingRenderQuery& pending_render,
512 const TRenderAggDataMap& merged_data)
override;
514 void insert_data(
const TSessionId& session,
const TInsertData& insert_data)
override;
515 void insert_chunks(
const TSessionId& session,
516 const TInsertChunks& insert_chunks)
override;
517 void checkpoint(
const TSessionId& session,
const int32_t table_id)
override;
519 void get_roles(std::vector<std::string>& _return,
const TSessionId& session)
override;
520 bool has_role(
const TSessionId& sessionId,
521 const std::string& granteeName,
522 const std::string& roleName)
override;
523 bool has_object_privilege(
const TSessionId& sessionId,
524 const std::string& granteeName,
525 const std::string& objectName,
527 const TDBObjectPermissions& permissions)
override;
528 void get_db_objects_for_grantee(std::vector<TDBObject>& _return,
529 const TSessionId& session,
530 const std::string& roleName)
override;
531 void get_db_object_privs(std::vector<TDBObject>& _return,
532 const TSessionId& session,
533 const std::string& objectName,
535 void get_all_roles_for_user(std::vector<std::string>& _return,
536 const TSessionId& session,
537 const std::string& granteeName)
override;
538 void get_all_effective_roles_for_user(std::vector<std::string>& _return,
539 const TSessionId& session,
540 const std::string& granteeName)
override;
541 std::vector<std::string> get_valid_groups(
const TSessionId& session,
542 int32_t dashboard_id,
543 std::vector<std::string> groups);
545 void set_license_key(TLicenseInfo& _return,
546 const TSessionId& session,
547 const std::string& key,
548 const std::string& nonce)
override;
549 void get_license_claims(TLicenseInfo& _return,
550 const TSessionId& session,
551 const std::string& nonce)
override;
558 const TSessionId& session)
override;
566 void register_runtime_extension_functions(
567 const TSessionId& session,
568 const std::vector<TUserDefinedFunction>& udfs,
569 const std::vector<TUserDefinedTableFunction>& udtfs,
570 const std::map<std::string, std::string>& device_ir_map)
override;
575 void get_function_names(std::vector<std::string>& _return,
576 const TSessionId& session)
override;
581 void get_runtime_function_names(std::vector<std::string>& _return,
582 const TSessionId& session)
override;
587 void get_function_details(std::vector<TUserDefinedFunction>& _return,
588 const TSessionId& session,
589 const std::vector<std::string>& udf_names)
override;
594 void get_table_function_names(std::vector<std::string>& _return,
595 const TSessionId& session)
override;
600 void get_runtime_table_function_names(std::vector<std::string>& _return,
601 const TSessionId& session)
override;
606 void get_table_function_details(std::vector<TUserDefinedTableFunction>& _return,
607 const TSessionId& session,
608 const std::vector<std::string>& udtf_names)
override;
613 void emergency_shutdown();
615 TSessionId getInvalidSessionId()
const;
617 void internal_connect(TSessionId& session,
618 const std::string& username,
619 const std::string& dbname);
621 bool isAggregator()
const;
653 template <
typename... ARGS>
655 return query_states_.create(std::forward<ARGS>(
args)...);
660 std::shared_ptr<Catalog_Namespace::SessionInfo> get_session_copy_ptr(
661 const TSessionId& session);
663 void get_tables_meta_impl(std::vector<TTableMeta>& _return,
666 const bool with_table_locks =
true);
669 void resizeDispatchQueue(
size_t queue_size);
673 std::shared_ptr<Catalog_Namespace::SessionInfo> get_session_ptr(
674 const TSessionId& session_id);
679 std::atomic<bool> initialized_{
false};
680 std::shared_ptr<Catalog_Namespace::SessionInfo> create_new_session(
682 const std::string& dbname,
684 std::shared_ptr<Catalog_Namespace::Catalog>
cat);
685 void connect_impl(TSessionId& session,
686 const std::string& passwd,
687 const std::string& dbname,
689 std::shared_ptr<Catalog_Namespace::Catalog>
cat,
691 void disconnect_impl(
const SessionMap::iterator& session_it,
693 void check_table_load_privileges(
const TSessionId& session,
694 const std::string& table_name);
696 const std::string& table_name);
697 void get_tables_impl(std::vector<std::string>& table_names,
700 const std::string& database_name = {});
701 void get_table_details_impl(TTableDetails& _return,
703 const std::string& table_name,
704 const bool get_system,
705 const bool get_physical,
706 const std::string& database_name = {});
707 void getAllRolesForUserImpl(
708 std::shared_ptr<Catalog_Namespace::SessionInfo const> session_ptr,
709 std::vector<std::string>& roles,
710 const TSessionId& sessionId,
711 const std::string& granteeName,
713 void check_read_only(
const std::string& str);
714 void check_session_exp_unsafe(
const SessionMap::iterator& session_it);
715 void validateGroups(
const std::vector<std::string>& groups);
717 const std::vector<int32_t>& dashboard_ids);
718 void shareOrUnshareDashboards(
const TSessionId& session,
719 const std::vector<int32_t>& dashboard_ids,
720 const std::vector<std::string>& groups,
721 const TDashboardPermissions& permissions,
722 const bool do_share);
729 std::shared_ptr<const Catalog_Namespace::SessionInfo> get_const_session_ptr(
730 const TSessionId& session);
732 template <
typename SESSION_MAP_LOCK>
733 SessionMap::iterator get_session_it_unsafe(
const TSessionId& session,
734 SESSION_MAP_LOCK& lock);
735 template <
typename SESSION_MAP_LOCK>
736 void expire_idle_sessions_unsafe(SESSION_MAP_LOCK& lock);
737 static void value_to_thrift_column(
const TargetValue& tv,
742 std::pair<TPlanResult, lockmgr::LockedTableDescriptors> parse_to_ra(
744 const std::string& query_str,
745 const std::vector<TFilterPushDownInfo>& filter_push_down_info,
746 const bool acquire_locks,
748 bool check_privileges =
true);
750 void sql_execute_local(
751 TQueryResult& _return,
753 const std::shared_ptr<Catalog_Namespace::SessionInfo> session_ptr,
754 const std::string& query_str,
755 const bool column_format,
756 const std::string& nonce,
757 const int32_t first_n,
758 const int32_t at_most_n,
759 const bool use_calcite);
761 int64_t process_deferred_copy_from(
const TSessionId& session_id);
763 static void convertData(TQueryResult& _return,
766 const bool column_format,
767 const int32_t first_n,
768 const int32_t at_most_n);
772 const bool column_format,
774 const int32_t first_n,
775 const int32_t at_most_n,
776 const bool use_calcite,
783 void execute_distributed_copy_statement(
787 TQueryResult validate_rel_alg(
const std::string& query_ra,
QueryStateProxy);
789 std::vector<PushedDownFilterInfo> execute_rel_alg(
792 const std::string& query_ra,
793 const bool column_format,
795 const int32_t first_n,
796 const int32_t at_most_n,
797 const bool just_validate,
798 const bool find_push_down_candidates,
800 const std::optional<size_t> executor_index = std::nullopt)
const;
802 void execute_rel_alg_with_filter_push_down(
805 std::string& query_ra,
806 const bool column_format,
808 const int32_t first_n,
809 const int32_t at_most_n,
810 const bool just_explain,
811 const bool just_calcite_explain,
812 const std::vector<PushedDownFilterInfo>& filter_push_down_requests);
814 void executeDdl(TQueryResult& _return,
815 const std::string& query_ra,
816 std::shared_ptr<Catalog_Namespace::SessionInfo const> session_ptr);
819 const std::string& query_ra,
820 std::shared_ptr<Catalog_Namespace::SessionInfo const> session_ptr);
824 TRowDescriptor fixup_row_descriptor(
const TRowDescriptor& row_desc,
828 char unescape_char(std::string str);
831 void check_geospatial_files(
const boost::filesystem::path file_path,
833 void render_rel_alg(TRenderResult& _return,
834 const std::string& query_ra,
835 const std::string& query_str,
837 const std::string& render_type,
838 const bool is_projection_query);
841 const std::string&
name,
842 const bool is_array);
844 static void convertExplain(TQueryResult& _return,
845 const ResultSet& results,
846 const bool column_format);
847 static void convertResult(TQueryResult& _return,
848 const ResultSet& results,
849 const bool column_format);
851 static void convertRows(TQueryResult& _return,
853 const std::vector<TargetMetaInfo>& targets,
854 const ResultSet& results,
855 const bool column_format,
856 const int32_t first_n,
857 const int32_t at_most_n);
863 const std::string& query_state_str,
864 TQueryResult& _return);
866 static void createSimpleResult(TQueryResult& _return,
867 const ResultSet& results,
868 const bool column_format,
869 const std::string
label);
871 std::vector<TargetMetaInfo> getTargetMetaInfo(
872 const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& targets)
const;
874 std::vector<std::string> getTargetNames(
875 const std::vector<TargetMetaInfo>& targets)
const;
877 std::vector<std::string> getTargetNames(
878 const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& targets)
const;
880 void get_completion_hints_unsorted(std::vector<TCompletionHint>& hints,
881 std::vector<std::string>& visible_tables,
883 const std::string& sql,
885 void get_token_based_completions(std::vector<TCompletionHint>& hints,
887 std::vector<std::string>& visible_tables,
888 const std::string& sql,
891 std::unordered_map<std::string, std::unordered_set<std::string>>
892 fill_column_names_by_table(std::vector<std::string>& table_names,
895 TDashboard get_dashboard_impl(
896 const std::shared_ptr<Catalog_Namespace::SessionInfo const>& session_ptr,
899 const bool populate_state =
true);
902 const TDBObjectPermissions& permissions);
904 const TDBObjectPermissions& permission);
906 const TDBObjectPermissions& permissions);
908 const TDBObjectPermissions& permissions);
910 const TDBObjectPermissions& permissions);
915 std::unordered_set<std::string> get_uc_compatible_table_names_by_column(
916 const std::unordered_set<std::string>& uc_column_names,
917 std::vector<std::string>& table_names,
920 std::unique_ptr<lockmgr::AbstractLockContainer<const TableDescriptor*>>
921 prepare_loader_generic(
923 const std::string& table_name,
925 std::unique_ptr<import_export::Loader>* loader,
926 std::vector<std::unique_ptr<import_export::TypedImportBuffer>>* import_buffers,
927 const std::vector<std::string>& column_names,
928 std::string load_type);
931 const TSessionId& session,
933 std::vector<std::unique_ptr<import_export::TypedImportBuffer>>& import_buffers,
937 const std::string& table_name,
938 bool assign_render_groups);
940 void fillMissingBuffers(
941 const TSessionId& session,
943 std::vector<std::unique_ptr<import_export::TypedImportBuffer>>& import_buffers,
944 const std::list<const ColumnDescriptor*>& cds,
945 const std::vector<int>& desc_id_to_column_id,
947 const std::string& table_name,
948 bool assign_render_groups);
971 const std::string& libgeos_so_filename_;
989 std::optional<DeferredCopyFromState>
operator()(
const std::string& session_id) {
990 std::lock_guard<std::mutex> map_lock(deferred_copy_from_mutex);
991 auto itr = was_deferred_copy_from.find(session_id);
992 if (itr == was_deferred_copy_from.end()) {
999 std::lock_guard<std::mutex> map_lock(deferred_copy_from_mutex);
1000 const auto ret = was_deferred_copy_from.insert(std::make_pair(session_id, state));
1004 void remove(
const std::string& session_id) {
1005 std::lock_guard<std::mutex> map_lock(deferred_copy_from_mutex);
1006 was_deferred_copy_from.erase(session_id);
1016 std::string base_path,
1017 std::string query_file_path);
1023 std::map<const std::string, const PermissionFuncPtr> permissionFuncMap_ = {
1024 {
"database"s, has_database_permission},
1025 {
"dashboard"s, has_dashboard_permission},
1026 {
"table"s, has_table_permission},
1027 {
"view"s, has_view_permission},
1028 {
"server"s, has_server_permission}};
1032 template <
typename STMT_TYPE>
1034 using namespace Parser;
1035 auto is_match = [&](
auto session_it) {
1037 return boost::iequals(name,
1038 session_it->second->getCatalog().getCurrentDB().dbName);
1040 return boost::iequals(name, session_it->second->get_currentUser().userName);
1044 auto check_and_remove_sessions = [&]() {
1045 for (
auto it = sessions_.begin(); it != sessions_.end();) {
1047 it = sessions_.erase(it);
1053 check_and_remove_sessions();
1056 std::string
const createInMemoryCalciteSession(
1057 const std::shared_ptr<Catalog_Namespace::Catalog>& catalog_ptr);
1059 void removeInMemoryCalciteSession(
const std::string& session_id);
1062 std::shared_ptr<Catalog_Namespace::SessionInfo const> session_ptr);
1068 std::shared_ptr<Catalog_Namespace::SessionInfo const> session_ptr);
1070 void get_queries_info(std::vector<TQueryInfo>& _return,
1071 const TSessionId& session)
override;
1075 const std::string& target_session);
1077 void alterSystemClear(
const std::string& sesson_id,
1079 const std::string& cache_type,
1080 int64_t& execution_time_ms);
1082 void alterSession(
const std::string& sesson_id,
1084 const std::pair<std::string, std::string>& session_parameter,
1085 int64_t& execution_time_ms);
1091 void loadTableBinaryColumnarInternal(
1092 const TSessionId& session,
1093 const std::string& table_name,
1094 const std::vector<TColumn>& cols,
1095 const std::vector<std::string>& column_names,
1096 const AssignRenderGroupsMode assign_render_groups_mode);
1100 using RenderGroupAssignmentColumnMap =
1101 std::unordered_map<std::string,
1102 std::unique_ptr<import_export::RenderGroupAnalyzer>>;
1104 std::unordered_map<std::string, RenderGroupAssignmentColumnMap>;
1106 std::unordered_map<TSessionId, RenderGroupAssignmentTableMap>;
1111 void importGeoTableGlobFilterSort(
const TSessionId& session,
1112 const std::string& table_name,
1113 const std::string& file_name,
1115 const TRowDescriptor& row_desc,
1116 const TCreateParams& create_params);
1118 void importGeoTableSingle(
const TSessionId& session,
1119 const std::string& table_name,
1120 const std::string& file_name,
1122 const TRowDescriptor& row_desc,
1123 const TCreateParams& create_params);
std::lock_guard< T > lock_guard
Classes used to wrap parser calls for calcite redirection.
std::vector< LeafHostInfo > string_leaves_
auto get_users(SysCatalog &syscat, std::unique_ptr< SqliteConnector > &sqliteConnector, const int32_t dbId=-1)
const std::vector< std::string > & clang_options_
boost::filesystem::path import_path_
std::unique_ptr< QueryDispatchQueue > dispatch_queue_
std::vector< std::unique_ptr< lockmgr::AbstractLockContainer< const TableDescriptor * >>> LockedTableDescriptors
std::unordered_map< std::string, RenderGroupAssignmentColumnMap > RenderGroupAssignmentTableMap
const std::string & udf_filename_
const int render_oom_retry_threshold_
class for a per-database catalog. also includes metadata for the current database and the current use...
void run_warmup_queries(std::shared_ptr< DBHandler > handler, std::string base_path, std::string query_file_path)
std::mutex handle_to_dev_ptr_mutex_
std::mutex deferred_copy_from_mutex
std::shared_ptr< query_state::QueryState > create_query_state(ARGS &&...args)
static thread_local std::string client_address
std::mutex render_group_assignment_mutex_
void add(const std::string &session_id, const DeferredCopyFromState &state)
bool user_can_access_table(const Catalog_Namespace::SessionInfo &session_info, const TableDescriptor *td, const AccessPrivileges access_priv)
bool(*)(const AccessPrivileges &, const TDBObjectPermissions &) PermissionFuncPtr
static thread_local ClientProtocol client_protocol
LeafAggregator leaf_aggregator_
std::unordered_map< TSessionId, RenderGroupAssignmentTableMap > RenderGroupAnalyzerSessionMap
const unsigned renderer_vulkan_timeout_
const std::string base_data_path_
heavyai::unique_lock< heavyai::shared_mutex > write_lock
const size_t render_mem_bytes_
std::map< TSessionId, std::shared_ptr< Catalog_Namespace::SessionInfo >> SessionMap
DeferredCopyFromSessions deferred_copy_from_sessions
const bool renderer_use_ppll_polys_
import_export::CopyParams copy_params
This file contains the class specification and related data structures for Catalog.
static size_t max_bytes_for_thrift()
query_state::QueryStates query_states_
Supported runtime functions management and retrieval.
const size_t reserved_gpu_mem_
std::optional< DeferredCopyFromState > operator()(const std::string &session_id)
Classes representing a parse tree.
const bool render_compositor_use_last_gpu_
void check_not_info_schema_db(const std::string &db_name, bool throw_db_exception)
const int max_session_duration_
ExecutorDeviceType executor_device_type_
std::vector< LeafHostInfo > db_leaves_
const File_Namespace::DiskCacheConfig & disk_cache_config_
const std::string & clang_path_
std::unique_lock< T > unique_lock
std::unique_ptr< RenderHandler > render_handler_
Checked json field retrieval.
std::shared_ptr< QueryEngine > query_engine_
SystemParameters & system_parameters_
const size_t num_reader_threads_
specifies the content in-memory of a row in the column metadata table
const bool enable_auto_clear_render_mem_
const bool renderer_prefer_igpu_
std::map< std::string, std::string > get_device_parameters(bool cpu_only)
bool is_info_schema_db(const std::string &db_name)
const bool allow_loop_joins_
heavyai::shared_mutex sessions_mutex_
std::unique_ptr< HeavyDBAggHandler > agg_handler_
heavyai::shared_mutex custom_expressions_mutex_
const bool enable_rendering_
std::unordered_map< std::string, DeferredCopyFromState > was_deferred_copy_from
const bool intel_jit_profile_
std::shared_ptr< Calcite > calcite_
std::shared_ptr< Data_Namespace::DataMgr > data_mgr_
bool process(std::shared_ptr<::apache::thrift::protocol::TProtocol > in, std::shared_ptr<::apache::thrift::protocol::TProtocol > out, void *connectionContext) override
const bool legacy_syntax_
std::unique_ptr< HeavyDBLeafHandler > leaf_handler_
const int idle_session_duration_
std::unordered_map< std::string, std::string > ipc_handle_to_dev_ptr_
RenderGroupAnalyzerSessionMap render_group_assignment_map_
const size_t max_concurrent_render_sessions_
boost::variant< ScalarTargetValue, ArrayTargetValue, GeoTargetValue, GeoTargetValuePtr > TargetValue
std::shared_timed_mutex shared_mutex
const AuthMetadata & authMetadata_
TrackingProcessor(std::shared_ptr< HeavyIf > handler, const bool check_origin)
void invalidate_sessions(std::string &name, STMT_TYPE *stmt)
std::default_random_engine random_gen_
std::uniform_int_distribution< int64_t > session_id_dist_