28 #include <gperftools/heap-profiler.h>
29 #endif // HAVE_PROFILER
63 #include <sys/types.h>
64 #include <thrift/server/TServer.h>
65 #include <thrift/transport/THttpClient.h>
66 #include <thrift/transport/TSocket.h>
67 #include <thrift/transport/TTransport.h>
69 #include <boost/algorithm/string.hpp>
70 #include <boost/algorithm/string/replace.hpp>
71 #include <boost/algorithm/string/trim.hpp>
72 #include <boost/filesystem.hpp>
73 #include <boost/make_shared.hpp>
74 #include <boost/noncopyable.hpp>
75 #include <boost/none_t.hpp>
76 #include <boost/optional.hpp>
77 #include <boost/program_options.hpp>
78 #include <boost/tokenizer.hpp>
90 #include <unordered_map>
92 #include "gen-cpp/Heavy.h"
93 #include "gen-cpp/extension_functions_types.h"
95 using namespace std::string_literals;
103 using SessionMap = std::map<TSessionId, std::shared_ptr<Catalog_Namespace::SessionInfo>>;
107 namespace dbhandler {
111 bool throw_db_exception =
false);
117 : HeavyProcessor(handler), check_origin_(check_origin) {}
119 bool process(std::shared_ptr<::apache::thrift::protocol::TProtocol> in,
120 std::shared_ptr<::apache::thrift::protocol::TProtocol> out,
121 void* connectionContext)
override {
122 using namespace ::apache::thrift;
124 auto transport = in->getTransport();
125 if (transport && check_origin_) {
126 static std::mutex processor_mutex;
128 const auto origin_str = transport->getOrigin();
129 std::vector<std::string> origins;
130 boost::split(origins, origin_str, boost::is_any_of(
","));
131 if (origins.empty()) {
136 auto trimmed_origin = origins.front();
137 boost::algorithm::trim(trimmed_origin);
140 if (dynamic_cast<transport::THttpTransport*>(transport.get())) {
142 }
else if (dynamic_cast<transport::TBufferedTransport*>(transport.get())) {
151 return HeavyProcessor::process(in, out, connectionContext);
161 namespace File_Namespace {
162 struct DiskCacheConfig;
167 DBHandler(
const std::vector<LeafHostInfo>& db_leaves,
168 const std::vector<LeafHostInfo>& string_leaves,
169 const std::string& base_data_path,
170 const bool allow_multifrag,
171 const bool jit_debug,
172 const bool intel_jit_profile,
173 const bool read_only,
174 const bool allow_loop_joins,
175 const bool enable_rendering,
176 const bool renderer_prefer_igpu,
177 const unsigned renderer_vulkan_timeout_ms,
178 const bool renderer_use_parallel_executors,
179 const bool enable_auto_clear_render_mem,
180 const int render_oom_retry_threshold,
181 const size_t render_mem_bytes,
182 const size_t max_concurrent_render_sessions,
183 const size_t reserved_gpu_mem,
184 const bool render_compositor_use_last_gpu,
185 const size_t num_reader_threads,
188 const bool legacy_syntax,
189 const int idle_session_duration,
190 const int max_session_duration,
191 const std::string& udf_filename,
192 const std::string& clang_path,
193 const std::vector<std::string>& clang_options,
195 const std::string& libgeos_so_filename,
197 #ifdef HAVE_TORCH_TFS
198 const std::string& torch_lib_path,
201 const bool is_new_db);
202 void initialize(
const bool is_new_db);
206 return 2 * 1000 * 1000 * 1000LL;
214 void krb5_connect(TKrb5Session& session,
215 const std::string& token,
216 const std::string& dbname)
override;
218 void connect(TSessionId& session,
219 const std::string& username,
220 const std::string& passwd,
221 const std::string& dbname)
override;
222 void disconnect(
const TSessionId& session)
override;
223 void switch_database(
const TSessionId& session,
const std::string& dbname)
override;
224 void clone_session(TSessionId& session2,
const TSessionId& session1)
override;
225 void get_server_status(TServerStatus& _return,
const TSessionId& session)
override;
226 void get_status(std::vector<TServerStatus>& _return,
227 const TSessionId& session)
override;
228 void get_hardware_info(TClusterHardwareInfo& _return,
229 const TSessionId& session)
override;
233 void get_tables(std::vector<std::string>& _return,
const TSessionId& session)
override;
234 void get_tables_for_database(std::vector<std::string>& _return,
235 const TSessionId& session,
236 const std::string& database_name)
override;
237 void get_physical_tables(std::vector<std::string>& _return,
238 const TSessionId& session)
override;
239 void get_views(std::vector<std::string>& _return,
const TSessionId& session)
override;
240 void get_tables_meta(std::vector<TTableMeta>& _return,
241 const TSessionId& session)
override;
242 void get_table_details(TTableDetails& _return,
243 const TSessionId& session,
244 const std::string& table_name)
override;
245 void get_table_details_for_database(TTableDetails& _return,
246 const TSessionId& session,
247 const std::string& table_name,
248 const std::string& database_name)
override;
249 void get_internal_table_details(TTableDetails& _return,
250 const TSessionId& session,
251 const std::string& table_name,
252 const bool include_system_columns)
override;
253 void get_internal_table_details_for_database(TTableDetails& _return,
254 const TSessionId& session,
255 const std::string& table_name,
256 const std::string& database_name)
override;
257 void get_users(std::vector<std::string>& _return,
const TSessionId& session)
override;
258 void get_databases(std::vector<TDBInfo>& _return,
const TSessionId& session)
override;
260 void get_version(std::string& _return)
override;
261 void start_heap_profile(
const TSessionId& session)
override;
262 void stop_heap_profile(
const TSessionId& session)
override;
263 void get_heap_profile(std::string& _return,
const TSessionId& session)
override;
264 void get_memory(std::vector<TNodeMemoryInfo>& _return,
265 const TSessionId& session,
266 const std::string& memory_level)
override;
267 void clear_cpu_memory(
const TSessionId& session)
override;
268 void clear_gpu_memory(
const TSessionId& session)
override;
269 void clearRenderMemory(
const TSessionId& session);
272 void pause_executor_queue(
273 const TSessionId& session);
274 void resume_executor_queue(
275 const TSessionId& session);
277 void set_cur_session(
const TSessionId& parent_session,
278 const TSessionId& leaf_session,
279 const std::string& start_time_str,
280 const std::string&
label,
281 bool for_running_query_kernel)
override;
282 void invalidate_cur_session(
const TSessionId& parent_session,
283 const TSessionId& leaf_session,
284 const std::string& start_time_str,
285 const std::string&
label,
286 bool for_running_query_kernel)
override;
287 void set_table_epoch(
const TSessionId& session,
290 const int new_epoch)
override;
291 void set_table_epoch_by_name(
const TSessionId& session,
292 const std::string& table_name,
293 const int new_epoch)
override;
294 int32_t get_table_epoch(
const TSessionId& session,
296 const int32_t table_id)
override;
297 int32_t get_table_epoch_by_name(
const TSessionId& session,
298 const std::string& table_name)
override;
299 void get_table_epochs(std::vector<TTableEpochInfo>& _return,
300 const TSessionId& session,
302 const int32_t table_id)
override;
303 void set_table_epochs(
const TSessionId& session,
305 const std::vector<TTableEpochInfo>& table_epochs)
override;
307 void get_session_info(TSessionInfo& _return,
const TSessionId& session)
override;
309 void set_leaf_info(
const TSessionId& session,
const TLeafInfo& info)
override;
312 const TSessionId& session,
313 const std::string& query,
314 const bool column_format,
315 const int32_t first_n,
316 const int32_t at_most_n,
319 void sql_execute(TQueryResult& _return,
320 const TSessionId& session,
321 const std::string& query,
322 const bool column_format,
323 const std::string& nonce,
324 const int32_t first_n,
325 const int32_t at_most_n)
override;
326 void get_completion_hints(std::vector<TCompletionHint>& hints,
327 const TSessionId& session,
328 const std::string& sql,
329 const int cursor)
override;
331 void sql_execute_df(TDataFrame& _return,
332 const TSessionId& session,
333 const std::string& query,
335 const int32_t device_id,
336 const int32_t first_n,
338 void sql_execute_gdf(TDataFrame& _return,
339 const TSessionId& session,
340 const std::string& query,
341 const int32_t device_id,
342 const int32_t first_n)
override;
343 void deallocate_df(
const TSessionId& session,
344 const TDataFrame& df,
346 const int32_t device_id)
override;
347 void interrupt(
const TSessionId& query_session,
348 const TSessionId& interrupt_session)
override;
349 void sql_validate(TRowDescriptor& _return,
350 const TSessionId& session,
351 const std::string& query)
override;
353 void set_execution_mode(
const TSessionId& session,
355 void render_vega(TRenderResult& _return,
356 const TSessionId& session,
357 const int64_t widget_id,
358 const std::string& vega_json,
359 const int32_t compression_level,
360 const std::string& nonce)
override;
361 void get_result_row_for_pixel(
362 TPixelTableRowResult& _return,
363 const TSessionId& session,
364 const int64_t widget_id,
366 const std::map<std::string, std::vector<std::string>>& table_col_names,
367 const bool column_format,
368 const int32_t pixel_radius,
369 const std::string& nonce)
override;
372 int32_t create_custom_expression(
const TSessionId& session,
373 const TCustomExpression& custom_expression)
override;
374 void get_custom_expressions(std::vector<TCustomExpression>& _return,
375 const TSessionId& session)
override;
376 void update_custom_expression(
const TSessionId& session,
378 const std::string& expression_json)
override;
379 void delete_custom_expressions(
const TSessionId& session,
380 const std::vector<int32_t>& custom_expression_ids,
381 const bool do_soft_delete)
override;
384 void get_dashboard(TDashboard& _return,
385 const TSessionId& session,
386 const int32_t dashboard_id)
override;
387 void get_dashboards(std::vector<TDashboard>& _return,
388 const TSessionId& session)
override;
390 const std::string& dashboard_name,
391 const std::string& dashboard_state,
392 const std::string& image_hash,
393 const std::string& dashboard_metadata)
override;
394 void replace_dashboard(
const TSessionId& session,
395 const int32_t dashboard_id,
396 const std::string& dashboard_name,
397 const std::string& dashboard_owner,
398 const std::string& dashboard_state,
399 const std::string& image_hash,
400 const std::string& dashboard_metadata)
override;
401 void delete_dashboard(
const TSessionId& session,
const int32_t dashboard_id)
override;
402 void share_dashboards(
const TSessionId& session,
403 const std::vector<int32_t>& dashboard_ids,
404 const std::vector<std::string>& groups,
405 const TDashboardPermissions& permissions)
override;
406 void delete_dashboards(
const TSessionId& session,
407 const std::vector<int32_t>& dashboard_ids)
override;
408 void share_dashboard(
const TSessionId& session,
409 const int32_t dashboard_id,
410 const std::vector<std::string>& groups,
411 const std::vector<std::string>& objects,
412 const TDashboardPermissions& permissions,
413 const bool grant_role)
override;
414 void unshare_dashboards(
const TSessionId& session,
415 const std::vector<int32_t>& dashboard_ids,
416 const std::vector<std::string>& groups,
417 const TDashboardPermissions& permissions)
override;
418 void unshare_dashboard(
const TSessionId& session,
419 const int32_t dashboard_id,
420 const std::vector<std::string>& groups,
421 const std::vector<std::string>& objects,
422 const TDashboardPermissions& permissions)
override;
423 void get_dashboard_grantees(std::vector<TDashboardGrantees>& _return,
424 const TSessionId& session,
425 const int32_t dashboard_id)
override;
427 void get_link_view(TFrontendView& _return,
428 const TSessionId& session,
429 const std::string& link)
override;
430 void create_link(std::string& _return,
431 const TSessionId& session,
432 const std::string& view_state,
433 const std::string& view_metadata)
override;
435 void load_table_binary(
const TSessionId& session,
436 const std::string& table_name,
437 const std::vector<TRow>&
rows,
438 const std::vector<std::string>& column_names)
override;
440 void load_table_binary_columnar(
const TSessionId& session,
441 const std::string& table_name,
442 const std::vector<TColumn>& cols,
443 const std::vector<std::string>& column_names)
override;
444 void load_table_binary_arrow(
const TSessionId& session,
445 const std::string& table_name,
446 const std::string& arrow_stream,
447 const bool use_column_names)
override;
449 void load_table(
const TSessionId& session,
450 const std::string& table_name,
451 const std::vector<TStringRow>&
rows,
452 const std::vector<std::string>& column_names)
override;
453 void detect_column_types(TDetectResult& _return,
454 const TSessionId& session,
455 const std::string& file_name,
456 const TCopyParams& copy_params)
override;
458 const std::string& table_name,
459 const TRowDescriptor& row_desc,
460 const TCreateParams& create_params)
override;
461 void import_table(
const TSessionId& session,
462 const std::string& table_name,
463 const std::string& file_name,
464 const TCopyParams& copy_params)
override;
465 void import_geo_table(
const TSessionId& session,
466 const std::string& table_name,
467 const std::string& file_name,
468 const TCopyParams& copy_params,
469 const TRowDescriptor& row_desc,
470 const TCreateParams& create_params)
override;
471 void import_table_status(TImportStatus& _return,
472 const TSessionId& session,
473 const std::string& import_id)
override;
474 void get_first_geo_file_in_archive(std::string& _return,
475 const TSessionId& session,
476 const std::string& archive_path,
477 const TCopyParams& copy_params)
override;
478 void get_all_files_in_archive(std::vector<std::string>& _return,
479 const TSessionId& session,
480 const std::string& archive_path,
481 const TCopyParams& copy_params)
override;
482 void get_layers_in_geo_file(std::vector<TGeoFileLayerInfo>& _return,
483 const TSessionId& session,
484 const std::string& file_name,
485 const TCopyParams& copy_params)
override;
487 int64_t query_get_outer_fragment_count(
const TSessionId& session,
488 const std::string& select_query)
override;
490 void check_table_consistency(TTableMeta& _return,
491 const TSessionId& session,
492 const int32_t table_id)
override;
493 void start_query(TPendingQuery& _return,
494 const TSessionId& leaf_session,
495 const TSessionId& parent_session,
496 const std::string& serialized_rel_alg_dag,
497 const std::string& start_time_str,
498 const bool just_explain,
499 const std::vector<int64_t>& outer_fragment_indices)
override;
500 void execute_query_step(TStepResult& _return,
501 const TPendingQuery& pending_query,
502 const TSubqueryId subquery_id,
503 const std::string& start_time_str)
override;
504 void broadcast_serialized_rows(
const TSerializedRows& serialized_rows,
505 const TRowDescriptor& row_desc,
506 const TQueryId query_id,
507 const TSubqueryId subquery_id,
508 const bool is_final_subquery_result)
override;
510 void start_render_query(TPendingRenderQuery& _return,
511 const TSessionId& session,
512 const int64_t widget_id,
513 const int16_t node_idx,
514 const std::string& vega_json)
override;
515 void execute_next_render_step(TRenderStepResult& _return,
516 const TPendingRenderQuery& pending_render,
517 const TRenderAggDataMap& merged_data)
override;
519 void insert_data(
const TSessionId& session,
const TInsertData& insert_data)
override;
520 void insert_chunks(
const TSessionId& session,
521 const TInsertChunks& insert_chunks)
override;
522 void checkpoint(
const TSessionId& session,
const int32_t table_id)
override;
524 void get_roles(std::vector<std::string>& _return,
const TSessionId& session)
override;
525 bool has_role(
const TSessionId& sessionId,
526 const std::string& granteeName,
527 const std::string& roleName)
override;
528 bool has_object_privilege(
const TSessionId& sessionId,
529 const std::string& granteeName,
530 const std::string& objectName,
532 const TDBObjectPermissions& permissions)
override;
533 void get_db_objects_for_grantee(std::vector<TDBObject>& _return,
534 const TSessionId& session,
535 const std::string& roleName)
override;
536 void get_db_object_privs(std::vector<TDBObject>& _return,
537 const TSessionId& session,
538 const std::string& objectName,
540 void get_all_roles_for_user(std::vector<std::string>& _return,
541 const TSessionId& session,
542 const std::string& granteeName)
override;
543 void get_all_effective_roles_for_user(std::vector<std::string>& _return,
544 const TSessionId& session,
545 const std::string& granteeName)
override;
546 std::vector<std::string> get_valid_groups(
const TSessionId& session,
547 int32_t dashboard_id,
548 std::vector<std::string> groups);
550 void set_license_key(TLicenseInfo& _return,
551 const TSessionId& session,
552 const std::string& key,
553 const std::string& nonce)
override;
554 void get_license_claims(TLicenseInfo& _return,
555 const TSessionId& session,
556 const std::string& nonce)
override;
563 const TSessionId& session)
override;
571 void register_runtime_extension_functions(
572 const TSessionId& session,
573 const std::vector<TUserDefinedFunction>& udfs,
574 const std::vector<TUserDefinedTableFunction>& udtfs,
575 const std::map<std::string, std::string>& device_ir_map)
override;
580 void get_function_names(std::vector<std::string>& _return,
581 const TSessionId& session)
override;
586 void get_runtime_function_names(std::vector<std::string>& _return,
587 const TSessionId& session)
override;
592 void get_function_details(std::vector<TUserDefinedFunction>& _return,
593 const TSessionId& session,
594 const std::vector<std::string>& udf_names)
override;
599 void get_table_function_names(std::vector<std::string>& _return,
600 const TSessionId& session)
override;
605 void get_runtime_table_function_names(std::vector<std::string>& _return,
606 const TSessionId& session)
override;
611 void get_table_function_details(std::vector<TUserDefinedTableFunction>& _return,
612 const TSessionId& session,
613 const std::vector<std::string>& udtf_names)
override;
618 void emergency_shutdown();
620 TSessionId getInvalidSessionId()
const;
622 void internal_connect(TSessionId& session,
623 const std::string& username,
624 const std::string& dbname);
626 bool isAggregator()
const;
628 bool checkInMemorySystemTableQuery(
629 const std::unordered_set<shared::TableKey>& tables_selected_from)
const;
661 template <
typename... ARGS>
663 return query_states_.create(std::forward<ARGS>(
args)...);
669 void get_tables_meta_impl(std::vector<TTableMeta>& _return,
672 const bool with_table_locks =
true);
675 void resizeDispatchQueue(
size_t queue_size);
679 std::shared_ptr<Catalog_Namespace::SessionInfo> get_session_ptr(
680 const TSessionId& session_id);
685 std::atomic<bool> initialized_{
false};
686 void init_executor_resource_mgr();
687 void validate_configurations();
688 std::shared_ptr<Catalog_Namespace::SessionInfo> create_new_session(
690 const std::string& dbname,
692 std::shared_ptr<Catalog_Namespace::Catalog>
cat);
693 void connect_impl(TSessionId& session,
694 const std::string& passwd,
695 const std::string& dbname,
697 std::shared_ptr<Catalog_Namespace::Catalog>
cat,
701 const std::string& table_name);
702 void get_tables_impl(std::vector<std::string>& table_names,
705 const std::string& database_name = {});
706 void get_table_details_impl(TTableDetails& _return,
708 const std::string& table_name,
709 const bool get_system,
710 const bool get_physical,
711 const std::string& database_name = {});
712 void getAllRolesForUserImpl(
713 std::shared_ptr<Catalog_Namespace::SessionInfo const> session_ptr,
714 std::vector<std::string>& roles,
715 const std::string& granteeName,
717 void check_read_only(
const std::string& str);
718 void validateGroups(
const std::vector<std::string>& groups);
720 const std::vector<int32_t>& dashboard_ids);
721 void shareOrUnshareDashboards(
const TSessionId& session,
722 const std::vector<int32_t>& dashboard_ids,
723 const std::vector<std::string>& groups,
724 const TDashboardPermissions& permissions,
725 const bool do_share);
727 static void value_to_thrift_column(
const TargetValue& tv,
732 std::pair<TPlanResult, lockmgr::LockedTableDescriptors> parse_to_ra(
734 const std::string& query_str,
735 const std::vector<TFilterPushDownInfo>& filter_push_down_info,
736 const bool acquire_locks,
738 bool check_privileges =
true);
740 void sql_execute_local(
741 TQueryResult& _return,
743 const std::shared_ptr<Catalog_Namespace::SessionInfo> session_ptr,
744 const std::string& query_str,
745 const bool column_format,
746 const std::string& nonce,
747 const int32_t first_n,
748 const int32_t at_most_n,
749 const bool use_calcite);
751 int64_t process_deferred_copy_from(
const TSessionId& session_id);
753 static void convertData(TQueryResult& _return,
756 const bool column_format,
757 const int32_t first_n,
758 const int32_t at_most_n);
762 const bool column_format,
764 const int32_t first_n,
765 const int32_t at_most_n,
766 const bool use_calcite,
773 void execute_distributed_copy_statement(
777 TPlanResult processCalciteRequest(
779 const std::shared_ptr<Catalog_Namespace::Catalog>&
cat,
780 const std::string& query_str,
781 const std::vector<TFilterPushDownInfo>& filter_push_down_info,
783 const bool check_privileges);
785 TRowDescriptor validateRelAlg(
const std::string& query_ra,
788 void dispatch_query_task(std::shared_ptr<QueryDispatchQueue::Task> query_task,
789 const bool is_update_delete);
791 std::vector<PushedDownFilterInfo> execute_rel_alg(
794 const std::string& query_ra,
795 const bool column_format,
797 const int32_t first_n,
798 const int32_t at_most_n,
799 const bool just_validate,
800 const bool find_push_down_candidates,
802 const std::optional<size_t> executor_index = std::nullopt)
const;
804 void execute_rel_alg_with_filter_push_down(
807 std::string& query_ra,
808 const bool column_format,
810 const int32_t first_n,
811 const int32_t at_most_n,
812 const bool just_explain,
813 const bool just_calcite_explain,
814 const std::vector<PushedDownFilterInfo>& filter_push_down_requests);
816 void executeDdl(TQueryResult& _return,
817 const std::string& query_ra,
818 std::shared_ptr<Catalog_Namespace::SessionInfo const> session_ptr);
821 const std::string& query_ra,
822 std::shared_ptr<Catalog_Namespace::SessionInfo const> session_ptr);
829 char unescape_char(std::string str);
832 void check_geospatial_files(
const boost::filesystem::path file_path,
834 void render_rel_alg(TRenderResult& _return,
835 const std::string& query_ra,
836 const std::string& query_str,
838 const std::string& render_type,
839 const bool is_projection_query);
842 const std::string&
name,
843 const bool is_array);
845 static void convertExplain(TQueryResult& _return,
846 const ResultSet& results,
847 const bool column_format);
848 static void convertResult(TQueryResult& _return,
849 const ResultSet& results,
850 const bool column_format);
852 static void convertRows(TQueryResult& _return,
854 const std::vector<TargetMetaInfo>& targets,
855 const ResultSet& results,
856 const bool column_format,
857 const int32_t first_n,
858 const int32_t at_most_n);
864 const std::string& query_state_str,
865 TQueryResult& _return);
867 static void createSimpleResult(TQueryResult& _return,
868 const ResultSet& results,
869 const bool column_format,
870 const std::string
label);
872 std::vector<TargetMetaInfo> getTargetMetaInfo(
873 const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& targets)
const;
875 std::vector<std::string> getTargetNames(
876 const std::vector<TargetMetaInfo>& targets)
const;
878 std::vector<std::string> getTargetNames(
879 const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& targets)
const;
881 void get_completion_hints_unsorted(std::vector<TCompletionHint>& hints,
882 std::vector<std::string>& visible_tables,
884 const std::string& sql,
886 void get_token_based_completions(std::vector<TCompletionHint>& hints,
888 std::vector<std::string>& visible_tables,
889 const std::string& sql,
892 std::unordered_map<std::string, std::unordered_set<std::string>>
893 fill_column_names_by_table(std::vector<std::string>& table_names,
896 TDashboard get_dashboard_impl(
897 const std::shared_ptr<Catalog_Namespace::SessionInfo const>& session_ptr,
900 const bool populate_state =
true);
903 const TDBObjectPermissions& permissions);
905 const TDBObjectPermissions& permission);
907 const TDBObjectPermissions& permissions);
909 const TDBObjectPermissions& permissions);
911 const TDBObjectPermissions& permissions);
916 std::unordered_set<std::string> get_uc_compatible_table_names_by_column(
917 const std::unordered_set<std::string>& uc_column_names,
918 std::vector<std::string>& table_names,
921 std::unique_ptr<lockmgr::AbstractLockContainer<const TableDescriptor*>>
922 prepare_loader_generic(
924 const std::string& table_name,
926 std::unique_ptr<import_export::Loader>* loader,
927 std::vector<std::unique_ptr<import_export::TypedImportBuffer>>* import_buffers,
928 const std::vector<std::string>& column_names,
929 std::string load_type);
932 const TSessionId& session,
934 std::vector<std::unique_ptr<import_export::TypedImportBuffer>>& import_buffers,
938 const std::string& table_name);
940 void fillMissingBuffers(
941 const TSessionId& session,
943 std::vector<std::unique_ptr<import_export::TypedImportBuffer>>& import_buffers,
944 const std::list<const ColumnDescriptor*>& cds,
945 const std::vector<int>& desc_id_to_column_id,
947 const std::string& table_name);
973 const std::string& libgeos_so_filename_;
975 #ifdef HAVE_TORCH_TFS
976 const std::string& torch_lib_path_;
982 int32_t max_num_sessions_{-1};
996 std::optional<DeferredCopyFromState>
operator()(
const std::string& session_id) {
997 std::lock_guard<std::mutex> map_lock(deferred_copy_from_mutex);
998 auto itr = was_deferred_copy_from.find(session_id);
999 if (itr == was_deferred_copy_from.end()) {
1000 return std::nullopt;
1006 std::lock_guard<std::mutex> map_lock(deferred_copy_from_mutex);
1007 const auto ret = was_deferred_copy_from.insert(std::make_pair(session_id, state));
1011 void remove(
const std::string& session_id) {
1012 std::lock_guard<std::mutex> map_lock(deferred_copy_from_mutex);
1013 was_deferred_copy_from.erase(session_id);
1023 std::string base_path,
1024 std::string query_file_path);
1030 std::map<const std::string, const PermissionFuncPtr> permissionFuncMap_ = {
1031 {
"database"s, has_database_permission},
1032 {
"dashboard"s, has_dashboard_permission},
1033 {
"table"s, has_table_permission},
1034 {
"view"s, has_view_permission},
1035 {
"server"s, has_server_permission}};
1039 std::string
const createInMemoryCalciteSession(
1040 const std::shared_ptr<Catalog_Namespace::Catalog>& catalog_ptr);
1041 void removeInMemoryCalciteSession(
const std::string& session_id);
1044 std::shared_ptr<Catalog_Namespace::SessionInfo const> session_ptr);
1050 std::shared_ptr<Catalog_Namespace::SessionInfo const> session_ptr);
1052 void get_queries_info(std::vector<TQueryInfo>& _return,
1053 const TSessionId& session)
override;
1057 const std::string& target_session);
1059 void alterSystemClear(
const std::string& sesson_id,
1061 const std::string& cache_type,
1062 int64_t& execution_time_ms);
1064 void alterSession(
const std::string& sesson_id,
1066 const std::pair<std::string, std::string>& session_parameter,
1067 int64_t& execution_time_ms);
1073 void importGeoTableGlobFilterSort(
const TSessionId& session,
1074 const std::string& table_name,
1075 const std::string& file_name,
1077 const TRowDescriptor& row_desc,
1078 const TCreateParams& create_params);
1080 void importGeoTableSingle(
const TSessionId& session,
1081 const std::string& table_name,
1082 const std::string& file_name,
1084 const TRowDescriptor& row_desc,
1085 const TCreateParams& create_params);
1087 void resetSessionsStore();
std::lock_guard< T > lock_guard
Classes used to wrap parser calls for calcite redirection.
std::vector< LeafHostInfo > string_leaves_
auto get_users(SysCatalog &syscat, std::unique_ptr< SqliteConnector > &sqliteConnector, const int32_t dbId=-1)
const std::vector< std::string > & clang_options_
boost::filesystem::path import_path_
std::unique_ptr< QueryDispatchQueue > dispatch_queue_
std::vector< std::unique_ptr< lockmgr::AbstractLockContainer< const TableDescriptor * >>> LockedTableDescriptors
const bool renderer_use_parallel_executors_
const std::string & udf_filename_
const int render_oom_retry_threshold_
class for a per-database catalog. also includes metadata for the current database and the current use...
void run_warmup_queries(std::shared_ptr< DBHandler > handler, std::string base_path, std::string query_file_path)
std::mutex handle_to_dev_ptr_mutex_
std::mutex deferred_copy_from_mutex
std::shared_ptr< query_state::QueryState > create_query_state(ARGS &&...args)
static thread_local std::string client_address
void add(const std::string &session_id, const DeferredCopyFromState &state)
bool user_can_access_table(const Catalog_Namespace::SessionInfo &session_info, const TableDescriptor *td, const AccessPrivileges access_priv)
bool(*)(const AccessPrivileges &, const TDBObjectPermissions &) PermissionFuncPtr
static thread_local ClientProtocol client_protocol
LeafAggregator leaf_aggregator_
const unsigned renderer_vulkan_timeout_
const std::string base_data_path_
const size_t render_mem_bytes_
std::map< TSessionId, std::shared_ptr< Catalog_Namespace::SessionInfo >> SessionMap
DeferredCopyFromSessions deferred_copy_from_sessions
import_export::CopyParams copy_params
std::unordered_map< std::string, Catalog_Namespace::SessionInfoPtr > calcite_sessions_
This file contains the class specification and related data structures for Catalog.
static size_t max_bytes_for_thrift()
query_state::QueryStates query_states_
Supported runtime functions management and retrieval.
const size_t reserved_gpu_mem_
std::optional< DeferredCopyFromState > operator()(const std::string &session_id)
Classes representing a parse tree.
const bool render_compositor_use_last_gpu_
void check_not_info_schema_db(const std::string &db_name, bool throw_db_exception)
const int max_session_duration_
ExecutorDeviceType executor_device_type_
std::vector< LeafHostInfo > db_leaves_
const File_Namespace::DiskCacheConfig & disk_cache_config_
const std::string & clang_path_
std::unique_ptr< RenderHandler > render_handler_
Checked json field retrieval.
std::shared_ptr< QueryEngine > query_engine_
SystemParameters & system_parameters_
const size_t num_reader_threads_
specifies the content in-memory of a row in the column metadata table
const bool enable_auto_clear_render_mem_
const bool renderer_prefer_igpu_
heavyai::shared_mutex calcite_sessions_mtx_
std::map< std::string, std::string > get_device_parameters(bool cpu_only)
bool is_info_schema_db(const std::string &db_name)
const bool allow_loop_joins_
heavyai::shared_mutex sessions_mutex_
std::unique_ptr< HeavyDBAggHandler > agg_handler_
heavyai::shared_mutex custom_expressions_mutex_
const bool enable_rendering_
std::unordered_map< std::string, DeferredCopyFromState > was_deferred_copy_from
const bool intel_jit_profile_
std::unique_ptr< Catalog_Namespace::SessionsStore > sessions_store_
std::shared_ptr< Calcite > calcite_
std::shared_ptr< Data_Namespace::DataMgr > data_mgr_
bool process(std::shared_ptr<::apache::thrift::protocol::TProtocol > in, std::shared_ptr<::apache::thrift::protocol::TProtocol > out, void *connectionContext) override
const bool legacy_syntax_
std::unique_ptr< HeavyDBLeafHandler > leaf_handler_
const int idle_session_duration_
std::unordered_map< std::string, std::string > ipc_handle_to_dev_ptr_
const size_t max_concurrent_render_sessions_
boost::variant< ScalarTargetValue, ArrayTargetValue, GeoTargetValue, GeoTargetValuePtr > TargetValue
std::shared_timed_mutex shared_mutex
const AuthMetadata & authMetadata_
TrackingProcessor(std::shared_ptr< HeavyIf > handler, const bool check_origin)
std::default_random_engine random_gen_
std::uniform_int_distribution< int64_t > session_id_dist_
std::shared_ptr< SessionInfo > SessionInfoPtr