30 #include <gperftools/heap-profiler.h>
31 #endif // HAVE_PROFILER
33 #include "MapDRelease.h"
36 #include "gen-cpp/CalciteServer.h"
55 #include "Parser/parser.h"
78 #include <sys/types.h>
82 #include <boost/algorithm/string.hpp>
83 #include <boost/filesystem.hpp>
84 #include <boost/make_shared.hpp>
85 #include <boost/process/search_path.hpp>
86 #include <boost/program_options.hpp>
87 #include <boost/regex.hpp>
88 #include <boost/tokenizer.hpp>
101 #include <arrow/api.h>
102 #include <arrow/io/api.h>
103 #include <arrow/ipc/api.h>
107 #define ENABLE_GEO_IMPORT_COLUMN_MATCHING 0
112 #define INVALID_SESSION_ID ""
114 #define THROW_MAPD_EXCEPTION(errstr) \
116 TOmniSciException ex; \
117 ex.error_msg = errstr; \
118 LOG(ERROR) << ex.error_msg; \
129 auto session_it = session_map.find(session);
130 if (session_it == session_map.end()) {
144 const TSessionId& session,
145 mapd_shared_lock<mapd_shared_mutex>&
read_lock) {
149 }
catch (
const ForceDisconnect& e) {
161 const TSessionId& session,
162 mapd_unique_lock<mapd_shared_mutex>&
write_lock) {
166 }
catch (
const ForceDisconnect& e) {
174 extern std::unique_ptr<std::string> g_libgeos_so_filename;
178 const std::vector<LeafHostInfo>& string_leaves,
179 const std::string& base_data_path,
181 const bool allow_multifrag,
182 const bool jit_debug,
183 const bool intel_jit_profile,
184 const bool read_only,
185 const bool allow_loop_joins,
186 const bool enable_rendering,
187 const bool renderer_use_vulkan_driver,
188 const bool enable_auto_clear_render_mem,
189 const int render_oom_retry_threshold,
190 const size_t render_mem_bytes,
191 const size_t max_concurrent_render_sessions,
194 const size_t reserved_gpu_mem,
195 const bool render_compositor_use_last_gpu,
196 const size_t num_reader_threads,
199 const bool legacy_syntax,
200 const int idle_session_duration,
201 const int max_session_duration,
202 const bool enable_runtime_udf_registration,
203 const std::string& udf_filename,
204 const std::string& clang_path,
205 const std::vector<std::string>& clang_options,
207 const std::string& libgeos_so_filename,
210 : leaf_aggregator_(db_leaves)
211 , string_leaves_(string_leaves)
212 , base_data_path_(base_data_path)
213 , random_gen_(std::random_device{}())
214 , session_id_dist_(0, INT32_MAX)
215 , jit_debug_(jit_debug)
216 , intel_jit_profile_(intel_jit_profile)
217 , allow_multifrag_(allow_multifrag)
218 , read_only_(read_only)
219 , allow_loop_joins_(allow_loop_joins)
220 , authMetadata_(authMetadata)
221 , system_parameters_(system_parameters)
222 , legacy_syntax_(legacy_syntax)
224 std::make_unique<QueryDispatchQueue>(system_parameters.num_executors))
225 , super_user_rights_(
false)
226 , idle_session_duration_(idle_session_duration * 60)
227 , max_session_duration_(max_session_duration * 60)
228 , runtime_udf_registration_enabled_(enable_runtime_udf_registration) {
232 bool is_rendering_enabled = enable_rendering;
236 is_rendering_enabled =
false;
238 cpu_mode_only_ =
true;
242 cpu_mode_only_ =
false;
245 LOG(
ERROR) <<
"This build isn't CUDA enabled, will run on CPU";
246 cpu_mode_only_ =
true;
247 is_rendering_enabled =
false;
250 }
catch (
const std::exception& e) {
251 LOG(
FATAL) <<
"Failed to executor device type: " << e.what();
254 const auto data_path = boost::filesystem::path(base_data_path_) /
"mapd_data";
257 size_t total_reserved = reserved_gpu_mem;
258 if (is_rendering_enabled) {
259 total_reserved += render_mem_bytes;
262 std::unique_ptr<CudaMgr_Namespace::CudaMgr> cuda_mgr;
264 if (!cpu_mode_only_ || is_rendering_enabled) {
266 cuda_mgr = std::make_unique<CudaMgr_Namespace::CudaMgr>(num_gpus, start_gpu);
267 }
catch (
const std::exception& e) {
268 LOG(
ERROR) <<
"Unable to instantiate CudaMgr, falling back to CPU-only mode. "
270 cpu_mode_only_ =
true;
271 is_rendering_enabled =
false;
284 }
catch (
const std::exception& e) {
285 LOG(
FATAL) <<
"Failed to initialize data manager: " << e.what();
288 std::string udf_ast_filename(
"");
291 if (!udf_filename.empty()) {
292 const auto cuda_mgr = data_mgr_->getCudaMgr();
294 cuda_mgr ? cuda_mgr->getDeviceArch()
296 UdfCompiler compiler(udf_filename, device_arch, clang_path, clang_options);
297 int compile_result = compiler.compileUdf();
299 if (compile_result == 0) {
300 udf_ast_filename = compiler.getAstFileName();
303 }
catch (
const std::exception& e) {
304 LOG(
FATAL) <<
"Failed to initialize UDF compiler: " << e.what();
309 std::make_shared<Calcite>(system_parameters, base_data_path_, udf_ast_filename);
310 }
catch (
const std::exception& e) {
311 LOG(
FATAL) <<
"Failed to initialize Calcite server: " << e.what();
316 if (!udf_filename.empty()) {
319 }
catch (
const std::exception& e) {
320 LOG(
FATAL) <<
"Failed to initialize extension functions: " << e.what();
325 }
catch (
const std::exception& e) {
326 LOG(
FATAL) <<
"Failed to initialize table functions factory: " << e.what();
332 std::vector<TUserDefinedFunction> udfs = {};
333 calcite_->setRuntimeExtensionFunctions(udfs, udtfs,
false);
334 }
catch (
const std::exception& e) {
335 LOG(
FATAL) <<
"Failed to register compile-time table functions: " << e.what();
338 if (!data_mgr_->gpusPresent() && !cpu_mode_only_) {
340 LOG(
ERROR) <<
"No GPUs detected, falling back to CPU mode";
341 cpu_mode_only_ =
true;
344 switch (executor_device_type_) {
346 LOG(
INFO) <<
"Started in GPU mode" << std::endl;
349 LOG(
INFO) <<
"Started in CPU mode" << std::endl;
354 SysCatalog::instance().init(base_data_path_,
361 }
catch (
const std::exception& e) {
362 LOG(
FATAL) <<
"Failed to initialize system catalog: " << e.what();
365 import_path_ = boost::filesystem::path(base_data_path_) /
"mapd_import";
366 start_time_ = std::time(
nullptr);
368 if (is_rendering_enabled) {
372 max_concurrent_render_sessions,
373 render_compositor_use_last_gpu,
377 system_parameters_));
378 }
catch (
const std::exception& e) {
379 LOG(
ERROR) <<
"Backend rendering disabled: " << e.what();
383 if (leaf_aggregator_.leafCount() > 0) {
386 }
catch (
const std::exception& e) {
387 LOG(
ERROR) <<
"Distributed aggregator support disabled: " << e.what();
392 }
catch (
const std::exception& e) {
393 LOG(
ERROR) <<
"Distributed leaf support disabled: " << e.what();
398 if (!libgeos_so_filename.empty()) {
399 g_libgeos_so_filename.reset(
new std::string(libgeos_so_filename));
400 LOG(
INFO) <<
"Overriding default geos library with '" + *g_libgeos_so_filename +
"'";
408 const std::string& query_str,
409 std::list<std::unique_ptr<Parser::Stmt>>& parse_trees) {
410 int num_parse_errors = 0;
411 std::string last_parsed;
413 num_parse_errors = parser.parse(query_str, parse_trees, last_parsed);
414 if (num_parse_errors > 0) {
415 throw std::runtime_error(
"Syntax error at: " + last_parsed);
417 if (parse_trees.size() > 1) {
418 throw std::runtime_error(
"multiple SQL statements not allowed");
419 }
else if (parse_trees.size() != 1) {
420 throw std::runtime_error(
"empty SQL statment not allowed");
430 const std::shared_ptr<Catalog_Namespace::Catalog>& catalog_ptr) {
437 std::string session_id;
442 calcite_->getInternalSessionProxyUserName(),
443 calcite_->getInternalSessionProxyPassword(),
447 const auto emplace_ret =
449 std::make_shared<Catalog_Namespace::SessionInfo>(
451 CHECK(emplace_ret.second);
465 const auto it =
sessions_.find(session_id);
472 const std::string& username,
473 const std::string& dbname) {
475 std::string username2 = username;
476 std::string dbname2 = dbname;
478 std::shared_ptr<Catalog>
cat =
nullptr;
481 SysCatalog::instance().login(dbname2, username2, std::string(), user_meta,
false);
482 }
catch (std::exception& e) {
489 std::vector<DBObject> dbObjects;
490 dbObjects.push_back(dbObject);
491 if (!SysCatalog::instance().checkPrivileges(user_meta, dbObjects)) {
493 " is not allowed to access database " + dbname2 +
".");
495 connect_impl(session, std::string(), dbname2, user_meta, cat, stdlog);
499 const std::string& inputToken,
500 const std::string& dbname) {
505 const std::string& username,
506 const std::string& passwd,
507 const std::string& dbname) {
510 std::string username2 = username;
511 std::string dbname2 = dbname;
513 std::shared_ptr<Catalog>
cat =
nullptr;
515 cat = SysCatalog::instance().login(
517 }
catch (std::exception& e) {
518 stdlog.appendNameValuePairs(
"user", username,
"db", dbname,
"exception", e.what());
525 std::vector<DBObject> dbObjects;
526 dbObjects.push_back(dbObject);
527 if (!SysCatalog::instance().checkPrivileges(user_meta, dbObjects)) {
528 stdlog.appendNameValuePairs(
529 "user", username,
"db", dbname,
"exception",
"Missing Privileges");
531 " is not allowed to access database " + dbname2 +
".");
533 connect_impl(session, passwd, dbname2, user_meta, cat, stdlog);
544 SysCatalog::instance().check_for_session_encryption(passwd, session);
549 const std::string& dbname,
551 std::shared_ptr<Catalog>
cat) {
555 std::pair<SessionMap::iterator, bool> emplace_retval =
557 std::make_shared<Catalog_Namespace::SessionInfo>(
559 CHECK(emplace_retval.second);
560 auto& session_ptr = emplace_retval.first->second;
566 const std::string& passwd,
567 const std::string& dbname,
569 std::shared_ptr<Catalog>
cat,
589 ? std::vector<std::string>{{
"super"}}
590 : SysCatalog::instance().getRoles(
601 stdlog.setSessionInfo(session_it->second);
602 const auto dbname = session_it->second->getCatalog().getCurrentDB().dbName;
604 LOG(
INFO) <<
"User " << session_it->second->get_currentUser().userLoggable()
605 <<
" disconnected from database " << dbname
606 <<
" with public_session_id: " << session_it->second->get_public_session_id();
612 mapd_unique_lock<mapd_shared_mutex>&
write_lock) {
615 const auto session_id = session_it->second->get_session_id();
633 std::string dbname2 = dbname;
636 std::shared_ptr<Catalog>
cat = SysCatalog::instance().switchDatabase(
637 dbname2, session_it->second->get_currentUser().userName);
638 session_it->second->set_catalog_ptr(cat);
643 }
catch (std::exception& e) {
656 session_it->second->get_currentUser();
657 std::shared_ptr<Catalog>
cat = session_it->second->get_catalog_ptr();
663 }
catch (std::exception& e) {
669 const TSessionId& interrupt_session) {
681 auto&
cat = session_it->second.get()->getCatalog();
682 const auto dbname =
cat.getCurrentDB().dbName;
689 VLOG(1) <<
"Received interrupt: "
690 <<
"Session " << *session_it->second <<
", Executor " << executor
692 << session_it->second->get_currentUser().userLoggable() <<
", Database "
693 << dbname << std::endl;
698 executor->interrupt(query_session, interrupt_session);
700 LOG(
INFO) <<
"User " << session_it->second->get_currentUser().userLoggable()
701 <<
" interrupted session with database " << dbname << std::endl;
711 _return.rendering_enabled = rendering_enabled;
712 _return.poly_rendering_enabled = rendering_enabled;
719 const TSessionId& session) {
726 ret.rendering_enabled = rendering_enabled;
727 ret.poly_rendering_enabled = rendering_enabled;
738 ret.role = TRole::type::SERVER;
741 _return.push_back(ret);
744 _return.insert(_return.end(), leaf_status.begin(), leaf_status.end());
749 const TSessionId& session) {
753 const auto cuda_mgr =
data_mgr_->getCudaMgr();
755 ret.num_gpu_hw = cuda_mgr->getDeviceCount();
756 ret.start_gpu = cuda_mgr->getStartGpu();
757 if (ret.start_gpu >= 0) {
758 ret.num_gpu_allocated = cuda_mgr->getDeviceCount() - cuda_mgr->getStartGpu();
761 for (int16_t device_id = 0; device_id < ret.num_gpu_hw; device_id++) {
762 TGpuSpecification gpu_spec;
763 auto deviceProperties = cuda_mgr->getDeviceProperties(device_id);
764 gpu_spec.num_sm = deviceProperties->numMPs;
765 gpu_spec.clock_frequency_kHz = deviceProperties->clockKhz;
766 gpu_spec.memory = deviceProperties->globalMem;
767 gpu_spec.compute_capability_major = deviceProperties->computeMajor;
768 gpu_spec.compute_capability_minor = deviceProperties->computeMinor;
769 ret.gpu_info.push_back(gpu_spec);
774 ret.num_cpu_hw = std::thread::hardware_concurrency();
778 _return.hardware_info.push_back(ret);
780 ret.host_name =
"aggregator";
782 _return.hardware_info.insert(_return.hardware_info.end(),
783 leaf_hardware.hardware_info.begin(),
784 leaf_hardware.hardware_info.end());
790 auto stdlog =
STDLOG(session_ptr);
792 auto user_metadata = session_ptr->get_currentUser();
794 _return.user = user_metadata.userName;
795 _return.database = session_ptr->getCatalog().getCurrentDB().dbName;
796 _return.start_time = session_ptr->get_start_time();
797 _return.is_super = user_metadata.isSuper;
805 const auto array_tv = boost::get<ArrayTargetValue>(&tv);
807 bool is_null = !array_tv->is_initialized();
809 const auto& vec = array_tv->get();
810 for (
const auto& elem_tv : vec) {
814 column.data.arr_col.push_back(tColumn);
815 column.nulls.push_back(is_null && !ti.
get_notnull());
817 const auto scalar_tv = boost::get<ScalarTargetValue>(&tv);
819 auto s_n = boost::get<NullableString>(scalar_tv);
820 auto s = boost::get<std::string>(s_n);
822 column.data.str_col.push_back(*s);
824 column.data.str_col.emplace_back(
"");
825 auto null_p = boost::get<void*>(s_n);
826 CHECK(null_p && !*null_p);
830 const auto array_tv = boost::get<ArrayTargetValue>(&tv);
832 bool is_null = !array_tv->is_initialized();
836 const auto& vec = array_tv->get();
837 for (
const auto& elem_tv : vec) {
840 column.data.arr_col.push_back(tColumn);
841 column.nulls.push_back(
false);
844 column.data.arr_col.push_back(tColumn);
845 column.nulls.push_back(is_null && !ti.
get_notnull());
850 const auto scalar_tv = boost::get<ScalarTargetValue>(&tv);
852 if (boost::get<int64_t>(scalar_tv)) {
853 int64_t data = *(boost::get<int64_t>(scalar_tv));
856 double val =
static_cast<double>(data);
858 val /= pow(10.0, std::abs(ti.
get_scale()));
860 column.data.real_col.push_back(val);
862 column.data.int_col.push_back(data);
891 column.nulls.push_back(
false);
893 }
else if (boost::get<double>(scalar_tv)) {
894 double data = *(boost::get<double>(scalar_tv));
895 column.data.real_col.push_back(data);
901 }
else if (boost::get<float>(scalar_tv)) {
903 float data = *(boost::get<float>(scalar_tv));
904 column.data.real_col.push_back(data);
906 }
else if (boost::get<NullableString>(scalar_tv)) {
907 auto s_n = boost::get<NullableString>(scalar_tv);
908 auto s = boost::get<std::string>(s_n);
910 column.data.str_col.push_back(*s);
912 column.data.str_col.emplace_back(
"");
913 auto null_p = boost::get<void*>(s_n);
914 CHECK(null_p && !*null_p);
925 const auto scalar_tv = boost::get<ScalarTargetValue>(&tv);
928 const auto array_tv = boost::get<ArrayTargetValue>(&tv);
930 if (array_tv->is_initialized()) {
931 const auto& vec = array_tv->get();
932 for (
const auto& elem_tv : vec) {
934 datum.val.arr_val.push_back(scalar_col_val);
937 datum.is_null =
false;
939 datum.is_null =
true;
943 if (boost::get<int64_t>(scalar_tv)) {
944 int64_t data = *(boost::get<int64_t>(scalar_tv));
947 double val =
static_cast<double>(data);
949 val /= pow(10.0, std::abs(ti.
get_scale()));
951 datum.val.real_val = val;
953 datum.val.int_val = data;
967 datum.is_null = (datum.val.int_val ==
NULL_INT);
972 datum.is_null = (datum.val.int_val ==
NULL_BIGINT);
979 datum.is_null = (datum.val.int_val ==
NULL_BIGINT);
982 datum.is_null =
false;
984 }
else if (boost::get<double>(scalar_tv)) {
985 datum.val.real_val = *(boost::get<double>(scalar_tv));
987 datum.is_null = (datum.val.real_val ==
NULL_FLOAT);
989 datum.is_null = (datum.val.real_val ==
NULL_DOUBLE);
991 }
else if (boost::get<float>(scalar_tv)) {
993 datum.val.real_val = *(boost::get<float>(scalar_tv));
994 datum.is_null = (datum.val.real_val ==
NULL_FLOAT);
995 }
else if (boost::get<NullableString>(scalar_tv)) {
996 auto s_n = boost::get<NullableString>(scalar_tv);
997 auto s = boost::get<std::string>(s_n);
999 datum.val.str_val = *s;
1001 auto null_p = boost::get<void*>(s_n);
1002 CHECK(null_p && !*null_p);
1012 const TSessionId& session,
1013 const std::string& query_str,
1014 const bool column_format,
1015 const std::string& nonce,
1016 const int32_t first_n,
1017 const int32_t at_most_n) {
1020 auto stdlog =
STDLOG(session_ptr, query_state);
1022 stdlog.appendNameValuePairs(
"nonce", nonce);
1026 ScopeGuard reset_was_geo_copy_from = [
this, &session_ptr] {
1030 if (first_n >= 0 && at_most_n >= 0) {
1032 std::string(
"At most one of first_n and at_most_n can be set"));
1041 query_state->createQueryStateProxy(),
1042 query_state->getQueryStr(),
1049 _return.nonce = nonce;
1053 query_state->createQueryStateProxy(),
1056 session_ptr->get_executor_device_type(),
1064 if (
auto geo_copy_from_state =
1074 TCreateParams create_params;
1075 if (geo_copy_from_state->geo_copy_from_partitions ==
"REPLICATED") {
1076 create_params.is_replicated =
true;
1083 geo_copy_from_state->geo_copy_from_table,
1084 geo_copy_from_state->geo_copy_from_file_name,
1090 std::string debug_json = timer.stopAndGetJson();
1091 if (!debug_json.empty()) {
1092 _return.__set_debug(std::move(debug_json));
1094 stdlog.appendNameValuePairs(
1095 "execution_time_ms",
1096 _return.execution_time_ms,
1098 stdlog.duration<std::chrono::milliseconds>());
1101 }
catch (
const std::exception& e) {
1102 if (strstr(e.what(),
"java.lang.NullPointerException")) {
1104 "query failed from broken view or other schema related issue");
1105 }
else if (strstr(e.what(),
"Parse failed: Encountered \";\"")) {
1107 }
else if (strstr(e.what(),
1108 "Parse failed: Encountered \"<EOF>\" at line 0, column 0")) {
1117 const TSessionId& session,
1118 const std::string& query_str,
1120 const int32_t device_id,
1121 const int32_t first_n,
1125 auto stdlog =
STDLOG(session_ptr, query_state);
1127 if (device_type == TDeviceType::GPU) {
1128 const auto executor_device_type = session_ptr->get_executor_device_type();
1131 std::string(
"Exception: GPU mode is not allowed in this session"));
1136 if (device_id < 0 || device_id >=
data_mgr_->getCudaMgr()->getDeviceCount()) {
1138 std::string(
"Exception: invalid device_id or unavailable GPU with this ID"));
1141 _return.execution_time_ms = 0;
1143 mapd_shared_lock<mapd_shared_mutex> executeReadLock(
1146 auto query_state_proxy = query_state->createQueryStateProxy();
1149 if (!pw.is_ddl && !pw.is_update_dml &&
1151 std::string query_ra;
1155 std::tie(result, locks) =
1157 query_ra = result.plan_result;
1160 if (pw.isCalciteExplain()) {
1161 throw std::runtime_error(
"explain is not unsupported by current thrift API");
1165 auto submitted_time = std::chrono::system_clock::now();
1166 query_state_proxy.getQueryState().setQuerySubmittedTime(submitted_time);
1167 executor->enrollQuerySession(session_ptr->get_session_id(),
1171 QuerySessionStatus::QueryStatus::PENDING_QUEUE);
1179 static_cast<size_t>(device_id),
1184 }
catch (std::exception& e) {
1188 "Exception: DDL or update DML are not unsupported by current thrift API");
1192 const TSessionId& session,
1193 const std::string& query_str,
1194 const int32_t device_id,
1195 const int32_t first_n) {
1203 TArrowTransport::SHARED_MEMORY);
1208 const TDataFrame& df,
1210 const int32_t device_id) {
1212 std::string serialized_cuda_handle =
"";
1213 if (device_type == TDeviceType::GPU) {
1216 TOmniSciException ex;
1217 ex.error_msg = std::string(
1218 "Exception: current data frame handle is not bookkept or been inserted "
1226 std::vector<char> sm_handle(df.sm_handle.begin(), df.sm_handle.end());
1227 std::vector<char> df_handle(df.df_handle.begin(), df.df_handle.end());
1229 sm_handle, df.sm_size, df_handle, df.df_size, serialized_cuda_handle};
1242 boost::regex copy_to{R
"(COPY\s*\(([^#])(.+)\)\s+TO\s)",
1243 boost::regex::extended | boost::regex::icase};
1246 what.position(), what.length(),
"COPY (#~#" + what[1] + what[2] +
"#~#) TO ");
1253 const TSessionId& session,
1254 const std::string& query_str) {
1259 stdlog.setQueryState(query_state);
1264 throw std::runtime_error(
"Can only validate SELECT statements.");
1267 const auto execute_read_lock = mapd_shared_lock<mapd_shared_mutex>(
1271 TPlanResult parse_result;
1273 std::tie(parse_result, locks) =
parse_to_ra(query_state->createQueryStateProxy(),
1274 query_state->getQueryStr(),
1279 const auto query_ra = parse_result.plan_result;
1283 query_state->getConstSessionInfo()->getCatalog());
1284 }
catch (
const std::exception& e) {
1300 const std::string& sql) {
1301 boost::regex id_regex{R
"(([[:alnum:]]|_|\.)+)",
1302 boost::regex::extended | boost::regex::icase};
1303 boost::sregex_token_iterator tok_it(sql.begin(), sql.end(), id_regex, 0);
1304 boost::sregex_token_iterator end;
1305 std::unordered_set<std::string> uc_column_names;
1306 std::unordered_set<std::string> uc_column_table_qualifiers;
1307 for (; tok_it != end; ++tok_it) {
1308 std::string column_name = *tok_it;
1309 std::vector<std::string> column_tokens;
1310 boost::split(column_tokens, column_name, boost::is_any_of(
"."));
1311 if (column_tokens.size() == 2) {
1313 uc_column_table_qualifiers.insert(
to_upper(column_tokens.front()));
1315 uc_column_names.insert(
to_upper(column_name));
1318 return {uc_column_names, uc_column_table_qualifiers};
1324 const TSessionId& session,
1325 const std::string& sql,
1328 std::vector<std::string> visible_tables;
1332 proj_tokens.uc_column_names, visible_tables, stdlog);
1334 compatible_table_names.insert(proj_tokens.uc_column_table_qualifiers.begin(),
1335 proj_tokens.uc_column_table_qualifiers.end());
1340 [&compatible_table_names](
const TCompletionHint& lhs,
const TCompletionHint& rhs) {
1341 if (lhs.type == TCompletionHintType::TABLE &&
1342 rhs.type == TCompletionHintType::TABLE) {
1345 if (compatible_table_names.find(
to_upper(lhs.hints.back())) !=
1346 compatible_table_names.end() &&
1347 compatible_table_names.find(
to_upper(rhs.hints.back())) ==
1348 compatible_table_names.end()) {
1352 return lhs.type < rhs.type;
1357 std::vector<std::string>& visible_tables,
1359 const std::string& sql,
1366 calcite_->getCompletionHints(session_info, visible_tables, sql, cursor));
1367 }
catch (
const std::exception& e) {
1368 TOmniSciException ex;
1369 ex.error_msg =
"Exception: " + std::string(e.what());
1373 boost::regex from_expr{R
"(\s+from\s+)", boost::regex::extended | boost::regex::icase};
1374 const size_t length_to_cursor =
1375 cursor < 0 ? sql.size() : std::min(sql.size(),
static_cast<size_t>(cursor));
1377 if (boost::regex_search(sql.cbegin(), sql.cbegin() + length_to_cursor, from_expr)) {
1386 std::vector<std::string>& visible_tables,
1387 const std::string& sql,
1389 const auto last_word =
1391 boost::regex select_expr{R
"(\s*select\s+)",
1392 boost::regex::extended | boost::regex::icase};
1393 const size_t length_to_cursor =
1394 cursor < 0 ? sql.size() : std::min(sql.size(),
static_cast<size_t>(cursor));
1397 if (boost::regex_search(sql.cbegin(), sql.cbegin() + length_to_cursor, select_expr)) {
1408 const std::string kFromKeyword{
"FROM"};
1409 if (boost::istarts_with(kFromKeyword, last_word)) {
1410 TCompletionHint keyword_hint;
1411 keyword_hint.type = TCompletionHintType::KEYWORD;
1412 keyword_hint.replaced = last_word;
1413 keyword_hint.hints.emplace_back(kFromKeyword);
1414 hints.push_back(keyword_hint);
1417 const std::string kSelectKeyword{
"SELECT"};
1418 if (boost::istarts_with(kSelectKeyword, last_word)) {
1419 TCompletionHint keyword_hint;
1420 keyword_hint.type = TCompletionHintType::KEYWORD;
1421 keyword_hint.replaced = last_word;
1422 keyword_hint.hints.emplace_back(kSelectKeyword);
1423 hints.push_back(keyword_hint);
1428 std::unordered_map<std::string, std::unordered_set<std::string>>
1431 std::unordered_map<std::string, std::unordered_set<std::string>> column_names_by_table;
1432 for (
auto it = table_names.begin(); it != table_names.end();) {
1433 TTableDetails table_details;
1436 }
catch (
const TOmniSciException& e) {
1438 it = table_names.erase(it);
1441 for (
const auto& column_type : table_details.row_desc) {
1442 column_names_by_table[*it].emplace(column_type.col_name);
1446 return column_names_by_table;
1455 const std::unordered_set<std::string>& uc_column_names,
1456 std::vector<std::string>& table_names,
1458 std::unordered_set<std::string> compatible_table_names_by_column;
1459 for (
auto it = table_names.begin(); it != table_names.end();) {
1460 TTableDetails table_details;
1463 }
catch (
const TOmniSciException& e) {
1465 it = table_names.erase(it);
1468 for (
const auto& column_type : table_details.row_desc) {
1469 if (uc_column_names.find(
to_upper(column_type.col_name)) != uc_column_names.end()) {
1470 compatible_table_names_by_column.emplace(
to_upper(*it));
1476 return compatible_table_names_by_column;
1482 auto execute_rel_alg_task = std::make_shared<QueryDispatchQueue::Task>(
1483 [
this, &
result, &query_state_proxy, &query_ra](
const size_t executor_index) {
1498 auto result_future = execute_rel_alg_task->get_future();
1499 result_future.get();
1505 auto session_ptr = stdlog.getConstSessionInfo();
1506 if (!session_ptr->get_currentUser().isSuper) {
1510 SysCatalog::instance().getRoles(session_ptr->get_currentUser().userName,
1511 session_ptr->getCatalog().getCurrentDB().dbId);
1513 roles = SysCatalog::instance().getRoles(
1514 false,
true, session_ptr->get_currentUser().userName);
1519 const std::string& granteeName,
1520 const std::string& roleName) {
1522 const auto session_ptr = stdlog.getConstSessionInfo();
1523 const auto current_user = session_ptr->get_currentUser();
1524 if (!current_user.isSuper) {
1525 if (
const auto* user = SysCatalog::instance().getUserGrantee(granteeName);
1526 user && current_user.userName != granteeName) {
1528 }
else if (!SysCatalog::instance().isRoleGrantedToGrantee(
1529 current_user.userName, granteeName,
true)) {
1531 "Only super users can check roles assignment that have not been directly "
1532 "granted to a user.");
1535 return SysCatalog::instance().isRoleGrantedToGrantee(granteeName, roleName,
false);
1540 TDBObject outObject;
1541 outObject.objectName = inObject.
getName();
1542 outObject.grantee = roleName;
1593 const int type_val =
static_cast<int>(inObject.
getType());
1594 CHECK(type_val >= 0 && type_val < 6);
1600 const TDBObjectPermissions& permissions) {
1601 if (!permissions.__isset.database_permissions_) {
1604 auto perms = permissions.database_permissions_;
1607 (perms.view_sql_editor_ &&
1617 const TDBObjectPermissions& permissions) {
1618 if (!permissions.__isset.table_permissions_) {
1621 auto perms = permissions.table_permissions_;
1637 const TDBObjectPermissions& permissions) {
1638 if (!permissions.__isset.dashboard_permissions_) {
1641 auto perms = permissions.dashboard_permissions_;
1653 const TDBObjectPermissions& permissions) {
1654 if (!permissions.__isset.view_permissions_) {
1657 auto perms = permissions.view_permissions_;
1671 const TDBObjectPermissions& permissions) {
1672 CHECK(permissions.__isset.server_permissions_);
1673 auto perms = permissions.server_permissions_;
1684 const std::string& granteeName,
1685 const std::string& objectName,
1687 const TDBObjectPermissions& permissions) {
1689 auto session_ptr = stdlog.getConstSessionInfo();
1690 auto const&
cat = session_ptr->getCatalog();
1691 auto const& current_user = session_ptr->get_currentUser();
1692 if (!current_user.isSuper && !SysCatalog::instance().isRoleGrantedToGrantee(
1693 current_user.userName, granteeName,
false)) {
1695 "Users except superusers can only check privileges for self or roles granted "
1700 if (SysCatalog::instance().getMetadataForUser(granteeName, user_meta) &&
1704 Grantee* grnt = SysCatalog::instance().getGrantee(granteeName);
1709 std::string func_name;
1710 switch (objectType) {
1713 func_name =
"database";
1717 func_name =
"table";
1721 func_name =
"dashboard";
1729 func_name =
"server";
1734 DBObject req_object(objectName, type);
1738 if (grantee_object) {
1748 const TSessionId& sessionId,
1749 const std::string& roleName) {
1751 auto session_ptr = stdlog.getConstSessionInfo();
1752 auto const& user = session_ptr->get_currentUser();
1753 if (!user.isSuper &&
1754 !SysCatalog::instance().isRoleGrantedToGrantee(user.userName, roleName,
false)) {
1757 auto* rl = SysCatalog::instance().getGrantee(roleName);
1759 auto dbId = session_ptr->getCatalog().getCurrentDB().dbId;
1760 for (
auto& dbObject : *rl->getDbObjects(
true)) {
1761 if (dbObject.first.dbId != dbId) {
1767 TDBObjectsForRole.push_back(tdbObject);
1775 const TSessionId& sessionId,
1776 const std::string& objectName,
1779 auto session_ptr = stdlog.getConstSessionInfo();
1801 DBObject object_to_find(objectName, object_type);
1806 if (objectName ==
"") {
1807 object_to_find =
DBObject(-1, object_type);
1809 object_to_find =
DBObject(std::stoi(objectName), object_type);
1812 !objectName.empty()) {
1814 auto const&
cat = session_ptr->getCatalog();
1815 auto td =
cat.getMetadataForTable(objectName,
false);
1818 object_to_find =
DBObject(objectName, object_type);
1821 object_to_find.
loadKey(session_ptr->getCatalog());
1822 }
catch (
const std::exception&) {
1827 DBObject object_to_find_dblevel(
"", object_type);
1828 object_to_find_dblevel.
loadKey(session_ptr->getCatalog());
1830 if (session_ptr->get_currentUser().isSuper) {
1834 session_ptr->get_currentUser().userId};
1835 dbObj.setName(
"super");
1836 TDBObjects.push_back(
1840 std::vector<std::string> grantees =
1841 SysCatalog::instance().getRoles(
true,
1842 session_ptr->get_currentUser().isSuper,
1843 session_ptr->get_currentUser().userName);
1844 for (
const auto& grantee : grantees) {
1846 auto* gr = SysCatalog::instance().getGrantee(grantee);
1847 if (gr && (object_found = gr->findDbObject(object_to_find.
getObjectKey(),
true))) {
1852 (object_found = gr->findDbObject(object_to_find_dblevel.
getObjectKey(),
true))) {
1859 const TSessionId& sessionId,
1860 const std::string& granteeName) {
1862 auto session_ptr = stdlog.getConstSessionInfo();
1863 auto* grantee = SysCatalog::instance().getGrantee(granteeName);
1865 if (session_ptr->get_currentUser().isSuper) {
1866 roles = grantee->getRoles();
1867 }
else if (grantee->isUser()) {
1868 if (session_ptr->get_currentUser().userName == granteeName) {
1869 roles = grantee->getRoles();
1872 "Only a superuser is authorized to request list of roles granted to "
1877 CHECK(!grantee->isUser());
1880 if (SysCatalog::instance().isRoleGrantedToGrantee(
1881 session_ptr->get_currentUser().userName, granteeName,
false)) {
1882 roles = grantee->getRoles();
1894 const std::map<std::string, std::vector<std::string>>& table_col_names) {
1895 std::ostringstream oss;
1896 for (
const auto& [table_name, col_names] : table_col_names) {
1897 oss <<
":" << table_name;
1898 for (
const auto& col_name : col_names) {
1899 oss <<
"," << col_name;
1907 TPixelTableRowResult& _return,
1908 const TSessionId& session,
1909 const int64_t widget_id,
1910 const TPixel& pixel,
1911 const std::map<std::string, std::vector<std::string>>& table_col_names,
1912 const bool column_format,
1913 const int32_t pixel_radius,
1914 const std::string& nonce) {
1931 auto session_ptr = stdlog.getSessionInfo();
1945 }
catch (std::exception& e) {
1952 TColumnType col_type;
1976 col_type.col_type.comp_param = 0;
1983 col_type.col_type.comp_param = dd->dictNBits;
1985 col_type.col_type.comp_param =
1995 const TSessionId& session,
1996 const std::string& table_name) {
2003 const TSessionId& session,
2004 const std::string& table_name) {
2012 const std::string& table_name,
2013 const bool get_system,
2014 const bool get_physical) {
2017 auto&
cat = session_info->getCatalog();
2018 const auto td_with_lock =
2020 cat, table_name,
false);
2021 const auto td = td_with_lock();
2024 bool have_privileges_on_view_sources =
true;
2032 const auto query_ra =
parse_to_ra(query_state->createQueryStateProxy(),
2033 query_state->getQueryStr(),
2039 calcite_->checkAccessedObjectsPrivileges(query_state->createQueryStateProxy(),
2041 }
catch (
const std::runtime_error&) {
2042 have_privileges_on_view_sources =
false;
2046 query_state->createQueryStateProxy());
2050 throw std::runtime_error(
2051 "Unable to access view " + table_name +
2052 ". The view may not exist, or the logged in user may not "
2053 "have permission to access the view.");
2055 }
catch (
const std::exception& e) {
2056 throw std::runtime_error(
"View '" + table_name +
2057 "' query has failed with an error: '" +
2058 std::string(e.what()) +
2059 "'.\nThe view must be dropped and re-created to "
2060 "resolve the error. \nQuery:\n" +
2061 query_state->getQueryStr());
2065 const auto col_descriptors =
2066 cat.getAllColumnMetadataForTable(td->tableId, get_system,
true, get_physical);
2067 const auto deleted_cd =
cat.getDeletedColumn(td);
2068 for (
const auto cd : col_descriptors) {
2069 if (cd == deleted_cd) {
2075 throw std::runtime_error(
2076 "Unable to access table " + table_name +
2077 ". The table may not exist, or the logged in user may not "
2078 "have permission to access the table.");
2081 _return.fragment_size = td->maxFragRows;
2082 _return.page_size = td->fragPageSize;
2083 _return.max_rows = td->maxRows;
2085 (have_privileges_on_view_sources ? td->viewSQL
2086 :
"[Not enough privileges to see the view SQL]");
2087 _return.shard_count = td->nShards;
2088 _return.key_metainfo = td->keyMetainfo;
2090 _return.partition_detail =
2091 td->partitions.empty()
2092 ? TPartitionDetail::DEFAULT
2094 ? TPartitionDetail::REPLICATED
2095 : (td->partitions ==
"SHARDED" ? TPartitionDetail::SHARDED
2096 : TPartitionDetail::OTHER));
2097 }
catch (
const std::runtime_error& e) {
2103 const TSessionId& session,
2104 const std::string& link) {
2107 auto session_ptr = stdlog.getConstSessionInfo();
2108 auto const&
cat = session_ptr->getCatalog();
2113 _return.view_state = ld->viewState;
2114 _return.view_name = ld->link;
2115 _return.update_time = ld->updateTime;
2116 _return.view_metadata = ld->viewMetadata;
2125 if (user_metadata.isSuper) {
2131 std::vector<DBObject> privObjects = {dbObject};
2133 return SysCatalog::instance().hasAnyPrivileges(user_metadata, privObjects);
2144 const TSessionId& session) {
2152 const TSessionId& session) {
2159 const TSessionId& session) {
2168 const bool with_table_locks) {
2170 const auto tables =
cat.getAllTableMetadata();
2171 _return.reserve(
tables.size());
2173 for (
const auto td :
tables) {
2174 if (td->shard >= 0) {
2184 ret.table_name = td->tableName;
2185 ret.is_view = td->isView;
2187 ret.shard_count = td->nShards;
2188 ret.max_rows = td->maxRows;
2189 ret.table_id = td->tableId;
2191 std::vector<TTypeInfo> col_types;
2192 std::vector<std::string> col_names;
2193 size_t num_cols = 0;
2196 TPlanResult parse_result;
2200 const auto query_ra = parse_result.plan_result;
2213 num_cols = result.row_set.row_desc.size();
2214 for (
const auto& col : result.row_set.row_desc) {
2215 if (col.is_physical) {
2219 col_types.push_back(col.col_type);
2220 col_names.push_back(col.col_name);
2222 }
catch (std::exception& e) {
2223 LOG(
WARNING) <<
"get_tables_meta: Ignoring broken view: " << td->tableName;
2228 const auto col_descriptors =
2229 cat.getAllColumnMetadataForTable(td->tableId,
false,
true,
false);
2230 const auto deleted_cd =
cat.getDeletedColumn(td);
2231 for (
const auto cd : col_descriptors) {
2232 if (cd == deleted_cd) {
2236 col_names.push_back(cd->columnName);
2238 num_cols = col_descriptors.size();
2242 }
catch (
const std::runtime_error& e) {
2247 ret.num_cols = num_cols;
2248 std::copy(col_types.begin(), col_types.end(), std::back_inserter(ret.col_types));
2249 std::copy(col_names.begin(), col_names.end(), std::back_inserter(ret.col_names));
2251 _return.push_back(ret);
2256 const TSessionId& session) {
2259 auto session_ptr = stdlog.getConstSessionInfo();
2261 stdlog.setQueryState(query_state);
2263 auto execute_read_lock = mapd_shared_lock<mapd_shared_mutex>(
2269 }
catch (
const std::exception& e) {
2275 const TSessionId& session) {
2278 auto session_ptr = stdlog.getConstSessionInfo();
2279 std::list<Catalog_Namespace::UserMetadata> user_list;
2281 if (!session_ptr->get_currentUser().isSuper) {
2282 user_list = SysCatalog::instance().getAllUserMetadata(
2283 session_ptr->getCatalog().getCurrentDB().dbId);
2285 user_list = SysCatalog::instance().getAllUserMetadata();
2287 for (
auto u : user_list) {
2288 user_names.push_back(u.userName);
2299 auto session_ptr = stdlog.getConstSessionInfo();
2300 if (!session_ptr->get_currentUser().isSuper) {
2305 }
catch (
const std::exception& e) {
2320 auto session_ptr = stdlog.getConstSessionInfo();
2321 if (!session_ptr->get_currentUser().isSuper) {
2326 }
catch (
const std::exception& e) {
2343 const TSessionId& session,
2344 const std::string& memory_level) {
2347 std::vector<Data_Namespace::MemoryInfo> internal_memory;
2349 if (!memory_level.compare(
"gpu")) {
2359 for (
auto memInfo : internal_memory) {
2360 TNodeMemoryInfo nodeInfo;
2364 nodeInfo.page_size = memInfo.pageSize;
2365 nodeInfo.max_num_pages = memInfo.maxNumPages;
2366 nodeInfo.num_pages_allocated = memInfo.numPageAllocated;
2367 nodeInfo.is_allocation_capped = memInfo.isAllocationCapped;
2368 for (
auto gpu : memInfo.nodeMemoryData) {
2370 md.slab = gpu.slabNum;
2371 md.start_page = gpu.startPage;
2372 md.num_pages = gpu.numPages;
2373 md.touch = gpu.touch;
2374 md.chunk_key.insert(md.chunk_key.end(), gpu.chunk_key.begin(), gpu.chunk_key.end());
2376 nodeInfo.node_memory_data.push_back(md);
2378 _return.push_back(nodeInfo);
2381 std::vector<TNodeMemoryInfo> leafSummary =
2383 _return.insert(_return.begin(), leafSummary.begin(), leafSummary.end());
2390 auto session_ptr = stdlog.getConstSessionInfo();
2391 const auto& user = session_ptr->get_currentUser();
2393 SysCatalog::instance().getDatabaseListForUser(user);
2394 for (
auto& db : dbs) {
2396 dbinfo.db_name = std::move(db.dbName);
2397 dbinfo.db_owner = std::move(db.dbOwnerName);
2398 dbinfos.push_back(std::move(dbinfo));
2412 }
catch (
const TOmniSciException& e) {
2413 LOG(
INFO) <<
"Aggregator failed to set execution mode: " << e.error_msg;
2424 throw std::runtime_error(
"Cannot import a sharded table directly to a leaf");
2429 const std::vector<std::string>& column_names) {
2430 std::unordered_set<std::string> unique_names;
2431 for (
const auto&
name : column_names) {
2433 if (unique_names.find(lower_name) != unique_names.end()) {
2436 unique_names.insert(lower_name);
2439 for (
const auto& cd : descs) {
2440 auto iter = unique_names.find(
to_lower(cd->columnName));
2441 if (iter != unique_names.end()) {
2442 unique_names.erase(iter);
2445 if (!unique_names.empty()) {
2455 const std::vector<std::string>& column_names) {
2456 std::vector<int> desc_to_column_ids;
2457 if (column_names.empty()) {
2459 for (
const auto& cd : descs) {
2460 if (!cd->isGeoPhyCol) {
2461 desc_to_column_ids.push_back(col_idx);
2466 for (
const auto& cd : descs) {
2467 if (!cd->isGeoPhyCol) {
2469 for (
size_t j = 0;
j < column_names.size(); ++
j) {
2472 desc_to_column_ids.push_back(j);
2477 if (!cd->columnType.get_notnull()) {
2478 desc_to_column_ids.push_back(-1);
2481 "' cannot be omitted due to NOT NULL constraint");
2487 return desc_to_column_ids;
2493 const std::string& table_name,
2494 const std::vector<TRow>& rows,
2495 const std::vector<std::string>& column_names) {
2499 auto session_ptr = stdlog.getConstSessionInfo();
2501 auto&
cat = session_ptr->getCatalog();
2502 const auto td_with_lock =
2504 cat, table_name,
true);
2505 const auto td = td_with_lock();
2515 std::unique_ptr<import_export::Loader> loader;
2521 auto col_descs = loader->get_column_descs();
2526 if (column_names.empty() &&
2527 rows.front().cols.size() !=
2528 static_cast<size_t>(td->nColumns) - (td->hasDeletedCol ? 2 : 1)) {
2530 }
else if (!column_names.empty() && rows.front().cols.size() != column_names.size()) {
2532 "Number of columns specified does not match the "
2533 "number of columns given (" +
2537 std::vector<std::unique_ptr<import_export::TypedImportBuffer>> import_buffers;
2538 for (
auto cd : col_descs) {
2539 import_buffers.push_back(std::unique_ptr<import_export::TypedImportBuffer>(
2543 empty_value.is_null =
true;
2544 size_t rows_completed = 0;
2545 for (
auto const& row : rows) {
2548 for (
auto cd : col_descs) {
2549 int mapped_idx = desc_id_to_column_id[col_idx];
2550 if (mapped_idx != -1) {
2551 import_buffers[col_idx]->add_value(
2552 cd, row.cols[mapped_idx], row.cols[mapped_idx].is_null);
2554 import_buffers[col_idx]->add_value(cd, empty_value,
true);
2559 }
catch (
const std::exception& e) {
2560 for (
size_t col_idx_to_pop = 0; col_idx_to_pop < col_idx; ++col_idx_to_pop) {
2561 import_buffers[col_idx_to_pop]->pop_value();
2563 LOG(
ERROR) <<
"Input exception thrown: " << e.what()
2564 <<
". Row discarded, issue at column : " << (col_idx + 1)
2565 <<
" data :" << row;
2569 session_ptr->getCatalog(), table_name);
2570 loader->load(import_buffers, rows_completed);
2571 }
catch (
const std::exception& e) {
2576 std::unique_ptr<lockmgr::AbstractLockContainer<const TableDescriptor*>>
2579 const std::string& table_name,
2581 std::unique_ptr<import_export::Loader>* loader,
2582 std::vector<std::unique_ptr<import_export::TypedImportBuffer>>* import_buffers,
2583 const std::vector<std::string>& column_names) {
2584 if (num_cols == 0) {
2589 std::make_unique<lockmgr::TableSchemaLockContainer<lockmgr::ReadLock>>(
2591 cat, table_name,
true));
2592 const auto td = (*td_with_lock)();
2605 auto col_descs = (*loader)->get_column_descs();
2607 if (column_names.empty()) {
2610 auto geo_physical_cols = std::count_if(
2611 col_descs.begin(), col_descs.end(), [](
auto cd) {
return cd->isGeoPhyCol; });
2612 const auto num_table_cols =
static_cast<size_t>(td->nColumns) - geo_physical_cols -
2613 (td->hasDeletedCol ? 2 : 1);
2614 if (num_cols != num_table_cols) {
2615 throw std::runtime_error(
"Number of columns to load (" +
std::to_string(num_cols) +
2616 ") does not match number of columns in table " +
2620 }
else if (num_cols != column_names.size()) {
2622 "Number of columns specified does not match the "
2623 "number of columns given (" +
2627 return std::move(td_with_lock);
2633 if (!column.nulls.empty()) {
2634 return column.nulls.size();
2639 return column.data.int_col.size() + column.data.arr_col.size() +
2640 column.data.real_col.size() + column.data.str_col.size();
2647 const std::string& table_name,
2648 const std::vector<TColumn>& cols,
2649 const std::vector<std::string>& column_names) {
2652 auto session_ptr = stdlog.getConstSessionInfo();
2654 auto const&
cat = session_ptr->getCatalog();
2655 std::unique_ptr<import_export::Loader> loader;
2656 std::vector<std::unique_ptr<import_export::TypedImportBuffer>> import_buffers;
2658 *session_ptr, table_name, cols.size(), &loader, &import_buffers, column_names);
2660 session_ptr->getCatalog(), table_name);
2661 auto desc_id_to_column_id =
2664 size_t import_idx = 0;
2666 TColumn empty_column;
2667 empty_column.nulls.resize(num_rows,
true);
2668 empty_column.data.arr_col.resize(num_rows);
2669 empty_column.data.int_col.resize(num_rows);
2670 empty_column.data.str_col.resize(num_rows);
2671 empty_column.data.real_col.resize(num_rows);
2673 size_t skip_physical_cols = 0;
2674 for (
auto cd : loader->get_column_descs()) {
2675 if (skip_physical_cols > 0) {
2676 if (!cd->isGeoPhyCol) {
2677 throw std::runtime_error(
"Unexpected physical column");
2679 skip_physical_cols--;
2682 int mapped_idx = desc_id_to_column_id[import_idx];
2683 const TColumn& value = mapped_idx == -1 ? empty_column : cols[mapped_idx];
2684 size_t col_rows = import_buffers[col_idx]->add_values(cd, value);
2685 if (col_rows != num_rows) {
2686 std::ostringstream oss;
2687 oss <<
"load_table_binary_columnar: Inconsistent number of rows in column "
2688 << cd->columnName <<
" , expecting " << num_rows <<
" rows, column "
2689 << col_idx <<
" has " << col_rows <<
" rows";
2696 if (cd->columnType.is_geometry()) {
2697 auto geo_col_idx = col_idx - 1;
2698 const auto wkt_or_wkb_hex_column =
2699 import_buffers[geo_col_idx]->getGeoStringBuffer();
2700 std::vector<std::vector<double>> coords_column, bounds_column;
2701 std::vector<std::vector<int>> ring_sizes_column, poly_rings_column;
2702 int render_group = 0;
2704 if (num_rows != wkt_or_wkb_hex_column->size() ||
2712 std::ostringstream oss;
2713 oss <<
"load_table_binary_columnar: Invalid geometry in column "
2728 skip_physical_cols = cd->columnType.get_physical_cols();
2733 }
catch (
const std::exception& e) {
2734 std::ostringstream oss;
2735 oss <<
"load_table_binary_columnar: Input exception thrown: " << e.what()
2736 <<
". Issue at column : " << (col_idx + 1) <<
". Import aborted";
2739 loader->load(import_buffers, num_rows);
2744 #define ARROW_THRIFT_THROW_NOT_OK(s) \
2746 ::arrow::Status _s = (s); \
2747 if (UNLIKELY(!_s.ok())) { \
2748 TOmniSciException ex; \
2749 ex.error_msg = _s.ToString(); \
2750 LOG(ERROR) << s.ToString(); \
2761 auto stream_buffer =
2762 std::make_shared<arrow::Buffer>(
reinterpret_cast<const uint8_t*
>(stream.c_str()),
2763 static_cast<int64_t>(stream.size()));
2765 arrow::io::BufferReader buf_reader(stream_buffer);
2766 std::shared_ptr<arrow::RecordBatchReader> batch_reader;
2768 arrow::ipc::RecordBatchStreamReader::Open(&buf_reader));
2771 std::shared_ptr<arrow::RecordBatch> batch;
2774 if (batch ==
nullptr) {
2777 batches.emplace_back(std::move(batch));
2779 }
catch (
const std::exception& e) {
2780 LOG(
ERROR) <<
"Error parsing Arrow stream: " << e.what() <<
". Import aborted";
2788 const std::string& table_name,
2789 const std::string& arrow_stream,
2790 const bool use_column_names) {
2792 auto session_ptr = stdlog.getConstSessionInfo();
2796 if (batches.size() != 1) {
2799 std::shared_ptr<arrow::RecordBatch> batch = batches[0];
2800 std::unique_ptr<import_export::Loader> loader;
2801 std::vector<std::unique_ptr<import_export::TypedImportBuffer>> import_buffers;
2802 std::vector<std::string> column_names;
2803 if (use_column_names) {
2804 column_names = batch->schema()->field_names();
2808 static_cast<size_t>(batch->num_columns()),
2813 session_ptr->getCatalog(), table_name);
2814 auto desc_id_to_column_id =
2816 size_t num_rows = 0;
2818 std::shared_ptr<arrow::Array> empty_array;
2820 arrow::BooleanBuilder builder;
2823 auto status = builder.Finish(&empty_array);
2830 for (
auto cd : loader->get_column_descs()) {
2831 int mapped_idx = desc_id_to_column_id[col_idx];
2832 if (mapped_idx != -1) {
2833 auto& array = *batch->column(mapped_idx);
2835 num_rows = import_buffers[col_idx]->add_arrow_values(
2836 cd, array,
true, row_slice,
nullptr);
2839 num_rows = import_buffers[col_idx]->add_arrow_values(
2840 cd, *empty_array,
false, row_slice,
nullptr);
2844 }
catch (
const std::exception& e) {
2845 LOG(
ERROR) <<
"Input exception thrown: " << e.what()
2846 <<
". Issue at column : " << (col_idx + 1) <<
". Import aborted";
2851 loader->load(import_buffers, num_rows);
2855 const std::string& table_name,
2856 const std::vector<TStringRow>& rows,
2857 const std::vector<std::string>& column_names) {
2861 auto session_ptr = stdlog.getConstSessionInfo();
2863 auto&
cat = session_ptr->getCatalog();
2864 const auto td_with_lock =
2866 cat, table_name,
true);
2867 const auto td = td_with_lock();
2878 std::unique_ptr<import_export::Loader> loader;
2884 auto col_descs = loader->get_column_descs();
2887 if (column_names.empty()) {
2888 auto geo_physical_cols = std::count_if(
2889 col_descs.begin(), col_descs.end(), [](
auto cd) {
return cd->isGeoPhyCol; });
2892 if (rows.front().cols.size() !=
static_cast<size_t>(td->nColumns) -
2894 (td->hasDeletedCol ? 2 : 1)) {
2900 if (rows.front().cols.size() != column_names.size()) {
2902 "Number of columns specified does not match the "
2903 "number of columns given (" +
2909 std::vector<std::unique_ptr<import_export::TypedImportBuffer>> import_buffers;
2910 for (
auto cd : col_descs) {
2911 import_buffers.push_back(std::unique_ptr<import_export::TypedImportBuffer>(
2915 size_t rows_completed = 0;
2916 for (
auto const& row : rows) {
2917 size_t import_idx = 0;
2920 size_t skip_physical_cols = 0;
2921 for (
auto cd : col_descs) {
2922 if (skip_physical_cols > 0) {
2923 if (!cd->isGeoPhyCol) {
2924 throw std::runtime_error(
"Unexpected physical column");
2926 skip_physical_cols--;
2929 const std::string empty_val =
"";
2930 int mapped_idx = desc_id_to_column_id[import_idx];
2931 const std::string& value =
2932 mapped_idx == -1 ? empty_val : row.cols[mapped_idx].str_val;
2933 bool is_null = mapped_idx == -1 || row.cols[mapped_idx].is_null;
2934 import_buffers[col_idx]->add_value(cd, value, is_null, copy_params);
2938 if (cd->columnType.is_geometry()) {
2940 std::vector<double> coords, bounds;
2941 std::vector<int> ring_sizes, poly_rings;
2942 int render_group = 0;
2945 !is_null ? value : std::string(),
2952 throw std::runtime_error(
"Invalid geometry");
2954 if (cd->columnType.get_type() != ti.get_type()) {
2955 throw std::runtime_error(
"Geometry type mismatch");
2966 skip_physical_cols = cd->columnType.get_physical_cols();
2972 }
catch (
const std::exception& e) {
2973 for (
size_t col_idx_to_pop = 0; col_idx_to_pop < col_idx; ++col_idx_to_pop) {
2974 import_buffers[col_idx_to_pop]->pop_value();
2976 LOG(
ERROR) <<
"Input exception thrown: " << e.what()
2977 <<
". Row discarded, issue at column : " << (col_idx + 1)
2978 <<
" data :" << row;
2982 session_ptr->getCatalog(), table_name);
2983 loader->load(import_buffers, rows_completed);
2984 }
catch (
const std::exception& e) {
2991 if (str.size() == 2 && str[0] ==
'\\') {
2992 if (str[1] ==
't') {
2994 }
else if (str[1] ==
'n') {
2996 }
else if (str[1] ==
'0') {
2998 }
else if (str[1] ==
'\'') {
3000 }
else if (str[1] ==
'\\') {
3009 switch (cp.has_header) {
3010 case TImportHeaderRow::AUTODETECT:
3013 case TImportHeaderRow::NO_HEADER:
3016 case TImportHeaderRow::HAS_HEADER:
3024 copy_params.
quoted = cp.quoted;
3025 if (cp.delimiter.length() > 0) {
3030 if (cp.null_str.length() > 0) {
3031 copy_params.
null_str = cp.null_str;
3033 if (cp.quote.length() > 0) {
3036 if (cp.escape.length() > 0) {
3039 if (cp.line_delim.length() > 0) {
3042 if (cp.array_delim.length() > 0) {
3045 if (cp.array_begin.length() > 0) {
3048 if (cp.array_end.length() > 0) {
3051 if (cp.threads != 0) {
3052 copy_params.
threads = cp.threads;
3054 if (cp.s3_access_key.length() > 0) {
3057 if (cp.s3_secret_key.length() > 0) {
3060 if (cp.s3_region.length() > 0) {
3063 if (cp.s3_endpoint.length() > 0) {
3066 switch (cp.file_type) {
3067 case TFileType::POLYGON:
3070 case TFileType::DELIMITED:
3073 #ifdef ENABLE_IMPORT_PARQUET
3074 case TFileType::PARQUET:
3075 copy_params.
file_type = import_export::FileType::PARQUET;
3083 switch (cp.geo_coords_encoding) {
3084 case TEncodingType::GEOINT:
3087 case TEncodingType::NONE:
3096 switch (cp.geo_coords_type) {
3097 case TDatumType::GEOGRAPHY:
3100 case TDatumType::GEOMETRY:
3108 switch (cp.geo_coords_srid) {
3128 TCopyParams copy_params;
3130 copy_params.null_str = cp.
null_str;
3133 copy_params.has_header = TImportHeaderRow::AUTODETECT;
3136 copy_params.has_header = TImportHeaderRow::NO_HEADER;
3139 copy_params.has_header = TImportHeaderRow::HAS_HEADER;
3145 copy_params.quoted = cp.
quoted;
3146 copy_params.quote = cp.
quote;
3147 copy_params.escape = cp.
escape;
3152 copy_params.threads = cp.
threads;
3159 copy_params.file_type = TFileType::POLYGON;
3162 copy_params.file_type = TFileType::DELIMITED;
3167 copy_params.geo_coords_encoding = TEncodingType::GEOINT;
3170 copy_params.geo_coords_encoding = TEncodingType::NONE;
3176 copy_params.geo_coords_type = TDatumType::GEOGRAPHY;
3179 copy_params.geo_coords_type = TDatumType::GEOMETRY;
3200 if (boost::istarts_with(path,
"http://") || boost::istarts_with(path,
"https://")) {
3201 if (!gdal_network) {
3203 "HTTP geo file import not supported! Update to GDAL 2.2 or later!");
3206 path =
"/vsicurl/" + path;
3207 }
else if (boost::istarts_with(path,
"s3://")) {
3208 if (!gdal_network) {
3210 "S3 geo file import not supported! Update to GDAL 2.2 or later!");
3213 boost::replace_first(path,
"s3://",
"/vsis3/");
3219 if (boost::iends_with(path,
".gz") && !boost::iends_with(path,
".tar.gz")) {
3220 path =
"/vsigzip/" + path;
3226 if (boost::iends_with(path,
".zip")) {
3228 path =
"/vsizip/" + path;
3229 }
else if (boost::iends_with(path,
".tar") || boost::iends_with(path,
".tgz") ||
3230 boost::iends_with(path,
".tar.gz")) {
3232 path =
"/vsitar/" + path;
3237 std::string path(path_in);
3240 if (boost::istarts_with(path,
"/vsizip/")) {
3241 boost::replace_first(path,
"/vsizip/",
"");
3242 }
else if (boost::istarts_with(path,
"/vsitar/")) {
3243 boost::replace_first(path,
"/vsitar/",
"");
3244 }
else if (boost::istarts_with(path,
"/vsigzip/")) {
3245 boost::replace_first(path,
"/vsigzip/",
"");
3249 if (boost::istarts_with(path,
"/vsicurl/")) {
3250 boost::replace_first(path,
"/vsicurl/",
"");
3251 }
else if (boost::istarts_with(path,
"/vsis3/")) {
3252 boost::replace_first(path,
"/vsis3/",
"s3://");
3259 if (boost::istarts_with(path,
"s3://") || boost::istarts_with(path,
"http://") ||
3260 boost::istarts_with(path,
"https://")) {
3263 return !boost::filesystem::path(path).is_absolute();
3267 auto filename = boost::filesystem::path(path).filename().string();
3279 if (boost::iends_with(path,
".geojson.gz") || boost::iends_with(path,
".json.gz")) {
3283 if (boost::iends_with(path,
".shp") || boost::iends_with(path,
".geojson") ||
3284 boost::iends_with(path,
".json") || boost::iends_with(path,
".kml") ||
3285 boost::iends_with(path,
".kmz") || boost::iends_with(path,
".gdb") ||
3286 boost::iends_with(path,
".gdb.zip")) {
3296 if (boost::iends_with(path,
".zip") && !boost::iends_with(path,
".gdb.zip")) {
3298 }
else if (boost::iends_with(path,
".tar") || boost::iends_with(path,
".tgz") ||
3299 boost::iends_with(path,
".tar.gz")) {
3308 std::vector<std::string> files =
3312 LOG(
INFO) <<
"Found " << files.size() <<
" files in Archive "
3314 for (
const auto& file : files) {
3319 bool found_suitable_file =
false;
3320 std::string file_name;
3321 for (
const auto& file : files) {
3324 found_suitable_file =
true;
3330 if (!found_suitable_file) {
3331 LOG(
INFO) <<
"Failed to find any supported geo files in Archive: " +
3341 return (!boost::istarts_with(file_path,
"s3://") &&
3342 !boost::istarts_with(file_path,
"http://") &&
3343 !boost::istarts_with(file_path,
"https://"));
3354 const TSessionId& session,
3355 const std::string& file_name_in,
3356 const TCopyParams& cp) {
3363 std::string file_name{file_name_in};
3366 auto file_path =
import_path_ / picosha2::hash256_hex_string(session) /
3367 boost::filesystem::path(file_name).filename();
3368 file_name = file_path.string();
3388 if (geo_file.size()) {
3389 file_name = file_name + std::string(
"/") + geo_file;
3397 auto file_path = boost::filesystem::path(file_name);
3399 if (!boost::istarts_with(file_name,
"s3://")) {
3400 if (!boost::filesystem::path(file_name).is_absolute()) {
3401 file_path =
import_path_ / picosha2::hash256_hex_string(session) /
3402 boost::filesystem::path(file_name).filename();
3403 file_name = file_path.string();
3413 if (!boost::filesystem::exists(file_path)) {
3421 #ifdef ENABLE_IMPORT_PARQUET
3422 || (copy_params.
file_type == import_export::FileType::PARQUET)
3427 std::vector<EncodingType> best_encodings = detector.
best_encodings;
3428 std::vector<std::string> headers = detector.
get_headers();
3432 _return.row_set.row_desc.resize(best_types.size());
3433 for (
size_t col_idx = 0; col_idx < best_types.size(); col_idx++) {
3442 col.col_type.precision =
static_cast<int>(copy_params.
geo_coords_type);
3451 col.col_name = headers[col_idx];
3454 _return.row_set.row_desc[col_idx] = col;
3456 size_t num_samples = 100;
3460 for (
auto row : sample_data) {
3461 sample_row.cols.clear();
3462 for (
const auto& s : row) {
3465 td.is_null = s.empty();
3466 sample_row.cols.push_back(td);
3468 _return.row_set.rows.push_back(sample_row);
3476 file_path.string(), geoColumnName, copy_params);
3477 for (
auto cd : cds) {
3483 std::map<std::string, std::vector<std::string>> sample_data;
3485 file_path.string(), geoColumnName, sample_data, 100, copy_params);
3486 if (sample_data.size() > 0) {
3487 for (
size_t i = 0;
i < sample_data.begin()->second.size();
i++) {
3489 for (
auto cd : cds) {
3491 td.val.str_val = sample_data[cd.sourceName].at(
i);
3492 td.is_null = td.val.str_val.empty();
3493 sample_row.cols.push_back(td);
3495 _return.row_set.rows.push_back(sample_row);
3500 }
catch (
const std::exception& e) {
3506 const TSessionId& session,
3507 const int64_t widget_id,
3508 const std::string& vega_json,
3509 const int compression_level,
3510 const std::string& nonce) {
3514 "compression_level",
3521 stdlog.appendNameValuePairs(
"nonce", nonce);
3522 auto session_ptr = stdlog.getConstSessionInfo();
3531 std::make_shared<Catalog_Namespace::SessionInfo>(*session_ptr),
3536 }
catch (std::exception& e) {
3543 int32_t dashboard_id,
3548 object.loadKey(catalog);
3549 object.setPrivileges(requestedPermissions);
3550 std::vector<DBObject> privs = {
object};
3551 return SysCatalog::instance().checkPrivileges(user, privs);
3556 const TSessionId& session,
3557 const int32_t dashboard_id) {
3560 auto session_ptr = stdlog.getConstSessionInfo();
3561 auto const&
cat = session_ptr->getCatalog();
3563 auto dash =
cat.getMetadataForDashboard(dashboard_id);
3574 SysCatalog::instance().getMetadataForUserById(dash->userId, user_meta);
3579 const TSessionId& session) {
3582 auto session_ptr = stdlog.getConstSessionInfo();
3583 auto const&
cat = session_ptr->getCatalog();
3585 const auto dashes =
cat.getAllDashboardsMetadata();
3587 for (
const auto dash : dashes) {
3599 const std::shared_ptr<Catalog_Namespace::SessionInfo const>& session_ptr,
3602 const bool populate_state) {
3603 auto const&
cat = session_ptr->getCatalog();
3604 SysCatalog::instance().getMetadataForUserById(dash->
userId, user_meta);
3605 auto objects_list = SysCatalog::instance().getMetadataForObject(
3606 cat.getCurrentDB().dbId,
3609 TDashboard dashboard;
3617 dashboard.dashboard_owner = dash->
user;
3618 TDashboardPermissions perms;
3620 if (session_ptr->get_currentUser().isSuper) {
3621 perms.create_ =
true;
3622 perms.delete_ =
true;
3630 obj_to_find.loadKey(session_ptr->getCatalog());
3631 std::vector<std::string> grantees =
3632 SysCatalog::instance().getRoles(
true,
3633 session_ptr->get_currentUser().isSuper,
3634 session_ptr->get_currentUser().userName);
3635 for (
const auto& grantee : grantees) {
3637 auto* gr = SysCatalog::instance().getGrantee(grantee);
3638 if (gr && (object_found = gr->findDbObject(obj_to_find.getObjectKey(),
true))) {
3647 dashboard.dashboard_permissions = perms;
3648 if (objects_list.empty() ||
3649 (objects_list.size() == 1 && objects_list[0]->roleName == user_meta.
userName)) {
3650 dashboard.is_dash_shared =
false;
3652 dashboard.is_dash_shared =
true;
3658 const std::string& dashboard_name,
3659 const std::string& dashboard_state,
3660 const std::string& image_hash,
3661 const std::string& dashboard_metadata) {
3664 auto session_ptr = stdlog.getConstSessionInfo();
3666 auto&
cat = session_ptr->getCatalog();
3673 auto dash =
cat.getMetadataForDashboard(
3674 std::to_string(session_ptr->get_currentUser().userId), dashboard_name);
3684 dd.
userId = session_ptr->get_currentUser().userId;
3685 dd.
user = session_ptr->get_currentUser().userName;
3688 auto id =
cat.createDashboard(dd);
3690 SysCatalog::instance().createDBObject(
3693 }
catch (
const std::exception& e) {
3699 const int32_t dashboard_id,
3700 const std::string& dashboard_name,
3701 const std::string& dashboard_owner,
3702 const std::string& dashboard_state,
3703 const std::string& image_hash,
3704 const std::string& dashboard_metadata) {
3707 auto session_ptr = stdlog.getConstSessionInfo();
3710 auto&
cat = session_ptr->getCatalog();
3723 if (!SysCatalog::instance().getMetadataForUser(dashboard_owner, user)) {
3728 dd.
user = dashboard_owner;
3732 cat.replaceDashboard(dd);
3733 }
catch (
const std::exception& e) {
3743 const std::vector<int32_t>& dashboard_ids) {
3746 auto session_ptr = stdlog.getConstSessionInfo();
3748 auto&
cat = session_ptr->getCatalog();
3751 cat.deleteMetadataForDashboards(dashboard_ids, session_ptr->get_currentUser());
3752 }
catch (
const std::exception& e) {
3758 int32_t dashboard_id,
3759 std::vector<std::string> groups) {
3761 auto&
cat = session_info.getCatalog();
3762 auto dash =
cat.getMetadataForDashboard(dashboard_id);
3766 }
else if (session_info.get_currentUser().userId != dash->userId &&
3767 !session_info.get_currentUser().isSuper) {
3768 throw std::runtime_error(
3769 "User should be either owner of dashboard or super user to share/unshare it");
3771 std::vector<std::string> valid_groups;
3773 for (
auto& group : groups) {
3775 if (!SysCatalog::instance().getGrantee(group)) {
3777 }
else if (!user_meta.
isSuper) {
3778 valid_groups.push_back(group);
3781 return valid_groups;
3785 for (
auto const& group : groups) {
3786 if (!SysCatalog::instance().getGrantee(group)) {
3794 const std::vector<int32_t>& dashboard_ids) {
3796 std::map<std::string, std::list<int32_t>> errors;
3797 for (
auto const& dashboard_id : dashboard_ids) {
3798 auto dashboard =
cat.getMetadataForDashboard(dashboard_id);
3800 errors[
"Dashboard id does not exist"].push_back(dashboard_id);
3803 errors[
"User should be either owner of dashboard or super user to share/unshare it"]
3804 .push_back(dashboard_id);
3807 if (!errors.empty()) {
3808 std::stringstream error_stream;
3809 error_stream <<
"Share/Unshare dashboard(s) failed with error(s)\n";
3810 for (
const auto& [error, id_list] : errors) {
3811 error_stream <<
"Dashboard ids " <<
join(id_list,
", ") <<
": " << error <<
"\n";
3818 const std::vector<int32_t>& dashboard_ids,
3819 const std::vector<std::string>& groups,
3820 const TDashboardPermissions& permissions,
3821 const bool do_share) {
3824 check_read_only(do_share ?
"share_dashboards" :
"unshare_dashboards");
3825 if (!permissions.create_ && !permissions.delete_ && !permissions.edit_ &&
3826 !permissions.view_) {
3828 std::string(do_share ?
"grants" :
"revokes"));
3830 auto session_ptr = stdlog.getConstSessionInfo();
3831 auto const& catalog = session_ptr->getCatalog();
3832 auto& sys_catalog = SysCatalog::instance();
3835 std::vector<DBObject> batch_objects;
3836 for (
auto const& dashboard_id : dashboard_ids) {
3839 if (permissions.delete_) {
3842 if (permissions.create_) {
3845 if (permissions.edit_) {
3848 if (permissions.view_) {
3851 object.setPrivileges(privs);
3852 batch_objects.push_back(
object);
3855 sys_catalog.grantDBObjectPrivilegesBatch(groups, batch_objects, catalog);
3857 sys_catalog.revokeDBObjectPrivilegesBatch(groups, batch_objects, catalog);
3862 const std::vector<int32_t>& dashboard_ids,
3863 const std::vector<std::string>& groups,
3864 const TDashboardPermissions& permissions) {
3870 const int32_t dashboard_id,
3871 const std::vector<std::string>& groups,
3872 const std::vector<std::string>& objects,
3873 const TDashboardPermissions& permissions,
3874 const bool grant_role =
false) {
3879 const std::vector<int32_t>& dashboard_ids,
3880 const std::vector<std::string>& groups,
3881 const TDashboardPermissions& permissions) {
3886 const int32_t dashboard_id,
3887 const std::vector<std::string>& groups,
3888 const std::vector<std::string>& objects,
3889 const TDashboardPermissions& permissions) {
3894 std::vector<TDashboardGrantees>& dashboard_grantees,
3895 const TSessionId& session,
3896 int32_t dashboard_id) {
3899 auto session_ptr = stdlog.getConstSessionInfo();
3900 auto const&
cat = session_ptr->getCatalog();
3902 auto dash =
cat.getMetadataForDashboard(dashboard_id);
3906 }
else if (session_ptr->get_currentUser().userId != dash->userId &&
3907 !session_ptr->get_currentUser().isSuper) {
3909 "User should be either owner of dashboard or super user to access grantees");
3911 std::vector<ObjectRoleDescriptor*> objectsList;
3912 objectsList = SysCatalog::instance().getMetadataForObject(
3913 cat.getCurrentDB().dbId,
3918 SysCatalog::instance().getMetadataForUserById(dash->userId, user_meta);
3919 for (
auto object : objectsList) {
3920 if (user_meta.
userName == object->roleName) {
3924 TDashboardGrantees grantee;
3925 TDashboardPermissions perm;
3926 grantee.name =
object->roleName;
3927 grantee.is_user =
object->roleType;
3932 grantee.permissions = perm;
3933 dashboard_grantees.push_back(grantee);
3938 const TSessionId& session,
3939 const std::string& view_state,
3940 const std::string& view_metadata) {
3943 auto session_ptr = stdlog.getConstSessionInfo();
3945 auto&
cat = session_ptr->getCatalog();
3948 ld.
userId = session_ptr->get_currentUser().userId;
3953 _return =
cat.createLink(ld, 6);
3954 }
catch (
const std::exception& e) {
3960 const std::string&
name,
3961 const bool is_array) {
3964 ct.col_type.type =
type;
3965 ct.col_type.is_array = is_array;
3971 const std::list<std::string> shp_ext{
".shp",
".shx",
".dbf"};
3972 if (std::find(shp_ext.begin(),
3974 boost::algorithm::to_lower_copy(file_path.extension().string())) !=
3976 for (
auto ext : shp_ext) {
3977 auto aux_file = file_path;
3979 aux_file.replace_extension(boost::algorithm::to_upper_copy(ext)).string(),
3982 aux_file.replace_extension(ext).string(), copy_params)) {
3983 throw std::runtime_error(
"required file for shapefile does not exist: " +
3984 aux_file.filename().string());
3991 const std::string& table_name,
3992 const TRowDescriptor& rd,
3994 const TCreateParams& create_params) {
3995 auto stdlog =
STDLOG(
"table_name", table_name);
4011 std::string stmt{
"CREATE TABLE " + table_name};
4012 std::vector<std::string> col_stmts;
4014 for (
auto col : rds) {
4020 if (col.col_type.type == TDatumType::INTERVAL_DAY_TIME ||
4021 col.col_type.type == TDatumType::INTERVAL_YEAR_MONTH) {
4023 " for column: " + col.col_name);
4026 if (col.col_type.type == TDatumType::DECIMAL) {
4028 if (col.col_type.precision == 0 && col.col_type.scale == 0) {
4029 col.col_type.precision = 14;
4030 col.col_type.scale = 7;
4034 std::string col_stmt;
4035 col_stmt.append(col.col_name +
" " +
thrift_to_name(col.col_type));
4046 col_stmt.append(
"(" +
std::to_string(col.col_type.comp_param) +
")");
4048 }
else if (col.col_type.type == TDatumType::STR) {
4050 col_stmt.append(
" ENCODING NONE");
4051 }
else if (col.col_type.type == TDatumType::POINT ||
4052 col.col_type.type == TDatumType::LINESTRING ||
4053 col.col_type.type == TDatumType::POLYGON ||
4054 col.col_type.type == TDatumType::MULTIPOLYGON) {
4056 if (col.col_type.scale == 4326) {
4057 col_stmt.append(
" ENCODING NONE");
4060 col_stmts.push_back(col_stmt);
4065 if (create_params.is_replicated) {
4066 stmt.append(
" WITH (PARTITIONS = 'REPLICATED')");
4072 sql_execute(ret, session, stmt,
true,
"", -1, -1);
4076 const std::string& table_name,
4077 const std::string& file_name_in,
4078 const TCopyParams& cp) {
4082 auto session_ptr = stdlog.getConstSessionInfo();
4084 LOG(
INFO) <<
"import_table " << table_name <<
" from " << file_name_in;
4086 auto&
cat = session_ptr->getCatalog();
4087 const auto td_with_lock =
4090 const auto td = td_with_lock();
4094 std::string file_name{file_name_in};
4095 auto file_path = boost::filesystem::path(file_name);
4097 if (!boost::istarts_with(file_name,
"s3://")) {
4098 if (!boost::filesystem::path(file_name).is_absolute()) {
4099 file_path =
import_path_ / picosha2::hash256_hex_string(session) /
4100 boost::filesystem::path(file_name).filename();
4101 file_name = file_path.string();
4103 if (!boost::filesystem::exists(file_path)) {
4112 if (boost::filesystem::extension(file_path) ==
".tsv") {
4118 session_ptr->getCatalog(), table_name);
4119 std::unique_ptr<import_export::Importer> importer;
4130 std::cout <<
"Total Import Time: " << (double)ms / 1000.0 <<
" Seconds." << std::endl;
4131 }
catch (
const std::exception& e) {
4143 return (t == TDatumType::POLYGON || t == TDatumType::MULTIPOLYGON ||
4144 t == TDatumType::LINESTRING || t == TDatumType::POINT);
4147 #if ENABLE_GEO_IMPORT_COLUMN_MATCHING
4150 std::stringstream ss;
4155 std::string TTypeInfo_GeoSubTypeToString(
const int32_t p) {
4159 result =
"GEOGRAPHY";
4162 result =
"GEOMETRY";
4172 std::stringstream ss;
4181 #define THROW_COLUMN_ATTR_MISMATCH_EXCEPTION(attr, got, expected) \
4182 THROW_MAPD_EXCEPTION("Could not append geo file '" + file_path.filename().string() + \
4183 "' to table '" + table_name + "'. Column '" + cd->columnName + \
4184 "' " + attr + " mismatch (got '" + got + "', expected '" + \
4188 const std::string& table_name,
4189 const std::string& file_name_in,
4190 const TCopyParams& cp,
4191 const TRowDescriptor& row_desc,
4192 const TCreateParams& create_params) {
4195 auto session_ptr = stdlog.getConstSessionInfo();
4197 auto&
cat = session_ptr->getCatalog();
4201 std::string file_name{file_name_in};
4205 auto file_path =
import_path_ / picosha2::hash256_hex_string(session) /
4206 boost::filesystem::path(file_name).filename();
4207 file_name = file_path.string();
4225 if (geo_file.size()) {
4226 file_name = file_name + std::string(
"/") + geo_file;
4234 LOG(
INFO) <<
"import_geo_table: Original filename: " << file_name_in;
4235 LOG(
INFO) <<
"import_geo_table: Actual filename: " << file_name;
4238 auto file_path = boost::filesystem::path(file_name);
4246 }
catch (
const std::exception& e) {
4256 std::vector<import_export::Importer::GeoFileLayerInfo> layer_info;
4259 }
catch (
const std::exception& e) {
4264 using LayerNameToContentsMap =
4265 std::map<std::string, import_export::Importer::GeoFileLayerContents>;
4266 LayerNameToContentsMap load_layers;
4267 LOG(
INFO) <<
"import_geo_table: Found the following layers in the geo file:";
4268 for (
const auto& layer : layer_info) {
4269 switch (layer.contents) {
4271 LOG(
INFO) <<
"import_geo_table: '" << layer.name
4272 <<
"' (will import as geo table)";
4273 load_layers[layer.name] = layer.contents;
4276 LOG(
INFO) <<
"import_geo_table: '" << layer.name
4277 <<
"' (will import as regular table)";
4278 load_layers[layer.name] = layer.contents;
4281 LOG(
WARNING) <<
"import_geo_table: '" << layer.name
4282 <<
"' (will not import, unsupported geo type)";
4285 LOG(
INFO) <<
"import_geo_table: '" << layer.name <<
"' (ignoring, empty)";
4293 if (load_layers.size() == 0) {
4301 for (
const auto& layer : layer_info) {
4306 load_layers.clear();
4307 load_layers[layer.name] = layer.contents;
4310 }
else if (layer.contents ==
4314 "' has unsupported geo type!");
4315 }
else if (layer.contents ==
4330 if (row_desc.size() > 0 && load_layers.size() > 1) {
4332 "import_geo_table: Multi-layer geo import not yet supported from Immerse!");
4337 auto construct_layer_table_name = [&load_layers](
const std::string& table_name,
4338 const std::string& layer_name) {
4339 if (load_layers.size() > 1) {
4341 if (sanitized_layer_name != layer_name) {
4342 LOG(
INFO) <<
"import_geo_table: Using sanitized layer name '"
4343 << sanitized_layer_name <<
"' for table name";
4345 return table_name +
"_" + sanitized_layer_name;
4351 if (load_layers.size() > 1) {
4352 for (
const auto& layer : load_layers) {
4354 auto this_table_name = construct_layer_table_name(table_name, layer.first);
4357 if (
cat.getMetadataForTable(this_table_name)) {
4359 "' already exists, aborting!");
4366 std::vector<std::string> caught_exception_messages;
4369 double total_import_ms = 0.0;
4373 for (
const auto& layer : load_layers) {
4375 const auto& layer_name = layer.first;
4376 const auto& layer_contents = layer.second;
4381 auto this_table_name = construct_layer_table_name(table_name, layer_name);
4384 LOG(
INFO) <<
"import_geo_table: Creating table: " << this_table_name;
4388 if (row_desc.size() > 0) {
4399 TCopyParams cp_copy = cp;
4400 cp_copy.geo_layer_name = layer_name;
4401 cp_copy.file_type = TFileType::POLYGON;
4404 }
catch (
const std::exception& e) {
4406 caught_exception_messages.emplace_back(
4407 "Invalid/Unsupported Column Types in Layer '" + layer_name +
"':" + e.what());
4410 rd = cds.row_set.row_desc;
4416 create_table(session, this_table_name, rd, TFileType::POLYGON, create_params);
4417 }
catch (
const std::exception& e) {
4419 caught_exception_messages.emplace_back(
"Failed to create table for Layer '" +
4420 layer_name +
"':" + e.what());
4430 std::string exception_message =
4431 "Could not import geo file '" + file_path.filename().string() +
"' to table '" +
4432 this_table_name +
"'; table does not exist or failed to create.";
4433 caught_exception_messages.emplace_back(exception_message);
4439 const auto col_descriptors =
4440 cat.getAllColumnMetadataForTable(td->
tableId,
false,
false,
false);
4443 if (col_descriptors.size() != rd.size()) {
4445 std::string exception_message =
4446 "Could not append geo file '" + file_path.filename().string() +
"' to table '" +
4447 this_table_name +
"'. Column count mismatch (got " +
std::to_string(rd.size()) +
4449 caught_exception_messages.emplace_back(exception_message);
4456 for (
auto cd : col_descriptors) {
4458 std::string gname = rd[rd_index].col_name;
4459 std::string ename = cd->columnName;
4460 #if ENABLE_GEO_IMPORT_COLUMN_MATCHING
4461 TTypeInfo gti = rd[rd_index].col_type;
4463 TTypeInfo eti = cd_col_type.col_type;
4465 if (gname != ename) {
4469 rd[rd_index].col_name = gname;
4471 <<
"import_geo_table: Renaming incoming geo column to match existing "
4472 "legacy default geo column";
4477 #if ENABLE_GEO_IMPORT_COLUMN_MATCHING
4480 if (gti.type != eti.type) {
4482 "type", TTypeInfo_TypeToString(gti.type), TTypeInfo_TypeToString(eti.type));
4484 if (gti.is_array != eti.is_array) {
4488 if (gti.nullable != eti.nullable) {
4497 if (gti.precision != eti.precision) {
4500 TTypeInfo_GeoSubTypeToString(gti.precision),
4501 TTypeInfo_GeoSubTypeToString(eti.precision));
4503 if (gti.scale != eti.scale) {
4507 if (gti.encoding != eti.encoding) {
4508 LOG(
INFO) <<
"import_geo_table: Ignoring geo encoding mismatch";
4510 if (gti.comp_param != eti.comp_param) {
4511 LOG(
INFO) <<
"import_geo_table: Ignoring geo comp_param mismatch";
4517 if (gti.precision != eti.precision) {
4522 if (gti.scale != eti.scale) {
4526 if (gti.encoding != eti.encoding) {
4529 TTypeInfo_EncodingToString(gti.encoding),
4530 TTypeInfo_EncodingToString(eti.encoding));
4532 if (gti.comp_param != eti.comp_param) {
4541 }
catch (
const std::exception& e) {
4543 caught_exception_messages.emplace_back(e.what());
4547 std::map<std::string, std::string> colname_to_src;
4549 colname_to_src[
r.col_name] =
4555 }
catch (
const std::exception& e) {
4557 caught_exception_messages.emplace_back(e.what());
4568 bool have_geo_column_with_correct_name =
false;
4569 for (
const auto&
r : rd) {
4573 have_geo_column_with_correct_name =
true;
4575 CHECK(colname_to_src.find(
r.col_name) != colname_to_src.end());
4577 colname_to_src[
r.col_name] =
r.col_name;
4578 have_geo_column_with_correct_name =
true;
4582 if (!have_geo_column_with_correct_name) {
4583 std::string exception_message =
"Table " + this_table_name +
4584 " does not have a geo column with name '" +
4586 caught_exception_messages.emplace_back(exception_message);
4596 std::unique_ptr<import_export::Importer> importer;
4609 LOG(
INFO) <<
"Import of Layer '" << layer_name <<
"' took " << (double)ms / 1000.0
4611 total_import_ms += ms;
4612 }
catch (
const std::exception& e) {
4613 std::string exception_message =
4614 "Import of Layer '" + this_table_name +
"' failed: " + e.what();
4615 caught_exception_messages.emplace_back(exception_message);
4621 if (caught_exception_messages.size()) {
4623 std::string combined_exception_message =
"Failed to import geo file:\n";
4624 for (
const auto& message : caught_exception_messages) {
4625 combined_exception_message += message +
"\n";
4630 LOG(
INFO) <<
"Import Successful!";
4631 LOG(
INFO) <<
"Total Import Time: " << total_import_ms / 1000.0 <<
"s";
4635 #undef THROW_COLUMN_ATTR_MISMATCH_EXCEPTION
4638 const TSessionId& session,
4639 const std::string& import_id) {
4643 _return.elapsed = is.elapsed.count();
4644 _return.rows_completed = is.rows_completed;
4645 _return.rows_estimated = is.rows_estimated;
4646 _return.rows_rejected = is.rows_rejected;
4650 const TSessionId& session,
4651 const std::string& archive_path_in,
4652 const TCopyParams& copy_params) {
4657 std::string archive_path(archive_path_in);
4661 auto file_path =
import_path_ / picosha2::hash256_hex_string(session) /
4662 boost::filesystem::path(archive_path).filename();
4663 archive_path = file_path.string();
4676 std::string geo_file =
4679 if (geo_file.size()) {
4681 _return = archive_path_in + std::string(
"/") + geo_file;
4684 _return = archive_path_in;
4688 _return = archive_path_in;
4693 const TSessionId& session,
4694 const std::string& archive_path_in,
4695 const TCopyParams& copy_params) {
4700 std::string archive_path(archive_path_in);
4703 auto file_path =
import_path_ / picosha2::hash256_hex_string(session) /
4704 boost::filesystem::path(archive_path).filename();
4705 archive_path = file_path.string();
4721 for (
auto& s : _return) {
4722 s = archive_path_in +
'/' + s;
4728 const TSessionId& session,
4729 const std::string& file_name_in,
4730 const TCopyParams& cp) {
4734 std::string file_name(file_name_in);
4741 auto file_path =
import_path_ / picosha2::hash256_hex_string(session) /
4742 boost::filesystem::path(file_name).filename();
4743 file_name = file_path.string();
4762 if (geo_file.size()) {
4763 file_name = file_name + std::string(
"/") + geo_file;
4776 auto internal_layer_info =
4780 for (
const auto& internal_layer : internal_layer_info) {
4781 TGeoFileLayerInfo layer;
4782 layer.name = internal_layer.name;
4783 switch (internal_layer.contents) {
4785 layer.contents = TGeoFileLayerContents::EMPTY;
4788 layer.contents = TGeoFileLayerContents::GEO;
4791 layer.contents = TGeoFileLayerContents::NON_GEO;
4794 layer.contents = TGeoFileLayerContents::UNSUPPORTED_GEO;
4799 _return.emplace_back(layer);
4805 #ifdef HAVE_PROFILER
4806 if (IsHeapProfilerRunning()) {
4809 HeapProfilerStart(
"omnisci");
4812 #endif // HAVE_PROFILER
4817 #ifdef HAVE_PROFILER
4818 if (!IsHeapProfilerRunning()) {
4824 #endif // HAVE_PROFILER
4829 #ifdef HAVE_PROFILER
4830 if (!IsHeapProfilerRunning()) {
4833 auto profile_buff = GetHeapProfile();
4834 profile = profile_buff;
4838 #endif // HAVE_PROFILER
4843 if (session_it->second.use_count() > 2 ||
4849 time_t last_used_time = session_it->second->get_last_used_time();
4850 time_t
start_time = session_it->second->get_start_time();
4851 const auto current_session_duration = time(0) - last_used_time;
4853 LOG(
INFO) <<
"Session " << session_it->second->get_public_session_id()
4854 <<
" idle duration " << current_session_duration
4856 <<
" seconds. Invalidating session.";
4857 throw ForceDisconnect(
"Idle Session Timeout. User should re-authenticate.");
4859 const auto total_session_duration = time(0) -
start_time;
4861 LOG(
INFO) <<
"Session " << session_it->second->get_public_session_id()
4862 <<
" total duration " << total_session_duration
4863 <<
" seconds exceeds maximum total session duration "
4865 throw ForceDisconnect(
"Maximum active Session Timeout. User should re-authenticate.");
4870 const TSessionId& session) {
4880 const TSessionId& session) {
4889 return std::make_shared<Catalog_Namespace::SessionInfo>(session_info_ref);
4893 const TSessionId& session_id) {
4902 if (session_id.empty()) {
4911 const std::string& table_name) {
4917 std::vector<DBObject> privObjects;
4918 privObjects.push_back(dbObject);
4919 if (!SysCatalog::instance().checkPrivileges(user_metadata, privObjects)) {
4921 user_metadata.userLoggable() +
4922 " has no insert privileges for table " + table_name +
".");
4927 const std::string& table_name) {
4936 case TExecuteMode::GPU:
4938 TOmniSciException e;
4939 e.error_msg =
"Cannot switch to GPU mode in a server started in CPU-only mode.";
4943 LOG(
INFO) <<
"User " << user_name <<
" sets GPU mode.";
4945 case TExecuteMode::CPU:
4947 LOG(
INFO) <<
"User " << user_name <<
" sets CPU mode.";
4953 TQueryResult& _return,
4955 const std::string& query_ra,
4956 const bool column_format,
4958 const int32_t first_n,
4959 const int32_t at_most_n,
4960 const bool just_validate,
4961 const bool find_push_down_candidates,
4963 const std::optional<size_t> executor_index)
const {
4991 auto validate_or_explain_query =
5002 find_push_down_candidates,
5011 ExecutionResult result{std::make_shared<ResultSet>(std::vector<TargetInfo>{},
5020 result = ra_executor.executeRelAlgQuery(co, eo, explain_info.
explain_plan,
nullptr);
5023 _return.execution_time_ms -= result.getRows()->getQueueTime();
5024 VLOG(1) <<
cat.getDataMgr().getSystemMemoryUsage();
5025 const auto& filter_push_down_info = result.getPushedDownFilterInfo();
5026 if (!filter_push_down_info.empty()) {
5027 return filter_push_down_info;
5034 result.getTargetsMeta(),
5044 const std::string& query_ra,
5048 const size_t device_id,
5049 const int32_t first_n,
5089 ExecutionResult result{std::make_shared<ResultSet>(std::vector<TargetInfo>{},
5098 [&]() { result = ra_executor.executeRelAlgQuery(co, eo,
false,
nullptr); });
5099 _return.execution_time_ms -= result.getRows()->getQueueTime();
5100 const auto rs = result.getRows();
5101 const auto converter =
5102 std::make_unique<ArrowResultSetConverter>(rs,
5110 _return.arrow_conversion_time_ms +=
5113 std::string(arrow_result.sm_handle.begin(), arrow_result.sm_handle.end());
5114 _return.sm_size = arrow_result.sm_size;
5116 std::string(arrow_result.df_handle.begin(), arrow_result.df_handle.end());
5118 std::string(arrow_result.df_buffer.begin(), arrow_result.df_buffer.end());
5123 std::make_pair(_return.df_handle, arrow_result.serialized_cuda_handle));
5125 _return.df_size = arrow_result.df_size;
5129 const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& targets)
const {
5130 std::vector<TargetMetaInfo>
result;
5131 for (
const auto& target : targets) {
5133 CHECK(target->get_expr());
5134 result.emplace_back(target->get_resname(), target->get_expr()->get_type_info());
5140 const std::vector<std::shared_ptr<Analyzer::TargetEntry>>& targets)
const {
5141 std::vector<std::string> names;
5142 for (
const auto& target : targets) {
5144 CHECK(target->get_expr());
5145 names.push_back(target->get_resname());
5151 const std::vector<TargetMetaInfo>& targets)
const {
5152 std::vector<std::string> names;
5153 for (
const auto& target : targets) {
5154 names.push_back(target.get_resname());
5161 const std::vector<TargetMetaInfo>& targets,
5162 const ResultSet& results,
5163 const bool column_format,
5164 const int32_t first_n,
5165 const int32_t at_most_n)
const {
5169 if (column_format) {
5170 _return.row_set.is_columnar =
true;
5171 std::vector<TColumn> tcolumns(results.colCount());
5172 while (first_n == -1 || fetched < first_n) {
5173 const auto crt_row = results.getNextRow(
true,
true);
5174 if (crt_row.empty()) {
5178 if (at_most_n >= 0 && fetched > at_most_n) {
5182 for (
size_t i = 0;
i < results.colCount(); ++
i) {
5183 const auto agg_result = crt_row[
i];
5187 for (
size_t i = 0;
i < results.colCount(); ++
i) {
5188 _return.row_set.columns.push_back(tcolumns[
i]);
5191 _return.row_set.is_columnar =
false;
5192 while (first_n == -1 || fetched < first_n) {
5193 const auto crt_row = results.getNextRow(
true,
true);
5194 if (crt_row.empty()) {
5198 if (at_most_n >= 0 && fetched > at_most_n) {
5203 trow.cols.reserve(results.colCount());
5204 for (
size_t i = 0;
i < results.colCount(); ++
i) {
5205 const auto agg_result = crt_row[
i];
5206 trow.cols.push_back(
value_to_thrift(agg_result, targets[
i].get_type_info()));
5208 _return.row_set.rows.push_back(trow);
5215 TRowDescriptor fixedup_row_desc;
5216 for (
const TColumnType& col_desc : row_desc) {
5217 auto fixedup_col_desc = col_desc;
5218 if (col_desc.col_type.encoding == TEncodingType::DICT &&
5219 col_desc.col_type.comp_param > 0) {
5221 fixedup_col_desc.col_type.comp_param = dd->
dictNBits;
5223 fixedup_row_desc.push_back(fixedup_col_desc);
5226 return fixedup_row_desc;
5231 const ResultSet& results,
5232 const bool column_format,
5233 const std::string
label)
const {
5234 CHECK_EQ(
size_t(1), results.rowCount());
5235 TColumnType proj_info;
5236 proj_info.col_name =
label;
5237 proj_info.col_type.type = TDatumType::STR;
5238 proj_info.col_type.nullable =
false;
5239 proj_info.col_type.is_array =
false;
5240 _return.row_set.row_desc.push_back(proj_info);
5241 const auto crt_row = results.getNextRow(
true,
true);
5242 const auto tv = crt_row[0];
5243 CHECK(results.getNextRow(
true,
true).empty());
5244 const auto scalar_tv = boost::get<ScalarTargetValue>(&tv);
5246 const auto s_n = boost::get<NullableString>(scalar_tv);
5248 const auto s = boost::get<std::string>(s_n);
5250 if (column_format) {
5252 tcol.data.str_col.push_back(*s);
5253 tcol.nulls.push_back(
false);
5254 _return.row_set.is_columnar =
true;
5255 _return.row_set.columns.push_back(tcol);
5258 explanation.val.str_val = *s;
5259 explanation.is_null =
false;
5261 trow.cols.push_back(explanation);
5262 _return.row_set.is_columnar =
false;
5263 _return.row_set.rows.push_back(trow);
5268 const ResultSet& results,
5269 const bool column_format)
const {
5274 const ResultSet& results,
5275 const bool column_format)
const {
5285 std::vector<DBObject> privObjects;
5289 privObjects.push_back(dbObject);
5290 return SysCatalog::instance().checkPrivileges(session_info.
get_currentUser(),
5301 if (rename_db_stmt) {
5306 if (drop_user_stmt) {
5311 if (rename_user_stmt) {
5319 const bool column_format,
5320 const std::string& nonce,
5322 const int32_t first_n,
5323 const int32_t at_most_n) {
5328 _return.nonce = nonce;
5329 _return.execution_time_ms = 0;
5333 auto&
cat = session_ptr->getCatalog();
5335 std::list<std::unique_ptr<Parser::Stmt>> parse_trees;
5337 mapd_unique_lock<mapd_shared_mutex> executeWriteLock;
5338 mapd_shared_lock<mapd_shared_mutex> executeReadLock;
5342 switch (pw.getQueryType()) {
5344 _return.query_type = TQueryType::READ;
5345 VLOG(1) <<
"query type: READ";
5349 _return.query_type = TQueryType::WRITE;
5350 VLOG(1) <<
"query type: WRITE";
5354 _return.query_type = TQueryType::SCHEMA_READ;
5355 VLOG(1) <<
"query type: SCHEMA READ";
5359 _return.query_type = TQueryType::SCHEMA_WRITE;
5360 VLOG(1) <<
"query type: SCHEMA WRITE";
5364 _return.query_type = TQueryType::UNKNOWN;
5369 if (pw.isCalcitePathPermissable(
read_only_)) {
5371 if (pw.isCalciteDdl()) {
5372 std::string query_ra;
5375 std::tie(result, locks) =
5377 query_ra = result.plan_result;
5384 executeReadLock = mapd_shared_lock<mapd_shared_mutex>(
5388 std::string query_ra;
5391 std::tie(result, locks) =
5393 query_ra = result.plan_result;
5396 std::string query_ra_calcite_explain;
5401 }
else if (pw.isCalciteExplain()) {
5403 std::string temp_query_str =
5404 query_str.substr(std::string(
"explain calcite ").length());
5406 CHECK(!locks.empty());
5407 query_ra_calcite_explain =
5411 const auto explain_info = pw.getExplainInfo();
5412 std::vector<PushedDownFilterInfo> filter_push_down_requests;
5413 auto execute_rel_alg_task = std::make_shared<QueryDispatchQueue::Task>(
5415 &filter_push_down_requests,
5419 &query_ra_calcite_explain,
5424 executor_device_type,
5426 at_most_n](
const size_t executor_index) {
5430 explain_info.justCalciteExplain() ? query_ra_calcite_explain : query_ra,
5432 executor_device_type,
5439 if (explain_info.justCalciteExplain() && filter_push_down_requests.empty()) {
5443 }
else if (!filter_push_down_requests.empty()) {
5444 CHECK(!locks.empty());
5449 executor_device_type,
5452 explain_info.justExplain(),
5453 explain_info.justCalciteExplain(),
5454 filter_push_down_requests);
5455 }
else if (explain_info.justCalciteExplain() &&
5456 filter_push_down_requests.empty()) {
5461 CHECK(!locks.empty());
5471 auto submitted_time = std::chrono::system_clock::now();
5472 query_state_proxy.getQueryState().setQuerySubmittedTime(submitted_time);
5473 executor->enrollQuerySession(session_ptr->get_session_id(),
5477 QuerySessionStatus::QueryStatus::PENDING_QUEUE);
5482 auto result_future = execute_rel_alg_task->get_future();
5483 result_future.get();
5485 }
else if (pw.is_optimize || pw.is_validate) {
5489 if (pw.is_optimize) {
5490 const auto optimize_stmt =
5492 CHECK(optimize_stmt);
5495 const auto td_with_lock =
5497 cat, optimize_stmt->getTableName());
5498 const auto td = td_with_lock();
5502 throw std::runtime_error(
"Table " + optimize_stmt->getTableName() +
5503 " does not exist.");
5506 throw std::runtime_error(
"OPTIMIZE TABLE command is not supported on views.");
5516 if (optimize_stmt->shouldVacuumDeletedRows()) {
5519 optimizer.recomputeMetadata();
5524 if (pw.is_validate) {
5526 if (!session_ptr->get_currentUser().isSuper) {
5527 throw std::runtime_error(
"Superuser is required to run VALIDATE");
5529 const auto validate_stmt =
5531 CHECK(validate_stmt);