OmniSciDB  8a228a1076
QueryRunner.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "QueryRunner.h"
18 
19 #include "Calcite/Calcite.h"
20 #include "Catalog/Catalog.h"
21 #include "DistributedLoader.h"
23 #include "Logger/Logger.h"
24 #include "Parser/ParserWrapper.h"
25 #include "Parser/parser.h"
31 #include "Shared/StringTransform.h"
33 #include "Shared/geosupport.h"
34 #include "Shared/import_helpers.h"
35 #include "bcrypt.h"
36 #include "gen-cpp/CalciteServer.h"
37 
38 #include <boost/filesystem/operations.hpp>
39 #include <csignal>
40 #include <random>
41 
42 #define CALCITEPORT 3279
43 
44 extern size_t g_leaf_count;
45 extern bool g_enable_filter_push_down;
46 
48 
49 extern bool g_serialize_temp_tables;
51 std::mutex calcite_lock;
52 
53 using namespace Catalog_Namespace;
54 namespace {
55 
56 std::shared_ptr<Calcite> g_calcite = nullptr;
57 
58 void calcite_shutdown_handler() noexcept {
59  if (g_calcite) {
60  g_calcite->close_calcite_server();
61  g_calcite.reset();
62  }
63 }
64 
65 void mapd_signal_handler(int signal_number) {
66  LOG(ERROR) << "Interrupt signal (" << signal_number << ") received.";
68  // shut down logging force a flush
70  // terminate program
71  if (signal_number == SIGTERM) {
72  std::exit(EXIT_SUCCESS);
73  } else {
74  std::exit(signal_number);
75  }
76 }
77 
79  std::signal(SIGTERM, mapd_signal_handler);
80  std::signal(SIGSEGV, mapd_signal_handler);
81  std::signal(SIGABRT, mapd_signal_handler);
82 }
83 
84 } // namespace
85 
86 namespace QueryRunner {
87 
88 std::unique_ptr<QueryRunner> QueryRunner::qr_instance_ = nullptr;
89 
90 query_state::QueryStates QueryRunner::query_states_;
91 
92 QueryRunner* QueryRunner::init(const char* db_path,
93  const std::string& user,
94  const std::string& pass,
95  const std::string& db_name,
96  const std::vector<LeafHostInfo>& string_servers,
97  const std::vector<LeafHostInfo>& leaf_servers,
98  const std::string& udf_filename,
99  bool uses_gpus,
100  const size_t max_gpu_mem,
101  const int reserved_gpu_mem,
102  const bool create_user,
103  const bool create_db) {
104  LOG_IF(FATAL, !leaf_servers.empty()) << "Distributed test runner not supported.";
105  CHECK(leaf_servers.empty());
106  qr_instance_.reset(new QueryRunner(db_path,
107  user,
108  pass,
109  db_name,
110  string_servers,
111  leaf_servers,
112  udf_filename,
113  uses_gpus,
114  max_gpu_mem,
115  reserved_gpu_mem,
116  create_user,
117  create_db));
118  return qr_instance_.get();
119 }
120 
121 QueryRunner::QueryRunner(const char* db_path,
122  const std::string& user_name,
123  const std::string& passwd,
124  const std::string& db_name,
125  const std::vector<LeafHostInfo>& string_servers,
126  const std::vector<LeafHostInfo>& leaf_servers,
127  const std::string& udf_filename,
128  bool uses_gpus,
129  const size_t max_gpu_mem,
130  const int reserved_gpu_mem,
131  const bool create_user,
132  const bool create_db)
133  : dispatch_queue_(std::make_unique<QueryDispatchQueue>(1)) {
135 
136  boost::filesystem::path base_path{db_path};
137  CHECK(boost::filesystem::exists(base_path));
138  auto system_db_file = base_path / "mapd_catalogs" / OMNISCI_DEFAULT_DB;
139  CHECK(boost::filesystem::exists(system_db_file));
140  auto data_dir = base_path / "mapd_data";
141  DiskCacheConfig disk_cache_config{(base_path / "omnisci_disk_cache").string()};
144 
147  g_calcite =
148  std::make_shared<Calcite>(-1, CALCITEPORT, db_path, 1024, 5000, true, udf_filename);
149  ExtensionFunctionsWhitelist::add(g_calcite->getExtensionFunctionWhitelist());
150  if (!udf_filename.empty()) {
151  ExtensionFunctionsWhitelist::addUdfs(g_calcite->getUserDefinedFunctionWhitelist());
152  }
153 
155 
156 #ifndef HAVE_CUDA
157  uses_gpus = false;
158 #endif
159  SystemParameters mapd_params;
160  mapd_params.gpu_buffer_mem_bytes = max_gpu_mem;
161  mapd_params.aggregator = !leaf_servers.empty();
162 
163  auto data_mgr = std::make_shared<Data_Namespace::DataMgr>(data_dir.string(),
164  mapd_params,
165  uses_gpus,
166  -1,
167  0,
168  reserved_gpu_mem,
169  0,
170  disk_cache_config);
171 
172  auto& sys_cat = Catalog_Namespace::SysCatalog::instance();
173 
174  sys_cat.init(base_path.string(),
175  data_mgr,
176  {},
177  g_calcite,
178  false,
179  mapd_params.aggregator,
180  string_servers);
181 
182  if (create_user) {
183  if (!sys_cat.getMetadataForUser(user_name, user)) {
184  sys_cat.createUser(user_name, passwd, false, "", true);
185  }
186  }
187  CHECK(sys_cat.getMetadataForUser(user_name, user));
188  CHECK(bcrypt_checkpw(passwd.c_str(), user.passwd_hash.c_str()) == 0);
189 
190  if (create_db) {
191  if (!sys_cat.getMetadataForDB(db_name, db)) {
192  sys_cat.createDatabase(db_name, user.userId);
193  }
194  }
195  CHECK(sys_cat.getMetadataForDB(db_name, db));
196  CHECK(user.isSuper || (user.userId == db.dbOwner));
197  auto cat = std::make_shared<Catalog_Namespace::Catalog>(
198  base_path.string(), db, data_mgr, string_servers, g_calcite, create_db);
199  Catalog_Namespace::Catalog::set(cat->getCurrentDB().dbName, cat);
200  session_info_ = std::make_unique<Catalog_Namespace::SessionInfo>(
201  cat, user, ExecutorDeviceType::GPU, "");
202 }
203 
204 void QueryRunner::resizeDispatchQueue(const size_t num_executors) {
205  dispatch_queue_ = std::make_unique<QueryDispatchQueue>(num_executors);
206 }
207 
208 QueryRunner::QueryRunner(std::unique_ptr<Catalog_Namespace::SessionInfo> session)
209  : session_info_(std::move(session))
210  , dispatch_queue_(std::make_unique<QueryDispatchQueue>(1)) {}
211 
212 std::shared_ptr<Catalog_Namespace::Catalog> QueryRunner::getCatalog() const {
213  CHECK(session_info_);
214  return session_info_->get_catalog_ptr();
215 }
216 
217 std::shared_ptr<Calcite> QueryRunner::getCalcite() const {
218  // TODO: Embed Calcite shared_ptr ownership in QueryRunner
219  return g_calcite;
220 }
221 
222 bool QueryRunner::gpusPresent() const {
223  CHECK(session_info_);
224  return session_info_->getCatalog().getDataMgr().gpusPresent();
225 }
226 
227 void QueryRunner::clearGpuMemory() const {
230 }
231 
232 void QueryRunner::clearCpuMemory() const {
235 }
236 
237 std::string apply_copy_to_shim(const std::string& query_str) {
238  auto result = query_str;
239  {
240  boost::regex copy_to{R"(COPY\s*\(([^#])(.+)\)\s+TO\s)",
241  boost::regex::extended | boost::regex::icase};
242  apply_shim(result, copy_to, [](std::string& result, const boost::smatch& what) {
243  result.replace(
244  what.position(), what.length(), "COPY (#~#" + what[1] + what[2] + "#~#) TO ");
245  });
246  }
247  return result;
248 }
249 
250 QueryHint QueryRunner::getParsedQueryHintofQuery(const std::string& query_str) {
251  CHECK(session_info_);
253  auto query_state = create_query_state(session_info_, query_str);
254  const auto& cat = session_info_->getCatalog();
256  auto calcite_mgr = cat.getCalciteMgr();
257  const auto query_ra = calcite_mgr
258  ->process(query_state->createQueryStateProxy(),
259  pg_shim(query_str),
260  {},
261  true,
262  false,
263  false,
264  true)
265  .plan_result;
266  auto ra_executor = RelAlgExecutor(executor.get(), cat, query_ra);
267  const auto& query_hints = ra_executor.getParsedQueryHints();
268  return query_hints;
269 }
270 
271 void QueryRunner::runDDLStatement(const std::string& stmt_str_in) {
272  CHECK(session_info_);
274 
275  std::string stmt_str = stmt_str_in;
276  // First remove special chars
277  boost::algorithm::trim_left_if(stmt_str, boost::algorithm::is_any_of("\n"));
278  // Then remove spaces
279  boost::algorithm::trim_left(stmt_str);
280 
281  ParserWrapper pw{stmt_str};
282  if (pw.is_copy_to) {
283  stmt_str = apply_copy_to_shim(stmt_str_in);
284  }
285 
286  auto query_state = create_query_state(session_info_, stmt_str);
287  auto stdlog = STDLOG(query_state);
288 
289  SQLParser parser;
290  std::list<std::unique_ptr<Parser::Stmt>> parse_trees;
291  std::string last_parsed;
292  CHECK_EQ(parser.parse(stmt_str, parse_trees, last_parsed), 0) << stmt_str_in;
293  CHECK_EQ(parse_trees.size(), size_t(1));
294  auto stmt = parse_trees.front().get();
295  auto ddl = dynamic_cast<Parser::DDLStmt*>(stmt);
296  CHECK(ddl);
297  ddl->execute(*session_info_);
298 }
299 
300 std::shared_ptr<ResultSet> QueryRunner::runSQL(const std::string& query_str,
301  const ExecutorDeviceType device_type,
302  const bool hoist_literals,
303  const bool allow_loop_joins) {
304  CHECK(session_info_);
306 
307  ParserWrapper pw{query_str};
308  if (pw.isCalcitePathPermissable()) {
309  const auto execution_result =
310  runSelectQuery(query_str, device_type, hoist_literals, allow_loop_joins);
311  VLOG(1) << session_info_->getCatalog().getDataMgr().getSystemMemoryUsage();
312  return execution_result->getRows();
313  }
314 
315  auto query_state = create_query_state(session_info_, query_str);
316  auto stdlog = STDLOG(query_state);
317 
318  SQLParser parser;
319  std::list<std::unique_ptr<Parser::Stmt>> parse_trees;
320  std::string last_parsed;
321  CHECK_EQ(parser.parse(query_str, parse_trees, last_parsed), 0) << query_str;
322  CHECK_EQ(parse_trees.size(), size_t(1));
323  auto stmt = parse_trees.front().get();
324  auto insert_values_stmt = dynamic_cast<InsertValuesStmt*>(stmt);
325  CHECK(insert_values_stmt);
326  insert_values_stmt->execute(*session_info_);
327  return nullptr;
328 }
329 
330 std::shared_ptr<Executor> QueryRunner::getExecutor() const {
331  CHECK(session_info_);
333  auto query_state = create_query_state(session_info_, "");
334  auto stdlog = STDLOG(query_state);
336  return executor;
337 }
338 
339 std::shared_ptr<ResultSet> QueryRunner::runSQLWithAllowingInterrupt(
340  const std::string& query_str,
341  std::shared_ptr<Executor> executor,
342  const std::string& session_id,
343  const ExecutorDeviceType device_type,
344  const unsigned interrupt_check_freq) {
345  CHECK(session_info_);
347  auto session_info =
348  std::make_shared<Catalog_Namespace::SessionInfo>(session_info_->get_catalog_ptr(),
349  session_info_->get_currentUser(),
351  session_id);
352  auto query_state = create_query_state(session_info, query_str);
353  auto stdlog = STDLOG(query_state);
354  const auto& cat = query_state->getConstSessionInfo()->getCatalog();
356 
358  true,
359  false,
360  true,
361  false,
362  false,
363  false,
364  false,
365  10000,
366  false,
367  false,
369  true,
370  interrupt_check_freq};
371  std::string query_ra{""};
372  {
373  // async query initiation for interrupt test
374  // incurs data race warning in TSAN since
375  // calcite_mgr is shared across multiple query threads
376  // so here we lock the manager during query parsing
377  std::lock_guard<std::mutex> calcite_lock_guard(calcite_lock);
378  auto calcite_mgr = cat.getCalciteMgr();
379  query_ra = calcite_mgr
380  ->process(query_state->createQueryStateProxy(),
381  pg_shim(query_state->getQueryStr()),
382  {},
383  true,
384  false,
385  false,
386  true)
387  .plan_result;
388  }
389  auto result = RelAlgExecutor(executor.get(), cat, query_ra, query_state)
390  .executeRelAlgQuery(co, eo, false, nullptr);
391  return result.getRows();
392 }
393 
394 std::vector<std::shared_ptr<ResultSet>> QueryRunner::runMultipleStatements(
395  const std::string& sql,
396  const ExecutorDeviceType dt) {
397  std::vector<std::shared_ptr<ResultSet>> results;
398  // TODO: Need to properly handle escaped semicolons instead of doing a naive split().
399  auto fields = split(sql, ";");
400  for (const auto& field : fields) {
401  auto text = strip(field) + ";";
402  if (text == ";") {
403  continue;
404  }
405  // TODO: Maybe remove this redundant parsing after enhancing Parser::Stmt?
406  SQLParser parser;
407  std::list<std::unique_ptr<Parser::Stmt>> parse_trees;
408  std::string last_parsed;
409  CHECK_EQ(parser.parse(text, parse_trees, last_parsed), 0);
410  CHECK_EQ(parse_trees.size(), size_t(1));
411  auto stmt = parse_trees.front().get();
412  Parser::DDLStmt* ddl = dynamic_cast<Parser::DDLStmt*>(stmt);
413  Parser::DMLStmt* dml = dynamic_cast<Parser::DMLStmt*>(stmt);
414  if (ddl != nullptr && dml == nullptr) {
415  runDDLStatement(text);
416  results.push_back(nullptr);
417  } else if (ddl == nullptr && dml != nullptr) {
418  results.push_back(runSQL(text, dt, true, true));
419  } else {
420  throw std::runtime_error("Unexpected SQL statement type: " + text);
421  }
422  }
423  return results;
424 }
425 
426 void QueryRunner::runImport(Parser::CopyTableStmt* import_stmt) {
427  CHECK(import_stmt);
428  import_stmt->execute(*session_info_);
429 }
430 
431 std::unique_ptr<import_export::Loader> QueryRunner::getLoader(
432  const TableDescriptor* td) const {
433  auto cat = getCatalog();
434  return std::make_unique<import_export::Loader>(*cat, td);
435 }
436 
437 namespace {
438 
439 std::shared_ptr<ExecutionResult> run_select_query_with_filter_push_down(
440  QueryStateProxy query_state_proxy,
441  const ExecutorDeviceType device_type,
442  const bool hoist_literals,
443  const bool allow_loop_joins,
444  const bool just_explain,
445  const bool with_filter_push_down) {
446  auto const& query_state = query_state_proxy.getQueryState();
447  const auto& cat = query_state.getConstSessionInfo()->getCatalog();
451 
453  true,
454  just_explain,
455  allow_loop_joins,
456  false,
457  false,
458  false,
459  false,
460  10000,
461  with_filter_push_down,
462  false,
464  false,
465  1000};
466  auto calcite_mgr = cat.getCalciteMgr();
467  const auto query_ra = calcite_mgr
468  ->process(query_state_proxy,
469  pg_shim(query_state.getQueryStr()),
470  {},
471  true,
472  false,
473  false,
474  true)
475  .plan_result;
476  auto ra_executor = RelAlgExecutor(executor.get(), cat, query_ra);
477  const auto& query_hints = ra_executor.getParsedQueryHints();
478  if (query_hints.cpu_mode) {
480  }
481  auto result = std::make_shared<ExecutionResult>(
482  ra_executor.executeRelAlgQuery(co, eo, false, nullptr));
483  const auto& filter_push_down_requests = result->getPushedDownFilterInfo();
484  if (!filter_push_down_requests.empty()) {
485  std::vector<TFilterPushDownInfo> filter_push_down_info;
486  for (const auto& req : filter_push_down_requests) {
487  TFilterPushDownInfo filter_push_down_info_for_request;
488  filter_push_down_info_for_request.input_prev = req.input_prev;
489  filter_push_down_info_for_request.input_start = req.input_start;
490  filter_push_down_info_for_request.input_next = req.input_next;
491  filter_push_down_info.push_back(filter_push_down_info_for_request);
492  }
493  const auto new_query_ra = calcite_mgr
494  ->process(query_state_proxy,
495  pg_shim(query_state.getQueryStr()),
496  filter_push_down_info,
497  true,
498  false,
499  false,
500  true)
501  .plan_result;
502  const ExecutionOptions eo_modified{eo.output_columnar_hint,
503  eo.allow_multifrag,
504  eo.just_explain,
505  eo.allow_loop_joins,
506  eo.with_watchdog,
507  eo.jit_debug,
508  eo.just_validate,
511  /*find_push_down_candidates=*/false,
512  /*just_calcite_explain=*/false,
515  auto new_ra_executor = RelAlgExecutor(executor.get(), cat, new_query_ra);
516  return std::make_shared<ExecutionResult>(
517  new_ra_executor.executeRelAlgQuery(co, eo_modified, false, nullptr));
518  } else {
519  return result;
520  }
521 }
522 
523 } // namespace
524 
525 std::shared_ptr<ExecutionResult> QueryRunner::runSelectQuery(
526  const std::string& query_str,
527  const ExecutorDeviceType device_type,
528  const bool hoist_literals,
529  const bool allow_loop_joins,
530  const bool just_explain) {
531  CHECK(session_info_);
533  auto query_state = create_query_state(session_info_, query_str);
534  auto stdlog = STDLOG(query_state);
536  return run_select_query_with_filter_push_down(query_state->createQueryStateProxy(),
537  device_type,
538  hoist_literals,
539  allow_loop_joins,
540  just_explain,
542  }
543 
544  const auto& cat = session_info_->getCatalog();
545 
546  std::shared_ptr<ExecutionResult> result;
547  auto query_launch_task =
548  std::make_shared<QueryDispatchQueue::Task>([&cat,
549  &query_str,
550  &device_type,
551  &allow_loop_joins,
552  &just_explain,
553  &query_state,
554  &result](const size_t worker_id) {
555  auto executor = Executor::getExecutor(worker_id);
558 
560  true,
561  just_explain,
562  allow_loop_joins,
563  false,
564  false,
565  false,
566  false,
567  10000,
568  false,
569  false,
571  false,
572  1000};
573  auto calcite_mgr = cat.getCalciteMgr();
574  const auto query_ra = calcite_mgr
575  ->process(query_state->createQueryStateProxy(),
576  pg_shim(query_str),
577  {},
578  true,
579  false,
581  true)
582  .plan_result;
583  auto ra_executor = RelAlgExecutor(executor.get(), cat, query_ra);
584  const auto& query_hints = ra_executor.getParsedQueryHints();
585  if (query_hints.cpu_mode) {
587  }
588  result = std::make_shared<ExecutionResult>(
589  ra_executor.executeRelAlgQuery(co, eo, false, nullptr));
590  });
591  CHECK(dispatch_queue_);
592  dispatch_queue_->submit(query_launch_task);
593  auto result_future = query_launch_task->get_future();
594  result_future.get();
595  CHECK(result);
596  return result;
597 }
598 
599 const std::shared_ptr<std::vector<int32_t>>& QueryRunner::getCachedJoinHashTable(
600  size_t idx) {
602 };
603 
604 const std::shared_ptr<std::vector<int8_t>>& QueryRunner::getCachedBaselineHashTable(
605  size_t idx) {
607 };
608 
609 size_t QueryRunner::getEntryCntCachedBaselineHashTable(size_t idx) {
611 }
612 
613 uint64_t QueryRunner::getNumberOfCachedJoinHashTables() {
615 };
616 
617 uint64_t QueryRunner::getNumberOfCachedBaselineJoinHashTables() {
619 };
620 
621 void QueryRunner::reset() {
622  qr_instance_.reset(nullptr);
624 }
625 
626 ImportDriver::ImportDriver(std::shared_ptr<Catalog_Namespace::Catalog> cat,
628  const ExecutorDeviceType dt)
629  : QueryRunner(std::make_unique<Catalog_Namespace::SessionInfo>(cat, user, dt, "")) {}
630 
631 void ImportDriver::importGeoTable(const std::string& file_path,
632  const std::string& table_name,
633  const bool compression,
634  const bool create_table,
635  const bool explode_collections) {
636  using namespace import_export;
637 
639  const std::string geo_column_name(OMNISCI_GEO_PREFIX);
640 
641  CopyParams copy_params;
642  if (compression) {
644  copy_params.geo_coords_comp_param = 32;
645  } else {
647  copy_params.geo_coords_comp_param = 0;
648  }
649  copy_params.geo_assign_render_groups = true;
650  copy_params.geo_explode_collections = explode_collections;
651 
652  auto cds = Importer::gdalToColumnDescriptors(file_path, geo_column_name, copy_params);
653  std::map<std::string, std::string> colname_to_src;
654  for (auto& cd : cds) {
655  const auto col_name_sanitized = ImportHelpers::sanitize_name(cd.columnName);
656  const auto ret =
657  colname_to_src.insert(std::make_pair(col_name_sanitized, cd.columnName));
658  CHECK(ret.second);
659  cd.columnName = col_name_sanitized;
660  }
661 
662  auto& cat = session_info_->getCatalog();
663 
664  if (create_table) {
665  const auto td = cat.getMetadataForTable(table_name);
666  if (td != nullptr) {
667  throw std::runtime_error("Error: Table " + table_name +
668  " already exists. Possible failure to correctly re-create "
669  "mapd_data directory.");
670  }
671  if (table_name != ImportHelpers::sanitize_name(table_name)) {
672  throw std::runtime_error("Invalid characters in table name: " + table_name);
673  }
674 
675  std::string stmt{"CREATE TABLE " + table_name};
676  std::vector<std::string> col_stmts;
677 
678  for (auto& cd : cds) {
679  if (cd.columnType.get_type() == SQLTypes::kINTERVAL_DAY_TIME ||
680  cd.columnType.get_type() == SQLTypes::kINTERVAL_YEAR_MONTH) {
681  throw std::runtime_error(
682  "Unsupported type: INTERVAL_DAY_TIME or INTERVAL_YEAR_MONTH for col " +
683  cd.columnName + " (table: " + table_name + ")");
684  }
685 
686  if (cd.columnType.get_type() == SQLTypes::kDECIMAL) {
687  if (cd.columnType.get_precision() == 0 && cd.columnType.get_scale() == 0) {
688  cd.columnType.set_precision(14);
689  cd.columnType.set_scale(7);
690  }
691  }
692 
693  std::string col_stmt;
694  col_stmt.append(cd.columnName + " " + cd.columnType.get_type_name() + " ");
695 
696  if (cd.columnType.get_compression() != EncodingType::kENCODING_NONE) {
697  col_stmt.append("ENCODING " + cd.columnType.get_compression_name() + " ");
698  } else {
699  if (cd.columnType.is_string()) {
700  col_stmt.append("ENCODING NONE");
701  } else if (cd.columnType.is_geometry()) {
702  if (cd.columnType.get_output_srid() == 4326) {
703  col_stmt.append("ENCODING NONE");
704  }
705  }
706  }
707  col_stmts.push_back(col_stmt);
708  }
709 
710  stmt.append(" (" + boost::algorithm::join(col_stmts, ",") + ");");
711  runDDLStatement(stmt);
712 
713  LOG(INFO) << "Created table: " << table_name;
714  } else {
715  LOG(INFO) << "Not creating table: " << table_name;
716  }
717 
718  const auto td = cat.getMetadataForTable(table_name);
719  if (td == nullptr) {
720  throw std::runtime_error("Error: Failed to create table " + table_name);
721  }
722 
723  import_export::Importer importer(cat, td, file_path, copy_params);
724  auto ms = measure<>::execution([&]() { importer.importGDAL(colname_to_src); });
725  LOG(INFO) << "Import Time for " << table_name << ": " << (double)ms / 1000.0 << " s";
726 }
727 
728 } // namespace QueryRunner
bool g_enable_calcite_view_optimize
Definition: QueryRunner.cpp:50
Classes used to wrap parser calls for calcite redirection.
static void addUdfs(const std::string &json_func_sigs)
#define CHECK_EQ(x, y)
Definition: Logger.h:205
#define CALCITEPORT
Definition: QueryRunner.cpp:42
static const std::shared_ptr< std::vector< int32_t > > & getCachedHashTable(size_t idx)
bool g_serialize_temp_tables
Definition: Catalog.cpp:95
ImportStatus importGDAL(std::map< std::string, std::string > colname_to_src)
Definition: Importer.cpp:4729
static const std::shared_ptr< std::vector< int8_t > > & getCachedHashTable(size_t idx)
ExecutorDeviceType
std::string strip(std::string_view str)
trim any whitespace from the left and right ends of a string
#define LOG(tag)
Definition: Logger.h:188
std::string join(T const &container, std::string const &delim)
void mapd_signal_handler(int signal_number)
Definition: QueryRunner.cpp:65
static void add(const std::string &json_func_sigs)
std::mutex calcite_lock
Definition: QueryRunner.cpp:51
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters system_parameters=SystemParameters())
Definition: Execute.cpp:148
ExecutorOptLevel opt_level
void set_once_fatal_func(FatalFunc fatal_func)
Definition: Logger.cpp:307
void execute(const Catalog_Namespace::SessionInfo &session) override
std::vector< std::string > split(std::string_view str, std::string_view delim, std::optional< size_t > maxsplit)
split apart a string into a vector of substrings
#define LOG_IF(severity, condition)
Definition: Logger.h:287
static void clearMemory(const Data_Namespace::MemoryLevel memory_level)
Definition: Execute.cpp:170
static uint64_t getNumberOfCachedHashTables()
ImportDriver(std::shared_ptr< Catalog_Namespace::Catalog > cat, const Catalog_Namespace::UserMetadata &user, const ExecutorDeviceType dt=ExecutorDeviceType::GPU)
This file contains the class specification and related data structures for Catalog.
const rapidjson::Value & field(const rapidjson::Value &obj, const char field[]) noexcept
Definition: JsonAccessors.h:31
bool g_enable_columnar_output
Definition: Execute.cpp:90
static SysCatalog & instance()
Definition: SysCatalog.h:288
const bool allow_multifrag
std::string cat(Ts &&... args)
const bool just_validate
void init(LogOptions const &log_opts)
Definition: Logger.cpp:280
QueryState & getQueryState()
Definition: QueryState.h:172
const std::string OMNISCI_DEFAULT_DB
Definition: SysCatalog.h:58
const bool with_dynamic_watchdog
const QueryHint getParsedQueryHints() const
const double gpu_input_mem_limit_percent
size_t g_leaf_count
Definition: ParserNode.cpp:70
bool g_enable_filter_push_down
Definition: Execute.cpp:86
virtual void runDDLStatement(const std::string &)
double g_gpu_mem_limit_percent
Definition: QueryRunner.cpp:47
std::string sanitize_name(const std::string &name)
ExecutorDeviceType device_type
void apply_shim(std::string &result, const boost::regex &reg_expr, const std::function< void(std::string &, const boost::smatch &)> &shim_fn)
void importGeoTable(const std::string &file_path, const std::string &table_name, const bool compression, const bool create_table, const bool explode_collections)
static uint64_t getNumberOfCachedHashTables()
static void set(const std::string &dbName, std::shared_ptr< Catalog > cat)
Definition: Catalog.cpp:3667
void shutdown()
Definition: Logger.cpp:314
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:187
void register_signal_handler(int signum, void(*handler)(int))
Definition: MapDServer.cpp:91
std::shared_ptr< Calcite > g_calcite
Definition: QueryRunner.cpp:56
static CompilationOptions defaults(const ExecutorDeviceType device_type=ExecutorDeviceType::GPU)
static size_t getEntryCntCachedHashTable(size_t idx)
const bool allow_loop_joins
#define CHECK(condition)
Definition: Logger.h:197
static TimeT::rep execution(F func, Args &&... args)
Definition: sample.cpp:29
std::shared_ptr< ExecutionResult > run_select_query_with_filter_push_down(QueryStateProxy query_state_proxy, const ExecutorDeviceType device_type, const bool hoist_literals, const bool allow_loop_joins, const bool just_explain, const bool with_filter_push_down)
specifies the content in-memory of a row in the table metadata table
const bool allow_runtime_query_interrupt
const unsigned dynamic_watchdog_time_limit
std::string pg_shim(const std::string &query)
#define STDLOG(...)
Definition: QueryState.h:225
#define VLOG(n)
Definition: Logger.h:291
const bool with_watchdog
std::atomic< bool > isSuper
Definition: SysCatalog.h:96
std::string apply_copy_to_shim(const std::string &query_str)
const std::string OMNISCI_GEO_PREFIX
Definition: geosupport.h:23
static const ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:303
EncodingType geo_coords_encoding
Definition: CopyParams.h:73