OmniSciDB  0b528656ed
QueryRunner.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "QueryRunner.h"
18 
19 #include "Calcite/Calcite.h"
20 #include "Catalog/Catalog.h"
21 #include "DistributedLoader.h"
23 #include "Parser/ParserWrapper.h"
24 #include "Parser/parser.h"
30 #include "Shared/Logger.h"
31 #include "Shared/StringTransform.h"
33 #include "Shared/geosupport.h"
34 #include "Shared/import_helpers.h"
35 #include "bcrypt.h"
36 #include "gen-cpp/CalciteServer.h"
37 
38 #include <boost/filesystem/operations.hpp>
39 #include <csignal>
40 #include <random>
41 
42 #define CALCITEPORT 3279
43 
44 extern size_t g_leaf_count;
45 extern bool g_enable_filter_push_down;
46 
48 
49 extern bool g_serialize_temp_tables;
50 std::mutex calcite_lock;
51 
52 using namespace Catalog_Namespace;
53 namespace {
54 
55 std::shared_ptr<Calcite> g_calcite = nullptr;
56 
57 void calcite_shutdown_handler() noexcept {
58  if (g_calcite) {
59  g_calcite->close_calcite_server();
60  g_calcite.reset();
61  }
62 }
63 
64 void mapd_signal_handler(int signal_number) {
65  LOG(ERROR) << "Interrupt signal (" << signal_number << ") received.";
67  // shut down logging force a flush
69  // terminate program
70  if (signal_number == SIGTERM) {
71  std::exit(EXIT_SUCCESS);
72  } else {
73  std::exit(signal_number);
74  }
75 }
76 
78  std::signal(SIGTERM, mapd_signal_handler);
79  std::signal(SIGSEGV, mapd_signal_handler);
80  std::signal(SIGABRT, mapd_signal_handler);
81 }
82 
83 } // namespace
84 
85 namespace QueryRunner {
86 
87 std::unique_ptr<QueryRunner> QueryRunner::qr_instance_ = nullptr;
88 
89 query_state::QueryStates QueryRunner::query_states_;
90 
91 QueryRunner* QueryRunner::init(const char* db_path,
92  const std::string& user,
93  const std::string& pass,
94  const std::string& db_name,
95  const std::vector<LeafHostInfo>& string_servers,
96  const std::vector<LeafHostInfo>& leaf_servers,
97  const std::string& udf_filename,
98  bool uses_gpus,
99  const size_t max_gpu_mem,
100  const int reserved_gpu_mem,
101  const bool create_user,
102  const bool create_db) {
103  LOG_IF(FATAL, !leaf_servers.empty()) << "Distributed test runner not supported.";
104  CHECK(leaf_servers.empty());
105  qr_instance_.reset(new QueryRunner(db_path,
106  user,
107  pass,
108  db_name,
109  string_servers,
110  leaf_servers,
111  udf_filename,
112  uses_gpus,
113  max_gpu_mem,
114  reserved_gpu_mem,
115  create_user,
116  create_db));
117  return qr_instance_.get();
118 }
119 
120 QueryRunner::QueryRunner(const char* db_path,
121  const std::string& user_name,
122  const std::string& passwd,
123  const std::string& db_name,
124  const std::vector<LeafHostInfo>& string_servers,
125  const std::vector<LeafHostInfo>& leaf_servers,
126  const std::string& udf_filename,
127  bool uses_gpus,
128  const size_t max_gpu_mem,
129  const int reserved_gpu_mem,
130  const bool create_user,
131  const bool create_db)
132  : dispatch_queue_(std::make_unique<QueryDispatchQueue>(1)) {
134 
135  boost::filesystem::path base_path{db_path};
136  CHECK(boost::filesystem::exists(base_path));
137  auto system_db_file = base_path / "mapd_catalogs" / OMNISCI_DEFAULT_DB;
138  CHECK(boost::filesystem::exists(system_db_file));
139  auto data_dir = base_path / "mapd_data";
142 
145  g_calcite =
146  std::make_shared<Calcite>(-1, CALCITEPORT, db_path, 1024, 5000, udf_filename);
147  ExtensionFunctionsWhitelist::add(g_calcite->getExtensionFunctionWhitelist());
148  if (!udf_filename.empty()) {
149  ExtensionFunctionsWhitelist::addUdfs(g_calcite->getUserDefinedFunctionWhitelist());
150  }
151 
153 
154 #ifndef HAVE_CUDA
155  uses_gpus = false;
156 #endif
157  SystemParameters mapd_params;
158  mapd_params.gpu_buffer_mem_bytes = max_gpu_mem;
159  mapd_params.aggregator = !leaf_servers.empty();
160 
161  auto data_mgr = std::make_shared<Data_Namespace::DataMgr>(
162  data_dir.string(), mapd_params, uses_gpus, -1, 0, reserved_gpu_mem);
163 
164  auto& sys_cat = Catalog_Namespace::SysCatalog::instance();
165 
166  sys_cat.init(base_path.string(),
167  data_mgr,
168  {},
169  g_calcite,
170  false,
171  mapd_params.aggregator,
172  string_servers);
173 
174  if (create_user) {
175  if (!sys_cat.getMetadataForUser(user_name, user)) {
176  sys_cat.createUser(user_name, passwd, false, "", true);
177  }
178  }
179  CHECK(sys_cat.getMetadataForUser(user_name, user));
180  CHECK(bcrypt_checkpw(passwd.c_str(), user.passwd_hash.c_str()) == 0);
181 
182  if (create_db) {
183  if (!sys_cat.getMetadataForDB(db_name, db)) {
184  sys_cat.createDatabase(db_name, user.userId);
185  }
186  }
187  CHECK(sys_cat.getMetadataForDB(db_name, db));
188  CHECK(user.isSuper || (user.userId == db.dbOwner));
189  auto cat = std::make_shared<Catalog_Namespace::Catalog>(
190  base_path.string(), db, data_mgr, string_servers, g_calcite, create_db);
191  Catalog_Namespace::Catalog::set(cat->getCurrentDB().dbName, cat);
192  session_info_ = std::make_unique<Catalog_Namespace::SessionInfo>(
193  cat, user, ExecutorDeviceType::GPU, "");
194 }
195 
196 void QueryRunner::resizeDispatchQueue(const size_t num_executors) {
197  dispatch_queue_ = std::make_unique<QueryDispatchQueue>(num_executors);
198 }
199 
200 QueryRunner::QueryRunner(std::unique_ptr<Catalog_Namespace::SessionInfo> session)
201  : session_info_(std::move(session))
202  , dispatch_queue_(std::make_unique<QueryDispatchQueue>(1)) {}
203 
204 std::shared_ptr<Catalog_Namespace::Catalog> QueryRunner::getCatalog() const {
205  CHECK(session_info_);
206  return session_info_->get_catalog_ptr();
207 }
208 
209 std::shared_ptr<Calcite> QueryRunner::getCalcite() const {
210  // TODO: Embed Calcite shared_ptr ownership in QueryRunner
211  return g_calcite;
212 }
213 
214 bool QueryRunner::gpusPresent() const {
215  CHECK(session_info_);
216  return session_info_->getCatalog().getDataMgr().gpusPresent();
217 }
218 
219 void QueryRunner::clearGpuMemory() const {
222 }
223 
224 void QueryRunner::clearCpuMemory() const {
227 }
228 
229 std::string apply_copy_to_shim(const std::string& query_str) {
230  auto result = query_str;
231  {
232  boost::regex copy_to{R"(COPY\s*\(([^#])(.+)\)\s+TO\s)",
233  boost::regex::extended | boost::regex::icase};
234  apply_shim(result, copy_to, [](std::string& result, const boost::smatch& what) {
235  result.replace(
236  what.position(), what.length(), "COPY (#~#" + what[1] + what[2] + "#~#) TO ");
237  });
238  }
239  return result;
240 }
241 
242 QueryHint QueryRunner::getParsedQueryHintofQuery(const std::string& query_str) {
243  CHECK(session_info_);
245  auto query_state = create_query_state(session_info_, query_str);
246  const auto& cat = session_info_->getCatalog();
248  auto calcite_mgr = cat.getCalciteMgr();
249  const auto query_ra = calcite_mgr
250  ->process(query_state->createQueryStateProxy(),
251  pg_shim(query_str),
252  {},
253  true,
254  false,
255  false,
256  true)
257  .plan_result;
258  auto ra_executor = RelAlgExecutor(executor.get(), cat, query_ra);
259  const auto& query_hints = ra_executor.getParsedQueryHints();
260  return query_hints;
261 }
262 
263 void QueryRunner::runDDLStatement(const std::string& stmt_str_in) {
264  CHECK(session_info_);
266 
267  std::string stmt_str = stmt_str_in;
268  // First remove special chars
269  boost::algorithm::trim_left_if(stmt_str, boost::algorithm::is_any_of("\n"));
270  // Then remove spaces
271  boost::algorithm::trim_left(stmt_str);
272 
273  ParserWrapper pw{stmt_str};
274  if (pw.is_copy_to) {
275  stmt_str = apply_copy_to_shim(stmt_str_in);
276  }
277 
278  auto query_state = create_query_state(session_info_, stmt_str);
279  auto stdlog = STDLOG(query_state);
280 
281  SQLParser parser;
282  std::list<std::unique_ptr<Parser::Stmt>> parse_trees;
283  std::string last_parsed;
284  CHECK_EQ(parser.parse(stmt_str, parse_trees, last_parsed), 0) << stmt_str_in;
285  CHECK_EQ(parse_trees.size(), size_t(1));
286  auto stmt = parse_trees.front().get();
287  auto ddl = dynamic_cast<Parser::DDLStmt*>(stmt);
288  CHECK(ddl);
289  ddl->execute(*session_info_);
290 }
291 
292 std::shared_ptr<ResultSet> QueryRunner::runSQL(const std::string& query_str,
293  const ExecutorDeviceType device_type,
294  const bool hoist_literals,
295  const bool allow_loop_joins) {
296  CHECK(session_info_);
298 
299  ParserWrapper pw{query_str};
300  if (pw.isCalcitePathPermissable()) {
301  const auto execution_result =
302  runSelectQuery(query_str, device_type, hoist_literals, allow_loop_joins);
303  VLOG(1) << session_info_->getCatalog().getDataMgr().getSystemMemoryUsage();
304  return execution_result->getRows();
305  }
306 
307  auto query_state = create_query_state(session_info_, query_str);
308  auto stdlog = STDLOG(query_state);
309 
310  SQLParser parser;
311  std::list<std::unique_ptr<Parser::Stmt>> parse_trees;
312  std::string last_parsed;
313  CHECK_EQ(parser.parse(query_str, parse_trees, last_parsed), 0) << query_str;
314  CHECK_EQ(parse_trees.size(), size_t(1));
315  auto stmt = parse_trees.front().get();
316  auto insert_values_stmt = dynamic_cast<InsertValuesStmt*>(stmt);
317  CHECK(insert_values_stmt);
318  insert_values_stmt->execute(*session_info_);
319  return nullptr;
320 }
321 
322 std::shared_ptr<Executor> QueryRunner::getExecutor() const {
323  CHECK(session_info_);
325  auto query_state = create_query_state(session_info_, "");
326  auto stdlog = STDLOG(query_state);
328  return executor;
329 }
330 
331 std::shared_ptr<ResultSet> QueryRunner::runSQLWithAllowingInterrupt(
332  const std::string& query_str,
333  std::shared_ptr<Executor> executor,
334  const std::string& session_id,
335  const ExecutorDeviceType device_type,
336  const unsigned interrupt_check_freq) {
337  CHECK(session_info_);
339  auto session_info =
340  std::make_shared<Catalog_Namespace::SessionInfo>(session_info_->get_catalog_ptr(),
341  session_info_->get_currentUser(),
343  session_id);
344  auto query_state = create_query_state(session_info, query_str);
345  auto stdlog = STDLOG(query_state);
346  const auto& cat = query_state->getConstSessionInfo()->getCatalog();
348 
350  true,
351  false,
352  true,
353  false,
354  false,
355  false,
356  false,
357  10000,
358  false,
359  false,
361  true,
362  interrupt_check_freq};
363  std::string query_ra{""};
364  {
365  // async query initiation for interrupt test
366  // incurs data race warning in TSAN since
367  // calcite_mgr is shared across multiple query threads
368  // so here we lock the manager during query parsing
369  std::lock_guard<std::mutex> calcite_lock_guard(calcite_lock);
370  auto calcite_mgr = cat.getCalciteMgr();
371  query_ra = calcite_mgr
372  ->process(query_state->createQueryStateProxy(),
373  pg_shim(query_state->getQueryStr()),
374  {},
375  true,
376  false,
377  false,
378  true)
379  .plan_result;
380  }
381  auto result = RelAlgExecutor(executor.get(), cat, query_ra, query_state)
382  .executeRelAlgQuery(co, eo, false, nullptr);
383  return result.getRows();
384 }
385 
386 std::vector<std::shared_ptr<ResultSet>> QueryRunner::runMultipleStatements(
387  const std::string& sql,
388  const ExecutorDeviceType dt) {
389  std::vector<std::shared_ptr<ResultSet>> results;
390  // TODO: Need to properly handle escaped semicolons instead of doing a naive split().
391  auto fields = split(sql, ";");
392  for (const auto& field : fields) {
393  auto text = strip(field) + ";";
394  if (text == ";") {
395  continue;
396  }
397  // TODO: Maybe remove this redundant parsing after enhancing Parser::Stmt?
398  SQLParser parser;
399  std::list<std::unique_ptr<Parser::Stmt>> parse_trees;
400  std::string last_parsed;
401  CHECK_EQ(parser.parse(text, parse_trees, last_parsed), 0);
402  CHECK_EQ(parse_trees.size(), size_t(1));
403  auto stmt = parse_trees.front().get();
404  Parser::DDLStmt* ddl = dynamic_cast<Parser::DDLStmt*>(stmt);
405  Parser::DMLStmt* dml = dynamic_cast<Parser::DMLStmt*>(stmt);
406  if (ddl != nullptr && dml == nullptr) {
407  runDDLStatement(text);
408  results.push_back(nullptr);
409  } else if (ddl == nullptr && dml != nullptr) {
410  results.push_back(runSQL(text, dt, true, true));
411  } else {
412  throw std::runtime_error("Unexpected SQL statement type: " + text);
413  }
414  }
415  return results;
416 }
417 
418 void QueryRunner::runImport(Parser::CopyTableStmt* import_stmt) {
419  CHECK(import_stmt);
420  import_stmt->execute(*session_info_);
421 }
422 
423 std::unique_ptr<import_export::Loader> QueryRunner::getLoader(
424  const TableDescriptor* td) const {
425  auto cat = getCatalog();
426  return std::make_unique<import_export::Loader>(*cat, td);
427 }
428 
429 namespace {
430 
431 std::shared_ptr<ExecutionResult> run_select_query_with_filter_push_down(
432  QueryStateProxy query_state_proxy,
433  const ExecutorDeviceType device_type,
434  const bool hoist_literals,
435  const bool allow_loop_joins,
436  const bool just_explain,
437  const bool with_filter_push_down) {
438  auto const& query_state = query_state_proxy.getQueryState();
439  const auto& cat = query_state.getConstSessionInfo()->getCatalog();
443 
445  true,
446  just_explain,
447  allow_loop_joins,
448  false,
449  false,
450  false,
451  false,
452  10000,
453  with_filter_push_down,
454  false,
456  false,
457  1000};
458  auto calcite_mgr = cat.getCalciteMgr();
459  const auto query_ra = calcite_mgr
460  ->process(query_state_proxy,
461  pg_shim(query_state.getQueryStr()),
462  {},
463  true,
464  false,
465  false,
466  true)
467  .plan_result;
468  auto ra_executor = RelAlgExecutor(executor.get(), cat, query_ra);
469  const auto& query_hints = ra_executor.getParsedQueryHints();
470  if (query_hints.cpu_mode) {
472  }
473  auto result = std::make_shared<ExecutionResult>(
474  ra_executor.executeRelAlgQuery(co, eo, false, nullptr));
475  const auto& filter_push_down_requests = result->getPushedDownFilterInfo();
476  if (!filter_push_down_requests.empty()) {
477  std::vector<TFilterPushDownInfo> filter_push_down_info;
478  for (const auto& req : filter_push_down_requests) {
479  TFilterPushDownInfo filter_push_down_info_for_request;
480  filter_push_down_info_for_request.input_prev = req.input_prev;
481  filter_push_down_info_for_request.input_start = req.input_start;
482  filter_push_down_info_for_request.input_next = req.input_next;
483  filter_push_down_info.push_back(filter_push_down_info_for_request);
484  }
485  const auto new_query_ra = calcite_mgr
486  ->process(query_state_proxy,
487  pg_shim(query_state.getQueryStr()),
488  filter_push_down_info,
489  true,
490  false,
491  false,
492  true)
493  .plan_result;
494  const ExecutionOptions eo_modified{eo.output_columnar_hint,
495  eo.allow_multifrag,
496  eo.just_explain,
497  eo.allow_loop_joins,
498  eo.with_watchdog,
499  eo.jit_debug,
500  eo.just_validate,
503  /*find_push_down_candidates=*/false,
504  /*just_calcite_explain=*/false,
507  auto new_ra_executor = RelAlgExecutor(executor.get(), cat, new_query_ra);
508  return std::make_shared<ExecutionResult>(
509  new_ra_executor.executeRelAlgQuery(co, eo_modified, false, nullptr));
510  } else {
511  return result;
512  }
513 }
514 
515 } // namespace
516 
517 std::shared_ptr<ExecutionResult> QueryRunner::runSelectQuery(
518  const std::string& query_str,
519  const ExecutorDeviceType device_type,
520  const bool hoist_literals,
521  const bool allow_loop_joins,
522  const bool just_explain) {
523  CHECK(session_info_);
525  auto query_state = create_query_state(session_info_, query_str);
526  auto stdlog = STDLOG(query_state);
528  return run_select_query_with_filter_push_down(query_state->createQueryStateProxy(),
529  device_type,
530  hoist_literals,
531  allow_loop_joins,
532  just_explain,
534  }
535 
536  const auto& cat = session_info_->getCatalog();
537 
538  std::shared_ptr<ExecutionResult> result;
539  auto query_launch_task =
540  std::make_shared<QueryDispatchQueue::Task>([&cat,
541  &query_str,
542  &device_type,
543  &allow_loop_joins,
544  &just_explain,
545  &query_state,
546  &result](const size_t worker_id) {
547  auto executor = Executor::getExecutor(worker_id);
550 
552  true,
553  just_explain,
554  allow_loop_joins,
555  false,
556  false,
557  false,
558  false,
559  10000,
560  false,
561  false,
563  false,
564  1000};
565  auto calcite_mgr = cat.getCalciteMgr();
566  const auto query_ra = calcite_mgr
567  ->process(query_state->createQueryStateProxy(),
568  pg_shim(query_str),
569  {},
570  true,
571  false,
572  false,
573  true)
574  .plan_result;
575  auto ra_executor = RelAlgExecutor(executor.get(), cat, query_ra);
576  const auto& query_hints = ra_executor.getParsedQueryHints();
577  if (query_hints.cpu_mode) {
579  }
580  result = std::make_shared<ExecutionResult>(
581  ra_executor.executeRelAlgQuery(co, eo, false, nullptr));
582  });
583  CHECK(dispatch_queue_);
584  dispatch_queue_->submit(query_launch_task);
585  auto result_future = query_launch_task->get_future();
586  result_future.get();
587  CHECK(result);
588  return result;
589 }
590 
591 const std::shared_ptr<std::vector<int32_t>>& QueryRunner::getCachedJoinHashTable(
592  size_t idx) {
594 };
595 
596 const std::shared_ptr<std::vector<int8_t>>& QueryRunner::getCachedBaselineHashTable(
597  size_t idx) {
599 };
600 
601 size_t QueryRunner::getEntryCntCachedBaselineHashTable(size_t idx) {
603 }
604 
605 uint64_t QueryRunner::getNumberOfCachedJoinHashTables() {
607 };
608 
609 uint64_t QueryRunner::getNumberOfCachedBaselineJoinHashTables() {
611 };
612 
613 void QueryRunner::reset() {
614  qr_instance_.reset(nullptr);
616 }
617 
618 ImportDriver::ImportDriver(std::shared_ptr<Catalog_Namespace::Catalog> cat,
620  const ExecutorDeviceType dt)
621  : QueryRunner(std::make_unique<Catalog_Namespace::SessionInfo>(cat, user, dt, "")) {}
622 
623 void ImportDriver::importGeoTable(const std::string& file_path,
624  const std::string& table_name,
625  const bool compression,
626  const bool create_table,
627  const bool explode_collections) {
628  using namespace import_export;
629 
631  const std::string geo_column_name(OMNISCI_GEO_PREFIX);
632 
633  CopyParams copy_params;
634  if (compression) {
636  copy_params.geo_coords_comp_param = 32;
637  } else {
639  copy_params.geo_coords_comp_param = 0;
640  }
641  copy_params.geo_assign_render_groups = true;
642  copy_params.geo_explode_collections = explode_collections;
643 
644  auto cds = Importer::gdalToColumnDescriptors(file_path, geo_column_name, copy_params);
645  std::map<std::string, std::string> colname_to_src;
646  for (auto& cd : cds) {
647  const auto col_name_sanitized = ImportHelpers::sanitize_name(cd.columnName);
648  const auto ret =
649  colname_to_src.insert(std::make_pair(col_name_sanitized, cd.columnName));
650  CHECK(ret.second);
651  cd.columnName = col_name_sanitized;
652  }
653 
654  auto& cat = session_info_->getCatalog();
655 
656  if (create_table) {
657  const auto td = cat.getMetadataForTable(table_name);
658  if (td != nullptr) {
659  throw std::runtime_error("Error: Table " + table_name +
660  " already exists. Possible failure to correctly re-create "
661  "mapd_data directory.");
662  }
663  if (table_name != ImportHelpers::sanitize_name(table_name)) {
664  throw std::runtime_error("Invalid characters in table name: " + table_name);
665  }
666 
667  std::string stmt{"CREATE TABLE " + table_name};
668  std::vector<std::string> col_stmts;
669 
670  for (auto& cd : cds) {
671  if (cd.columnType.get_type() == SQLTypes::kINTERVAL_DAY_TIME ||
672  cd.columnType.get_type() == SQLTypes::kINTERVAL_YEAR_MONTH) {
673  throw std::runtime_error(
674  "Unsupported type: INTERVAL_DAY_TIME or INTERVAL_YEAR_MONTH for col " +
675  cd.columnName + " (table: " + table_name + ")");
676  }
677 
678  if (cd.columnType.get_type() == SQLTypes::kDECIMAL) {
679  if (cd.columnType.get_precision() == 0 && cd.columnType.get_scale() == 0) {
680  cd.columnType.set_precision(14);
681  cd.columnType.set_scale(7);
682  }
683  }
684 
685  std::string col_stmt;
686  col_stmt.append(cd.columnName + " " + cd.columnType.get_type_name() + " ");
687 
688  if (cd.columnType.get_compression() != EncodingType::kENCODING_NONE) {
689  col_stmt.append("ENCODING " + cd.columnType.get_compression_name() + " ");
690  } else {
691  if (cd.columnType.is_string()) {
692  col_stmt.append("ENCODING NONE");
693  } else if (cd.columnType.is_geometry()) {
694  if (cd.columnType.get_output_srid() == 4326) {
695  col_stmt.append("ENCODING NONE");
696  }
697  }
698  }
699  col_stmts.push_back(col_stmt);
700  }
701 
702  stmt.append(" (" + boost::algorithm::join(col_stmts, ",") + ");");
703  runDDLStatement(stmt);
704 
705  LOG(INFO) << "Created table: " << table_name;
706  } else {
707  LOG(INFO) << "Not creating table: " << table_name;
708  }
709 
710  const auto td = cat.getMetadataForTable(table_name);
711  if (td == nullptr) {
712  throw std::runtime_error("Error: Failed to create table " + table_name);
713  }
714 
715  import_export::Importer importer(cat, td, file_path, copy_params);
716  auto ms = measure<>::execution([&]() { importer.importGDAL(colname_to_src); });
717  LOG(INFO) << "Import Time for " << table_name << ": " << (double)ms / 1000.0 << " s";
718 }
719 
720 } // namespace QueryRunner
Classes used to wrap parser calls for calcite redirection.
static void addUdfs(const std::string &json_func_sigs)
#define CHECK_EQ(x, y)
Definition: Logger.h:205
#define CALCITEPORT
Definition: QueryRunner.cpp:42
static const std::shared_ptr< std::vector< int32_t > > & getCachedHashTable(size_t idx)
bool g_serialize_temp_tables
Definition: Catalog.cpp:95
ImportStatus importGDAL(std::map< std::string, std::string > colname_to_src)
Definition: Importer.cpp:4708
static const std::shared_ptr< std::vector< int8_t > > & getCachedHashTable(size_t idx)
ExecutorDeviceType
std::string strip(std::string_view str)
trim any whitespace from the left and right ends of a string
#define LOG(tag)
Definition: Logger.h:188
std::string join(T const &container, std::string const &delim)
void mapd_signal_handler(int signal_number)
Definition: QueryRunner.cpp:64
static void add(const std::string &json_func_sigs)
std::mutex calcite_lock
Definition: QueryRunner.cpp:50
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters system_parameters=SystemParameters())
Definition: Execute.cpp:150
ExecutorOptLevel opt_level
void set_once_fatal_func(FatalFunc fatal_func)
Definition: Logger.cpp:303
void execute(const Catalog_Namespace::SessionInfo &session) override
std::vector< std::string > split(std::string_view str, std::string_view delim, std::optional< size_t > maxsplit)
split apart a string into a vector of substrings
#define LOG_IF(severity, condition)
Definition: Logger.h:287
static void clearMemory(const Data_Namespace::MemoryLevel memory_level)
Definition: Execute.cpp:172
static uint64_t getNumberOfCachedHashTables()
ImportDriver(std::shared_ptr< Catalog_Namespace::Catalog > cat, const Catalog_Namespace::UserMetadata &user, const ExecutorDeviceType dt=ExecutorDeviceType::GPU)
This file contains the class specification and related data structures for Catalog.
const rapidjson::Value & field(const rapidjson::Value &obj, const char field[]) noexcept
Definition: JsonAccessors.h:31
bool g_enable_columnar_output
Definition: Execute.cpp:90
static SysCatalog & instance()
Definition: SysCatalog.h:284
const bool allow_multifrag
std::string cat(Ts &&... args)
const bool just_validate
void init(LogOptions const &log_opts)
Definition: Logger.cpp:276
QueryState & getQueryState()
Definition: QueryState.h:172
const std::string OMNISCI_DEFAULT_DB
Definition: SysCatalog.h:58
const bool with_dynamic_watchdog
const QueryHint getParsedQueryHints() const
const double gpu_input_mem_limit_percent
size_t g_leaf_count
Definition: ParserNode.cpp:70
bool g_enable_filter_push_down
Definition: Execute.cpp:86
virtual void runDDLStatement(const std::string &)
double g_gpu_mem_limit_percent
Definition: QueryRunner.cpp:47
std::string sanitize_name(const std::string &name)
ExecutorDeviceType device_type
void apply_shim(std::string &result, const boost::regex &reg_expr, const std::function< void(std::string &, const boost::smatch &)> &shim_fn)
void importGeoTable(const std::string &file_path, const std::string &table_name, const bool compression, const bool create_table, const bool explode_collections)
static uint64_t getNumberOfCachedHashTables()
static void set(const std::string &dbName, std::shared_ptr< Catalog > cat)
Definition: Catalog.cpp:3638
void shutdown()
Definition: Logger.cpp:310
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:187
void register_signal_handler(int signum, void(*handler)(int))
Definition: MapDServer.cpp:91
std::shared_ptr< Calcite > g_calcite
Definition: QueryRunner.cpp:55
static CompilationOptions defaults(const ExecutorDeviceType device_type=ExecutorDeviceType::GPU)
static size_t getEntryCntCachedHashTable(size_t idx)
const bool allow_loop_joins
#define CHECK(condition)
Definition: Logger.h:197
static TimeT::rep execution(F func, Args &&... args)
Definition: sample.cpp:29
std::shared_ptr< ExecutionResult > run_select_query_with_filter_push_down(QueryStateProxy query_state_proxy, const ExecutorDeviceType device_type, const bool hoist_literals, const bool allow_loop_joins, const bool just_explain, const bool with_filter_push_down)
specifies the content in-memory of a row in the table metadata table
const bool allow_runtime_query_interrupt
const unsigned dynamic_watchdog_time_limit
static unsigned pass
Definition: testing.h:29
std::string pg_shim(const std::string &query)
#define STDLOG(...)
Definition: QueryState.h:225
#define VLOG(n)
Definition: Logger.h:291
const bool with_watchdog
std::atomic< bool > isSuper
Definition: SysCatalog.h:96
std::string apply_copy_to_shim(const std::string &query_str)
const std::string OMNISCI_GEO_PREFIX
Definition: geosupport.h:23
static const ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:325
EncodingType geo_coords_encoding
Definition: CopyParams.h:73