OmniSciDB  085a039ca4
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
QueryRunner.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "QueryRunner.h"
18 
19 #include "Calcite/Calcite.h"
20 #include "Catalog/Catalog.h"
22 #include "DistributedLoader.h"
23 #include "Geospatial/ColumnNames.h"
25 #include "Logger/Logger.h"
26 #include "Parser/ParserNode.h"
27 #include "Parser/ParserWrapper.h"
36 #include "Shared/StringTransform.h"
37 #include "Shared/SysDefinitions.h"
39 #include "Shared/import_helpers.h"
41 #include "gen-cpp/CalciteServer.h"
42 #include "include/bcrypt.h"
43 
44 #include <boost/filesystem/operations.hpp>
45 #include <csignal>
46 #include <random>
47 
48 #define CALCITEPORT 3279
49 
50 extern size_t g_leaf_count;
51 extern bool g_enable_filter_push_down;
52 
54 
55 extern bool g_serialize_temp_tables;
57 std::mutex calcite_lock;
58 
59 using namespace Catalog_Namespace;
60 namespace {
61 
62 std::shared_ptr<Calcite> g_calcite = nullptr;
63 
64 void calcite_shutdown_handler() noexcept {
65  if (g_calcite) {
66  g_calcite->close_calcite_server();
67  g_calcite.reset();
68  }
69 }
70 
74 }
75 
76 } // namespace
77 
78 namespace QueryRunner {
79 
80 std::unique_ptr<QueryRunner> QueryRunner::qr_instance_ = nullptr;
81 
82 query_state::QueryStates QueryRunner::query_states_;
83 
84 QueryRunner* QueryRunner::init(const char* db_path,
85  const std::string& udf_filename,
86  const size_t max_gpu_mem,
87  const int reserved_gpu_mem) {
88  return QueryRunner::init(db_path,
90  "HyperInteractive",
92  {},
93  {},
94  udf_filename,
95  true,
96  max_gpu_mem,
97  reserved_gpu_mem);
98 }
99 
101  const char* db_path,
102  const std::vector<LeafHostInfo>& string_servers,
103  const std::vector<LeafHostInfo>& leaf_servers) {
104  return QueryRunner::init(db_path,
106  "HyperInteractive",
108  string_servers,
109  leaf_servers,
110  "",
111  true,
112  0,
113  256 << 20,
114  false,
115  false,
116  disk_cache_config);
117 }
118 
119 QueryRunner* QueryRunner::init(const char* db_path,
120  const std::string& user,
121  const std::string& pass,
122  const std::string& db_name,
123  const std::vector<LeafHostInfo>& string_servers,
124  const std::vector<LeafHostInfo>& leaf_servers,
125  const std::string& udf_filename,
126  bool uses_gpus,
127  const size_t max_gpu_mem,
128  const int reserved_gpu_mem,
129  const bool create_user,
130  const bool create_db,
131  const File_Namespace::DiskCacheConfig* disk_cache_config) {
132  // Whitelist root path for tests by default
134  ddl_utils::FilePathWhitelist::initialize(db_path, "[\"/\"]", "[\"/\"]");
135  LOG_IF(FATAL, !leaf_servers.empty()) << "Distributed test runner not supported.";
136  CHECK(leaf_servers.empty());
137  qr_instance_.reset(new QueryRunner(db_path,
138  user,
139  pass,
140  db_name,
141  string_servers,
142  leaf_servers,
143  udf_filename,
144  uses_gpus,
145  max_gpu_mem,
146  reserved_gpu_mem,
147  create_user,
148  create_db,
149  disk_cache_config));
150  return qr_instance_.get();
151 }
152 
153 QueryRunner::QueryRunner(const char* db_path,
154  const std::string& user_name,
155  const std::string& passwd,
156  const std::string& db_name,
157  const std::vector<LeafHostInfo>& string_servers,
158  const std::vector<LeafHostInfo>& leaf_servers,
159  const std::string& udf_filename,
160  bool uses_gpus,
161  const size_t max_gpu_mem,
162  const int reserved_gpu_mem,
163  const bool create_user,
164  const bool create_db,
165  const File_Namespace::DiskCacheConfig* cache_config)
166  : dispatch_queue_(std::make_unique<QueryDispatchQueue>(1)) {
168  boost::filesystem::path base_path{db_path};
169  CHECK(boost::filesystem::exists(base_path));
170  auto system_db_file =
172  CHECK(boost::filesystem::exists(system_db_file));
173  auto data_dir = base_path / shared::kDataDirectoryName;
174  File_Namespace::DiskCacheConfig disk_cache_config{
175  (base_path / shared::kDefaultDiskCacheDirName).string(),
177  if (cache_config) {
178  disk_cache_config = *cache_config;
179  }
181 
184  g_calcite =
185  std::make_shared<Calcite>(-1, CALCITEPORT, db_path, 1024, 5000, true, udf_filename);
186  ExtensionFunctionsWhitelist::add(g_calcite->getExtensionFunctionWhitelist());
187  if (!udf_filename.empty()) {
188  ExtensionFunctionsWhitelist::addUdfs(g_calcite->getUserDefinedFunctionWhitelist());
189  }
190 
192  auto udtfs = ThriftSerializers::to_thrift(
194  std::vector<TUserDefinedFunction> udfs = {};
195  g_calcite->setRuntimeExtensionFunctions(udfs, udtfs, /*is_runtime=*/false);
196 
197  std::unique_ptr<CudaMgr_Namespace::CudaMgr> cuda_mgr;
198 #ifdef HAVE_CUDA
199  if (uses_gpus) {
200  cuda_mgr = std::make_unique<CudaMgr_Namespace::CudaMgr>(-1, 0);
201  }
202 #else
203  uses_gpus = false;
204 #endif
205  SystemParameters mapd_params;
206  mapd_params.gpu_buffer_mem_bytes = max_gpu_mem;
207  mapd_params.aggregator = !leaf_servers.empty();
208 
209  data_mgr_.reset(new Data_Namespace::DataMgr(data_dir.string(),
210  mapd_params,
211  std::move(cuda_mgr),
212  uses_gpus,
213  reserved_gpu_mem,
214  0,
215  disk_cache_config));
216 
217  auto& sys_cat = Catalog_Namespace::SysCatalog::instance();
218 
219  g_base_path = base_path.string();
220  sys_cat.init(g_base_path,
221  data_mgr_,
222  {},
223  g_calcite,
224  false,
225  mapd_params.aggregator,
226  string_servers);
227 
229 
230  if (create_user) {
231  if (!sys_cat.getMetadataForUser(user_name, user)) {
232  sys_cat.createUser(
233  user_name,
235  passwd, /*is_super=*/false, /*default_db=*/"", /*can_login=*/true},
236  g_read_only);
237  }
238  }
239  CHECK(sys_cat.getMetadataForUser(user_name, user));
240  CHECK(bcrypt_checkpw(passwd.c_str(), user.passwd_hash.c_str()) == 0);
241 
242  if (create_db) {
243  if (!sys_cat.getMetadataForDB(db_name, db_metadata_)) {
244  sys_cat.createDatabase(db_name, user.userId);
245  }
246  }
247  CHECK(sys_cat.getMetadataForDB(db_name, db_metadata_));
248  CHECK(user.isSuper || (user.userId == db_metadata_.dbOwner));
249  auto cat = sys_cat.getCatalog(db_metadata_, create_db);
250  CHECK(cat);
251  session_info_ = std::make_unique<Catalog_Namespace::SessionInfo>(
252  cat, user, ExecutorDeviceType::GPU, "");
253 }
254 
255 void QueryRunner::resizeDispatchQueue(const size_t num_executors) {
256  dispatch_queue_ = std::make_unique<QueryDispatchQueue>(num_executors);
257 }
258 
259 QueryRunner::QueryRunner(std::unique_ptr<Catalog_Namespace::SessionInfo> session)
260  : session_info_(std::move(session))
261  , dispatch_queue_(std::make_unique<QueryDispatchQueue>(1)) {}
262 
263 std::shared_ptr<Catalog_Namespace::Catalog> QueryRunner::getCatalog() const {
265  return session_info_->get_catalog_ptr();
266 }
267 
268 std::shared_ptr<Calcite> QueryRunner::getCalcite() const {
269  // TODO: Embed Calcite shared_ptr ownership in QueryRunner
270  return g_calcite;
271 }
272 
273 bool QueryRunner::gpusPresent() const {
275  return session_info_->getCatalog().getDataMgr().gpusPresent();
276 }
277 
278 void QueryRunner::clearGpuMemory() const {
281 }
282 
283 void QueryRunner::clearCpuMemory() const {
286 }
287 
288 std::vector<MemoryInfo> QueryRunner::getMemoryInfo(
289  const Data_Namespace::MemoryLevel memory_level) const {
291  return session_info_->getCatalog().getDataMgr().getMemoryInfo(memory_level);
292 }
293 
294 BufferPoolStats QueryRunner::getBufferPoolStats(
295  const Data_Namespace::MemoryLevel memory_level,
296  const bool current_db_only) const {
297  // Only works single-node for now
299  const std::vector<MemoryInfo> memory_infos =
300  session_info_->getCatalog().getDataMgr().getMemoryInfo(memory_level);
301  if (memory_level == Data_Namespace::MemoryLevel::CPU_LEVEL) {
302  CHECK_EQ(memory_infos.size(), static_cast<size_t>(1));
303  }
304  std::set<std::vector<int32_t>> chunk_keys;
305  std::set<std::vector<int32_t>> table_keys;
306  std::set<std::vector<int32_t>> column_keys;
307  std::set<std::vector<int32_t>> fragment_keys;
308  size_t total_num_buffers{
309  0}; // can be greater than chunk keys set size due to table replication
310  size_t total_num_bytes{0};
311  for (auto& pool_memory_info : memory_infos) {
312  const std::vector<MemoryData>& memory_data = pool_memory_info.nodeMemoryData;
313  for (auto& memory_datum : memory_data) {
314  total_num_buffers++;
315  const auto& chunk_key = memory_datum.chunk_key;
316  if (memory_datum.memStatus == Buffer_Namespace::MemStatus::FREE ||
317  chunk_key.size() < 4) {
318  continue;
319  }
320  if (current_db_only) {
321  if (chunk_key[0] != db_metadata_.dbId) {
322  continue;
323  }
324  }
325  total_num_bytes += (memory_datum.numPages * pool_memory_info.pageSize);
326  table_keys.insert({chunk_key[0], chunk_key[1]});
327  column_keys.insert({chunk_key[0], chunk_key[1], chunk_key[2]});
328  fragment_keys.insert({chunk_key[0], chunk_key[1], chunk_key[3]});
329  chunk_keys.insert(chunk_key);
330  }
331  }
332  return {total_num_buffers,
333  total_num_bytes,
334  table_keys.size(),
335  column_keys.size(),
336  fragment_keys.size(),
337  chunk_keys.size()};
338 }
339 
340 RegisteredQueryHint QueryRunner::getParsedQueryHint(const std::string& query_str) {
343  auto query_state = create_query_state(session_info_, query_str);
344  const auto& cat = session_info_->getCatalog();
346 
347  auto calcite_mgr = cat.getCalciteMgr();
348  const auto calciteQueryParsingOption =
349  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
350  const auto calciteOptimizationOption =
351  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {});
352  const auto query_ra = calcite_mgr
353  ->process(query_state->createQueryStateProxy(),
354  pg_shim(query_str),
355  calciteQueryParsingOption,
356  calciteOptimizationOption)
357  .plan_result;
358  auto ra_executor = RelAlgExecutor(executor.get(), cat, query_ra, query_state);
359  auto query_hints =
360  ra_executor.getParsedQueryHint(ra_executor.getRootRelAlgNodeShPtr().get());
361  return query_hints ? *query_hints : RegisteredQueryHint::defaults();
362 }
363 
364 std::shared_ptr<const RelAlgNode> QueryRunner::getRootNodeFromParsedQuery(
365  const std::string& query_str) {
368  auto query_state = create_query_state(session_info_, query_str);
369  const auto& cat = session_info_->getCatalog();
371 
372  auto calcite_mgr = cat.getCalciteMgr();
373  const auto calciteQueryParsingOption =
374  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
375  const auto calciteOptimizationOption =
376  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {});
377  const auto query_ra = calcite_mgr
378  ->process(query_state->createQueryStateProxy(),
379  pg_shim(query_str),
380  calciteQueryParsingOption,
381  calciteOptimizationOption)
382  .plan_result;
383  auto ra_executor = RelAlgExecutor(executor.get(), cat, query_ra, query_state);
384  return ra_executor.getRootRelAlgNodeShPtr();
385 }
386 
388  std::unordered_map<size_t, std::unordered_map<unsigned, RegisteredQueryHint>>>
389 QueryRunner::getParsedQueryHints(const std::string& query_str) {
392  auto query_state = create_query_state(session_info_, query_str);
393  const auto& cat = session_info_->getCatalog();
395  auto calcite_mgr = cat.getCalciteMgr();
396  const auto calciteQueryParsingOption =
397  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
398  const auto calciteOptimizationOption =
399  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {});
400  const auto query_ra = calcite_mgr
401  ->process(query_state->createQueryStateProxy(),
402  pg_shim(query_str),
403  calciteQueryParsingOption,
404  calciteOptimizationOption)
405  .plan_result;
406  auto ra_executor = RelAlgExecutor(executor.get(), cat, query_ra, query_state);
407  return ra_executor.getParsedQueryHints();
408 }
409 
410 std::optional<RegisteredQueryHint> QueryRunner::getParsedGlobalQueryHints(
411  const std::string& query_str) {
414  auto query_state = create_query_state(session_info_, query_str);
415  const auto& cat = session_info_->getCatalog();
417  auto calcite_mgr = cat.getCalciteMgr();
418  const auto calciteQueryParsingOption =
419  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
420  const auto calciteOptimizationOption =
421  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {});
422  const auto query_ra = calcite_mgr
423  ->process(query_state->createQueryStateProxy(),
424  pg_shim(query_str),
425  calciteQueryParsingOption,
426  calciteOptimizationOption)
427  .plan_result;
428  auto ra_executor = RelAlgExecutor(executor.get(), cat, query_ra, query_state);
429  return ra_executor.getGlobalQueryHint();
430 }
431 
432 RaExecutionSequence QueryRunner::getRaExecutionSequence(const std::string& query_str) {
435  auto query_state = create_query_state(session_info_, query_str);
436  const auto& cat = session_info_->getCatalog();
438  auto calcite_mgr = cat.getCalciteMgr();
439  const auto calciteQueryParsingOption =
440  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
441  const auto calciteOptimizationOption =
442  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {});
443  const auto query_ra = calcite_mgr
444  ->process(query_state->createQueryStateProxy(),
445  pg_shim(query_str),
446  calciteQueryParsingOption,
447  calciteOptimizationOption)
448  .plan_result;
449  auto ra_executor = RelAlgExecutor(executor.get(), cat, query_ra, query_state);
450  return ra_executor.getRaExecutionSequence(ra_executor.getRootRelAlgNodeShPtr().get(),
451  executor.get());
452 }
453 
454 // used to validate calcite ddl statements
455 void QueryRunner::validateDDLStatement(const std::string& stmt_str_in) {
457 
458  std::string stmt_str = stmt_str_in;
459  // First remove special chars
460  boost::algorithm::trim_left_if(stmt_str, boost::algorithm::is_any_of("\n"));
461  // Then remove spaces
462  boost::algorithm::trim_left(stmt_str);
463 
464  auto query_state = create_query_state(session_info_, stmt_str);
465  auto stdlog = STDLOG(query_state);
466 
467  const auto& cat = session_info_->getCatalog();
468  auto calcite_mgr = cat.getCalciteMgr();
469  const auto calciteQueryParsingOption =
470  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
471  const auto calciteOptimizationOption =
472  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {});
473  calcite_mgr->process(query_state->createQueryStateProxy(),
474  pg_shim(stmt_str),
475  calciteQueryParsingOption,
476  calciteOptimizationOption);
477 }
478 
479 std::shared_ptr<RelAlgTranslator> QueryRunner::getRelAlgTranslator(
480  const std::string& query_str,
481  Executor* executor) {
484  auto query_state = create_query_state(session_info_, query_str);
485  const auto& cat = session_info_->getCatalog();
486  auto calcite_mgr = cat.getCalciteMgr();
487  const auto calciteQueryParsingOption =
488  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
489  const auto calciteOptimizationOption =
490  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {});
491  const auto query_ra = calcite_mgr
492  ->process(query_state->createQueryStateProxy(),
493  pg_shim(query_str),
494  calciteQueryParsingOption,
495  calciteOptimizationOption)
496  .plan_result;
497  executor->setCatalog(&cat);
498  auto ra_executor = RelAlgExecutor(executor, cat, query_ra);
499  auto root_node_shared_ptr = ra_executor.getRootRelAlgNodeShPtr();
500  return ra_executor.getRelAlgTranslator(root_node_shared_ptr.get());
501 }
502 
503 QueryPlanDagInfo QueryRunner::getQueryInfoForDataRecyclerTest(
504  const std::string& query_str) {
507  auto query_state = create_query_state(session_info_, query_str);
508  const auto& cat = session_info_->getCatalog();
510  auto calcite_mgr = cat.getCalciteMgr();
511  const auto calciteQueryParsingOption =
512  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
513  const auto calciteOptimizationOption =
514  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {});
515  const auto query_ra = calcite_mgr
516  ->process(query_state->createQueryStateProxy(),
517  pg_shim(query_str),
518  calciteQueryParsingOption,
519  calciteOptimizationOption)
520  .plan_result;
521  executor->setCatalog(&cat);
522  auto ra_executor = RelAlgExecutor(executor.get(), cat, query_ra);
523  // note that we assume the test for data recycler that needs to have join_info
524  // does not contain any ORDER BY clause; this is necessary to create work_unit
525  // without actually performing the query
526  auto root_node_shared_ptr = ra_executor.getRootRelAlgNodeShPtr();
527  auto join_info = ra_executor.getJoinInfo(root_node_shared_ptr.get());
528  auto relAlgTranslator = ra_executor.getRelAlgTranslator(root_node_shared_ptr.get());
529  return {root_node_shared_ptr, join_info.first, join_info.second, relAlgTranslator};
530 }
531 
532 std::unique_ptr<Parser::Stmt> QueryRunner::createStatement(
533  const std::string& stmt_str_in) {
536 
537  std::string stmt_str = stmt_str_in;
538  // First remove special chars
539  boost::algorithm::trim_left_if(stmt_str, boost::algorithm::is_any_of("\n"));
540  // Then remove spaces
541  boost::algorithm::trim_left(stmt_str);
542 
543  ParserWrapper pw{stmt_str};
544 
545  auto query_state = create_query_state(session_info_, stmt_str);
546  auto stdlog = STDLOG(query_state);
547 
548  if (pw.isDdl()) {
549  const auto& cat = session_info_->getCatalog();
550  auto calcite_mgr = cat.getCalciteMgr();
551  const auto calciteQueryParsingOption =
552  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
553  const auto calciteOptimizationOption =
554  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {});
555  const auto query_json = calcite_mgr
556  ->process(query_state->createQueryStateProxy(),
557  pg_shim(stmt_str),
558  calciteQueryParsingOption,
559  calciteOptimizationOption)
560  .plan_result;
561  return Parser::create_stmt_for_json(query_json);
562  }
563 
564  // simply fail here as non-Calcite parsing is about to be removed
565  UNREACHABLE();
566  return nullptr;
567 }
568 
569 void QueryRunner::runDDLStatement(const std::string& stmt_str_in) {
572 
573  std::string stmt_str = stmt_str_in;
574  // First remove special chars
575  boost::algorithm::trim_left_if(stmt_str, boost::algorithm::is_any_of("\n"));
576  // Then remove spaces
577  boost::algorithm::trim_left(stmt_str);
578 
579  ParserWrapper pw{stmt_str};
580 
581  auto query_state = create_query_state(session_info_, stmt_str);
582  auto stdlog = STDLOG(query_state);
583 
584  if (pw.isDdl() || pw.getDMLType() == ParserWrapper::DMLType::Insert) {
585  const auto& cat = session_info_->getCatalog();
586  auto calcite_mgr = cat.getCalciteMgr();
587  const auto calciteQueryParsingOption =
588  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
589  const auto calciteOptimizationOption =
590  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {});
591  const auto query_ra = calcite_mgr
592  ->process(query_state->createQueryStateProxy(),
593  pg_shim(stmt_str),
594  calciteQueryParsingOption,
595  calciteOptimizationOption)
596  .plan_result;
597  if (pw.getDMLType() == ParserWrapper::DMLType::Insert) {
598  rapidjson::Document ddl_query;
599  ddl_query.Parse(query_ra);
600  CHECK(ddl_query.HasMember("payload"));
601  CHECK(ddl_query["payload"].IsObject());
602  auto stmt = Parser::InsertValuesStmt(ddl_query["payload"].GetObject());
603  stmt.execute(*session_info_);
604  return;
605  }
607  executor.execute();
608  return;
609  }
610 }
611 
612 std::shared_ptr<ResultSet> QueryRunner::runSQL(const std::string& query_str,
614  ExecutionOptions eo) {
617 
618  ParserWrapper pw{query_str};
619  if (pw.isCalcitePathPermissable()) {
620  if (pw.getDMLType() == ParserWrapper::DMLType::Insert) {
621  runDDLStatement(query_str);
622  return nullptr;
623  }
624  const auto execution_result = runSelectQuery(query_str, std::move(co), std::move(eo));
625  VLOG(1) << session_info_->getCatalog().getDataMgr().getSystemMemoryUsage();
626  return execution_result->getRows();
627  }
628 
629  UNREACHABLE();
630  return nullptr;
631 }
632 
633 std::shared_ptr<ResultSet> QueryRunner::runSQL(const std::string& query_str,
634  const ExecutorDeviceType device_type,
635  const bool hoist_literals,
636  const bool allow_loop_joins) {
637  auto co = CompilationOptions::defaults(device_type);
638  co.hoist_literals = hoist_literals;
639  return runSQL(
640  query_str, std::move(co), defaultExecutionOptionsForRunSQL(allow_loop_joins));
641 }
642 
643 ExecutionOptions QueryRunner::defaultExecutionOptionsForRunSQL(bool allow_loop_joins,
644  bool just_explain) {
645  return {g_enable_columnar_output,
646  false,
647  true,
648  just_explain,
649  allow_loop_joins,
650  false,
651  false,
652  false,
653  false,
654  10000,
655  false,
656  false,
658  false,
659  1000};
660 }
661 
662 std::shared_ptr<Executor> QueryRunner::getExecutor() const {
665  auto query_state = create_query_state(session_info_, "");
666  auto stdlog = STDLOG(query_state);
668  return executor;
669 }
670 
671 std::shared_ptr<ResultSet> QueryRunner::runSQLWithAllowingInterrupt(
672  const std::string& query_str,
673  const std::string& session_id,
674  const ExecutorDeviceType device_type,
675  const double running_query_check_freq,
676  const unsigned pending_query_check_freq) {
679  auto current_user = session_info_->get_currentUser();
680  auto session_info = std::make_shared<Catalog_Namespace::SessionInfo>(
681  session_info_->get_catalog_ptr(), current_user, device_type, session_id);
682  auto query_state = create_query_state(session_info, query_str);
683  auto stdlog = STDLOG(query_state);
684  const auto& cat = query_state->getConstSessionInfo()->getCatalog();
685  std::string query_ra{""};
686 
687  std::shared_ptr<ExecutionResult> result;
688  auto query_launch_task = std::make_shared<QueryDispatchQueue::Task>(
689  [&cat,
690  &query_ra,
691  &device_type,
692  &query_state,
693  &result,
694  &running_query_check_freq,
695  &pending_query_check_freq](const size_t worker_id) {
696  auto executor = Executor::getExecutor(worker_id);
698 
700  false,
701  true,
702  false,
703  true,
704  false,
705  false,
706  false,
707  false,
708  10000,
709  false,
710  false,
712  true,
713  running_query_check_freq,
714  pending_query_check_freq};
715  {
716  // async query initiation for interrupt test
717  // incurs data race warning in TSAN since
718  // calcite_mgr is shared across multiple query threads
719  // so here we lock the manager during query parsing
720  std::lock_guard<std::mutex> calcite_lock_guard(calcite_lock);
721  auto calcite_mgr = cat.getCalciteMgr();
722  const auto calciteQueryParsingOption =
723  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
724  const auto calciteOptimizationOption =
725  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {});
726  query_ra = calcite_mgr
727  ->process(query_state->createQueryStateProxy(),
728  pg_shim(query_state->getQueryStr()),
729  calciteQueryParsingOption,
730  calciteOptimizationOption)
731  .plan_result;
732  }
733  auto ra_executor = RelAlgExecutor(executor.get(), cat, query_ra, query_state);
734  result = std::make_shared<ExecutionResult>(
735  ra_executor.executeRelAlgQuery(co, eo, false, nullptr));
736  });
738  executor->enrollQuerySession(session_id,
739  query_str,
740  query_state->getQuerySubmittedTime(),
742  QuerySessionStatus::QueryStatus::PENDING_QUEUE);
744  dispatch_queue_->submit(query_launch_task, /*is_update_delete=*/false);
745  auto result_future = query_launch_task->get_future();
746  result_future.get();
747  CHECK(result);
748  return result->getRows();
749 }
750 
751 std::vector<std::shared_ptr<ResultSet>> QueryRunner::runMultipleStatements(
752  const std::string& sql,
753  const ExecutorDeviceType dt) {
754  std::vector<std::shared_ptr<ResultSet>> results;
755  // TODO: Need to properly handle escaped semicolons instead of doing a naive split().
756  auto fields = split(sql, ";");
757  for (const auto& field : fields) {
758  auto text = strip(field) + ";";
759  if (text == ";") {
760  continue;
761  }
762 
763  ParserWrapper pw{text};
764  if (pw.isDdl() || pw.getDMLType() == ParserWrapper::DMLType::Insert) {
765  runDDLStatement(text);
766  results.push_back(nullptr);
767  } else {
768  // is not DDL, then assume it's DML and try to execute
769  results.push_back(runSQL(text, dt, true, true));
770  }
771  }
772  return results;
773 }
774 
775 void QueryRunner::runImport(Parser::CopyTableStmt* import_stmt) {
776  CHECK(import_stmt);
777  import_stmt->execute(*session_info_);
778 }
779 
780 std::unique_ptr<import_export::Loader> QueryRunner::getLoader(
781  const TableDescriptor* td) const {
782  auto cat = getCatalog();
783  return std::make_unique<import_export::Loader>(*cat, td);
784 }
785 
786 namespace {
787 
788 std::shared_ptr<ExecutionResult> run_select_query_with_filter_push_down(
789  QueryStateProxy query_state_proxy,
790  const ExecutorDeviceType device_type,
791  const bool hoist_literals,
792  const bool allow_loop_joins,
793  const bool just_explain,
794  const ExecutorExplainType explain_type,
795  const bool with_filter_push_down) {
796  auto const& query_state = query_state_proxy.getQueryState();
797  const auto& cat = query_state.getConstSessionInfo()->getCatalog();
800  co.explain_type = explain_type;
801 
804  eo.just_explain = just_explain;
805  eo.allow_loop_joins = allow_loop_joins;
806  eo.find_push_down_candidates = with_filter_push_down;
808 
809  auto calcite_mgr = cat.getCalciteMgr();
810  const auto calciteQueryParsingOption =
811  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
812  const auto calciteOptimizationOption =
813  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {});
814  const auto query_ra = calcite_mgr
815  ->process(query_state_proxy,
816  pg_shim(query_state.getQueryStr()),
817  calciteQueryParsingOption,
818  calciteOptimizationOption)
819  .plan_result;
820  auto ra_executor = RelAlgExecutor(executor.get(), cat, query_ra);
821  auto result = std::make_shared<ExecutionResult>(
822  ra_executor.executeRelAlgQuery(co, eo, false, nullptr));
823  const auto& filter_push_down_requests = result->getPushedDownFilterInfo();
824  if (!filter_push_down_requests.empty()) {
825  std::vector<TFilterPushDownInfo> filter_push_down_info;
826  for (const auto& req : filter_push_down_requests) {
827  TFilterPushDownInfo filter_push_down_info_for_request;
828  filter_push_down_info_for_request.input_prev = req.input_prev;
829  filter_push_down_info_for_request.input_start = req.input_start;
830  filter_push_down_info_for_request.input_next = req.input_next;
831  filter_push_down_info.push_back(filter_push_down_info_for_request);
832  }
833  const auto calciteQueryParsingOption =
834  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
835  const auto calciteOptimizationOption = calcite_mgr->getCalciteOptimizationOption(
836  false, g_enable_watchdog, filter_push_down_info);
837  const auto new_query_ra = calcite_mgr
838  ->process(query_state_proxy,
839  pg_shim(query_state.getQueryStr()),
840  calciteQueryParsingOption,
841  calciteOptimizationOption)
842  .plan_result;
843  const ExecutionOptions eo_modified{eo.output_columnar_hint,
844  eo.keep_result,
845  eo.allow_multifrag,
846  eo.just_explain,
847  eo.allow_loop_joins,
848  eo.with_watchdog,
849  eo.jit_debug,
850  eo.just_validate,
853  /*find_push_down_candidates=*/false,
854  /*just_calcite_explain=*/false,
859  auto new_ra_executor = RelAlgExecutor(executor.get(), cat, new_query_ra);
860  return std::make_shared<ExecutionResult>(
861  new_ra_executor.executeRelAlgQuery(co, eo_modified, false, nullptr));
862  } else {
863  return result;
864  }
865 }
866 
867 } // namespace
868 
869 std::shared_ptr<ExecutionResult> QueryRunner::runSelectQuery(const std::string& query_str,
871  ExecutionOptions eo) {
874  auto query_state = create_query_state(session_info_, query_str);
875  auto stdlog = STDLOG(query_state);
877  return run_select_query_with_filter_push_down(query_state->createQueryStateProxy(),
878  co.device_type,
879  co.hoist_literals,
880  eo.allow_loop_joins,
881  eo.just_explain,
884  }
885 
886  const auto& cat = session_info_->getCatalog();
887 
888  std::shared_ptr<ExecutionResult> result;
889  auto query_launch_task =
890  std::make_shared<QueryDispatchQueue::Task>([&cat,
891  &query_str,
892  &co,
893  explain_type = this->explain_type_,
894  &eo,
895  &query_state,
896  &result](const size_t worker_id) {
897  auto executor = Executor::getExecutor(worker_id);
898  // TODO The next line should be deleted since it overwrites co, but then
899  // NycTaxiTest.RunSelectsEncodingDictWhereGreater fails due to co not getting
900  // reset to its default values.
901  co = CompilationOptions::defaults(co.device_type);
902  co.explain_type = explain_type;
903  auto calcite_mgr = cat.getCalciteMgr();
904  const auto calciteQueryParsingOption =
905  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
906  const auto calciteOptimizationOption = calcite_mgr->getCalciteOptimizationOption(
908  const auto query_ra = calcite_mgr
909  ->process(query_state->createQueryStateProxy(),
910  pg_shim(query_str),
911  calciteQueryParsingOption,
912  calciteOptimizationOption)
913  .plan_result;
914  auto ra_executor = RelAlgExecutor(executor.get(), cat, query_ra);
915  result = std::make_shared<ExecutionResult>(
916  ra_executor.executeRelAlgQuery(co, eo, false, nullptr));
917  });
919  dispatch_queue_->submit(query_launch_task, /*is_update_delete=*/false);
920  auto result_future = query_launch_task->get_future();
921  result_future.get();
922  CHECK(result);
923  return result;
924 }
925 
926 std::shared_ptr<ExecutionResult> QueryRunner::runSelectQuery(
927  const std::string& query_str,
928  const ExecutorDeviceType device_type,
929  const bool hoist_literals,
930  const bool allow_loop_joins,
931  const bool just_explain) {
932  auto co = CompilationOptions::defaults(device_type);
933  co.hoist_literals = hoist_literals;
934  return runSelectQuery(query_str,
935  std::move(co),
936  defaultExecutionOptionsForRunSQL(allow_loop_joins, just_explain));
937 }
938 
939 ExtractedQueryPlanDag QueryRunner::extractQueryPlanDag(const std::string& query_str) {
940  auto query_dag_info = getQueryInfoForDataRecyclerTest(query_str);
942  auto extracted_dag_info = QueryPlanDagExtractor::extractQueryPlanDag(
943  query_dag_info.root_node.get(), executor);
944  return extracted_dag_info;
945 }
946 
947 // this function exists to test data recycler
948 // specifically, it is tricky to get a hashtable cache key when we only know
949 // a target query sql in test code
950 // so this function utilizes an incorrect way to manipulate our hashtable recycler
951 // but provides the cached hashtable for performing the test
952 // a set "visited" contains cached hashtable keys that we have retrieved so far
953 // based on that, this function iterates hashtable cache and return a cached one
954 // when its hashtable cache key has not been visited yet
955 // for instance, if we call this funtion with an empty "visited" key, we return
956 // the first hashtable that its iterator visits
957 std::tuple<QueryPlanHash,
958  std::shared_ptr<HashTable>,
959  std::optional<HashtableCacheMetaInfo>>
960 QueryRunner::getCachedHashtableWithoutCacheKey(std::set<size_t>& visited,
961  CacheItemType hash_table_type,
962  DeviceIdentifier device_identifier) {
963  HashtableRecycler* hash_table_cache{nullptr};
964  switch (hash_table_type) {
966  hash_table_cache = PerfectJoinHashTable::getHashTableCache();
967  break;
968  }
970  hash_table_cache = BaselineJoinHashTable::getHashTableCache();
971  break;
972  }
974  hash_table_cache = OverlapsJoinHashTable::getHashTableCache();
975  break;
976  }
977  default: {
978  UNREACHABLE();
979  break;
980  }
981  }
982  CHECK(hash_table_cache);
983  return hash_table_cache->getCachedHashtableWithoutCacheKey(
984  visited, hash_table_type, device_identifier);
985 }
986 
987 std::shared_ptr<CacheItemMetric> QueryRunner::getCacheItemMetric(
988  QueryPlanHash cache_key,
989  CacheItemType hash_table_type,
990  DeviceIdentifier device_identifier) {
991  HashtableRecycler* hash_table_cache{nullptr};
992  switch (hash_table_type) {
994  hash_table_cache = PerfectJoinHashTable::getHashTableCache();
995  break;
996  }
998  hash_table_cache = BaselineJoinHashTable::getHashTableCache();
999  break;
1000  }
1002  hash_table_cache = OverlapsJoinHashTable::getHashTableCache();
1003  break;
1004  }
1005  default: {
1006  UNREACHABLE();
1007  break;
1008  }
1009  }
1010  CHECK(hash_table_cache);
1011  return hash_table_cache->getCachedItemMetric(
1012  hash_table_type, device_identifier, cache_key);
1013 }
1014 
1015 size_t QueryRunner::getNumberOfCachedItem(CacheItemStatus item_status,
1016  CacheItemType hash_table_type,
1017  bool with_overlaps_tuning_param) const {
1018  auto get_num_cached_auto_tuner_param = [&item_status]() {
1019  auto auto_tuner_cache = OverlapsJoinHashTable::getOverlapsTuningParamCache();
1020  CHECK(auto_tuner_cache);
1021  switch (item_status) {
1022  case CacheItemStatus::ALL: {
1023  return auto_tuner_cache->getCurrentNumCachedItems(
1026  }
1028  return auto_tuner_cache->getCurrentNumCleanCachedItems(
1031  }
1033  return auto_tuner_cache->getCurrentNumDirtyCachedItems(
1036  }
1037  default: {
1038  UNREACHABLE();
1039  return static_cast<size_t>(0);
1040  }
1041  }
1042  };
1043 
1044  auto get_num_cached_hashtable =
1045  [&item_status,
1046  &hash_table_type,
1047  &with_overlaps_tuning_param,
1048  &get_num_cached_auto_tuner_param](HashtableRecycler* hash_table_cache) {
1049  switch (item_status) {
1050  case CacheItemStatus::ALL: {
1051  if (with_overlaps_tuning_param) {
1052  // we assume additional consideration of turing param cache is only valid
1053  // for overlaps join hashtable
1054  CHECK_EQ(hash_table_type, CacheItemType::OVERLAPS_HT);
1055  return hash_table_cache->getCurrentNumCachedItems(
1056  hash_table_type, DataRecyclerUtil::CPU_DEVICE_IDENTIFIER) +
1057  get_num_cached_auto_tuner_param();
1058  }
1059  return hash_table_cache->getCurrentNumCachedItems(
1060  hash_table_type, DataRecyclerUtil::CPU_DEVICE_IDENTIFIER);
1061  }
1063  if (with_overlaps_tuning_param) {
1064  CHECK_EQ(hash_table_type, CacheItemType::OVERLAPS_HT);
1065  return hash_table_cache->getCurrentNumCleanCachedItems(
1066  hash_table_type, DataRecyclerUtil::CPU_DEVICE_IDENTIFIER) +
1067  get_num_cached_auto_tuner_param();
1068  }
1069  return hash_table_cache->getCurrentNumCleanCachedItems(
1070  hash_table_type, DataRecyclerUtil::CPU_DEVICE_IDENTIFIER);
1071  }
1073  if (with_overlaps_tuning_param) {
1074  CHECK_EQ(hash_table_type, CacheItemType::OVERLAPS_HT);
1075  return hash_table_cache->getCurrentNumDirtyCachedItems(
1076  hash_table_type, DataRecyclerUtil::CPU_DEVICE_IDENTIFIER) +
1077  get_num_cached_auto_tuner_param();
1078  }
1079  return hash_table_cache->getCurrentNumDirtyCachedItems(
1080  hash_table_type, DataRecyclerUtil::CPU_DEVICE_IDENTIFIER);
1081  }
1082  default: {
1083  UNREACHABLE();
1084  return static_cast<size_t>(0);
1085  }
1086  }
1087  };
1088 
1089  switch (hash_table_type) {
1091  auto hash_table_cache = PerfectJoinHashTable::getHashTableCache();
1092  CHECK(hash_table_cache);
1093  return get_num_cached_hashtable(hash_table_cache);
1094  }
1096  auto hash_table_cache = BaselineJoinHashTable::getHashTableCache();
1097  CHECK(hash_table_cache);
1098  return get_num_cached_hashtable(hash_table_cache);
1099  }
1101  auto hash_table_cache = OverlapsJoinHashTable::getHashTableCache();
1102  CHECK(hash_table_cache);
1103  return get_num_cached_hashtable(hash_table_cache);
1104  }
1106  return get_num_cached_auto_tuner_param();
1107  }
1108  default: {
1109  UNREACHABLE();
1110  return 0;
1111  }
1112  }
1113  return 0;
1114 }
1115 
1116 void QueryRunner::reset() {
1117  qr_instance_->query_engine_.reset();
1118  qr_instance_.reset(nullptr);
1120 }
1121 
1122 ImportDriver::ImportDriver(std::shared_ptr<Catalog_Namespace::Catalog> cat,
1123  const Catalog_Namespace::UserMetadata& user,
1124  const ExecutorDeviceType dt,
1125  const std::string session_id)
1126  : QueryRunner(
1127  std::make_unique<Catalog_Namespace::SessionInfo>(cat, user, dt, session_id)) {}
1128 
1129 void ImportDriver::importGeoTable(const std::string& file_path,
1130  const std::string& table_name,
1131  const bool compression,
1132  const bool create_table,
1133  const bool explode_collections) {
1134  using namespace import_export;
1135 
1136  static constexpr bool kIsGeoRaster{false};
1137 
1139 
1140  CopyParams copy_params;
1142  if (compression) {
1143  copy_params.geo_coords_encoding = EncodingType::kENCODING_GEOINT;
1144  copy_params.geo_coords_comp_param = 32;
1145  } else {
1146  copy_params.geo_coords_encoding = EncodingType::kENCODING_NONE;
1147  copy_params.geo_coords_comp_param = 0;
1148  }
1149  copy_params.geo_assign_render_groups = true;
1150  copy_params.geo_explode_collections = explode_collections;
1151 
1152  std::map<std::string, std::string> colname_to_src;
1153  auto& cat = session_info_->getCatalog();
1154  auto cds = Importer::gdalToColumnDescriptors(
1155  file_path, kIsGeoRaster, Geospatial::kGeoColumnName, copy_params);
1156 
1157  for (auto& cd : cds) {
1158  const auto col_name_sanitized = ImportHelpers::sanitize_name(cd.columnName);
1159  const auto ret =
1160  colname_to_src.insert(std::make_pair(col_name_sanitized, cd.columnName));
1161  CHECK(ret.second);
1162  cd.columnName = col_name_sanitized;
1163  }
1164 
1165  if (create_table) {
1166  const auto td = cat.getMetadataForTable(table_name);
1167  if (td != nullptr) {
1168  throw std::runtime_error(
1169  "Error: Table " + table_name +
1170  " already exists. Possible failure to correctly re-create " +
1171  shared::kDataDirectoryName + " directory.");
1172  }
1173  if (table_name != ImportHelpers::sanitize_name(table_name)) {
1174  throw std::runtime_error("Invalid characters in table name: " + table_name);
1175  }
1176 
1177  std::string stmt{"CREATE TABLE " + table_name};
1178  std::vector<std::string> col_stmts;
1179 
1180  for (auto& cd : cds) {
1181  if (cd.columnType.get_type() == SQLTypes::kINTERVAL_DAY_TIME ||
1182  cd.columnType.get_type() == SQLTypes::kINTERVAL_YEAR_MONTH) {
1183  throw std::runtime_error(
1184  "Unsupported type: INTERVAL_DAY_TIME or INTERVAL_YEAR_MONTH for col " +
1185  cd.columnName + " (table: " + table_name + ")");
1186  }
1187 
1188  if (cd.columnType.get_type() == SQLTypes::kDECIMAL) {
1189  if (cd.columnType.get_precision() == 0 && cd.columnType.get_scale() == 0) {
1190  cd.columnType.set_precision(14);
1191  cd.columnType.set_scale(7);
1192  }
1193  }
1194 
1195  std::string col_stmt;
1196  col_stmt.append(cd.columnName + " " + cd.columnType.get_type_name() + " ");
1197 
1198  if (cd.columnType.get_compression() != EncodingType::kENCODING_NONE) {
1199  col_stmt.append("ENCODING " + cd.columnType.get_compression_name() + " ");
1200  } else {
1201  if (cd.columnType.is_string()) {
1202  col_stmt.append("ENCODING NONE");
1203  } else if (cd.columnType.is_geometry()) {
1204  if (cd.columnType.get_output_srid() == 4326) {
1205  col_stmt.append("ENCODING NONE");
1206  }
1207  }
1208  }
1209  col_stmts.push_back(col_stmt);
1210  }
1211 
1212  stmt.append(" (" + boost::algorithm::join(col_stmts, ",") + ");");
1213  runDDLStatement(stmt);
1214 
1215  LOG(INFO) << "Created table: " << table_name;
1216  } else {
1217  LOG(INFO) << "Not creating table: " << table_name;
1218  }
1219 
1220  const auto td = cat.getMetadataForTable(table_name);
1221  if (td == nullptr) {
1222  throw std::runtime_error("Error: Failed to create table " + table_name);
1223  }
1224 
1225  import_export::Importer importer(cat, td, file_path, copy_params);
1226  auto ms = measure<>::execution(
1227  [&]() { importer.importGDAL(colname_to_src, session_info_.get(), kIsGeoRaster); });
1228  LOG(INFO) << "Import Time for " << table_name << ": " << (double)ms / 1000.0 << " s";
1229 }
1230 
1231 } // namespace QueryRunner
bool g_enable_calcite_view_optimize
Definition: QueryRunner.cpp:56
Classes used to wrap parser calls for calcite redirection.
static void addUdfs(const std::string &json_func_sigs)
#define CHECK_EQ(x, y)
Definition: Logger.h:231
ImportStatus importGDAL(const std::map< std::string, std::string > &colname_to_src, const Catalog_Namespace::SessionInfo *session_info, const bool is_raster)
Definition: Importer.cpp:5305
#define CALCITEPORT
Definition: QueryRunner.cpp:48
size_t DeviceIdentifier
Definition: DataRecycler.h:129
const std::string kDataDirectoryName
static std::vector< TableFunction > get_table_funcs()
static ExtractedQueryPlanDag extractQueryPlanDag(const RelAlgNode *top_node, Executor *executor)
std::string cat(Ts &&...args)
std::unique_ptr< QueryDispatchQueue > dispatch_queue_
Definition: QueryRunner.h:326
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
ExecutorDeviceType
static void initialize(const std::string &data_dir, const std::string &allowed_import_paths, const std::string &allowed_export_paths)
Definition: DdlUtils.cpp:827
const std::string kDefaultDiskCacheDirName
std::string strip(std::string_view str)
trim any whitespace from the left and right ends of a string
ImportDriver(std::shared_ptr< Catalog_Namespace::Catalog > cat, const Catalog_Namespace::UserMetadata &user, const ExecutorDeviceType dt=ExecutorDeviceType::GPU, const std::string session_id="")
#define LOG(tag)
Definition: Logger.h:217
ExecutorExplainType explain_type_
Definition: QueryRunner.h:322
std::string join(T const &container, std::string const &delim)
static void add(const std::string &json_func_sigs)
#define UNREACHABLE()
Definition: Logger.h:267
static HashtableRecycler * getHashTableCache()
std::mutex calcite_lock
Definition: QueryRunner.cpp:57
std::optional< std::unordered_map< size_t, std::unordered_map< unsigned, RegisteredQueryHint > > > getParsedQueryHints()
std::optional< RegisteredQueryHint > getParsedQueryHint(const RelAlgNode *node)
void set_once_fatal_func(FatalFunc fatal_func)
Definition: Logger.cpp:334
void execute(const Catalog_Namespace::SessionInfo &session) override
Catalog_Namespace::DBMetadata db_metadata_
Definition: QueryRunner.h:324
static ExecutionOptions defaultExecutionOptionsForRunSQL(bool allow_loop_joins=true, bool just_explain=false)
virtual std::shared_ptr< ResultSet > runSQL(const std::string &query_str, CompilationOptions co, ExecutionOptions eo)
const std::string kGeoColumnName
Definition: ColumnNames.h:23
std::vector< std::string > split(std::string_view str, std::string_view delim, std::optional< size_t > maxsplit)
split apart a string into a vector of substrings
double running_query_interrupt_freq
#define LOG_IF(severity, condition)
Definition: Logger.h:313
static void clearMemory(const Data_Namespace::MemoryLevel memory_level)
Definition: Execute.cpp:507
static void addShutdownCallback(std::function< void()> shutdown_callback)
std::shared_ptr< const RelAlgNode > getRootRelAlgNodeShPtr() const
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:483
std::shared_ptr< QueryEngine > query_engine_
Definition: QueryRunner.h:328
This file contains the class specification and related data structures for Catalog.
RaExecutionSequence getRaExecutionSequence(const RelAlgNode *root_node, Executor *executor)
const rapidjson::Value & field(const rapidjson::Value &obj, const char field[]) noexcept
Definition: JsonAccessors.h:31
bool g_enable_columnar_output
Definition: Execute.cpp:99
static SysCatalog & instance()
Definition: SysCatalog.h:337
Classes representing a parse tree.
CacheItemType
Definition: DataRecycler.h:38
const std::string kDefaultDbName
std::string g_base_path
Definition: SysCatalog.cpp:64
void init(LogOptions const &log_opts)
Definition: Logger.cpp:306
QueryState & getQueryState()
Definition: QueryState.h:181
std::unique_ptr< Parser::Stmt > create_stmt_for_json(const std::string &query_json)
static HashtableRecycler * getHashTableCache()
A container for relational algebra descriptors defining the execution order for a relational algebra ...
ExecutorExplainType explain_type
bool g_enable_watchdog
virtual void runDDLStatement(const std::string &)
static std::unique_ptr< QueryRunner > qr_instance_
Definition: QueryRunner.h:320
double g_gpu_mem_limit_percent
Definition: QueryRunner.cpp:53
std::string sanitize_name(const std::string &name)
import_export::SourceType source_type
Definition: CopyParams.h:57
static OverlapsTuningParamRecycler * getOverlapsTuningParamCache()
unsigned pending_query_interrupt_freq
bool g_serialize_temp_tables
Definition: Catalog.cpp:106
ExecutorDeviceType device_type
std::optional< RegisteredQueryHint > getGlobalQueryHint()
ExecutionResult execute()
void importGeoTable(const std::string &file_path, const std::string &table_name, const bool compression, const bool create_table, const bool explode_collections)
const std::string kRootUsername
static RegisteredQueryHint defaults()
Definition: QueryHint.h:237
TExtArgumentType::type to_thrift(const ExtArgumentType &t)
std::shared_ptr< ExecutionResult > run_select_query_with_filter_push_down(QueryStateProxy query_state_proxy, const ExecutorDeviceType device_type, const bool hoist_literals, const bool allow_loop_joins, const bool just_explain, const ExecutorExplainType explain_type, const bool with_filter_push_down)
bool g_read_only
Definition: File.cpp:40
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:325
size_t QueryPlanHash
virtual std::shared_ptr< ExecutionResult > runSelectQuery(const std::string &query_str, CompilationOptions co, ExecutionOptions eo)
std::shared_ptr< Calcite > g_calcite
Definition: QueryRunner.cpp:62
const std::string kCatalogDirectoryName
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:298
static CompilationOptions defaults(const ExecutorDeviceType device_type=ExecutorDeviceType::GPU)
bool g_enable_filter_push_down
Definition: Execute.cpp:95
QueryPlanDagInfo getQueryInfoForDataRecyclerTest(const std::string &)
#define CHECK(condition)
Definition: Logger.h:223
static HashtableRecycler * getHashTableCache()
double gpu_input_mem_limit_percent
static std::shared_ptr< QueryEngine > getInstance()
Definition: QueryEngine.h:61
Serializers for query engine types to/from thrift.
std::shared_ptr< Catalog_Namespace::Catalog > getCatalog() const
unsigned dynamic_watchdog_time_limit
std::shared_ptr< Data_Namespace::DataMgr > data_mgr_
Definition: QueryRunner.h:327
size_t g_leaf_count
Definition: ParserNode.cpp:78
static constexpr DeviceIdentifier CPU_DEVICE_IDENTIFIER
Definition: DataRecycler.h:136
static ExecutionOptions defaults()
ExecutorExplainType
std::string pg_shim(const std::string &query)
#define STDLOG(...)
Definition: QueryState.h:235
#define VLOG(n)
Definition: Logger.h:317
std::atomic< bool > isSuper
Definition: SysCatalog.h:107
static const ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:376