OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
QueryRunner::QueryRunner Class Reference

#include <QueryRunner.h>

+ Inheritance diagram for QueryRunner::QueryRunner:
+ Collaboration diagram for QueryRunner::QueryRunner:

Public Member Functions

std::shared_ptr
< Catalog_Namespace::SessionInfo
getSession () const
 
void addSessionId (const std::string &session_id, ExecutorDeviceType device_type=ExecutorDeviceType::GPU)
 
void clearSessionId ()
 
std::shared_ptr
< Catalog_Namespace::Catalog
getCatalog () const
 
std::shared_ptr< CalcitegetCalcite () const
 
std::shared_ptr< ExecutorgetExecutor () const
 
Catalog_Namespace::UserMetadatagetUserMetadata () const
 
bool gpusPresent () const
 
virtual void clearGpuMemory () const
 
virtual void clearCpuMemory () const
 
std::vector< MemoryInfogetMemoryInfo (const Data_Namespace::MemoryLevel memory_level) const
 
BufferPoolStats getBufferPoolStats (const Data_Namespace::MemoryLevel memory_level, const bool current_db_only) const
 
virtual std::unique_ptr
< Parser::Stmt
createStatement (const std::string &)
 
virtual void runDDLStatement (const std::string &)
 
virtual void validateDDLStatement (const std::string &)
 
virtual std::shared_ptr
< ResultSet
runSQL (const std::string &query_str, CompilationOptions co, ExecutionOptions eo)
 
virtual std::shared_ptr
< ExecutionResult
runSelectQuery (const std::string &query_str, CompilationOptions co, ExecutionOptions eo)
 
virtual std::shared_ptr
< ResultSet
runSQL (const std::string &query_str, const ExecutorDeviceType device_type, const bool hoist_literals=true, const bool allow_loop_joins=true)
 
virtual std::shared_ptr
< ExecutionResult
runSelectQuery (const std::string &query_str, const ExecutorDeviceType device_type, const bool hoist_literals, const bool allow_loop_joins, const bool just_explain=false)
 
virtual std::shared_ptr
< ResultSet
runSQLWithAllowingInterrupt (const std::string &query_str, const std::string &session_id, const ExecutorDeviceType device_type, const double running_query_check_freq=0.9, const unsigned pending_query_check_freq=1000)
 
virtual std::vector
< std::shared_ptr< ResultSet > > 
runMultipleStatements (const std::string &, const ExecutorDeviceType)
 
virtual void runImport (Parser::CopyTableStmt *import_stmt)
 
virtual std::unique_ptr
< import_export::Loader
getLoader (const TableDescriptor *td) const
 
RegisteredQueryHint getParsedQueryHint (const std::string &)
 
std::optional
< std::unordered_map< size_t,
std::unordered_map< unsigned,
RegisteredQueryHint > > > 
getParsedQueryHints (const std::string &query_str)
 
std::shared_ptr< const RelAlgNodegetRootNodeFromParsedQuery (const std::string &query_str)
 
std::optional
< RegisteredQueryHint
getParsedGlobalQueryHints (const std::string &query_str)
 
RaExecutionSequence getRaExecutionSequence (const std::string &query_str)
 
virtual std::shared_ptr
< ResultSet
getCalcitePlan (const std::string &query_str, bool enable_watchdog, bool is_explain_as_json_str, bool is_explain_detailed) const
 
std::tuple< QueryPlanHash,
std::shared_ptr< HashTable >
, std::optional
< HashtableCacheMetaInfo > > 
getCachedHashtableWithoutCacheKey (std::set< size_t > &visited, CacheItemType hash_table_type, DeviceIdentifier device_identifier)
 
std::shared_ptr< CacheItemMetricgetCacheItemMetric (QueryPlanHash cache_key, CacheItemType hash_table_type, DeviceIdentifier device_identifier)
 
size_t getNumberOfCachedItem (CacheItemStatus item_status, CacheItemType hash_table_type, bool with_bbox_intersect_tuning_param=false) const
 
void resizeDispatchQueue (const size_t num_executors)
 
QueryPlanDagInfo getQueryInfoForDataRecyclerTest (const std::string &)
 
std::shared_ptr< RelAlgTranslatorgetRelAlgTranslator (const std::string &, Executor *)
 
ExtractedQueryPlanDag extractQueryPlanDag (const std::string &)
 
std::unique_ptr< RelAlgDaggetRelAlgDag (const std::string &)
 
 QueryRunner (std::unique_ptr< Catalog_Namespace::SessionInfo > session)
 
virtual ~QueryRunner ()=default
 
void setExplainType (const ExecutorExplainType explain_type)
 

Static Public Member Functions

static QueryRunnerinit (const char *db_path, const std::string &udf_filename="", const size_t max_gpu_mem=0, const int reserved_gpu_mem=256<< 20)
 
static QueryRunnerinit (const File_Namespace::DiskCacheConfig *disk_cache_config, const char *db_path, const std::vector< LeafHostInfo > &string_servers={}, const std::vector< LeafHostInfo > &leaf_servers={})
 
static QueryRunnerinit (const char *db_path, const std::vector< LeafHostInfo > &string_servers, const std::vector< LeafHostInfo > &leaf_servers)
 
static QueryRunnerinit (const char *db_path, const std::string &user, const std::string &pass, const std::string &db_name, const std::vector< LeafHostInfo > &string_servers, const std::vector< LeafHostInfo > &leaf_servers, const std::string &udf_filename="", bool uses_gpus=true, const size_t max_gpu_mem=0, const int reserved_gpu_mem=256<< 20, const bool create_user=false, const bool create_db=false, const File_Namespace::DiskCacheConfig *config=nullptr)
 
static QueryRunnerinit (std::unique_ptr< Catalog_Namespace::SessionInfo > &session)
 
static QueryRunnerget ()
 
static void reset ()
 
static ExecutionOptions defaultExecutionOptionsForRunSQL (bool allow_loop_joins=true, bool just_explain=false)
 
template<typename... Ts>
static std::shared_ptr
< query_state::QueryState
create_query_state (Ts &&...args)
 

Static Public Attributes

static query_state::QueryStates query_states_
 

Protected Member Functions

 QueryRunner (const char *db_path, const std::string &user, const std::string &pass, const std::string &db_name, const std::vector< LeafHostInfo > &string_servers, const std::vector< LeafHostInfo > &leaf_servers, const std::string &udf_filename, bool uses_gpus, const size_t max_gpu_mem, const int reserved_gpu_mem, const bool create_user, const bool create_db, const File_Namespace::DiskCacheConfig *disk_cache_config=nullptr)
 

Protected Attributes

ExecutorExplainType explain_type_ = ExecutorExplainType::Default
 
Catalog_Namespace::DBMetadata db_metadata_
 
std::shared_ptr
< Catalog_Namespace::SessionInfo
session_info_
 
std::unique_ptr
< QueryDispatchQueue
dispatch_queue_
 
std::shared_ptr< QueryEnginequery_engine_
 

Static Protected Attributes

static std::unique_ptr
< QueryRunner
qr_instance_ = nullptr
 

Detailed Description

Definition at line 139 of file QueryRunner.h.

Constructor & Destructor Documentation

QueryRunner::QueryRunner::QueryRunner ( std::unique_ptr< Catalog_Namespace::SessionInfo session)

Referenced by init().

+ Here is the caller graph for this function:

virtual QueryRunner::QueryRunner::~QueryRunner ( )
virtualdefault
QueryRunner::QueryRunner::QueryRunner ( const char *  db_path,
const std::string &  user,
const std::string &  pass,
const std::string &  db_name,
const std::vector< LeafHostInfo > &  string_servers,
const std::vector< LeafHostInfo > &  leaf_servers,
const std::string &  udf_filename,
bool  uses_gpus,
const size_t  max_gpu_mem,
const int  reserved_gpu_mem,
const bool  create_user,
const bool  create_db,
const File_Namespace::DiskCacheConfig disk_cache_config = nullptr 
)
protected

Definition at line 156 of file QueryRunner.cpp.

References ExtensionFunctionsWhitelist::add(), ExtensionFunctionsWhitelist::addUdfs(), SystemParameters::aggregator, anonymous_namespace{QueryRunner.cpp}::calcite_shutdown_handler(), CALCITEPORT, cat(), CHECK, cpu_threads(), QueryEngine::createInstance(), db_metadata_, Catalog_Namespace::DBMetadata::dbOwner, File_Namespace::fsi, g_base_path, anonymous_namespace{QueryRunner.cpp}::g_calcite, g_enable_executor_resource_mgr, g_read_only, g_serialize_temp_tables, table_functions::TableFunctionsFactory::get_table_funcs(), GPU, SystemParameters::gpu_buffer_mem_bytes, Executor::init_resource_mgr(), table_functions::init_table_functions(), Catalog_Namespace::SysCatalog::instance(), Catalog_Namespace::UserMetadata::isSuper, shared::kCatalogDirectoryName, shared::kDataDirectoryName, shared::kDefaultDbName, shared::kDefaultDiskCacheDirName, RuntimeLibManager::loadRuntimeLibs(), RuntimeLibManager::loadTestRuntimeLibs(), Catalog_Namespace::UserMetadata::passwd_hash, query_engine_, session_info_, logger::set_once_fatal_func(), anonymous_namespace{QueryRunner.cpp}::setup_signal_handler(), ThriftSerializers::to_thrift(), and Catalog_Namespace::UserMetadata::userId.

169  : dispatch_queue_(std::make_unique<QueryDispatchQueue>(1)) {
171  boost::filesystem::path base_path{db_path};
172  CHECK(boost::filesystem::exists(base_path));
173  auto system_db_file =
175  CHECK(boost::filesystem::exists(system_db_file));
176  auto data_dir = base_path / shared::kDataDirectoryName;
177  File_Namespace::DiskCacheConfig disk_cache_config{
178  (base_path / shared::kDefaultDiskCacheDirName).string(),
180  if (cache_config) {
181  disk_cache_config = *cache_config;
182  }
184 
187  g_calcite =
188  std::make_shared<Calcite>(-1, CALCITEPORT, db_path, 1024, 5000, true, udf_filename);
189  ExtensionFunctionsWhitelist::add(g_calcite->getExtensionFunctionWhitelist());
190  if (!udf_filename.empty()) {
191  ExtensionFunctionsWhitelist::addUdfs(g_calcite->getUserDefinedFunctionWhitelist());
192  }
193 
195 #ifdef HAVE_RUNTIME_LIBS
198 #endif
199  auto udtfs = ThriftSerializers::to_thrift(
201  std::vector<TUserDefinedFunction> udfs = {};
202  g_calcite->setRuntimeExtensionFunctions(udfs, udtfs, /*is_runtime=*/false);
203 
204  std::unique_ptr<CudaMgr_Namespace::CudaMgr> cuda_mgr;
205 #ifdef HAVE_CUDA
206  if (uses_gpus) {
207  cuda_mgr = std::make_unique<CudaMgr_Namespace::CudaMgr>(-1, 0);
208  }
209 #else
210  uses_gpus = false;
211 #endif
212  const size_t num_gpus = static_cast<size_t>(cuda_mgr ? cuda_mgr->getDeviceCount() : 0);
213  SystemParameters mapd_params;
214  mapd_params.gpu_buffer_mem_bytes = max_gpu_mem;
215  mapd_params.aggregator = !leaf_servers.empty();
216 
217  auto& sys_cat = Catalog_Namespace::SysCatalog::instance();
218 
219  g_base_path = base_path.string();
220 
221  if (!sys_cat.isInitialized()) {
222  auto data_mgr = std::make_shared<Data_Namespace::DataMgr>(data_dir.string(),
223  mapd_params,
224  std::move(cuda_mgr),
225  uses_gpus,
226  reserved_gpu_mem,
227  0,
228  disk_cache_config);
229 
231  // With the exception of cpu_result_mem, the below values essentially mirror
232  // how ExecutorResourceMgr is initialized by DBHandler for normal DB operation.
233  // The static 4GB allowcation of CPU result memory is sufficient for our tests,
234  // and prevents variability based on the DBHandler approach to sizing as a fraction
235  // of CPU buffer pool mem size.
237  cpu_threads() /* num_cpu_slots */,
238  num_gpus /* num_gpu_slots */,
239  static_cast<size_t>(1UL << 32) /* cpu_result_mem */,
240  data_mgr->getCpuBufferPoolSize() /* cpu_buffer_pool_mem */,
241  data_mgr->getGpuBufferPoolSize() /* gpu_buffer_pool_mem */,
242  0.9 /* per_query_max_cpu_slots_ratio */,
243  1.0 /* per_query_max_cpu_result_mem_ratio */,
244  true /* allow_cpu_kernel_concurrency */,
245  true /* allow_cpu_gpu_kernel_concurrency */,
246  false /* allow_cpu_slot_oversubscription_concurrency */,
247  false /* allow_cpu_result_mem_oversubscription_concurrency */,
248  0.9 /* max_available_resource_use_ratio */);
249  }
250 
251  sys_cat.init(g_base_path,
252  data_mgr,
253  {},
254  g_calcite,
255  false,
256  mapd_params.aggregator,
257  string_servers);
258  }
259 
260  query_engine_ =
261  QueryEngine::createInstance(sys_cat.getDataMgr().getCudaMgr(), !uses_gpus);
262 
263  if (create_user) {
264  if (!sys_cat.getMetadataForUser(user_name, user)) {
265  sys_cat.createUser(
266  user_name,
268  passwd, /*is_super=*/false, /*default_db=*/"", /*can_login=*/true},
269  g_read_only);
270  }
271  }
272  CHECK(sys_cat.getMetadataForUser(user_name, user));
273  CHECK(bcrypt_checkpw(passwd.c_str(), user.passwd_hash.c_str()) == 0);
274 
275  if (create_db) {
276  if (!sys_cat.getMetadataForDB(db_name, db_metadata_)) {
277  sys_cat.createDatabase(db_name, user.userId);
278  }
279  }
280  CHECK(sys_cat.getMetadataForDB(db_name, db_metadata_));
281  CHECK(user.isSuper || (user.userId == db_metadata_.dbOwner));
282  auto cat = sys_cat.getCatalog(db_metadata_, create_db);
283  CHECK(cat);
284  session_info_ = std::make_unique<Catalog_Namespace::SessionInfo>(
285  cat, user, ExecutorDeviceType::GPU, "");
286 }
static void addUdfs(const std::string &json_func_sigs)
#define CALCITEPORT
Definition: QueryRunner.cpp:51
const std::string kDataDirectoryName
static std::vector< TableFunction > get_table_funcs()
static void loadTestRuntimeLibs()
std::string cat(Ts &&...args)
std::unique_ptr< QueryDispatchQueue > dispatch_queue_
Definition: QueryRunner.h:332
const std::string kDefaultDiskCacheDirName
static void loadRuntimeLibs(const std::string &torch_lib_path=std::string())
static void add(const std::string &json_func_sigs)
static void init_resource_mgr(const size_t num_cpu_slots, const size_t num_gpu_slots, const size_t cpu_result_mem, const size_t cpu_buffer_pool_mem, const size_t gpu_buffer_pool_mem, const double per_query_max_cpu_slots_ratio, const double per_query_max_cpu_result_mem_ratio, const bool allow_cpu_kernel_concurrency, const bool allow_cpu_gpu_kernel_concurrency, const bool allow_cpu_slot_oversubscription_concurrency, const bool allow_cpu_result_mem_oversubscription, const double max_available_resource_use_ratio)
Definition: Execute.cpp:5353
void set_once_fatal_func(FatalFunc fatal_func)
Definition: Logger.cpp:394
Catalog_Namespace::DBMetadata db_metadata_
Definition: QueryRunner.h:330
std::shared_ptr< QueryEngine > query_engine_
Definition: QueryRunner.h:333
bool g_enable_executor_resource_mgr
Definition: Execute.cpp:174
static SysCatalog & instance()
Definition: SysCatalog.h:343
const std::string kDefaultDbName
std::string g_base_path
Definition: SysCatalog.cpp:62
static std::shared_ptr< QueryEngine > createInstance(CudaMgr_Namespace::CudaMgr *cuda_mgr, bool cpu_only)
Definition: QueryEngine.h:97
bool g_serialize_temp_tables
Definition: Catalog.cpp:106
TExtArgumentType::type to_thrift(const ExtArgumentType &t)
bool g_read_only
Definition: heavyai_locks.h:21
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
std::shared_ptr< Calcite > g_calcite
Definition: QueryRunner.cpp:65
const std::string kCatalogDirectoryName
#define CHECK(condition)
Definition: Logger.h:291
void init_table_functions()
int cpu_threads()
Definition: thread_count.h:25
std::atomic< bool > isSuper
Definition: SysCatalog.h:107

+ Here is the call graph for this function:

Member Function Documentation

void QueryRunner::QueryRunner::addSessionId ( const std::string &  session_id,
ExecutorDeviceType  device_type = ExecutorDeviceType::GPU 
)
inline

Definition at line 194 of file QueryRunner.h.

References session_info_.

195  {
196  auto user_info = session_info_->get_currentUser();
197  session_info_ = std::make_unique<Catalog_Namespace::SessionInfo>(
198  session_info_->get_catalog_ptr(), user_info, device_type, session_id);
199  }
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
void QueryRunner::QueryRunner::clearCpuMemory ( ) const
virtual

Definition at line 316 of file QueryRunner.cpp.

References CHECK, Executor::clearMemory(), Data_Namespace::CPU_LEVEL, and Catalog_Namespace::SysCatalog::instance().

316  {
319 }
static void clearMemory(const Data_Namespace::MemoryLevel memory_level)
Definition: Execute.cpp:531
static SysCatalog & instance()
Definition: SysCatalog.h:343
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

void QueryRunner::QueryRunner::clearGpuMemory ( ) const
virtual

Definition at line 311 of file QueryRunner.cpp.

References CHECK, Executor::clearMemory(), Data_Namespace::GPU_LEVEL, and Catalog_Namespace::SysCatalog::instance().

311  {
314 }
static void clearMemory(const Data_Namespace::MemoryLevel memory_level)
Definition: Execute.cpp:531
static SysCatalog & instance()
Definition: SysCatalog.h:343
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

void QueryRunner::QueryRunner::clearSessionId ( )
inline

Definition at line 201 of file QueryRunner.h.

References session_info_.

201 { session_info_ = nullptr; }
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
template<typename... Ts>
static std::shared_ptr<query_state::QueryState> QueryRunner::QueryRunner::create_query_state ( Ts &&...  args)
inlinestatic

Definition at line 304 of file QueryRunner.h.

References run_benchmark_import::args, query_state::QueryStates::create(), and query_states_.

Referenced by createStatement(), getCalcitePlan(), getExecutor(), getParsedGlobalQueryHints(), getParsedQueryHint(), getParsedQueryHints(), getQueryInfoForDataRecyclerTest(), getRaExecutionSequence(), getRelAlgDag(), getRelAlgTranslator(), getRootNodeFromParsedQuery(), runDDLStatement(), runSelectQuery(), runSQLWithAllowingInterrupt(), and validateDDLStatement().

304  {
305  return query_states_.create(std::forward<Ts>(args)...);
306  }
static query_state::QueryStates query_states_
Definition: QueryRunner.h:301
CircleBuffer::value_type create(ARGS &&...args)
Definition: QueryState.h:194

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::unique_ptr< Parser::Stmt > QueryRunner::QueryRunner::createStatement ( const std::string &  stmt_str_in)
virtual

Definition at line 563 of file QueryRunner.cpp.

References cat(), CHECK, create_query_state(), Parser::create_stmt_for_json(), g_enable_watchdog, Catalog_Namespace::SysCatalog::instance(), pg_shim(), session_info_, STDLOG, and UNREACHABLE.

564  {
567 
568  std::string stmt_str = stmt_str_in;
569  // First remove special chars
570  boost::algorithm::trim_left_if(stmt_str, boost::algorithm::is_any_of("\n"));
571  // Then remove spaces
572  boost::algorithm::trim_left(stmt_str);
573 
574  ParserWrapper pw{stmt_str};
575 
576  auto query_state = create_query_state(session_info_, stmt_str);
577  auto stdlog = STDLOG(query_state);
578 
579  if (pw.is_ddl) {
580  const auto& cat = session_info_->getCatalog();
581  auto calcite_mgr = cat.getCalciteMgr();
582  const auto calciteQueryParsingOption =
583  calcite_mgr->getCalciteQueryParsingOption(true, false, true, false);
584  const auto calciteOptimizationOption =
585  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {}, false);
586  const auto query_json = calcite_mgr
587  ->process(query_state->createQueryStateProxy(),
588  pg_shim(stmt_str),
589  calciteQueryParsingOption,
590  calciteOptimizationOption)
591  .plan_result;
592  return Parser::create_stmt_for_json(query_json);
593  }
594 
595  // simply fail here as non-Calcite parsing is about to be removed
596  UNREACHABLE();
597  return nullptr;
598 }
std::string cat(Ts &&...args)
#define UNREACHABLE()
Definition: Logger.h:338
static SysCatalog & instance()
Definition: SysCatalog.h:343
std::unique_ptr< Parser::Stmt > create_stmt_for_json(const std::string &query_json)
bool g_enable_watchdog
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:304
#define CHECK(condition)
Definition: Logger.h:291
std::string pg_shim(const std::string &query)
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

ExecutionOptions QueryRunner::QueryRunner::defaultExecutionOptionsForRunSQL ( bool  allow_loop_joins = true,
bool  just_explain = false 
)
static

Definition at line 669 of file QueryRunner.cpp.

References g_enable_columnar_output, and g_gpu_mem_limit_percent.

Referenced by runSelectQuery(), and runSQL().

670  {
671  return {g_enable_columnar_output,
672  false,
673  true,
674  just_explain,
675  allow_loop_joins,
676  false,
677  false,
678  false,
679  false,
680  10000,
681  false,
682  false,
684  false,
685  0.5,
686  1000,
687  false};
688 }
bool g_enable_columnar_output
Definition: Execute.cpp:102
double g_gpu_mem_limit_percent
Definition: QueryRunner.cpp:56

+ Here is the caller graph for this function:

ExtractedQueryPlanDag QueryRunner::QueryRunner::extractQueryPlanDag ( const std::string &  query_str)

Definition at line 1002 of file QueryRunner.cpp.

References QueryPlanDagExtractor::extractQueryPlanDag(), Executor::getExecutor(), getQueryInfoForDataRecyclerTest(), and Executor::UNITARY_EXECUTOR_ID.

1002  {
1003  auto query_dag_info = getQueryInfoForDataRecyclerTest(query_str);
1005  auto extracted_dag_info = QueryPlanDagExtractor::extractQueryPlanDag(
1006  query_dag_info.root_node.get(), executor);
1007  return extracted_dag_info;
1008 }
static ExtractedQueryPlanDag extractQueryPlanDag(const RelAlgNode *top_node, Executor *executor)
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:509
QueryPlanDagInfo getQueryInfoForDataRecyclerTest(const std::string &)
static constexpr ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:423

+ Here is the call graph for this function:

static QueryRunner* QueryRunner::QueryRunner::get ( )
inlinestatic

Definition at line 181 of file QueryRunner.h.

References qr_instance_.

181  {
182  if (!qr_instance_) {
183  throw std::runtime_error("QueryRunner must be initialized before calling get().");
184  }
185  return qr_instance_.get();
186  }
static std::unique_ptr< QueryRunner > qr_instance_
Definition: QueryRunner.h:326
BufferPoolStats QueryRunner::QueryRunner::getBufferPoolStats ( const Data_Namespace::MemoryLevel  memory_level,
const bool  current_db_only 
) const

Definition at line 327 of file QueryRunner.cpp.

References CHECK, CHECK_EQ, Data_Namespace::CPU_LEVEL, db_metadata_, Catalog_Namespace::DBMetadata::dbId, Buffer_Namespace::FREE, Catalog_Namespace::SysCatalog::instance(), and session_info_.

329  {
330  // Only works single-node for now
332  const std::vector<MemoryInfo> memory_infos =
333  session_info_->getCatalog().getDataMgr().getMemoryInfo(memory_level);
334  if (memory_level == Data_Namespace::MemoryLevel::CPU_LEVEL) {
335  CHECK_EQ(memory_infos.size(), static_cast<size_t>(1));
336  }
337  std::set<std::vector<int32_t>> chunk_keys;
338  std::set<std::vector<int32_t>> table_keys;
339  std::set<std::vector<int32_t>> column_keys;
340  std::set<std::vector<int32_t>> fragment_keys;
341  size_t total_num_buffers{
342  0}; // can be greater than chunk keys set size due to table replication
343  size_t total_num_bytes{0};
344  for (auto& pool_memory_info : memory_infos) {
345  const std::vector<MemoryData>& memory_data = pool_memory_info.nodeMemoryData;
346  for (auto& memory_datum : memory_data) {
347  total_num_buffers++;
348  const auto& chunk_key = memory_datum.chunk_key;
349  if (memory_datum.memStatus == Buffer_Namespace::MemStatus::FREE ||
350  chunk_key.size() < 4) {
351  continue;
352  }
353  if (current_db_only) {
354  if (chunk_key[0] != db_metadata_.dbId) {
355  continue;
356  }
357  }
358  total_num_bytes += (memory_datum.numPages * pool_memory_info.pageSize);
359  table_keys.insert({chunk_key[0], chunk_key[1]});
360  column_keys.insert({chunk_key[0], chunk_key[1], chunk_key[2]});
361  fragment_keys.insert({chunk_key[0], chunk_key[1], chunk_key[3]});
362  chunk_keys.insert(chunk_key);
363  }
364  }
365  return {total_num_buffers,
366  total_num_bytes,
367  table_keys.size(),
368  column_keys.size(),
369  fragment_keys.size(),
370  chunk_keys.size()};
371 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
Catalog_Namespace::DBMetadata db_metadata_
Definition: QueryRunner.h:330
static SysCatalog & instance()
Definition: SysCatalog.h:343
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

std::tuple< QueryPlanHash, std::shared_ptr< HashTable >, std::optional< HashtableCacheMetaInfo > > QueryRunner::QueryRunner::getCachedHashtableWithoutCacheKey ( std::set< size_t > &  visited,
CacheItemType  hash_table_type,
DeviceIdentifier  device_identifier 
)

Definition at line 1062 of file QueryRunner.cpp.

References BASELINE_HT, BBOX_INTERSECT_HT, CHECK, BoundingBoxIntersectJoinHashTable::getHashTableCache(), PerfectJoinHashTable::getHashTableCache(), BaselineJoinHashTable::getHashTableCache(), PERFECT_HT, and UNREACHABLE.

1064  {
1065  HashtableRecycler* hash_table_cache{nullptr};
1066  switch (hash_table_type) {
1068  hash_table_cache = PerfectJoinHashTable::getHashTableCache();
1069  break;
1070  }
1072  hash_table_cache = BaselineJoinHashTable::getHashTableCache();
1073  break;
1074  }
1077  break;
1078  }
1079  default: {
1080  UNREACHABLE();
1081  break;
1082  }
1083  }
1084  CHECK(hash_table_cache);
1085  return hash_table_cache->getCachedHashtableWithoutCacheKey(
1086  visited, hash_table_type, device_identifier);
1087 }
#define UNREACHABLE()
Definition: Logger.h:338
static HashtableRecycler * getHashTableCache()
#define CHECK(condition)
Definition: Logger.h:291
static HashtableRecycler * getHashTableCache()

+ Here is the call graph for this function:

std::shared_ptr< CacheItemMetric > QueryRunner::QueryRunner::getCacheItemMetric ( QueryPlanHash  cache_key,
CacheItemType  hash_table_type,
DeviceIdentifier  device_identifier 
)

Definition at line 1089 of file QueryRunner.cpp.

References BASELINE_HT, BBOX_INTERSECT_HT, CHECK, BoundingBoxIntersectJoinHashTable::getHashTableCache(), PerfectJoinHashTable::getHashTableCache(), BaselineJoinHashTable::getHashTableCache(), PERFECT_HT, and UNREACHABLE.

1092  {
1093  HashtableRecycler* hash_table_cache{nullptr};
1094  switch (hash_table_type) {
1096  hash_table_cache = PerfectJoinHashTable::getHashTableCache();
1097  break;
1098  }
1100  hash_table_cache = BaselineJoinHashTable::getHashTableCache();
1101  break;
1102  }
1105  break;
1106  }
1107  default: {
1108  UNREACHABLE();
1109  break;
1110  }
1111  }
1112  CHECK(hash_table_cache);
1113  return hash_table_cache->getCachedItemMetric(
1114  hash_table_type, device_identifier, cache_key);
1115 }
#define UNREACHABLE()
Definition: Logger.h:338
static HashtableRecycler * getHashTableCache()
#define CHECK(condition)
Definition: Logger.h:291
static HashtableRecycler * getHashTableCache()

+ Here is the call graph for this function:

std::shared_ptr< Calcite > QueryRunner::QueryRunner::getCalcite ( ) const

Definition at line 301 of file QueryRunner.cpp.

References anonymous_namespace{QueryRunner.cpp}::g_calcite.

301  {
302  // TODO: Embed Calcite shared_ptr ownership in QueryRunner
303  return g_calcite;
304 }
std::shared_ptr< Calcite > g_calcite
Definition: QueryRunner.cpp:65
std::shared_ptr< ResultSet > QueryRunner::QueryRunner::getCalcitePlan ( const std::string &  query_str,
bool  enable_watchdog,
bool  is_explain_as_json_str,
bool  is_explain_detailed 
) const
virtual

Definition at line 884 of file QueryRunner.cpp.

References cat(), CHECK, create_query_state(), dispatch_queue_, g_enable_calcite_view_optimize, Executor::getExecutor(), Catalog_Namespace::SysCatalog::instance(), pg_shim(), run_benchmark_import::result, session_info_, STDLOG, and logger::thread_local_ids().

887  {
890  const auto& cat = session_info_->getCatalog();
891  auto query_state = create_query_state(session_info_, query_str);
892  auto stdlog = STDLOG(query_state);
893 
894  std::shared_ptr<ResultSet> result;
895  auto query_launch_task = std::make_shared<QueryDispatchQueue::Task>(
896  [&cat,
897  &query_str,
898  &enable_watchdog,
899  &is_explain_as_json_str,
900  &is_explain_detailed,
901  &query_state,
902  &result,
903  parent_thread_local_ids = logger::thread_local_ids()](const size_t worker_id) {
904  logger::LocalIdsScopeGuard lisg = parent_thread_local_ids.setNewThreadId();
905  auto executor = Executor::getExecutor(worker_id);
906  auto calcite_mgr = cat.getCalciteMgr();
907  // Calcite returns its plan as a form of `json_str` by default,
908  // so we set `is_explain` to TRUE if `!is_explain_as_json_str`
909  const auto calciteQueryParsingOption = calcite_mgr->getCalciteQueryParsingOption(
910  true, !is_explain_as_json_str, false, is_explain_detailed);
911  const auto calciteOptimizationOption = calcite_mgr->getCalciteOptimizationOption(
912  g_enable_calcite_view_optimize, enable_watchdog, {}, false);
913  const auto query_ra = calcite_mgr
914  ->process(query_state->createQueryStateProxy(),
915  pg_shim(query_str),
916  calciteQueryParsingOption,
917  calciteOptimizationOption)
918  .plan_result;
919  result = std::make_shared<ResultSet>(query_ra);
920  return result;
921  });
923  dispatch_queue_->submit(query_launch_task, /*is_update_delete=*/false);
924  auto result_future = query_launch_task->get_future();
925  result_future.get();
926  CHECK(result);
927  return result;
928 }
bool g_enable_calcite_view_optimize
Definition: QueryRunner.cpp:59
std::string cat(Ts &&...args)
std::unique_ptr< QueryDispatchQueue > dispatch_queue_
Definition: QueryRunner.h:332
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:509
static SysCatalog & instance()
Definition: SysCatalog.h:343
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:304
#define CHECK(condition)
Definition: Logger.h:291
std::string pg_shim(const std::string &query)
ThreadLocalIds thread_local_ids()
Definition: Logger.cpp:880
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

std::shared_ptr< Catalog_Namespace::Catalog > QueryRunner::QueryRunner::getCatalog ( ) const

Definition at line 296 of file QueryRunner.cpp.

References CHECK, and session_info_.

Referenced by getLoader().

296  {
298  return session_info_->get_catalog_ptr();
299 }
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the caller graph for this function:

std::shared_ptr< Executor > QueryRunner::QueryRunner::getExecutor ( ) const

Definition at line 690 of file QueryRunner.cpp.

References CHECK, create_query_state(), Executor::getExecutor(), Catalog_Namespace::SysCatalog::instance(), session_info_, STDLOG, and Executor::UNITARY_EXECUTOR_ID.

690  {
693  auto query_state = create_query_state(session_info_, "");
694  auto stdlog = STDLOG(query_state);
696  return executor;
697 }
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:509
static SysCatalog & instance()
Definition: SysCatalog.h:343
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:304
#define CHECK(condition)
Definition: Logger.h:291
static constexpr ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:423
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

std::unique_ptr< import_export::Loader > QueryRunner::QueryRunner::getLoader ( const TableDescriptor td) const
virtual

Definition at line 812 of file QueryRunner.cpp.

References cat(), and getCatalog().

813  {
814  auto cat = getCatalog();
815  return std::make_unique<import_export::Loader>(*cat, td);
816 }
std::string cat(Ts &&...args)
std::shared_ptr< Catalog_Namespace::Catalog > getCatalog() const

+ Here is the call graph for this function:

std::vector< MemoryInfo > QueryRunner::QueryRunner::getMemoryInfo ( const Data_Namespace::MemoryLevel  memory_level) const

Definition at line 321 of file QueryRunner.cpp.

References CHECK, Catalog_Namespace::SysCatalog::instance(), and session_info_.

322  {
324  return session_info_->getCatalog().getDataMgr().getMemoryInfo(memory_level);
325 }
static SysCatalog & instance()
Definition: SysCatalog.h:343
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

size_t QueryRunner::QueryRunner::getNumberOfCachedItem ( CacheItemStatus  item_status,
CacheItemType  hash_table_type,
bool  with_bbox_intersect_tuning_param = false 
) const

Definition at line 1117 of file QueryRunner.cpp.

References QueryRunner::ALL, BASELINE_HT, BBOX_INTERSECT_AUTO_TUNER_PARAM, BBOX_INTERSECT_HT, CHECK, CHECK_EQ, QueryRunner::CLEAN_ONLY, DataRecyclerUtil::CPU_DEVICE_IDENTIFIER, QueryRunner::DIRTY_ONLY, BoundingBoxIntersectJoinHashTable::getBoundingBoxIntersectTuningParamCache(), BoundingBoxIntersectJoinHashTable::getHashTableCache(), PerfectJoinHashTable::getHashTableCache(), BaselineJoinHashTable::getHashTableCache(), PERFECT_HT, and UNREACHABLE.

1119  {
1120  auto get_num_cached_auto_tuner_param = [&item_status]() {
1121  auto auto_tuner_cache =
1123  CHECK(auto_tuner_cache);
1124  switch (item_status) {
1125  case CacheItemStatus::ALL: {
1126  return auto_tuner_cache->getCurrentNumCachedItems(
1129  }
1131  return auto_tuner_cache->getCurrentNumCleanCachedItems(
1134  }
1136  return auto_tuner_cache->getCurrentNumDirtyCachedItems(
1139  }
1140  default: {
1141  UNREACHABLE();
1142  return static_cast<size_t>(0);
1143  }
1144  }
1145  };
1146 
1147  auto get_num_cached_hashtable =
1148  [&item_status,
1149  &hash_table_type,
1150  &with_bbox_intersect_tuning_param,
1151  &get_num_cached_auto_tuner_param](HashtableRecycler* hash_table_cache) {
1152  switch (item_status) {
1153  case CacheItemStatus::ALL: {
1154  if (with_bbox_intersect_tuning_param) {
1155  // we assume additional consideration of turing param cache is only valid
1156  // for bounding box intersection
1157  CHECK_EQ(hash_table_type, CacheItemType::BBOX_INTERSECT_HT);
1158  return hash_table_cache->getCurrentNumCachedItems(
1159  hash_table_type, DataRecyclerUtil::CPU_DEVICE_IDENTIFIER) +
1160  get_num_cached_auto_tuner_param();
1161  }
1162  return hash_table_cache->getCurrentNumCachedItems(
1163  hash_table_type, DataRecyclerUtil::CPU_DEVICE_IDENTIFIER);
1164  }
1166  if (with_bbox_intersect_tuning_param) {
1167  CHECK_EQ(hash_table_type, CacheItemType::BBOX_INTERSECT_HT);
1168  return hash_table_cache->getCurrentNumCleanCachedItems(
1169  hash_table_type, DataRecyclerUtil::CPU_DEVICE_IDENTIFIER) +
1170  get_num_cached_auto_tuner_param();
1171  }
1172  return hash_table_cache->getCurrentNumCleanCachedItems(
1173  hash_table_type, DataRecyclerUtil::CPU_DEVICE_IDENTIFIER);
1174  }
1176  if (with_bbox_intersect_tuning_param) {
1177  CHECK_EQ(hash_table_type, CacheItemType::BBOX_INTERSECT_HT);
1178  return hash_table_cache->getCurrentNumDirtyCachedItems(
1179  hash_table_type, DataRecyclerUtil::CPU_DEVICE_IDENTIFIER) +
1180  get_num_cached_auto_tuner_param();
1181  }
1182  return hash_table_cache->getCurrentNumDirtyCachedItems(
1183  hash_table_type, DataRecyclerUtil::CPU_DEVICE_IDENTIFIER);
1184  }
1185  default: {
1186  UNREACHABLE();
1187  return static_cast<size_t>(0);
1188  }
1189  }
1190  };
1191 
1192  switch (hash_table_type) {
1194  auto hash_table_cache = PerfectJoinHashTable::getHashTableCache();
1195  CHECK(hash_table_cache);
1196  return get_num_cached_hashtable(hash_table_cache);
1197  }
1199  auto hash_table_cache = BaselineJoinHashTable::getHashTableCache();
1200  CHECK(hash_table_cache);
1201  return get_num_cached_hashtable(hash_table_cache);
1202  }
1204  auto hash_table_cache = BoundingBoxIntersectJoinHashTable::getHashTableCache();
1205  CHECK(hash_table_cache);
1206  return get_num_cached_hashtable(hash_table_cache);
1207  }
1209  return get_num_cached_auto_tuner_param();
1210  }
1211  default: {
1212  UNREACHABLE();
1213  return 0;
1214  }
1215  }
1216  return 0;
1217 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
#define UNREACHABLE()
Definition: Logger.h:338
static HashtableRecycler * getHashTableCache()
#define CHECK(condition)
Definition: Logger.h:291
static HashtableRecycler * getHashTableCache()
static constexpr DeviceIdentifier CPU_DEVICE_IDENTIFIER
Definition: DataRecycler.h:136
static BoundingBoxIntersectTuningParamRecycler * getBoundingBoxIntersectTuningParamCache()

+ Here is the call graph for this function:

std::optional< RegisteredQueryHint > QueryRunner::QueryRunner::getParsedGlobalQueryHints ( const std::string &  query_str)

Definition at line 443 of file QueryRunner.cpp.

References cat(), CHECK, create_query_state(), g_enable_watchdog, Executor::getExecutor(), RelAlgExecutor::getGlobalQueryHint(), Catalog_Namespace::SysCatalog::instance(), pg_shim(), session_info_, and Executor::UNITARY_EXECUTOR_ID.

444  {
447  auto query_state = create_query_state(session_info_, query_str);
448  auto& cat = session_info_->getCatalog();
450  auto calcite_mgr = cat.getCalciteMgr();
451  const auto calciteQueryParsingOption =
452  calcite_mgr->getCalciteQueryParsingOption(true, false, true, false);
453  const auto calciteOptimizationOption =
454  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {}, false);
455  const auto query_ra = calcite_mgr
456  ->process(query_state->createQueryStateProxy(),
457  pg_shim(query_str),
458  calciteQueryParsingOption,
459  calciteOptimizationOption)
460  .plan_result;
461  auto ra_executor = RelAlgExecutor(executor.get(), query_ra, query_state);
462  return ra_executor.getGlobalQueryHint();
463 }
std::string cat(Ts &&...args)
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:509
static SysCatalog & instance()
Definition: SysCatalog.h:343
bool g_enable_watchdog
std::optional< RegisteredQueryHint > getGlobalQueryHint()
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:304
#define CHECK(condition)
Definition: Logger.h:291
static constexpr ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:423
std::string pg_shim(const std::string &query)

+ Here is the call graph for this function:

RegisteredQueryHint QueryRunner::QueryRunner::getParsedQueryHint ( const std::string &  query_str)

Definition at line 373 of file QueryRunner.cpp.

References cat(), CHECK, create_query_state(), RegisteredQueryHint::defaults(), g_enable_watchdog, Executor::getExecutor(), RelAlgExecutor::getParsedQueryHint(), Catalog_Namespace::SysCatalog::instance(), pg_shim(), session_info_, and Executor::UNITARY_EXECUTOR_ID.

373  {
376  auto query_state = create_query_state(session_info_, query_str);
377  auto& cat = session_info_->getCatalog();
379 
380  auto calcite_mgr = cat.getCalciteMgr();
381  const auto calciteQueryParsingOption =
382  calcite_mgr->getCalciteQueryParsingOption(true, false, true, false);
383  const auto calciteOptimizationOption =
384  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {}, false);
385  const auto query_ra = calcite_mgr
386  ->process(query_state->createQueryStateProxy(),
387  pg_shim(query_str),
388  calciteQueryParsingOption,
389  calciteOptimizationOption)
390  .plan_result;
391  auto ra_executor = RelAlgExecutor(executor.get(), query_ra, query_state);
392  auto query_hints =
393  ra_executor.getParsedQueryHint(ra_executor.getRootRelAlgNodeShPtr().get());
394  return query_hints ? *query_hints : RegisteredQueryHint::defaults();
395 }
std::string cat(Ts &&...args)
std::optional< RegisteredQueryHint > getParsedQueryHint(const RelAlgNode *node)
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:509
static SysCatalog & instance()
Definition: SysCatalog.h:343
bool g_enable_watchdog
static RegisteredQueryHint defaults()
Definition: QueryHint.h:364
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:304
#define CHECK(condition)
Definition: Logger.h:291
static constexpr ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:423
std::string pg_shim(const std::string &query)

+ Here is the call graph for this function:

std::optional< std::unordered_map< size_t, std::unordered_map< unsigned, RegisteredQueryHint > > > QueryRunner::QueryRunner::getParsedQueryHints ( const std::string &  query_str)

Definition at line 422 of file QueryRunner.cpp.

References cat(), CHECK, create_query_state(), g_enable_watchdog, Executor::getExecutor(), RelAlgExecutor::getParsedQueryHints(), Catalog_Namespace::SysCatalog::instance(), pg_shim(), session_info_, and Executor::UNITARY_EXECUTOR_ID.

422  {
425  auto query_state = create_query_state(session_info_, query_str);
426  auto& cat = session_info_->getCatalog();
428  auto calcite_mgr = cat.getCalciteMgr();
429  const auto calciteQueryParsingOption =
430  calcite_mgr->getCalciteQueryParsingOption(true, false, true, false);
431  const auto calciteOptimizationOption =
432  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {}, false);
433  const auto query_ra = calcite_mgr
434  ->process(query_state->createQueryStateProxy(),
435  pg_shim(query_str),
436  calciteQueryParsingOption,
437  calciteOptimizationOption)
438  .plan_result;
439  auto ra_executor = RelAlgExecutor(executor.get(), query_ra, query_state);
440  return ra_executor.getParsedQueryHints();
441 }
std::string cat(Ts &&...args)
std::optional< std::unordered_map< size_t, std::unordered_map< unsigned, RegisteredQueryHint > > > getParsedQueryHints()
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:509
static SysCatalog & instance()
Definition: SysCatalog.h:343
bool g_enable_watchdog
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:304
#define CHECK(condition)
Definition: Logger.h:291
static constexpr ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:423
std::string pg_shim(const std::string &query)

+ Here is the call graph for this function:

QueryPlanDagInfo QueryRunner::QueryRunner::getQueryInfoForDataRecyclerTest ( const std::string &  query_str)

Definition at line 535 of file QueryRunner.cpp.

References cat(), CHECK, create_query_state(), g_enable_watchdog, Executor::getExecutor(), RelAlgExecutor::getRootRelAlgNodeShPtr(), Catalog_Namespace::SysCatalog::instance(), pg_shim(), session_info_, and Executor::UNITARY_EXECUTOR_ID.

Referenced by extractQueryPlanDag().

536  {
539  auto query_state = create_query_state(session_info_, query_str);
540  auto& cat = session_info_->getCatalog();
542  auto calcite_mgr = cat.getCalciteMgr();
543  const auto calciteQueryParsingOption =
544  calcite_mgr->getCalciteQueryParsingOption(true, false, true, false);
545  const auto calciteOptimizationOption =
546  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {}, false);
547  const auto query_ra = calcite_mgr
548  ->process(query_state->createQueryStateProxy(),
549  pg_shim(query_str),
550  calciteQueryParsingOption,
551  calciteOptimizationOption)
552  .plan_result;
553  auto ra_executor = RelAlgExecutor(executor.get(), query_ra);
554  // note that we assume the test for data recycler that needs to have join_info
555  // does not contain any ORDER BY clause; this is necessary to create work_unit
556  // without actually performing the query
557  auto root_node_shared_ptr = ra_executor.getRootRelAlgNodeShPtr();
558  auto join_info = ra_executor.getJoinInfo(root_node_shared_ptr.get());
559  auto relAlgTranslator = ra_executor.getRelAlgTranslator(root_node_shared_ptr.get());
560  return {root_node_shared_ptr, join_info.first, join_info.second, relAlgTranslator};
561 }
std::string cat(Ts &&...args)
std::shared_ptr< const RelAlgNode > getRootRelAlgNodeShPtr() const
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:509
static SysCatalog & instance()
Definition: SysCatalog.h:343
bool g_enable_watchdog
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:304
#define CHECK(condition)
Definition: Logger.h:291
static constexpr ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:423
std::string pg_shim(const std::string &query)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

RaExecutionSequence QueryRunner::QueryRunner::getRaExecutionSequence ( const std::string &  query_str)

Definition at line 465 of file QueryRunner.cpp.

References cat(), CHECK, create_query_state(), g_enable_watchdog, Executor::getExecutor(), RelAlgExecutor::getRaExecutionSequence(), Catalog_Namespace::SysCatalog::instance(), pg_shim(), session_info_, and Executor::UNITARY_EXECUTOR_ID.

465  {
468  auto query_state = create_query_state(session_info_, query_str);
469  auto& cat = session_info_->getCatalog();
471  auto calcite_mgr = cat.getCalciteMgr();
472  const auto calciteQueryParsingOption =
473  calcite_mgr->getCalciteQueryParsingOption(true, false, true, false);
474  const auto calciteOptimizationOption =
475  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {}, false);
476  const auto query_ra = calcite_mgr
477  ->process(query_state->createQueryStateProxy(),
478  pg_shim(query_str),
479  calciteQueryParsingOption,
480  calciteOptimizationOption)
481  .plan_result;
482  auto ra_executor = RelAlgExecutor(executor.get(), query_ra, query_state);
483  return ra_executor.getRaExecutionSequence(ra_executor.getRootRelAlgNodeShPtr().get(),
484  executor.get());
485 }
std::string cat(Ts &&...args)
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:509
RaExecutionSequence getRaExecutionSequence(const RelAlgNode *root_node, Executor *executor)
static SysCatalog & instance()
Definition: SysCatalog.h:343
bool g_enable_watchdog
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:304
#define CHECK(condition)
Definition: Logger.h:291
static constexpr ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:423
std::string pg_shim(const std::string &query)

+ Here is the call graph for this function:

std::unique_ptr< RelAlgDag > QueryRunner::QueryRunner::getRelAlgDag ( const std::string &  query_str)

Definition at line 1010 of file QueryRunner.cpp.

References cat(), CHECK, create_query_state(), ExecutionOptions::defaults(), dispatch_queue_, g_enable_calcite_view_optimize, g_enable_watchdog, Executor::getExecutor(), RelAlgExecutor::getOwnedRelAlgDag(), Catalog_Namespace::SysCatalog::instance(), pg_shim(), session_info_, STDLOG, and logger::thread_local_ids().

1010  {
1013  auto query_state = create_query_state(session_info_, query_str);
1014  auto stdlog = STDLOG(query_state);
1015  auto& cat = session_info_->getCatalog();
1016 
1017  std::unique_ptr<RelAlgDag> rel_alg_dag;
1018  auto query_launch_task = std::make_shared<QueryDispatchQueue::Task>(
1019  [&cat,
1020  &query_str,
1021  &query_state,
1022  &rel_alg_dag,
1023  parent_thread_local_ids = logger::thread_local_ids()](const size_t worker_id) {
1024  logger::LocalIdsScopeGuard lisg = parent_thread_local_ids.setNewThreadId();
1025  auto executor = Executor::getExecutor(worker_id);
1026  auto eo = ExecutionOptions::defaults();
1027  auto calcite_mgr = cat.getCalciteMgr();
1028  const auto calciteQueryParsingOption =
1029  calcite_mgr->getCalciteQueryParsingOption(true, false, true, false);
1030  const auto calciteOptimizationOption = calcite_mgr->getCalciteOptimizationOption(
1032  const auto query_ra = calcite_mgr
1033  ->process(query_state->createQueryStateProxy(),
1034  pg_shim(query_str),
1035  calciteQueryParsingOption,
1036  calciteOptimizationOption)
1037  .plan_result;
1038  auto ra_executor = RelAlgExecutor(executor.get(), query_ra);
1039  rel_alg_dag = ra_executor.getOwnedRelAlgDag();
1040  });
1042  dispatch_queue_->submit(query_launch_task, /*is_update_delete=*/false);
1043  auto result_future = query_launch_task->get_future();
1044  result_future.get();
1045  CHECK(rel_alg_dag);
1046  return rel_alg_dag;
1047 }
bool g_enable_calcite_view_optimize
Definition: QueryRunner.cpp:59
std::string cat(Ts &&...args)
std::unique_ptr< QueryDispatchQueue > dispatch_queue_
Definition: QueryRunner.h:332
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:509
static SysCatalog & instance()
Definition: SysCatalog.h:343
bool g_enable_watchdog
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:304
#define CHECK(condition)
Definition: Logger.h:291
std::unique_ptr< RelAlgDag > getOwnedRelAlgDag()
static ExecutionOptions defaults()
std::string pg_shim(const std::string &query)
ThreadLocalIds thread_local_ids()
Definition: Logger.cpp:880
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

std::shared_ptr< RelAlgTranslator > QueryRunner::QueryRunner::getRelAlgTranslator ( const std::string &  query_str,
Executor executor 
)

Definition at line 512 of file QueryRunner.cpp.

References cat(), CHECK, create_query_state(), g_enable_watchdog, Catalog_Namespace::SysCatalog::instance(), pg_shim(), and session_info_.

514  {
517  auto query_state = create_query_state(session_info_, query_str);
518  auto& cat = session_info_->getCatalog();
519  auto calcite_mgr = cat.getCalciteMgr();
520  const auto calciteQueryParsingOption =
521  calcite_mgr->getCalciteQueryParsingOption(true, false, true, false);
522  const auto calciteOptimizationOption =
523  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {}, false);
524  const auto query_ra = calcite_mgr
525  ->process(query_state->createQueryStateProxy(),
526  pg_shim(query_str),
527  calciteQueryParsingOption,
528  calciteOptimizationOption)
529  .plan_result;
530  auto ra_executor = RelAlgExecutor(executor, query_ra);
531  auto root_node_shared_ptr = ra_executor.getRootRelAlgNodeShPtr();
532  return ra_executor.getRelAlgTranslator(root_node_shared_ptr.get());
533 }
std::string cat(Ts &&...args)
static SysCatalog & instance()
Definition: SysCatalog.h:343
bool g_enable_watchdog
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:304
#define CHECK(condition)
Definition: Logger.h:291
std::string pg_shim(const std::string &query)

+ Here is the call graph for this function:

std::shared_ptr< const RelAlgNode > QueryRunner::QueryRunner::getRootNodeFromParsedQuery ( const std::string &  query_str)

Definition at line 397 of file QueryRunner.cpp.

References cat(), CHECK, create_query_state(), g_enable_watchdog, Executor::getExecutor(), RelAlgExecutor::getRootRelAlgNodeShPtr(), Catalog_Namespace::SysCatalog::instance(), pg_shim(), session_info_, and Executor::UNITARY_EXECUTOR_ID.

398  {
401  auto query_state = create_query_state(session_info_, query_str);
402  auto& cat = session_info_->getCatalog();
404 
405  auto calcite_mgr = cat.getCalciteMgr();
406  const auto calciteQueryParsingOption =
407  calcite_mgr->getCalciteQueryParsingOption(true, false, true, false);
408  const auto calciteOptimizationOption =
409  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {}, false);
410  const auto query_ra = calcite_mgr
411  ->process(query_state->createQueryStateProxy(),
412  pg_shim(query_str),
413  calciteQueryParsingOption,
414  calciteOptimizationOption)
415  .plan_result;
416  auto ra_executor = RelAlgExecutor(executor.get(), query_ra, query_state);
417  return ra_executor.getRootRelAlgNodeShPtr();
418 }
std::string cat(Ts &&...args)
std::shared_ptr< const RelAlgNode > getRootRelAlgNodeShPtr() const
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:509
static SysCatalog & instance()
Definition: SysCatalog.h:343
bool g_enable_watchdog
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:304
#define CHECK(condition)
Definition: Logger.h:291
static constexpr ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:423
std::string pg_shim(const std::string &query)

+ Here is the call graph for this function:

std::shared_ptr<Catalog_Namespace::SessionInfo> QueryRunner::QueryRunner::getSession ( ) const
inline

Definition at line 190 of file QueryRunner.h.

References session_info_.

190  {
191  return session_info_;
192  }
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
Catalog_Namespace::UserMetadata& QueryRunner::QueryRunner::getUserMetadata ( ) const
bool QueryRunner::QueryRunner::gpusPresent ( ) const

Definition at line 306 of file QueryRunner.cpp.

References CHECK, and session_info_.

306  {
308  return session_info_->getCatalog().getDataMgr().gpusPresent();
309 }
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
#define CHECK(condition)
Definition: Logger.h:291
QueryRunner * QueryRunner::QueryRunner::init ( const char *  db_path,
const std::string &  udf_filename = "",
const size_t  max_gpu_mem = 0,
const int  reserved_gpu_mem = 256 << 20 
)
static

Definition at line 87 of file QueryRunner.cpp.

References logger::init(), shared::kDefaultDbName, and shared::kRootUsername.

90  {
91  return QueryRunner::init(db_path,
93  "HyperInteractive",
95  {},
96  {},
97  udf_filename,
98  true,
99  max_gpu_mem,
100  reserved_gpu_mem);
101 }
const std::string kDefaultDbName
void init(LogOptions const &log_opts)
Definition: Logger.cpp:364
const std::string kRootUsername

+ Here is the call graph for this function:

QueryRunner * QueryRunner::QueryRunner::init ( const File_Namespace::DiskCacheConfig disk_cache_config,
const char *  db_path,
const std::vector< LeafHostInfo > &  string_servers = {},
const std::vector< LeafHostInfo > &  leaf_servers = {} 
)
static

Definition at line 103 of file QueryRunner.cpp.

References logger::init(), shared::kDefaultDbName, and shared::kRootUsername.

106  {
107  return QueryRunner::init(db_path,
109  "HyperInteractive",
111  string_servers,
112  leaf_servers,
113  "",
114  true,
115  0,
116  256 << 20,
117  false,
118  false,
119  disk_cache_config);
120 }
const std::string kDefaultDbName
void init(LogOptions const &log_opts)
Definition: Logger.cpp:364
const std::string kRootUsername

+ Here is the call graph for this function:

static QueryRunner* QueryRunner::QueryRunner::init ( const char *  db_path,
const std::vector< LeafHostInfo > &  string_servers,
const std::vector< LeafHostInfo > &  leaf_servers 
)
inlinestatic

Definition at line 151 of file QueryRunner.h.

References logger::init(), shared::kDefaultDbName, and shared::kRootUsername.

153  {
154  return QueryRunner::init(db_path,
156  "HyperInteractive",
158  string_servers,
159  leaf_servers);
160  }
const std::string kDefaultDbName
void init(LogOptions const &log_opts)
Definition: Logger.cpp:364
const std::string kRootUsername

+ Here is the call graph for this function:

QueryRunner * QueryRunner::QueryRunner::init ( const char *  db_path,
const std::string &  user,
const std::string &  pass,
const std::string &  db_name,
const std::vector< LeafHostInfo > &  string_servers,
const std::vector< LeafHostInfo > &  leaf_servers,
const std::string &  udf_filename = "",
bool  uses_gpus = true,
const size_t  max_gpu_mem = 0,
const int  reserved_gpu_mem = 256 << 20,
const bool  create_user = false,
const bool  create_db = false,
const File_Namespace::DiskCacheConfig config = nullptr 
)
static

Definition at line 122 of file QueryRunner.cpp.

References CHECK, ddl_utils::FilePathWhitelist::clear(), logger::FATAL, ddl_utils::FilePathWhitelist::initialize(), and LOG_IF.

134  {
135  // Whitelist root path for tests by default
137  ddl_utils::FilePathWhitelist::initialize(db_path, "[\"/\"]", "[\"/\"]");
138  LOG_IF(FATAL, !leaf_servers.empty()) << "Distributed test runner not supported.";
139  CHECK(leaf_servers.empty());
140  qr_instance_.reset(new QueryRunner(db_path,
141  user,
142  pass,
143  db_name,
144  string_servers,
145  leaf_servers,
146  udf_filename,
147  uses_gpus,
148  max_gpu_mem,
149  reserved_gpu_mem,
150  create_user,
151  create_db,
152  disk_cache_config));
153  return qr_instance_.get();
154 }
static void initialize(const std::string &data_dir, const std::string &allowed_import_paths, const std::string &allowed_export_paths)
Definition: DdlUtils.cpp:878
#define LOG_IF(severity, condition)
Definition: Logger.h:384
QueryRunner(std::unique_ptr< Catalog_Namespace::SessionInfo > session)
static std::unique_ptr< QueryRunner > qr_instance_
Definition: QueryRunner.h:326
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

static QueryRunner* QueryRunner::QueryRunner::init ( std::unique_ptr< Catalog_Namespace::SessionInfo > &  session)
inlinestatic

Definition at line 176 of file QueryRunner.h.

References qr_instance_, and QueryRunner().

176  {
177  qr_instance_.reset(new QueryRunner(std::move(session)));
178  return qr_instance_.get();
179  }
QueryRunner(std::unique_ptr< Catalog_Namespace::SessionInfo > session)
static std::unique_ptr< QueryRunner > qr_instance_
Definition: QueryRunner.h:326

+ Here is the call graph for this function:

void QueryRunner::QueryRunner::reset ( )
static

Definition at line 1219 of file QueryRunner.cpp.

References anonymous_namespace{QueryRunner.cpp}::calcite_shutdown_handler(), and qr_instance_.

1219  {
1220  qr_instance_->query_engine_.reset();
1221  qr_instance_.reset(nullptr);
1223 }
static std::unique_ptr< QueryRunner > qr_instance_
Definition: QueryRunner.h:326

+ Here is the call graph for this function:

void QueryRunner::QueryRunner::resizeDispatchQueue ( const size_t  num_executors)

Definition at line 288 of file QueryRunner.cpp.

References dispatch_queue_.

288  {
289  dispatch_queue_ = std::make_unique<QueryDispatchQueue>(num_executors);
290 }
std::unique_ptr< QueryDispatchQueue > dispatch_queue_
Definition: QueryRunner.h:332
void QueryRunner::QueryRunner::runDDLStatement ( const std::string &  stmt_str_in)
virtual

Definition at line 600 of file QueryRunner.cpp.

References cat(), CHECK, create_query_state(), DdlCommandExecutor::execute(), g_enable_watchdog, ParserWrapper::Insert, Catalog_Namespace::SysCatalog::instance(), pg_shim(), session_info_, and STDLOG.

Referenced by QueryRunner::ImportDriver::importGeoTable(), runMultipleStatements(), and runSQL().

600  {
603 
604  std::string stmt_str = stmt_str_in;
605  // First remove special chars
606  boost::algorithm::trim_left_if(stmt_str, boost::algorithm::is_any_of("\n"));
607  // Then remove spaces
608  boost::algorithm::trim_left(stmt_str);
609 
610  ParserWrapper pw{stmt_str};
611 
612  auto query_state = create_query_state(session_info_, stmt_str);
613  auto stdlog = STDLOG(query_state);
614 
615  if (pw.is_ddl || pw.getDMLType() == ParserWrapper::DMLType::Insert) {
616  auto& cat = session_info_->getCatalog();
617  auto calcite_mgr = cat.getCalciteMgr();
618  const auto calciteQueryParsingOption =
619  calcite_mgr->getCalciteQueryParsingOption(true, false, true, false);
620  const auto calciteOptimizationOption =
621  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {}, false);
622  const auto query_ra = calcite_mgr
623  ->process(query_state->createQueryStateProxy(),
624  pg_shim(stmt_str),
625  calciteQueryParsingOption,
626  calciteOptimizationOption)
627  .plan_result;
628  if (pw.getDMLType() == ParserWrapper::DMLType::Insert) {
629  rapidjson::Document ddl_query;
630  ddl_query.Parse(query_ra);
631  CHECK(ddl_query.HasMember("payload"));
632  CHECK(ddl_query["payload"].IsObject());
633  auto stmt = Parser::InsertValuesStmt(cat, ddl_query["payload"].GetObject());
634  stmt.execute(*session_info_, false /* read only */);
635  return;
636  }
638  executor.execute(false /* read only */);
639  return;
640  }
641 }
std::string cat(Ts &&...args)
static SysCatalog & instance()
Definition: SysCatalog.h:343
bool g_enable_watchdog
ExecutionResult execute(bool read_only_mode)
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:304
#define CHECK(condition)
Definition: Logger.h:291
std::string pg_shim(const std::string &query)
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void QueryRunner::QueryRunner::runImport ( Parser::CopyTableStmt import_stmt)
virtual

Definition at line 807 of file QueryRunner.cpp.

References CHECK, Parser::CopyTableStmt::execute(), and session_info_.

807  {
808  CHECK(import_stmt);
809  import_stmt->execute(*session_info_, false /* read only */);
810 }
void execute(const Catalog_Namespace::SessionInfo &session, bool read_only_mode) override
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

std::vector< std::shared_ptr< ResultSet > > QueryRunner::QueryRunner::runMultipleStatements ( const std::string &  sql,
const ExecutorDeviceType  dt 
)
virtual

Definition at line 783 of file QueryRunner.cpp.

References field(), ParserWrapper::Insert, runDDLStatement(), runSQL(), split(), and strip().

785  {
786  std::vector<std::shared_ptr<ResultSet>> results;
787  // TODO: Need to properly handle escaped semicolons instead of doing a naive split().
788  auto fields = split(sql, ";");
789  for (const auto& field : fields) {
790  auto text = strip(field) + ";";
791  if (text == ";") {
792  continue;
793  }
794 
795  ParserWrapper pw{text};
796  if (pw.is_ddl || pw.getDMLType() == ParserWrapper::DMLType::Insert) {
797  runDDLStatement(text);
798  results.push_back(nullptr);
799  } else {
800  // is not DDL, then assume it's DML and try to execute
801  results.push_back(runSQL(text, dt, true, true));
802  }
803  }
804  return results;
805 }
std::string strip(std::string_view str)
trim any whitespace from the left and right ends of a string
virtual std::shared_ptr< ResultSet > runSQL(const std::string &query_str, CompilationOptions co, ExecutionOptions eo)
std::vector< std::string > split(std::string_view str, std::string_view delim, std::optional< size_t > maxsplit)
split apart a string into a vector of substrings
const rapidjson::Value & field(const rapidjson::Value &obj, const char field[]) noexcept
Definition: JsonAccessors.h:33
virtual void runDDLStatement(const std::string &)

+ Here is the call graph for this function:

std::shared_ptr< ExecutionResult > QueryRunner::QueryRunner::runSelectQuery ( const std::string &  query_str,
CompilationOptions  co,
ExecutionOptions  eo 
)
virtual

Definition at line 930 of file QueryRunner.cpp.

References ExecutionOptions::allow_loop_joins, cat(), CHECK, create_query_state(), CompilationOptions::defaults(), CompilationOptions::device_type, dispatch_queue_, explain_type_, g_enable_calcite_view_optimize, g_enable_filter_push_down, g_enable_watchdog, Executor::getExecutor(), CompilationOptions::hoist_literals, Catalog_Namespace::SysCatalog::instance(), ExecutionOptions::just_explain, pg_shim(), run_benchmark_import::result, QueryRunner::anonymous_namespace{QueryRunner.cpp}::run_select_query_with_filter_push_down(), session_info_, STDLOG, and logger::thread_local_ids().

Referenced by runSelectQuery(), and runSQL().

932  {
935  auto query_state = create_query_state(session_info_, query_str);
936  auto stdlog = STDLOG(query_state);
938  return run_select_query_with_filter_push_down(query_state->createQueryStateProxy(),
939  co.device_type,
940  co.hoist_literals,
941  eo.allow_loop_joins,
942  eo.just_explain,
945  }
946 
947  auto& cat = session_info_->getCatalog();
948 
949  std::shared_ptr<ExecutionResult> result;
950  auto query_launch_task = std::make_shared<QueryDispatchQueue::Task>(
951  [&cat,
952  &query_str,
953  &co,
954  explain_type = this->explain_type_,
955  &eo,
956  &query_state,
957  &result,
958  parent_thread_local_ids = logger::thread_local_ids()](const size_t worker_id) {
959  logger::LocalIdsScopeGuard lisg = parent_thread_local_ids.setNewThreadId();
960  auto executor = Executor::getExecutor(worker_id);
961  // TODO The next line should be deleted since it overwrites co, but then
962  // NycTaxiTest.RunSelectsEncodingDictWhereGreater fails due to co not getting
963  // reset to its default values.
964  co = CompilationOptions::defaults(co.device_type);
965  co.explain_type = explain_type;
966  auto calcite_mgr = cat.getCalciteMgr();
967  const auto calciteQueryParsingOption =
968  calcite_mgr->getCalciteQueryParsingOption(true, false, true, false);
969  const auto calciteOptimizationOption = calcite_mgr->getCalciteOptimizationOption(
971  const auto query_ra = calcite_mgr
972  ->process(query_state->createQueryStateProxy(),
973  pg_shim(query_str),
974  calciteQueryParsingOption,
975  calciteOptimizationOption)
976  .plan_result;
977  auto ra_executor = RelAlgExecutor(executor.get(), query_ra);
978  result = std::make_shared<ExecutionResult>(
979  ra_executor.executeRelAlgQuery(co, eo, false, false, nullptr));
980  });
982  dispatch_queue_->submit(query_launch_task, /*is_update_delete=*/false);
983  auto result_future = query_launch_task->get_future();
984  result_future.get();
985  CHECK(result);
986  return result;
987 }
bool g_enable_calcite_view_optimize
Definition: QueryRunner.cpp:59
std::string cat(Ts &&...args)
std::unique_ptr< QueryDispatchQueue > dispatch_queue_
Definition: QueryRunner.h:332
ExecutorExplainType explain_type_
Definition: QueryRunner.h:328
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:509
static SysCatalog & instance()
Definition: SysCatalog.h:343
bool g_enable_watchdog
ExecutorDeviceType device_type
std::shared_ptr< ExecutionResult > run_select_query_with_filter_push_down(QueryStateProxy query_state_proxy, const ExecutorDeviceType device_type, const bool hoist_literals, const bool allow_loop_joins, const bool just_explain, const ExecutorExplainType explain_type, const bool with_filter_push_down)
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:304
static CompilationOptions defaults(const ExecutorDeviceType device_type=ExecutorDeviceType::GPU)
bool g_enable_filter_push_down
Definition: Execute.cpp:98
#define CHECK(condition)
Definition: Logger.h:291
std::string pg_shim(const std::string &query)
ThreadLocalIds thread_local_ids()
Definition: Logger.cpp:880
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::shared_ptr< ExecutionResult > QueryRunner::QueryRunner::runSelectQuery ( const std::string &  query_str,
const ExecutorDeviceType  device_type,
const bool  hoist_literals,
const bool  allow_loop_joins,
const bool  just_explain = false 
)
virtual

Definition at line 989 of file QueryRunner.cpp.

References defaultExecutionOptionsForRunSQL(), CompilationOptions::defaults(), and runSelectQuery().

994  {
995  auto co = CompilationOptions::defaults(device_type);
996  co.hoist_literals = hoist_literals;
997  return runSelectQuery(query_str,
998  std::move(co),
999  defaultExecutionOptionsForRunSQL(allow_loop_joins, just_explain));
1000 }
static ExecutionOptions defaultExecutionOptionsForRunSQL(bool allow_loop_joins=true, bool just_explain=false)
virtual std::shared_ptr< ExecutionResult > runSelectQuery(const std::string &query_str, CompilationOptions co, ExecutionOptions eo)
static CompilationOptions defaults(const ExecutorDeviceType device_type=ExecutorDeviceType::GPU)

+ Here is the call graph for this function:

std::shared_ptr< ResultSet > QueryRunner::QueryRunner::runSQL ( const std::string &  query_str,
CompilationOptions  co,
ExecutionOptions  eo 
)
virtual

Definition at line 643 of file QueryRunner.cpp.

References CHECK, ParserWrapper::Insert, Catalog_Namespace::SysCatalog::instance(), runDDLStatement(), runSelectQuery(), session_info_, and VLOG.

Referenced by runMultipleStatements(), and runSQL().

645  {
648 
649  ParserWrapper pw{query_str};
650  if (pw.getDMLType() == ParserWrapper::DMLType::Insert) {
651  runDDLStatement(query_str);
652  return nullptr;
653  }
654  const auto execution_result = runSelectQuery(query_str, std::move(co), std::move(eo));
655  VLOG(1) << session_info_->getCatalog().getDataMgr().getSystemMemoryUsage();
656  return execution_result->getRows();
657 }
static SysCatalog & instance()
Definition: SysCatalog.h:343
virtual void runDDLStatement(const std::string &)
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
virtual std::shared_ptr< ExecutionResult > runSelectQuery(const std::string &query_str, CompilationOptions co, ExecutionOptions eo)
#define CHECK(condition)
Definition: Logger.h:291
#define VLOG(n)
Definition: Logger.h:388

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::shared_ptr< ResultSet > QueryRunner::QueryRunner::runSQL ( const std::string &  query_str,
const ExecutorDeviceType  device_type,
const bool  hoist_literals = true,
const bool  allow_loop_joins = true 
)
virtual

Definition at line 659 of file QueryRunner.cpp.

References defaultExecutionOptionsForRunSQL(), CompilationOptions::defaults(), and runSQL().

662  {
663  auto co = CompilationOptions::defaults(device_type);
664  co.hoist_literals = hoist_literals;
665  return runSQL(
666  query_str, std::move(co), defaultExecutionOptionsForRunSQL(allow_loop_joins));
667 }
static ExecutionOptions defaultExecutionOptionsForRunSQL(bool allow_loop_joins=true, bool just_explain=false)
virtual std::shared_ptr< ResultSet > runSQL(const std::string &query_str, CompilationOptions co, ExecutionOptions eo)
static CompilationOptions defaults(const ExecutorDeviceType device_type=ExecutorDeviceType::GPU)

+ Here is the call graph for this function:

std::shared_ptr< ResultSet > QueryRunner::QueryRunner::runSQLWithAllowingInterrupt ( const std::string &  query_str,
const std::string &  session_id,
const ExecutorDeviceType  device_type,
const double  running_query_check_freq = 0.9,
const unsigned  pending_query_check_freq = 1000 
)
virtual

Definition at line 699 of file QueryRunner.cpp.

References calcite_lock, cat(), CHECK, create_query_state(), CompilationOptions::defaults(), dispatch_queue_, g_enable_columnar_output, g_enable_watchdog, g_gpu_mem_limit_percent, Executor::getExecutor(), Catalog_Namespace::SysCatalog::instance(), pg_shim(), run_benchmark_import::result, session_info_, STDLOG, logger::thread_local_ids(), and Executor::UNITARY_EXECUTOR_ID.

704  {
707  auto current_user = session_info_->get_currentUser();
708  auto session_info = std::make_shared<Catalog_Namespace::SessionInfo>(
709  session_info_->get_catalog_ptr(), current_user, device_type, session_id);
710  auto query_state = create_query_state(session_info, query_str);
711  auto stdlog = STDLOG(query_state);
712  auto& cat = query_state->getConstSessionInfo()->getCatalog();
713  std::string query_ra{""};
714 
715  std::shared_ptr<ExecutionResult> result;
716  auto query_launch_task = std::make_shared<QueryDispatchQueue::Task>(
717  [&cat,
718  &query_ra,
719  &device_type,
720  &query_state,
721  &result,
722  &running_query_check_freq,
723  &pending_query_check_freq,
724  parent_thread_local_ids = logger::thread_local_ids()](const size_t worker_id) {
725  logger::LocalIdsScopeGuard lisg = parent_thread_local_ids.setNewThreadId();
726  auto executor = Executor::getExecutor(worker_id);
728 
730  false,
731  true,
732  false,
733  true,
734  false,
735  false,
736  false,
737  false,
738  10000,
739  false,
740  false,
742  true,
743  running_query_check_freq,
744  pending_query_check_freq,
745  false};
746  {
747  // async query initiation for interrupt test
748  // incurs data race warning in TSAN since
749  // calcite_mgr is shared across multiple query threads
750  // so here we lock the manager during query parsing
751  std::lock_guard<std::mutex> calcite_lock_guard(calcite_lock);
752  auto calcite_mgr = cat.getCalciteMgr();
753  const auto calciteQueryParsingOption =
754  calcite_mgr->getCalciteQueryParsingOption(true, false, true, false);
755  const auto calciteOptimizationOption =
756  calcite_mgr->getCalciteOptimizationOption(
757  false, g_enable_watchdog, {}, false);
758  query_ra = calcite_mgr
759  ->process(query_state->createQueryStateProxy(),
760  pg_shim(query_state->getQueryStr()),
761  calciteQueryParsingOption,
762  calciteOptimizationOption)
763  .plan_result;
764  }
765  auto ra_executor = RelAlgExecutor(executor.get(), query_ra, query_state);
766  result = std::make_shared<ExecutionResult>(
767  ra_executor.executeRelAlgQuery(co, eo, false, false, nullptr));
768  });
770  executor->enrollQuerySession(session_id,
771  query_str,
772  query_state->getQuerySubmittedTime(),
774  QuerySessionStatus::QueryStatus::PENDING_QUEUE);
776  dispatch_queue_->submit(query_launch_task, /*is_update_delete=*/false);
777  auto result_future = query_launch_task->get_future();
778  result_future.get();
779  CHECK(result);
780  return result->getRows();
781 }
std::string cat(Ts &&...args)
std::unique_ptr< QueryDispatchQueue > dispatch_queue_
Definition: QueryRunner.h:332
std::mutex calcite_lock
Definition: QueryRunner.cpp:60
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:509
bool g_enable_columnar_output
Definition: Execute.cpp:102
static SysCatalog & instance()
Definition: SysCatalog.h:343
bool g_enable_watchdog
double g_gpu_mem_limit_percent
Definition: QueryRunner.cpp:56
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:304
static CompilationOptions defaults(const ExecutorDeviceType device_type=ExecutorDeviceType::GPU)
#define CHECK(condition)
Definition: Logger.h:291
static constexpr ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:423
std::string pg_shim(const std::string &query)
ThreadLocalIds thread_local_ids()
Definition: Logger.cpp:880
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

void QueryRunner::QueryRunner::setExplainType ( const ExecutorExplainType  explain_type)
inline

Definition at line 308 of file QueryRunner.h.

References explain_type_.

308  {
309  explain_type_ = explain_type;
310  }
ExecutorExplainType explain_type_
Definition: QueryRunner.h:328
void QueryRunner::QueryRunner::validateDDLStatement ( const std::string &  stmt_str_in)
virtual

Definition at line 488 of file QueryRunner.cpp.

References cat(), CHECK, create_query_state(), g_enable_watchdog, pg_shim(), session_info_, and STDLOG.

488  {
490 
491  std::string stmt_str = stmt_str_in;
492  // First remove special chars
493  boost::algorithm::trim_left_if(stmt_str, boost::algorithm::is_any_of("\n"));
494  // Then remove spaces
495  boost::algorithm::trim_left(stmt_str);
496 
497  auto query_state = create_query_state(session_info_, stmt_str);
498  auto stdlog = STDLOG(query_state);
499 
500  auto& cat = session_info_->getCatalog();
501  auto calcite_mgr = cat.getCalciteMgr();
502  const auto calciteQueryParsingOption =
503  calcite_mgr->getCalciteQueryParsingOption(true, false, true, false);
504  const auto calciteOptimizationOption =
505  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {}, false);
506  calcite_mgr->process(query_state->createQueryStateProxy(),
507  pg_shim(stmt_str),
508  calciteQueryParsingOption,
509  calciteOptimizationOption);
510 }
std::string cat(Ts &&...args)
bool g_enable_watchdog
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:331
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:304
#define CHECK(condition)
Definition: Logger.h:291
std::string pg_shim(const std::string &query)
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

Member Data Documentation

Catalog_Namespace::DBMetadata QueryRunner::QueryRunner::db_metadata_
protected

Definition at line 330 of file QueryRunner.h.

Referenced by getBufferPoolStats(), and QueryRunner().

std::unique_ptr<QueryDispatchQueue> QueryRunner::QueryRunner::dispatch_queue_
protected
ExecutorExplainType QueryRunner::QueryRunner::explain_type_ = ExecutorExplainType::Default
protected

Definition at line 328 of file QueryRunner.h.

Referenced by runSelectQuery(), and setExplainType().

std::unique_ptr< QueryRunner > QueryRunner::QueryRunner::qr_instance_ = nullptr
staticprotected

Definition at line 326 of file QueryRunner.h.

Referenced by get(), init(), and reset().

std::shared_ptr<QueryEngine> QueryRunner::QueryRunner::query_engine_
protected

Definition at line 333 of file QueryRunner.h.

Referenced by QueryRunner().

query_state::QueryStates QueryRunner::QueryRunner::query_states_
static

Definition at line 301 of file QueryRunner.h.

Referenced by create_query_state().


The documentation for this class was generated from the following files: