OmniSciDB  c1a53651b2
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
QueryRunner::QueryRunner Class Reference

#include <QueryRunner.h>

+ Inheritance diagram for QueryRunner::QueryRunner:
+ Collaboration diagram for QueryRunner::QueryRunner:

Public Member Functions

std::shared_ptr
< Catalog_Namespace::SessionInfo
getSession () const
 
void addSessionId (const std::string &session_id, ExecutorDeviceType device_type=ExecutorDeviceType::GPU)
 
void clearSessionId ()
 
std::shared_ptr
< Catalog_Namespace::Catalog
getCatalog () const
 
std::shared_ptr< CalcitegetCalcite () const
 
std::shared_ptr< ExecutorgetExecutor () const
 
Catalog_Namespace::UserMetadatagetUserMetadata () const
 
bool gpusPresent () const
 
virtual void clearGpuMemory () const
 
virtual void clearCpuMemory () const
 
std::vector< MemoryInfogetMemoryInfo (const Data_Namespace::MemoryLevel memory_level) const
 
BufferPoolStats getBufferPoolStats (const Data_Namespace::MemoryLevel memory_level, const bool current_db_only) const
 
virtual std::unique_ptr
< Parser::Stmt
createStatement (const std::string &)
 
virtual void runDDLStatement (const std::string &)
 
virtual void validateDDLStatement (const std::string &)
 
virtual std::shared_ptr
< ResultSet
runSQL (const std::string &query_str, CompilationOptions co, ExecutionOptions eo)
 
virtual std::shared_ptr
< ExecutionResult
runSelectQuery (const std::string &query_str, CompilationOptions co, ExecutionOptions eo)
 
virtual std::shared_ptr
< ResultSet
runSQL (const std::string &query_str, const ExecutorDeviceType device_type, const bool hoist_literals=true, const bool allow_loop_joins=true)
 
virtual std::shared_ptr
< ExecutionResult
runSelectQuery (const std::string &query_str, const ExecutorDeviceType device_type, const bool hoist_literals, const bool allow_loop_joins, const bool just_explain=false)
 
virtual std::shared_ptr
< ResultSet
runSQLWithAllowingInterrupt (const std::string &query_str, const std::string &session_id, const ExecutorDeviceType device_type, const double running_query_check_freq=0.9, const unsigned pending_query_check_freq=1000)
 
virtual std::vector
< std::shared_ptr< ResultSet > > 
runMultipleStatements (const std::string &, const ExecutorDeviceType)
 
virtual void runImport (Parser::CopyTableStmt *import_stmt)
 
virtual std::unique_ptr
< import_export::Loader
getLoader (const TableDescriptor *td) const
 
RegisteredQueryHint getParsedQueryHint (const std::string &)
 
std::optional
< std::unordered_map< size_t,
std::unordered_map< unsigned,
RegisteredQueryHint > > > 
getParsedQueryHints (const std::string &query_str)
 
std::shared_ptr< const RelAlgNodegetRootNodeFromParsedQuery (const std::string &query_str)
 
std::optional
< RegisteredQueryHint
getParsedGlobalQueryHints (const std::string &query_str)
 
RaExecutionSequence getRaExecutionSequence (const std::string &query_str)
 
virtual std::shared_ptr
< ResultSet
getCalcitePlan (const std::string &query_str, bool enable_watchdog, bool as_json_str) const
 
std::tuple< QueryPlanHash,
std::shared_ptr< HashTable >
, std::optional
< HashtableCacheMetaInfo > > 
getCachedHashtableWithoutCacheKey (std::set< size_t > &visited, CacheItemType hash_table_type, DeviceIdentifier device_identifier)
 
std::shared_ptr< CacheItemMetricgetCacheItemMetric (QueryPlanHash cache_key, CacheItemType hash_table_type, DeviceIdentifier device_identifier)
 
size_t getNumberOfCachedItem (CacheItemStatus item_status, CacheItemType hash_table_type, bool with_overlaps_tuning_param=false) const
 
void resizeDispatchQueue (const size_t num_executors)
 
QueryPlanDagInfo getQueryInfoForDataRecyclerTest (const std::string &)
 
std::shared_ptr< RelAlgTranslatorgetRelAlgTranslator (const std::string &, Executor *)
 
ExtractedQueryPlanDag extractQueryPlanDag (const std::string &)
 
std::unique_ptr< RelAlgDaggetRelAlgDag (const std::string &)
 
 QueryRunner (std::unique_ptr< Catalog_Namespace::SessionInfo > session)
 
virtual ~QueryRunner ()=default
 
void setExplainType (const ExecutorExplainType explain_type)
 

Static Public Member Functions

static QueryRunnerinit (const char *db_path, const std::string &udf_filename="", const size_t max_gpu_mem=0, const int reserved_gpu_mem=256<< 20)
 
static QueryRunnerinit (const File_Namespace::DiskCacheConfig *disk_cache_config, const char *db_path, const std::vector< LeafHostInfo > &string_servers={}, const std::vector< LeafHostInfo > &leaf_servers={})
 
static QueryRunnerinit (const char *db_path, const std::vector< LeafHostInfo > &string_servers, const std::vector< LeafHostInfo > &leaf_servers)
 
static QueryRunnerinit (const char *db_path, const std::string &user, const std::string &pass, const std::string &db_name, const std::vector< LeafHostInfo > &string_servers, const std::vector< LeafHostInfo > &leaf_servers, const std::string &udf_filename="", bool uses_gpus=true, const size_t max_gpu_mem=0, const int reserved_gpu_mem=256<< 20, const bool create_user=false, const bool create_db=false, const File_Namespace::DiskCacheConfig *config=nullptr)
 
static QueryRunnerinit (std::unique_ptr< Catalog_Namespace::SessionInfo > &session)
 
static QueryRunnerget ()
 
static void reset ()
 
static ExecutionOptions defaultExecutionOptionsForRunSQL (bool allow_loop_joins=true, bool just_explain=false)
 
template<typename... Ts>
static std::shared_ptr
< query_state::QueryState
create_query_state (Ts &&...args)
 

Static Public Attributes

static query_state::QueryStates query_states_
 

Protected Member Functions

 QueryRunner (const char *db_path, const std::string &user, const std::string &pass, const std::string &db_name, const std::vector< LeafHostInfo > &string_servers, const std::vector< LeafHostInfo > &leaf_servers, const std::string &udf_filename, bool uses_gpus, const size_t max_gpu_mem, const int reserved_gpu_mem, const bool create_user, const bool create_db, const File_Namespace::DiskCacheConfig *disk_cache_config=nullptr)
 

Protected Attributes

ExecutorExplainType explain_type_ = ExecutorExplainType::Default
 
Catalog_Namespace::DBMetadata db_metadata_
 
std::shared_ptr
< Catalog_Namespace::SessionInfo
session_info_
 
std::unique_ptr
< QueryDispatchQueue
dispatch_queue_
 
std::shared_ptr< QueryEnginequery_engine_
 

Static Protected Attributes

static std::unique_ptr
< QueryRunner
qr_instance_ = nullptr
 

Detailed Description

Definition at line 139 of file QueryRunner.h.

Constructor & Destructor Documentation

QueryRunner::QueryRunner::QueryRunner ( std::unique_ptr< Catalog_Namespace::SessionInfo session)

Referenced by init().

+ Here is the caller graph for this function:

virtual QueryRunner::QueryRunner::~QueryRunner ( )
virtualdefault
QueryRunner::QueryRunner::QueryRunner ( const char *  db_path,
const std::string &  user,
const std::string &  pass,
const std::string &  db_name,
const std::vector< LeafHostInfo > &  string_servers,
const std::vector< LeafHostInfo > &  leaf_servers,
const std::string &  udf_filename,
bool  uses_gpus,
const size_t  max_gpu_mem,
const int  reserved_gpu_mem,
const bool  create_user,
const bool  create_db,
const File_Namespace::DiskCacheConfig disk_cache_config = nullptr 
)
protected

Definition at line 153 of file QueryRunner.cpp.

References ExtensionFunctionsWhitelist::add(), ExtensionFunctionsWhitelist::addUdfs(), SystemParameters::aggregator, anonymous_namespace{QueryRunner.cpp}::calcite_shutdown_handler(), CALCITEPORT, cat(), CHECK, QueryEngine::createInstance(), db_metadata_, Catalog_Namespace::DBMetadata::dbOwner, File_Namespace::fsi, g_base_path, anonymous_namespace{QueryRunner.cpp}::g_calcite, g_read_only, g_serialize_temp_tables, table_functions::TableFunctionsFactory::get_table_funcs(), GPU, SystemParameters::gpu_buffer_mem_bytes, table_functions::TableFunctionsFactory::init(), Catalog_Namespace::SysCatalog::instance(), Catalog_Namespace::UserMetadata::isSuper, shared::kCatalogDirectoryName, shared::kDataDirectoryName, shared::kDefaultDbName, shared::kDefaultDiskCacheDirName, Catalog_Namespace::UserMetadata::passwd_hash, query_engine_, session_info_, logger::set_once_fatal_func(), anonymous_namespace{QueryRunner.cpp}::setup_signal_handler(), ThriftSerializers::to_thrift(), and Catalog_Namespace::UserMetadata::userId.

166  : dispatch_queue_(std::make_unique<QueryDispatchQueue>(1)) {
168  boost::filesystem::path base_path{db_path};
169  CHECK(boost::filesystem::exists(base_path));
170  auto system_db_file =
172  CHECK(boost::filesystem::exists(system_db_file));
173  auto data_dir = base_path / shared::kDataDirectoryName;
174  File_Namespace::DiskCacheConfig disk_cache_config{
175  (base_path / shared::kDefaultDiskCacheDirName).string(),
177  if (cache_config) {
178  disk_cache_config = *cache_config;
179  }
181 
184  g_calcite =
185  std::make_shared<Calcite>(-1, CALCITEPORT, db_path, 1024, 5000, true, udf_filename);
186  ExtensionFunctionsWhitelist::add(g_calcite->getExtensionFunctionWhitelist());
187  if (!udf_filename.empty()) {
188  ExtensionFunctionsWhitelist::addUdfs(g_calcite->getUserDefinedFunctionWhitelist());
189  }
190 
192  auto udtfs = ThriftSerializers::to_thrift(
194  std::vector<TUserDefinedFunction> udfs = {};
195  g_calcite->setRuntimeExtensionFunctions(udfs, udtfs, /*is_runtime=*/false);
196 
197  std::unique_ptr<CudaMgr_Namespace::CudaMgr> cuda_mgr;
198 #ifdef HAVE_CUDA
199  if (uses_gpus) {
200  cuda_mgr = std::make_unique<CudaMgr_Namespace::CudaMgr>(-1, 0);
201  }
202 #else
203  uses_gpus = false;
204 #endif
205  SystemParameters mapd_params;
206  mapd_params.gpu_buffer_mem_bytes = max_gpu_mem;
207  mapd_params.aggregator = !leaf_servers.empty();
208 
209  auto& sys_cat = Catalog_Namespace::SysCatalog::instance();
210 
211  g_base_path = base_path.string();
212 
213  if (!sys_cat.isInitialized()) {
214  auto data_mgr = std::make_shared<Data_Namespace::DataMgr>(data_dir.string(),
215  mapd_params,
216  std::move(cuda_mgr),
217  uses_gpus,
218  reserved_gpu_mem,
219  0,
220  disk_cache_config);
221  sys_cat.init(g_base_path,
222  data_mgr,
223  {},
224  g_calcite,
225  false,
226  mapd_params.aggregator,
227  string_servers);
228  }
229 
230  query_engine_ =
231  QueryEngine::createInstance(sys_cat.getDataMgr().getCudaMgr(), !uses_gpus);
232 
233  if (create_user) {
234  if (!sys_cat.getMetadataForUser(user_name, user)) {
235  sys_cat.createUser(
236  user_name,
238  passwd, /*is_super=*/false, /*default_db=*/"", /*can_login=*/true},
239  g_read_only);
240  }
241  }
242  CHECK(sys_cat.getMetadataForUser(user_name, user));
243  CHECK(bcrypt_checkpw(passwd.c_str(), user.passwd_hash.c_str()) == 0);
244 
245  if (create_db) {
246  if (!sys_cat.getMetadataForDB(db_name, db_metadata_)) {
247  sys_cat.createDatabase(db_name, user.userId);
248  }
249  }
250  CHECK(sys_cat.getMetadataForDB(db_name, db_metadata_));
251  CHECK(user.isSuper || (user.userId == db_metadata_.dbOwner));
252  auto cat = sys_cat.getCatalog(db_metadata_, create_db);
253  CHECK(cat);
254  session_info_ = std::make_unique<Catalog_Namespace::SessionInfo>(
255  cat, user, ExecutorDeviceType::GPU, "");
256 }
static void addUdfs(const std::string &json_func_sigs)
#define CALCITEPORT
Definition: QueryRunner.cpp:48
const std::string kDataDirectoryName
static std::vector< TableFunction > get_table_funcs()
std::string cat(Ts &&...args)
std::unique_ptr< QueryDispatchQueue > dispatch_queue_
Definition: QueryRunner.h:331
const std::string kDefaultDiskCacheDirName
static void add(const std::string &json_func_sigs)
void set_once_fatal_func(FatalFunc fatal_func)
Definition: Logger.cpp:390
Catalog_Namespace::DBMetadata db_metadata_
Definition: QueryRunner.h:329
std::shared_ptr< QueryEngine > query_engine_
Definition: QueryRunner.h:332
static SysCatalog & instance()
Definition: SysCatalog.h:343
const std::string kDefaultDbName
std::string g_base_path
Definition: SysCatalog.cpp:62
static std::shared_ptr< QueryEngine > createInstance(CudaMgr_Namespace::CudaMgr *cuda_mgr, bool cpu_only)
Definition: QueryEngine.h:89
bool g_serialize_temp_tables
Definition: Catalog.cpp:106
TExtArgumentType::type to_thrift(const ExtArgumentType &t)
bool g_read_only
Definition: File.cpp:40
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
std::shared_ptr< Calcite > g_calcite
Definition: QueryRunner.cpp:62
const std::string kCatalogDirectoryName
#define CHECK(condition)
Definition: Logger.h:291
std::atomic< bool > isSuper
Definition: SysCatalog.h:107

+ Here is the call graph for this function:

Member Function Documentation

void QueryRunner::QueryRunner::addSessionId ( const std::string &  session_id,
ExecutorDeviceType  device_type = ExecutorDeviceType::GPU 
)
inline

Definition at line 194 of file QueryRunner.h.

References session_info_.

195  {
196  auto user_info = session_info_->get_currentUser();
197  session_info_ = std::make_unique<Catalog_Namespace::SessionInfo>(
198  session_info_->get_catalog_ptr(), user_info, device_type, session_id);
199  }
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
void QueryRunner::QueryRunner::clearCpuMemory ( ) const
virtual

Definition at line 286 of file QueryRunner.cpp.

References CHECK, Executor::clearMemory(), Data_Namespace::CPU_LEVEL, and Catalog_Namespace::SysCatalog::instance().

286  {
289 }
static void clearMemory(const Data_Namespace::MemoryLevel memory_level)
Definition: Execute.cpp:497
static SysCatalog & instance()
Definition: SysCatalog.h:343
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

void QueryRunner::QueryRunner::clearGpuMemory ( ) const
virtual

Definition at line 281 of file QueryRunner.cpp.

References CHECK, Executor::clearMemory(), Data_Namespace::GPU_LEVEL, and Catalog_Namespace::SysCatalog::instance().

281  {
284 }
static void clearMemory(const Data_Namespace::MemoryLevel memory_level)
Definition: Execute.cpp:497
static SysCatalog & instance()
Definition: SysCatalog.h:343
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

void QueryRunner::QueryRunner::clearSessionId ( )
inline

Definition at line 201 of file QueryRunner.h.

References session_info_.

201 { session_info_ = nullptr; }
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
template<typename... Ts>
static std::shared_ptr<query_state::QueryState> QueryRunner::QueryRunner::create_query_state ( Ts &&...  args)
inlinestatic

Definition at line 303 of file QueryRunner.h.

References run_benchmark_import::args, query_state::QueryStates::create(), and query_states_.

Referenced by createStatement(), getCalcitePlan(), getExecutor(), getParsedGlobalQueryHints(), getParsedQueryHint(), getParsedQueryHints(), getQueryInfoForDataRecyclerTest(), getRaExecutionSequence(), getRelAlgDag(), getRelAlgTranslator(), getRootNodeFromParsedQuery(), runDDLStatement(), runSelectQuery(), runSQLWithAllowingInterrupt(), and validateDDLStatement().

303  {
304  return query_states_.create(std::forward<Ts>(args)...);
305  }
static query_state::QueryStates query_states_
Definition: QueryRunner.h:300
CircleBuffer::value_type create(ARGS &&...args)
Definition: QueryState.h:194

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::unique_ptr< Parser::Stmt > QueryRunner::QueryRunner::createStatement ( const std::string &  stmt_str_in)
virtual

Definition at line 533 of file QueryRunner.cpp.

References cat(), CHECK, create_query_state(), Parser::create_stmt_for_json(), g_enable_watchdog, Catalog_Namespace::SysCatalog::instance(), pg_shim(), session_info_, STDLOG, and UNREACHABLE.

534  {
537 
538  std::string stmt_str = stmt_str_in;
539  // First remove special chars
540  boost::algorithm::trim_left_if(stmt_str, boost::algorithm::is_any_of("\n"));
541  // Then remove spaces
542  boost::algorithm::trim_left(stmt_str);
543 
544  ParserWrapper pw{stmt_str};
545 
546  auto query_state = create_query_state(session_info_, stmt_str);
547  auto stdlog = STDLOG(query_state);
548 
549  if (pw.is_ddl) {
550  const auto& cat = session_info_->getCatalog();
551  auto calcite_mgr = cat.getCalciteMgr();
552  const auto calciteQueryParsingOption =
553  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
554  const auto calciteOptimizationOption =
555  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {}, false);
556  const auto query_json = calcite_mgr
557  ->process(query_state->createQueryStateProxy(),
558  pg_shim(stmt_str),
559  calciteQueryParsingOption,
560  calciteOptimizationOption)
561  .plan_result;
562  return Parser::create_stmt_for_json(query_json);
563  }
564 
565  // simply fail here as non-Calcite parsing is about to be removed
566  UNREACHABLE();
567  return nullptr;
568 }
std::string cat(Ts &&...args)
#define UNREACHABLE()
Definition: Logger.h:337
static SysCatalog & instance()
Definition: SysCatalog.h:343
std::unique_ptr< Parser::Stmt > create_stmt_for_json(const std::string &query_json)
bool g_enable_watchdog
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:303
#define CHECK(condition)
Definition: Logger.h:291
std::string pg_shim(const std::string &query)
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

ExecutionOptions QueryRunner::QueryRunner::defaultExecutionOptionsForRunSQL ( bool  allow_loop_joins = true,
bool  just_explain = false 
)
static

Definition at line 639 of file QueryRunner.cpp.

References g_enable_columnar_output, and g_gpu_mem_limit_percent.

Referenced by runSelectQuery(), and runSQL().

640  {
641  return {g_enable_columnar_output,
642  false,
643  true,
644  just_explain,
645  allow_loop_joins,
646  false,
647  false,
648  false,
649  false,
650  10000,
651  false,
652  false,
654  false,
655  1000,
656  false};
657 }
bool g_enable_columnar_output
Definition: Execute.cpp:99
double g_gpu_mem_limit_percent
Definition: QueryRunner.cpp:53

+ Here is the caller graph for this function:

ExtractedQueryPlanDag QueryRunner::QueryRunner::extractQueryPlanDag ( const std::string &  query_str)

Definition at line 967 of file QueryRunner.cpp.

References QueryPlanDagExtractor::extractQueryPlanDag(), Executor::getExecutor(), getQueryInfoForDataRecyclerTest(), and Executor::UNITARY_EXECUTOR_ID.

967  {
968  auto query_dag_info = getQueryInfoForDataRecyclerTest(query_str);
970  auto extracted_dag_info = QueryPlanDagExtractor::extractQueryPlanDag(
971  query_dag_info.root_node.get(), executor);
972  return extracted_dag_info;
973 }
static ExtractedQueryPlanDag extractQueryPlanDag(const RelAlgNode *top_node, Executor *executor)
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:475
QueryPlanDagInfo getQueryInfoForDataRecyclerTest(const std::string &)
static const ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:373

+ Here is the call graph for this function:

static QueryRunner* QueryRunner::QueryRunner::get ( )
inlinestatic

Definition at line 181 of file QueryRunner.h.

References qr_instance_.

181  {
182  if (!qr_instance_) {
183  throw std::runtime_error("QueryRunner must be initialized before calling get().");
184  }
185  return qr_instance_.get();
186  }
static std::unique_ptr< QueryRunner > qr_instance_
Definition: QueryRunner.h:325
BufferPoolStats QueryRunner::QueryRunner::getBufferPoolStats ( const Data_Namespace::MemoryLevel  memory_level,
const bool  current_db_only 
) const

Definition at line 297 of file QueryRunner.cpp.

References CHECK, CHECK_EQ, Data_Namespace::CPU_LEVEL, db_metadata_, Catalog_Namespace::DBMetadata::dbId, Buffer_Namespace::FREE, Catalog_Namespace::SysCatalog::instance(), and session_info_.

299  {
300  // Only works single-node for now
302  const std::vector<MemoryInfo> memory_infos =
303  session_info_->getCatalog().getDataMgr().getMemoryInfo(memory_level);
304  if (memory_level == Data_Namespace::MemoryLevel::CPU_LEVEL) {
305  CHECK_EQ(memory_infos.size(), static_cast<size_t>(1));
306  }
307  std::set<std::vector<int32_t>> chunk_keys;
308  std::set<std::vector<int32_t>> table_keys;
309  std::set<std::vector<int32_t>> column_keys;
310  std::set<std::vector<int32_t>> fragment_keys;
311  size_t total_num_buffers{
312  0}; // can be greater than chunk keys set size due to table replication
313  size_t total_num_bytes{0};
314  for (auto& pool_memory_info : memory_infos) {
315  const std::vector<MemoryData>& memory_data = pool_memory_info.nodeMemoryData;
316  for (auto& memory_datum : memory_data) {
317  total_num_buffers++;
318  const auto& chunk_key = memory_datum.chunk_key;
319  if (memory_datum.memStatus == Buffer_Namespace::MemStatus::FREE ||
320  chunk_key.size() < 4) {
321  continue;
322  }
323  if (current_db_only) {
324  if (chunk_key[0] != db_metadata_.dbId) {
325  continue;
326  }
327  }
328  total_num_bytes += (memory_datum.numPages * pool_memory_info.pageSize);
329  table_keys.insert({chunk_key[0], chunk_key[1]});
330  column_keys.insert({chunk_key[0], chunk_key[1], chunk_key[2]});
331  fragment_keys.insert({chunk_key[0], chunk_key[1], chunk_key[3]});
332  chunk_keys.insert(chunk_key);
333  }
334  }
335  return {total_num_buffers,
336  total_num_bytes,
337  table_keys.size(),
338  column_keys.size(),
339  fragment_keys.size(),
340  chunk_keys.size()};
341 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
Catalog_Namespace::DBMetadata db_metadata_
Definition: QueryRunner.h:329
static SysCatalog & instance()
Definition: SysCatalog.h:343
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

std::tuple< QueryPlanHash, std::shared_ptr< HashTable >, std::optional< HashtableCacheMetaInfo > > QueryRunner::QueryRunner::getCachedHashtableWithoutCacheKey ( std::set< size_t > &  visited,
CacheItemType  hash_table_type,
DeviceIdentifier  device_identifier 
)

Definition at line 1027 of file QueryRunner.cpp.

References BASELINE_HT, CHECK, OverlapsJoinHashTable::getHashTableCache(), PerfectJoinHashTable::getHashTableCache(), BaselineJoinHashTable::getHashTableCache(), OVERLAPS_HT, PERFECT_HT, and UNREACHABLE.

1029  {
1030  HashtableRecycler* hash_table_cache{nullptr};
1031  switch (hash_table_type) {
1033  hash_table_cache = PerfectJoinHashTable::getHashTableCache();
1034  break;
1035  }
1037  hash_table_cache = BaselineJoinHashTable::getHashTableCache();
1038  break;
1039  }
1041  hash_table_cache = OverlapsJoinHashTable::getHashTableCache();
1042  break;
1043  }
1044  default: {
1045  UNREACHABLE();
1046  break;
1047  }
1048  }
1049  CHECK(hash_table_cache);
1050  return hash_table_cache->getCachedHashtableWithoutCacheKey(
1051  visited, hash_table_type, device_identifier);
1052 }
#define UNREACHABLE()
Definition: Logger.h:337
static HashtableRecycler * getHashTableCache()
static HashtableRecycler * getHashTableCache()
#define CHECK(condition)
Definition: Logger.h:291
static HashtableRecycler * getHashTableCache()

+ Here is the call graph for this function:

std::shared_ptr< CacheItemMetric > QueryRunner::QueryRunner::getCacheItemMetric ( QueryPlanHash  cache_key,
CacheItemType  hash_table_type,
DeviceIdentifier  device_identifier 
)

Definition at line 1054 of file QueryRunner.cpp.

References BASELINE_HT, CHECK, OverlapsJoinHashTable::getHashTableCache(), PerfectJoinHashTable::getHashTableCache(), BaselineJoinHashTable::getHashTableCache(), OVERLAPS_HT, PERFECT_HT, and UNREACHABLE.

1057  {
1058  HashtableRecycler* hash_table_cache{nullptr};
1059  switch (hash_table_type) {
1061  hash_table_cache = PerfectJoinHashTable::getHashTableCache();
1062  break;
1063  }
1065  hash_table_cache = BaselineJoinHashTable::getHashTableCache();
1066  break;
1067  }
1069  hash_table_cache = OverlapsJoinHashTable::getHashTableCache();
1070  break;
1071  }
1072  default: {
1073  UNREACHABLE();
1074  break;
1075  }
1076  }
1077  CHECK(hash_table_cache);
1078  return hash_table_cache->getCachedItemMetric(
1079  hash_table_type, device_identifier, cache_key);
1080 }
#define UNREACHABLE()
Definition: Logger.h:337
static HashtableRecycler * getHashTableCache()
static HashtableRecycler * getHashTableCache()
#define CHECK(condition)
Definition: Logger.h:291
static HashtableRecycler * getHashTableCache()

+ Here is the call graph for this function:

std::shared_ptr< Calcite > QueryRunner::QueryRunner::getCalcite ( ) const

Definition at line 271 of file QueryRunner.cpp.

References anonymous_namespace{QueryRunner.cpp}::g_calcite.

271  {
272  // TODO: Embed Calcite shared_ptr ownership in QueryRunner
273  return g_calcite;
274 }
std::shared_ptr< Calcite > g_calcite
Definition: QueryRunner.cpp:62
std::shared_ptr< ResultSet > QueryRunner::QueryRunner::getCalcitePlan ( const std::string &  query_str,
bool  enable_watchdog,
bool  as_json_str 
) const
virtual

Definition at line 853 of file QueryRunner.cpp.

References cat(), CHECK, create_query_state(), dispatch_queue_, g_enable_calcite_view_optimize, Executor::getExecutor(), Catalog_Namespace::SysCatalog::instance(), pg_shim(), run_benchmark_import::result, session_info_, STDLOG, and logger::thread_local_ids().

855  {
858  const auto& cat = session_info_->getCatalog();
859  auto query_state = create_query_state(session_info_, query_str);
860  auto stdlog = STDLOG(query_state);
861 
862  std::shared_ptr<ResultSet> result;
863  auto query_launch_task = std::make_shared<QueryDispatchQueue::Task>(
864  [&cat,
865  &query_str,
866  &enable_watchdog,
867  &as_json_str,
868  &query_state,
869  &result,
870  parent_thread_local_ids = logger::thread_local_ids()](const size_t worker_id) {
871  logger::LocalIdsScopeGuard lisg = parent_thread_local_ids.setNewThreadId();
872  auto executor = Executor::getExecutor(worker_id);
873  auto calcite_mgr = cat.getCalciteMgr();
874  const auto calciteQueryParsingOption =
875  calcite_mgr->getCalciteQueryParsingOption(true, as_json_str, false);
876  const auto calciteOptimizationOption = calcite_mgr->getCalciteOptimizationOption(
877  g_enable_calcite_view_optimize, enable_watchdog, {}, false);
878  const auto query_ra = calcite_mgr
879  ->process(query_state->createQueryStateProxy(),
880  pg_shim(query_str),
881  calciteQueryParsingOption,
882  calciteOptimizationOption)
883  .plan_result;
884  result = std::make_shared<ResultSet>(query_ra);
885  return result;
886  });
888  dispatch_queue_->submit(query_launch_task, /*is_update_delete=*/false);
889  auto result_future = query_launch_task->get_future();
890  result_future.get();
891  CHECK(result);
892  return result;
893 }
bool g_enable_calcite_view_optimize
Definition: QueryRunner.cpp:56
std::string cat(Ts &&...args)
std::unique_ptr< QueryDispatchQueue > dispatch_queue_
Definition: QueryRunner.h:331
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:475
static SysCatalog & instance()
Definition: SysCatalog.h:343
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:303
#define CHECK(condition)
Definition: Logger.h:291
std::string pg_shim(const std::string &query)
ThreadLocalIds thread_local_ids()
Definition: Logger.cpp:874
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

std::shared_ptr< Catalog_Namespace::Catalog > QueryRunner::QueryRunner::getCatalog ( ) const

Definition at line 266 of file QueryRunner.cpp.

References CHECK, and session_info_.

Referenced by getLoader().

266  {
268  return session_info_->get_catalog_ptr();
269 }
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the caller graph for this function:

std::shared_ptr< Executor > QueryRunner::QueryRunner::getExecutor ( ) const

Definition at line 659 of file QueryRunner.cpp.

References CHECK, create_query_state(), Executor::getExecutor(), Catalog_Namespace::SysCatalog::instance(), session_info_, STDLOG, and Executor::UNITARY_EXECUTOR_ID.

659  {
662  auto query_state = create_query_state(session_info_, "");
663  auto stdlog = STDLOG(query_state);
665  return executor;
666 }
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:475
static SysCatalog & instance()
Definition: SysCatalog.h:343
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:303
#define CHECK(condition)
Definition: Logger.h:291
#define STDLOG(...)
Definition: QueryState.h:234
static const ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:373

+ Here is the call graph for this function:

std::unique_ptr< import_export::Loader > QueryRunner::QueryRunner::getLoader ( const TableDescriptor td) const
virtual

Definition at line 781 of file QueryRunner.cpp.

References cat(), and getCatalog().

782  {
783  auto cat = getCatalog();
784  return std::make_unique<import_export::Loader>(*cat, td);
785 }
std::string cat(Ts &&...args)
std::shared_ptr< Catalog_Namespace::Catalog > getCatalog() const

+ Here is the call graph for this function:

std::vector< MemoryInfo > QueryRunner::QueryRunner::getMemoryInfo ( const Data_Namespace::MemoryLevel  memory_level) const

Definition at line 291 of file QueryRunner.cpp.

References CHECK, Catalog_Namespace::SysCatalog::instance(), and session_info_.

292  {
294  return session_info_->getCatalog().getDataMgr().getMemoryInfo(memory_level);
295 }
static SysCatalog & instance()
Definition: SysCatalog.h:343
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

size_t QueryRunner::QueryRunner::getNumberOfCachedItem ( CacheItemStatus  item_status,
CacheItemType  hash_table_type,
bool  with_overlaps_tuning_param = false 
) const

Definition at line 1082 of file QueryRunner.cpp.

References QueryRunner::ALL, BASELINE_HT, CHECK, CHECK_EQ, QueryRunner::CLEAN_ONLY, DataRecyclerUtil::CPU_DEVICE_IDENTIFIER, QueryRunner::DIRTY_ONLY, OverlapsJoinHashTable::getHashTableCache(), PerfectJoinHashTable::getHashTableCache(), BaselineJoinHashTable::getHashTableCache(), OverlapsJoinHashTable::getOverlapsTuningParamCache(), OVERLAPS_AUTO_TUNER_PARAM, OVERLAPS_HT, PERFECT_HT, and UNREACHABLE.

1084  {
1085  auto get_num_cached_auto_tuner_param = [&item_status]() {
1086  auto auto_tuner_cache = OverlapsJoinHashTable::getOverlapsTuningParamCache();
1087  CHECK(auto_tuner_cache);
1088  switch (item_status) {
1089  case CacheItemStatus::ALL: {
1090  return auto_tuner_cache->getCurrentNumCachedItems(
1093  }
1095  return auto_tuner_cache->getCurrentNumCleanCachedItems(
1098  }
1100  return auto_tuner_cache->getCurrentNumDirtyCachedItems(
1103  }
1104  default: {
1105  UNREACHABLE();
1106  return static_cast<size_t>(0);
1107  }
1108  }
1109  };
1110 
1111  auto get_num_cached_hashtable =
1112  [&item_status,
1113  &hash_table_type,
1114  &with_overlaps_tuning_param,
1115  &get_num_cached_auto_tuner_param](HashtableRecycler* hash_table_cache) {
1116  switch (item_status) {
1117  case CacheItemStatus::ALL: {
1118  if (with_overlaps_tuning_param) {
1119  // we assume additional consideration of turing param cache is only valid
1120  // for overlaps join hashtable
1121  CHECK_EQ(hash_table_type, CacheItemType::OVERLAPS_HT);
1122  return hash_table_cache->getCurrentNumCachedItems(
1123  hash_table_type, DataRecyclerUtil::CPU_DEVICE_IDENTIFIER) +
1124  get_num_cached_auto_tuner_param();
1125  }
1126  return hash_table_cache->getCurrentNumCachedItems(
1127  hash_table_type, DataRecyclerUtil::CPU_DEVICE_IDENTIFIER);
1128  }
1130  if (with_overlaps_tuning_param) {
1131  CHECK_EQ(hash_table_type, CacheItemType::OVERLAPS_HT);
1132  return hash_table_cache->getCurrentNumCleanCachedItems(
1133  hash_table_type, DataRecyclerUtil::CPU_DEVICE_IDENTIFIER) +
1134  get_num_cached_auto_tuner_param();
1135  }
1136  return hash_table_cache->getCurrentNumCleanCachedItems(
1137  hash_table_type, DataRecyclerUtil::CPU_DEVICE_IDENTIFIER);
1138  }
1140  if (with_overlaps_tuning_param) {
1141  CHECK_EQ(hash_table_type, CacheItemType::OVERLAPS_HT);
1142  return hash_table_cache->getCurrentNumDirtyCachedItems(
1143  hash_table_type, DataRecyclerUtil::CPU_DEVICE_IDENTIFIER) +
1144  get_num_cached_auto_tuner_param();
1145  }
1146  return hash_table_cache->getCurrentNumDirtyCachedItems(
1147  hash_table_type, DataRecyclerUtil::CPU_DEVICE_IDENTIFIER);
1148  }
1149  default: {
1150  UNREACHABLE();
1151  return static_cast<size_t>(0);
1152  }
1153  }
1154  };
1155 
1156  switch (hash_table_type) {
1158  auto hash_table_cache = PerfectJoinHashTable::getHashTableCache();
1159  CHECK(hash_table_cache);
1160  return get_num_cached_hashtable(hash_table_cache);
1161  }
1163  auto hash_table_cache = BaselineJoinHashTable::getHashTableCache();
1164  CHECK(hash_table_cache);
1165  return get_num_cached_hashtable(hash_table_cache);
1166  }
1168  auto hash_table_cache = OverlapsJoinHashTable::getHashTableCache();
1169  CHECK(hash_table_cache);
1170  return get_num_cached_hashtable(hash_table_cache);
1171  }
1173  return get_num_cached_auto_tuner_param();
1174  }
1175  default: {
1176  UNREACHABLE();
1177  return 0;
1178  }
1179  }
1180  return 0;
1181 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
#define UNREACHABLE()
Definition: Logger.h:337
static HashtableRecycler * getHashTableCache()
static HashtableRecycler * getHashTableCache()
static OverlapsTuningParamRecycler * getOverlapsTuningParamCache()
#define CHECK(condition)
Definition: Logger.h:291
static HashtableRecycler * getHashTableCache()
static constexpr DeviceIdentifier CPU_DEVICE_IDENTIFIER
Definition: DataRecycler.h:136

+ Here is the call graph for this function:

std::optional< RegisteredQueryHint > QueryRunner::QueryRunner::getParsedGlobalQueryHints ( const std::string &  query_str)

Definition at line 413 of file QueryRunner.cpp.

References cat(), CHECK, create_query_state(), g_enable_watchdog, Executor::getExecutor(), RelAlgExecutor::getGlobalQueryHint(), Catalog_Namespace::SysCatalog::instance(), pg_shim(), session_info_, and Executor::UNITARY_EXECUTOR_ID.

414  {
417  auto query_state = create_query_state(session_info_, query_str);
418  auto& cat = session_info_->getCatalog();
420  auto calcite_mgr = cat.getCalciteMgr();
421  const auto calciteQueryParsingOption =
422  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
423  const auto calciteOptimizationOption =
424  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {}, false);
425  const auto query_ra = calcite_mgr
426  ->process(query_state->createQueryStateProxy(),
427  pg_shim(query_str),
428  calciteQueryParsingOption,
429  calciteOptimizationOption)
430  .plan_result;
431  auto ra_executor = RelAlgExecutor(executor.get(), query_ra, query_state);
432  return ra_executor.getGlobalQueryHint();
433 }
std::string cat(Ts &&...args)
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:475
static SysCatalog & instance()
Definition: SysCatalog.h:343
bool g_enable_watchdog
std::optional< RegisteredQueryHint > getGlobalQueryHint()
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:303
#define CHECK(condition)
Definition: Logger.h:291
std::string pg_shim(const std::string &query)
static const ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:373

+ Here is the call graph for this function:

RegisteredQueryHint QueryRunner::QueryRunner::getParsedQueryHint ( const std::string &  query_str)

Definition at line 343 of file QueryRunner.cpp.

References cat(), CHECK, create_query_state(), RegisteredQueryHint::defaults(), g_enable_watchdog, Executor::getExecutor(), RelAlgExecutor::getParsedQueryHint(), Catalog_Namespace::SysCatalog::instance(), pg_shim(), session_info_, and Executor::UNITARY_EXECUTOR_ID.

343  {
346  auto query_state = create_query_state(session_info_, query_str);
347  auto& cat = session_info_->getCatalog();
349 
350  auto calcite_mgr = cat.getCalciteMgr();
351  const auto calciteQueryParsingOption =
352  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
353  const auto calciteOptimizationOption =
354  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {}, false);
355  const auto query_ra = calcite_mgr
356  ->process(query_state->createQueryStateProxy(),
357  pg_shim(query_str),
358  calciteQueryParsingOption,
359  calciteOptimizationOption)
360  .plan_result;
361  auto ra_executor = RelAlgExecutor(executor.get(), query_ra, query_state);
362  auto query_hints =
363  ra_executor.getParsedQueryHint(ra_executor.getRootRelAlgNodeShPtr().get());
364  return query_hints ? *query_hints : RegisteredQueryHint::defaults();
365 }
std::string cat(Ts &&...args)
std::optional< RegisteredQueryHint > getParsedQueryHint(const RelAlgNode *node)
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:475
static SysCatalog & instance()
Definition: SysCatalog.h:343
bool g_enable_watchdog
static RegisteredQueryHint defaults()
Definition: QueryHint.h:329
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:303
#define CHECK(condition)
Definition: Logger.h:291
std::string pg_shim(const std::string &query)
static const ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:373

+ Here is the call graph for this function:

std::optional< std::unordered_map< size_t, std::unordered_map< unsigned, RegisteredQueryHint > > > QueryRunner::QueryRunner::getParsedQueryHints ( const std::string &  query_str)

Definition at line 392 of file QueryRunner.cpp.

References cat(), CHECK, create_query_state(), g_enable_watchdog, Executor::getExecutor(), RelAlgExecutor::getParsedQueryHints(), Catalog_Namespace::SysCatalog::instance(), pg_shim(), session_info_, and Executor::UNITARY_EXECUTOR_ID.

392  {
395  auto query_state = create_query_state(session_info_, query_str);
396  auto& cat = session_info_->getCatalog();
398  auto calcite_mgr = cat.getCalciteMgr();
399  const auto calciteQueryParsingOption =
400  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
401  const auto calciteOptimizationOption =
402  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {}, false);
403  const auto query_ra = calcite_mgr
404  ->process(query_state->createQueryStateProxy(),
405  pg_shim(query_str),
406  calciteQueryParsingOption,
407  calciteOptimizationOption)
408  .plan_result;
409  auto ra_executor = RelAlgExecutor(executor.get(), query_ra, query_state);
410  return ra_executor.getParsedQueryHints();
411 }
std::string cat(Ts &&...args)
std::optional< std::unordered_map< size_t, std::unordered_map< unsigned, RegisteredQueryHint > > > getParsedQueryHints()
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:475
static SysCatalog & instance()
Definition: SysCatalog.h:343
bool g_enable_watchdog
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:303
#define CHECK(condition)
Definition: Logger.h:291
std::string pg_shim(const std::string &query)
static const ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:373

+ Here is the call graph for this function:

QueryPlanDagInfo QueryRunner::QueryRunner::getQueryInfoForDataRecyclerTest ( const std::string &  query_str)

Definition at line 505 of file QueryRunner.cpp.

References cat(), CHECK, create_query_state(), g_enable_watchdog, Executor::getExecutor(), RelAlgExecutor::getRootRelAlgNodeShPtr(), Catalog_Namespace::SysCatalog::instance(), pg_shim(), session_info_, and Executor::UNITARY_EXECUTOR_ID.

Referenced by extractQueryPlanDag().

506  {
509  auto query_state = create_query_state(session_info_, query_str);
510  auto& cat = session_info_->getCatalog();
512  auto calcite_mgr = cat.getCalciteMgr();
513  const auto calciteQueryParsingOption =
514  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
515  const auto calciteOptimizationOption =
516  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {}, false);
517  const auto query_ra = calcite_mgr
518  ->process(query_state->createQueryStateProxy(),
519  pg_shim(query_str),
520  calciteQueryParsingOption,
521  calciteOptimizationOption)
522  .plan_result;
523  auto ra_executor = RelAlgExecutor(executor.get(), query_ra);
524  // note that we assume the test for data recycler that needs to have join_info
525  // does not contain any ORDER BY clause; this is necessary to create work_unit
526  // without actually performing the query
527  auto root_node_shared_ptr = ra_executor.getRootRelAlgNodeShPtr();
528  auto join_info = ra_executor.getJoinInfo(root_node_shared_ptr.get());
529  auto relAlgTranslator = ra_executor.getRelAlgTranslator(root_node_shared_ptr.get());
530  return {root_node_shared_ptr, join_info.first, join_info.second, relAlgTranslator};
531 }
std::string cat(Ts &&...args)
std::shared_ptr< const RelAlgNode > getRootRelAlgNodeShPtr() const
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:475
static SysCatalog & instance()
Definition: SysCatalog.h:343
bool g_enable_watchdog
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:303
#define CHECK(condition)
Definition: Logger.h:291
std::string pg_shim(const std::string &query)
static const ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:373

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

RaExecutionSequence QueryRunner::QueryRunner::getRaExecutionSequence ( const std::string &  query_str)

Definition at line 435 of file QueryRunner.cpp.

References cat(), CHECK, create_query_state(), g_enable_watchdog, Executor::getExecutor(), RelAlgExecutor::getRaExecutionSequence(), Catalog_Namespace::SysCatalog::instance(), pg_shim(), session_info_, and Executor::UNITARY_EXECUTOR_ID.

435  {
438  auto query_state = create_query_state(session_info_, query_str);
439  auto& cat = session_info_->getCatalog();
441  auto calcite_mgr = cat.getCalciteMgr();
442  const auto calciteQueryParsingOption =
443  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
444  const auto calciteOptimizationOption =
445  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {}, false);
446  const auto query_ra = calcite_mgr
447  ->process(query_state->createQueryStateProxy(),
448  pg_shim(query_str),
449  calciteQueryParsingOption,
450  calciteOptimizationOption)
451  .plan_result;
452  auto ra_executor = RelAlgExecutor(executor.get(), query_ra, query_state);
453  return ra_executor.getRaExecutionSequence(ra_executor.getRootRelAlgNodeShPtr().get(),
454  executor.get());
455 }
std::string cat(Ts &&...args)
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:475
RaExecutionSequence getRaExecutionSequence(const RelAlgNode *root_node, Executor *executor)
static SysCatalog & instance()
Definition: SysCatalog.h:343
bool g_enable_watchdog
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:303
#define CHECK(condition)
Definition: Logger.h:291
std::string pg_shim(const std::string &query)
static const ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:373

+ Here is the call graph for this function:

std::unique_ptr< RelAlgDag > QueryRunner::QueryRunner::getRelAlgDag ( const std::string &  query_str)

Definition at line 975 of file QueryRunner.cpp.

References cat(), CHECK, create_query_state(), ExecutionOptions::defaults(), dispatch_queue_, g_enable_calcite_view_optimize, g_enable_watchdog, Executor::getExecutor(), RelAlgExecutor::getOwnedRelAlgDag(), Catalog_Namespace::SysCatalog::instance(), pg_shim(), session_info_, STDLOG, and logger::thread_local_ids().

975  {
978  auto query_state = create_query_state(session_info_, query_str);
979  auto stdlog = STDLOG(query_state);
980  auto& cat = session_info_->getCatalog();
981 
982  std::unique_ptr<RelAlgDag> rel_alg_dag;
983  auto query_launch_task = std::make_shared<QueryDispatchQueue::Task>(
984  [&cat,
985  &query_str,
986  &query_state,
987  &rel_alg_dag,
988  parent_thread_local_ids = logger::thread_local_ids()](const size_t worker_id) {
989  logger::LocalIdsScopeGuard lisg = parent_thread_local_ids.setNewThreadId();
990  auto executor = Executor::getExecutor(worker_id);
991  auto eo = ExecutionOptions::defaults();
992  auto calcite_mgr = cat.getCalciteMgr();
993  const auto calciteQueryParsingOption =
994  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
995  const auto calciteOptimizationOption = calcite_mgr->getCalciteOptimizationOption(
997  const auto query_ra = calcite_mgr
998  ->process(query_state->createQueryStateProxy(),
999  pg_shim(query_str),
1000  calciteQueryParsingOption,
1001  calciteOptimizationOption)
1002  .plan_result;
1003  auto ra_executor = RelAlgExecutor(executor.get(), query_ra);
1004  rel_alg_dag = ra_executor.getOwnedRelAlgDag();
1005  });
1007  dispatch_queue_->submit(query_launch_task, /*is_update_delete=*/false);
1008  auto result_future = query_launch_task->get_future();
1009  result_future.get();
1010  CHECK(rel_alg_dag);
1011  return rel_alg_dag;
1012 }
bool g_enable_calcite_view_optimize
Definition: QueryRunner.cpp:56
std::string cat(Ts &&...args)
std::unique_ptr< QueryDispatchQueue > dispatch_queue_
Definition: QueryRunner.h:331
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:475
static SysCatalog & instance()
Definition: SysCatalog.h:343
bool g_enable_watchdog
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:303
#define CHECK(condition)
Definition: Logger.h:291
std::unique_ptr< RelAlgDag > getOwnedRelAlgDag()
static ExecutionOptions defaults()
std::string pg_shim(const std::string &query)
ThreadLocalIds thread_local_ids()
Definition: Logger.cpp:874
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

std::shared_ptr< RelAlgTranslator > QueryRunner::QueryRunner::getRelAlgTranslator ( const std::string &  query_str,
Executor executor 
)

Definition at line 482 of file QueryRunner.cpp.

References cat(), CHECK, create_query_state(), g_enable_watchdog, Catalog_Namespace::SysCatalog::instance(), pg_shim(), and session_info_.

484  {
487  auto query_state = create_query_state(session_info_, query_str);
488  auto& cat = session_info_->getCatalog();
489  auto calcite_mgr = cat.getCalciteMgr();
490  const auto calciteQueryParsingOption =
491  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
492  const auto calciteOptimizationOption =
493  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {}, false);
494  const auto query_ra = calcite_mgr
495  ->process(query_state->createQueryStateProxy(),
496  pg_shim(query_str),
497  calciteQueryParsingOption,
498  calciteOptimizationOption)
499  .plan_result;
500  auto ra_executor = RelAlgExecutor(executor, query_ra);
501  auto root_node_shared_ptr = ra_executor.getRootRelAlgNodeShPtr();
502  return ra_executor.getRelAlgTranslator(root_node_shared_ptr.get());
503 }
std::string cat(Ts &&...args)
static SysCatalog & instance()
Definition: SysCatalog.h:343
bool g_enable_watchdog
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:303
#define CHECK(condition)
Definition: Logger.h:291
std::string pg_shim(const std::string &query)

+ Here is the call graph for this function:

std::shared_ptr< const RelAlgNode > QueryRunner::QueryRunner::getRootNodeFromParsedQuery ( const std::string &  query_str)

Definition at line 367 of file QueryRunner.cpp.

References cat(), CHECK, create_query_state(), g_enable_watchdog, Executor::getExecutor(), RelAlgExecutor::getRootRelAlgNodeShPtr(), Catalog_Namespace::SysCatalog::instance(), pg_shim(), session_info_, and Executor::UNITARY_EXECUTOR_ID.

368  {
371  auto query_state = create_query_state(session_info_, query_str);
372  auto& cat = session_info_->getCatalog();
374 
375  auto calcite_mgr = cat.getCalciteMgr();
376  const auto calciteQueryParsingOption =
377  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
378  const auto calciteOptimizationOption =
379  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {}, false);
380  const auto query_ra = calcite_mgr
381  ->process(query_state->createQueryStateProxy(),
382  pg_shim(query_str),
383  calciteQueryParsingOption,
384  calciteOptimizationOption)
385  .plan_result;
386  auto ra_executor = RelAlgExecutor(executor.get(), query_ra, query_state);
387  return ra_executor.getRootRelAlgNodeShPtr();
388 }
std::string cat(Ts &&...args)
std::shared_ptr< const RelAlgNode > getRootRelAlgNodeShPtr() const
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:475
static SysCatalog & instance()
Definition: SysCatalog.h:343
bool g_enable_watchdog
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:303
#define CHECK(condition)
Definition: Logger.h:291
std::string pg_shim(const std::string &query)
static const ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:373

+ Here is the call graph for this function:

std::shared_ptr<Catalog_Namespace::SessionInfo> QueryRunner::QueryRunner::getSession ( ) const
inline

Definition at line 190 of file QueryRunner.h.

References session_info_.

190  {
191  return session_info_;
192  }
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
Catalog_Namespace::UserMetadata& QueryRunner::QueryRunner::getUserMetadata ( ) const
bool QueryRunner::QueryRunner::gpusPresent ( ) const

Definition at line 276 of file QueryRunner.cpp.

References CHECK, and session_info_.

276  {
278  return session_info_->getCatalog().getDataMgr().gpusPresent();
279 }
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
#define CHECK(condition)
Definition: Logger.h:291
QueryRunner * QueryRunner::QueryRunner::init ( const char *  db_path,
const std::string &  udf_filename = "",
const size_t  max_gpu_mem = 0,
const int  reserved_gpu_mem = 256 << 20 
)
static

Definition at line 84 of file QueryRunner.cpp.

References logger::init(), shared::kDefaultDbName, and shared::kRootUsername.

87  {
88  return QueryRunner::init(db_path,
90  "HyperInteractive",
92  {},
93  {},
94  udf_filename,
95  true,
96  max_gpu_mem,
97  reserved_gpu_mem);
98 }
const std::string kDefaultDbName
void init(LogOptions const &log_opts)
Definition: Logger.cpp:360
const std::string kRootUsername

+ Here is the call graph for this function:

QueryRunner * QueryRunner::QueryRunner::init ( const File_Namespace::DiskCacheConfig disk_cache_config,
const char *  db_path,
const std::vector< LeafHostInfo > &  string_servers = {},
const std::vector< LeafHostInfo > &  leaf_servers = {} 
)
static

Definition at line 100 of file QueryRunner.cpp.

References logger::init(), shared::kDefaultDbName, and shared::kRootUsername.

103  {
104  return QueryRunner::init(db_path,
106  "HyperInteractive",
108  string_servers,
109  leaf_servers,
110  "",
111  true,
112  0,
113  256 << 20,
114  false,
115  false,
116  disk_cache_config);
117 }
const std::string kDefaultDbName
void init(LogOptions const &log_opts)
Definition: Logger.cpp:360
const std::string kRootUsername

+ Here is the call graph for this function:

static QueryRunner* QueryRunner::QueryRunner::init ( const char *  db_path,
const std::vector< LeafHostInfo > &  string_servers,
const std::vector< LeafHostInfo > &  leaf_servers 
)
inlinestatic

Definition at line 151 of file QueryRunner.h.

References logger::init(), shared::kDefaultDbName, and shared::kRootUsername.

153  {
154  return QueryRunner::init(db_path,
156  "HyperInteractive",
158  string_servers,
159  leaf_servers);
160  }
const std::string kDefaultDbName
void init(LogOptions const &log_opts)
Definition: Logger.cpp:360
const std::string kRootUsername

+ Here is the call graph for this function:

QueryRunner * QueryRunner::QueryRunner::init ( const char *  db_path,
const std::string &  user,
const std::string &  pass,
const std::string &  db_name,
const std::vector< LeafHostInfo > &  string_servers,
const std::vector< LeafHostInfo > &  leaf_servers,
const std::string &  udf_filename = "",
bool  uses_gpus = true,
const size_t  max_gpu_mem = 0,
const int  reserved_gpu_mem = 256 << 20,
const bool  create_user = false,
const bool  create_db = false,
const File_Namespace::DiskCacheConfig config = nullptr 
)
static

Definition at line 119 of file QueryRunner.cpp.

References CHECK, ddl_utils::FilePathWhitelist::clear(), logger::FATAL, ddl_utils::FilePathWhitelist::initialize(), and LOG_IF.

131  {
132  // Whitelist root path for tests by default
134  ddl_utils::FilePathWhitelist::initialize(db_path, "[\"/\"]", "[\"/\"]");
135  LOG_IF(FATAL, !leaf_servers.empty()) << "Distributed test runner not supported.";
136  CHECK(leaf_servers.empty());
137  qr_instance_.reset(new QueryRunner(db_path,
138  user,
139  pass,
140  db_name,
141  string_servers,
142  leaf_servers,
143  udf_filename,
144  uses_gpus,
145  max_gpu_mem,
146  reserved_gpu_mem,
147  create_user,
148  create_db,
149  disk_cache_config));
150  return qr_instance_.get();
151 }
static void initialize(const std::string &data_dir, const std::string &allowed_import_paths, const std::string &allowed_export_paths)
Definition: DdlUtils.cpp:841
#define LOG_IF(severity, condition)
Definition: Logger.h:383
QueryRunner(std::unique_ptr< Catalog_Namespace::SessionInfo > session)
static std::unique_ptr< QueryRunner > qr_instance_
Definition: QueryRunner.h:325
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

static QueryRunner* QueryRunner::QueryRunner::init ( std::unique_ptr< Catalog_Namespace::SessionInfo > &  session)
inlinestatic

Definition at line 176 of file QueryRunner.h.

References qr_instance_, and QueryRunner().

176  {
177  qr_instance_.reset(new QueryRunner(std::move(session)));
178  return qr_instance_.get();
179  }
QueryRunner(std::unique_ptr< Catalog_Namespace::SessionInfo > session)
static std::unique_ptr< QueryRunner > qr_instance_
Definition: QueryRunner.h:325

+ Here is the call graph for this function:

void QueryRunner::QueryRunner::reset ( )
static

Definition at line 1183 of file QueryRunner.cpp.

References anonymous_namespace{QueryRunner.cpp}::calcite_shutdown_handler(), and qr_instance_.

1183  {
1184  qr_instance_->query_engine_.reset();
1185  qr_instance_.reset(nullptr);
1187 }
static std::unique_ptr< QueryRunner > qr_instance_
Definition: QueryRunner.h:325

+ Here is the call graph for this function:

void QueryRunner::QueryRunner::resizeDispatchQueue ( const size_t  num_executors)

Definition at line 258 of file QueryRunner.cpp.

References dispatch_queue_.

258  {
259  dispatch_queue_ = std::make_unique<QueryDispatchQueue>(num_executors);
260 }
std::unique_ptr< QueryDispatchQueue > dispatch_queue_
Definition: QueryRunner.h:331
void QueryRunner::QueryRunner::runDDLStatement ( const std::string &  stmt_str_in)
virtual

Definition at line 570 of file QueryRunner.cpp.

References cat(), CHECK, create_query_state(), DdlCommandExecutor::execute(), g_enable_watchdog, ParserWrapper::Insert, Catalog_Namespace::SysCatalog::instance(), pg_shim(), session_info_, and STDLOG.

Referenced by QueryRunner::ImportDriver::importGeoTable(), runMultipleStatements(), and runSQL().

570  {
573 
574  std::string stmt_str = stmt_str_in;
575  // First remove special chars
576  boost::algorithm::trim_left_if(stmt_str, boost::algorithm::is_any_of("\n"));
577  // Then remove spaces
578  boost::algorithm::trim_left(stmt_str);
579 
580  ParserWrapper pw{stmt_str};
581 
582  auto query_state = create_query_state(session_info_, stmt_str);
583  auto stdlog = STDLOG(query_state);
584 
585  if (pw.is_ddl || pw.getDMLType() == ParserWrapper::DMLType::Insert) {
586  auto& cat = session_info_->getCatalog();
587  auto calcite_mgr = cat.getCalciteMgr();
588  const auto calciteQueryParsingOption =
589  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
590  const auto calciteOptimizationOption =
591  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {}, false);
592  const auto query_ra = calcite_mgr
593  ->process(query_state->createQueryStateProxy(),
594  pg_shim(stmt_str),
595  calciteQueryParsingOption,
596  calciteOptimizationOption)
597  .plan_result;
598  if (pw.getDMLType() == ParserWrapper::DMLType::Insert) {
599  rapidjson::Document ddl_query;
600  ddl_query.Parse(query_ra);
601  CHECK(ddl_query.HasMember("payload"));
602  CHECK(ddl_query["payload"].IsObject());
603  auto stmt = Parser::InsertValuesStmt(ddl_query["payload"].GetObject());
604  stmt.execute(*session_info_, false /* read only */);
605  return;
606  }
608  executor.execute(false /* read only */);
609  return;
610  }
611 }
std::string cat(Ts &&...args)
static SysCatalog & instance()
Definition: SysCatalog.h:343
bool g_enable_watchdog
ExecutionResult execute(bool read_only_mode)
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:303
#define CHECK(condition)
Definition: Logger.h:291
std::string pg_shim(const std::string &query)
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void QueryRunner::QueryRunner::runImport ( Parser::CopyTableStmt import_stmt)
virtual

Definition at line 776 of file QueryRunner.cpp.

References CHECK, Parser::CopyTableStmt::execute(), and session_info_.

776  {
777  CHECK(import_stmt);
778  import_stmt->execute(*session_info_, false /* read only */);
779 }
void execute(const Catalog_Namespace::SessionInfo &session, bool read_only_mode) override
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

std::vector< std::shared_ptr< ResultSet > > QueryRunner::QueryRunner::runMultipleStatements ( const std::string &  sql,
const ExecutorDeviceType  dt 
)
virtual

Definition at line 752 of file QueryRunner.cpp.

References field(), ParserWrapper::Insert, runDDLStatement(), runSQL(), split(), and strip().

754  {
755  std::vector<std::shared_ptr<ResultSet>> results;
756  // TODO: Need to properly handle escaped semicolons instead of doing a naive split().
757  auto fields = split(sql, ";");
758  for (const auto& field : fields) {
759  auto text = strip(field) + ";";
760  if (text == ";") {
761  continue;
762  }
763 
764  ParserWrapper pw{text};
765  if (pw.is_ddl || pw.getDMLType() == ParserWrapper::DMLType::Insert) {
766  runDDLStatement(text);
767  results.push_back(nullptr);
768  } else {
769  // is not DDL, then assume it's DML and try to execute
770  results.push_back(runSQL(text, dt, true, true));
771  }
772  }
773  return results;
774 }
std::string strip(std::string_view str)
trim any whitespace from the left and right ends of a string
virtual std::shared_ptr< ResultSet > runSQL(const std::string &query_str, CompilationOptions co, ExecutionOptions eo)
std::vector< std::string > split(std::string_view str, std::string_view delim, std::optional< size_t > maxsplit)
split apart a string into a vector of substrings
const rapidjson::Value & field(const rapidjson::Value &obj, const char field[]) noexcept
Definition: JsonAccessors.h:31
virtual void runDDLStatement(const std::string &)

+ Here is the call graph for this function:

std::shared_ptr< ExecutionResult > QueryRunner::QueryRunner::runSelectQuery ( const std::string &  query_str,
CompilationOptions  co,
ExecutionOptions  eo 
)
virtual

Definition at line 895 of file QueryRunner.cpp.

References ExecutionOptions::allow_loop_joins, cat(), CHECK, create_query_state(), CompilationOptions::defaults(), CompilationOptions::device_type, dispatch_queue_, explain_type_, g_enable_calcite_view_optimize, g_enable_filter_push_down, g_enable_watchdog, Executor::getExecutor(), CompilationOptions::hoist_literals, Catalog_Namespace::SysCatalog::instance(), ExecutionOptions::just_explain, pg_shim(), run_benchmark_import::result, QueryRunner::anonymous_namespace{QueryRunner.cpp}::run_select_query_with_filter_push_down(), session_info_, STDLOG, and logger::thread_local_ids().

Referenced by runSelectQuery(), and runSQL().

897  {
900  auto query_state = create_query_state(session_info_, query_str);
901  auto stdlog = STDLOG(query_state);
903  return run_select_query_with_filter_push_down(query_state->createQueryStateProxy(),
904  co.device_type,
905  co.hoist_literals,
906  eo.allow_loop_joins,
907  eo.just_explain,
910  }
911 
912  auto& cat = session_info_->getCatalog();
913 
914  std::shared_ptr<ExecutionResult> result;
915  auto query_launch_task = std::make_shared<QueryDispatchQueue::Task>(
916  [&cat,
917  &query_str,
918  &co,
919  explain_type = this->explain_type_,
920  &eo,
921  &query_state,
922  &result,
923  parent_thread_local_ids = logger::thread_local_ids()](const size_t worker_id) {
924  logger::LocalIdsScopeGuard lisg = parent_thread_local_ids.setNewThreadId();
925  auto executor = Executor::getExecutor(worker_id);
926  // TODO The next line should be deleted since it overwrites co, but then
927  // NycTaxiTest.RunSelectsEncodingDictWhereGreater fails due to co not getting
928  // reset to its default values.
929  co = CompilationOptions::defaults(co.device_type);
930  co.explain_type = explain_type;
931  auto calcite_mgr = cat.getCalciteMgr();
932  const auto calciteQueryParsingOption =
933  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
934  const auto calciteOptimizationOption = calcite_mgr->getCalciteOptimizationOption(
936  const auto query_ra = calcite_mgr
937  ->process(query_state->createQueryStateProxy(),
938  pg_shim(query_str),
939  calciteQueryParsingOption,
940  calciteOptimizationOption)
941  .plan_result;
942  auto ra_executor = RelAlgExecutor(executor.get(), query_ra);
943  result = std::make_shared<ExecutionResult>(
944  ra_executor.executeRelAlgQuery(co, eo, false, nullptr));
945  });
947  dispatch_queue_->submit(query_launch_task, /*is_update_delete=*/false);
948  auto result_future = query_launch_task->get_future();
949  result_future.get();
950  CHECK(result);
951  return result;
952 }
bool g_enable_calcite_view_optimize
Definition: QueryRunner.cpp:56
std::string cat(Ts &&...args)
std::unique_ptr< QueryDispatchQueue > dispatch_queue_
Definition: QueryRunner.h:331
ExecutorExplainType explain_type_
Definition: QueryRunner.h:327
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:475
static SysCatalog & instance()
Definition: SysCatalog.h:343
bool g_enable_watchdog
ExecutorDeviceType device_type
std::shared_ptr< ExecutionResult > run_select_query_with_filter_push_down(QueryStateProxy query_state_proxy, const ExecutorDeviceType device_type, const bool hoist_literals, const bool allow_loop_joins, const bool just_explain, const ExecutorExplainType explain_type, const bool with_filter_push_down)
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:303
static CompilationOptions defaults(const ExecutorDeviceType device_type=ExecutorDeviceType::GPU)
bool g_enable_filter_push_down
Definition: Execute.cpp:95
#define CHECK(condition)
Definition: Logger.h:291
std::string pg_shim(const std::string &query)
ThreadLocalIds thread_local_ids()
Definition: Logger.cpp:874
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::shared_ptr< ExecutionResult > QueryRunner::QueryRunner::runSelectQuery ( const std::string &  query_str,
const ExecutorDeviceType  device_type,
const bool  hoist_literals,
const bool  allow_loop_joins,
const bool  just_explain = false 
)
virtual

Definition at line 954 of file QueryRunner.cpp.

References defaultExecutionOptionsForRunSQL(), CompilationOptions::defaults(), and runSelectQuery().

959  {
960  auto co = CompilationOptions::defaults(device_type);
961  co.hoist_literals = hoist_literals;
962  return runSelectQuery(query_str,
963  std::move(co),
964  defaultExecutionOptionsForRunSQL(allow_loop_joins, just_explain));
965 }
static ExecutionOptions defaultExecutionOptionsForRunSQL(bool allow_loop_joins=true, bool just_explain=false)
virtual std::shared_ptr< ExecutionResult > runSelectQuery(const std::string &query_str, CompilationOptions co, ExecutionOptions eo)
static CompilationOptions defaults(const ExecutorDeviceType device_type=ExecutorDeviceType::GPU)

+ Here is the call graph for this function:

std::shared_ptr< ResultSet > QueryRunner::QueryRunner::runSQL ( const std::string &  query_str,
CompilationOptions  co,
ExecutionOptions  eo 
)
virtual

Definition at line 613 of file QueryRunner.cpp.

References CHECK, ParserWrapper::Insert, Catalog_Namespace::SysCatalog::instance(), runDDLStatement(), runSelectQuery(), session_info_, and VLOG.

Referenced by runMultipleStatements(), and runSQL().

615  {
618 
619  ParserWrapper pw{query_str};
620  if (pw.getDMLType() == ParserWrapper::DMLType::Insert) {
621  runDDLStatement(query_str);
622  return nullptr;
623  }
624  const auto execution_result = runSelectQuery(query_str, std::move(co), std::move(eo));
625  VLOG(1) << session_info_->getCatalog().getDataMgr().getSystemMemoryUsage();
626  return execution_result->getRows();
627 }
static SysCatalog & instance()
Definition: SysCatalog.h:343
virtual void runDDLStatement(const std::string &)
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
virtual std::shared_ptr< ExecutionResult > runSelectQuery(const std::string &query_str, CompilationOptions co, ExecutionOptions eo)
#define CHECK(condition)
Definition: Logger.h:291
#define VLOG(n)
Definition: Logger.h:387

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::shared_ptr< ResultSet > QueryRunner::QueryRunner::runSQL ( const std::string &  query_str,
const ExecutorDeviceType  device_type,
const bool  hoist_literals = true,
const bool  allow_loop_joins = true 
)
virtual

Definition at line 629 of file QueryRunner.cpp.

References defaultExecutionOptionsForRunSQL(), CompilationOptions::defaults(), and runSQL().

632  {
633  auto co = CompilationOptions::defaults(device_type);
634  co.hoist_literals = hoist_literals;
635  return runSQL(
636  query_str, std::move(co), defaultExecutionOptionsForRunSQL(allow_loop_joins));
637 }
static ExecutionOptions defaultExecutionOptionsForRunSQL(bool allow_loop_joins=true, bool just_explain=false)
virtual std::shared_ptr< ResultSet > runSQL(const std::string &query_str, CompilationOptions co, ExecutionOptions eo)
static CompilationOptions defaults(const ExecutorDeviceType device_type=ExecutorDeviceType::GPU)

+ Here is the call graph for this function:

std::shared_ptr< ResultSet > QueryRunner::QueryRunner::runSQLWithAllowingInterrupt ( const std::string &  query_str,
const std::string &  session_id,
const ExecutorDeviceType  device_type,
const double  running_query_check_freq = 0.9,
const unsigned  pending_query_check_freq = 1000 
)
virtual

Definition at line 668 of file QueryRunner.cpp.

References calcite_lock, cat(), CHECK, create_query_state(), CompilationOptions::defaults(), dispatch_queue_, g_enable_columnar_output, g_enable_watchdog, g_gpu_mem_limit_percent, Executor::getExecutor(), Catalog_Namespace::SysCatalog::instance(), pg_shim(), run_benchmark_import::result, session_info_, STDLOG, logger::thread_local_ids(), and Executor::UNITARY_EXECUTOR_ID.

673  {
676  auto current_user = session_info_->get_currentUser();
677  auto session_info = std::make_shared<Catalog_Namespace::SessionInfo>(
678  session_info_->get_catalog_ptr(), current_user, device_type, session_id);
679  auto query_state = create_query_state(session_info, query_str);
680  auto stdlog = STDLOG(query_state);
681  auto& cat = query_state->getConstSessionInfo()->getCatalog();
682  std::string query_ra{""};
683 
684  std::shared_ptr<ExecutionResult> result;
685  auto query_launch_task = std::make_shared<QueryDispatchQueue::Task>(
686  [&cat,
687  &query_ra,
688  &device_type,
689  &query_state,
690  &result,
691  &running_query_check_freq,
692  &pending_query_check_freq,
693  parent_thread_local_ids = logger::thread_local_ids()](const size_t worker_id) {
694  logger::LocalIdsScopeGuard lisg = parent_thread_local_ids.setNewThreadId();
695  auto executor = Executor::getExecutor(worker_id);
697 
699  false,
700  true,
701  false,
702  true,
703  false,
704  false,
705  false,
706  false,
707  10000,
708  false,
709  false,
711  true,
712  running_query_check_freq,
713  pending_query_check_freq,
714  false};
715  {
716  // async query initiation for interrupt test
717  // incurs data race warning in TSAN since
718  // calcite_mgr is shared across multiple query threads
719  // so here we lock the manager during query parsing
720  std::lock_guard<std::mutex> calcite_lock_guard(calcite_lock);
721  auto calcite_mgr = cat.getCalciteMgr();
722  const auto calciteQueryParsingOption =
723  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
724  const auto calciteOptimizationOption =
725  calcite_mgr->getCalciteOptimizationOption(
726  false, g_enable_watchdog, {}, false);
727  query_ra = calcite_mgr
728  ->process(query_state->createQueryStateProxy(),
729  pg_shim(query_state->getQueryStr()),
730  calciteQueryParsingOption,
731  calciteOptimizationOption)
732  .plan_result;
733  }
734  auto ra_executor = RelAlgExecutor(executor.get(), query_ra, query_state);
735  result = std::make_shared<ExecutionResult>(
736  ra_executor.executeRelAlgQuery(co, eo, false, nullptr));
737  });
739  executor->enrollQuerySession(session_id,
740  query_str,
741  query_state->getQuerySubmittedTime(),
743  QuerySessionStatus::QueryStatus::PENDING_QUEUE);
745  dispatch_queue_->submit(query_launch_task, /*is_update_delete=*/false);
746  auto result_future = query_launch_task->get_future();
747  result_future.get();
748  CHECK(result);
749  return result->getRows();
750 }
std::string cat(Ts &&...args)
std::unique_ptr< QueryDispatchQueue > dispatch_queue_
Definition: QueryRunner.h:331
std::mutex calcite_lock
Definition: QueryRunner.cpp:57
static std::shared_ptr< Executor > getExecutor(const ExecutorId id, const std::string &debug_dir="", const std::string &debug_file="", const SystemParameters &system_parameters=SystemParameters())
Definition: Execute.cpp:475
bool g_enable_columnar_output
Definition: Execute.cpp:99
static SysCatalog & instance()
Definition: SysCatalog.h:343
bool g_enable_watchdog
double g_gpu_mem_limit_percent
Definition: QueryRunner.cpp:53
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:303
static CompilationOptions defaults(const ExecutorDeviceType device_type=ExecutorDeviceType::GPU)
#define CHECK(condition)
Definition: Logger.h:291
std::string pg_shim(const std::string &query)
ThreadLocalIds thread_local_ids()
Definition: Logger.cpp:874
#define STDLOG(...)
Definition: QueryState.h:234
static const ExecutorId UNITARY_EXECUTOR_ID
Definition: Execute.h:373

+ Here is the call graph for this function:

void QueryRunner::QueryRunner::setExplainType ( const ExecutorExplainType  explain_type)
inline

Definition at line 307 of file QueryRunner.h.

References explain_type_.

307  {
308  explain_type_ = explain_type;
309  }
ExecutorExplainType explain_type_
Definition: QueryRunner.h:327
void QueryRunner::QueryRunner::validateDDLStatement ( const std::string &  stmt_str_in)
virtual

Definition at line 458 of file QueryRunner.cpp.

References cat(), CHECK, create_query_state(), g_enable_watchdog, pg_shim(), session_info_, and STDLOG.

458  {
460 
461  std::string stmt_str = stmt_str_in;
462  // First remove special chars
463  boost::algorithm::trim_left_if(stmt_str, boost::algorithm::is_any_of("\n"));
464  // Then remove spaces
465  boost::algorithm::trim_left(stmt_str);
466 
467  auto query_state = create_query_state(session_info_, stmt_str);
468  auto stdlog = STDLOG(query_state);
469 
470  auto& cat = session_info_->getCatalog();
471  auto calcite_mgr = cat.getCalciteMgr();
472  const auto calciteQueryParsingOption =
473  calcite_mgr->getCalciteQueryParsingOption(true, false, true);
474  const auto calciteOptimizationOption =
475  calcite_mgr->getCalciteOptimizationOption(false, g_enable_watchdog, {}, false);
476  calcite_mgr->process(query_state->createQueryStateProxy(),
477  pg_shim(stmt_str),
478  calciteQueryParsingOption,
479  calciteOptimizationOption);
480 }
std::string cat(Ts &&...args)
bool g_enable_watchdog
std::shared_ptr< Catalog_Namespace::SessionInfo > session_info_
Definition: QueryRunner.h:330
static std::shared_ptr< query_state::QueryState > create_query_state(Ts &&...args)
Definition: QueryRunner.h:303
#define CHECK(condition)
Definition: Logger.h:291
std::string pg_shim(const std::string &query)
#define STDLOG(...)
Definition: QueryState.h:234

+ Here is the call graph for this function:

Member Data Documentation

Catalog_Namespace::DBMetadata QueryRunner::QueryRunner::db_metadata_
protected

Definition at line 329 of file QueryRunner.h.

Referenced by getBufferPoolStats(), and QueryRunner().

std::unique_ptr<QueryDispatchQueue> QueryRunner::QueryRunner::dispatch_queue_
protected
ExecutorExplainType QueryRunner::QueryRunner::explain_type_ = ExecutorExplainType::Default
protected

Definition at line 327 of file QueryRunner.h.

Referenced by runSelectQuery(), and setExplainType().

std::unique_ptr< QueryRunner > QueryRunner::QueryRunner::qr_instance_ = nullptr
staticprotected

Definition at line 325 of file QueryRunner.h.

Referenced by get(), init(), and reset().

std::shared_ptr<QueryEngine> QueryRunner::QueryRunner::query_engine_
protected

Definition at line 332 of file QueryRunner.h.

Referenced by QueryRunner().

query_state::QueryStates QueryRunner::QueryRunner::query_states_
static

Definition at line 300 of file QueryRunner.h.

Referenced by create_query_state().


The documentation for this class was generated from the following files: