EmbeddedDatabase::DBEngineImpl Class Reference

Inheritance diagram for EmbeddedDatabase::DBEngineImpl:

Collaboration diagram for EmbeddedDatabase::DBEngineImpl:

Public Member Functions
	DBEngineImpl ()
	~DBEngineImpl ()
bool	init (const std::string &cmd_line)
std::shared_ptr< CursorImpl >	sql_execute_dbe (const TSessionId &session_id, const std::string &query_str, const bool column_format, const int32_t first_n, const int32_t at_most_n)
void	executeDDL (const std::string &query)
void	importArrowTable (const std::string &name, std::shared_ptr< arrow::Table > &table, uint64_t fragment_size)
std::shared_ptr< CursorImpl >	executeDML (const std::string &query)
std::shared_ptr< CursorImpl >	executeRA (const std::string &query)
std::vector< std::string >	getTables ()
std::vector< ColumnDetails >	getTableDetails (const std::string &table_name)
bool	setDatabase (std::string &db_name)
bool	login (std::string &db_name, std::string &user_name, const std::string &password)
Public Member Functions inherited from EmbeddedDatabase::DBEngine
virtual	~DBEngine ()
void	executeDDL (const std::string &query)
std::shared_ptr< Cursor >	executeDML (const std::string &query)
std::shared_ptr< Cursor >	executeRA (const std::string &query)
void	importArrowTable (const std::string &name, std::shared_ptr< arrow::Table > &table, uint64_t fragment_size=0)
std::vector< std::string >	getTables ()
std::vector< ColumnDetails >	getTableDetails (const std::string &table_name)
bool	setDatabase (std::string &db_name)
bool	login (std::string &db_name, std::string &user_name, const std::string &password)

Protected Member Functions
void	reset ()
bool	catalogExists (const std::string &base_path)
void	cleanCatalog (const std::string &base_path)
std::string	createCatalog (const std::string &base_path)
Protected Member Functions inherited from EmbeddedDatabase::DBEngine
	DBEngine ()
	DBEngine (const DBEngine &)=delete
DBEngine &	operator= (const DBEngine &)=delete

Private Attributes
std::string	base_path_
std::string	session_id_
mapd::shared_ptr< DBHandler >	db_handler_
bool	is_temp_db_
std::string	udf_filename_
std::vector< std::string >	system_folders_

Additional Inherited Members
Static Public Member Functions inherited from EmbeddedDatabase::DBEngine
static std::shared_ptr< DBEngine >	create (const std::string &cmd_line)

Detailed Description

DBEngine internal implementation

Definition at line 98 of file DBEngine.cpp.

Constructor & Destructor Documentation

EmbeddedDatabase::DBEngineImpl::DBEngineImpl ( )

inline

Definition at line 100 of file DBEngine.cpp.

: is_temp_db_(false) {}

EmbeddedDatabase::DBEngineImpl::~DBEngineImpl ( )

inline

Definition at line 102 of file DBEngine.cpp.

References reset().

{ reset(); }

Here is the call graph for this function:

Member Function Documentation

bool EmbeddedDatabase::DBEngineImpl::catalogExists ( const std::string &  base_path )

inlineprotected

Definition at line 360 of file DBEngine.cpp.

References system_folders_.

Referenced by init().

                                                 {
    if (!boost::filesystem::exists(base_path)) {
      return false;
    }
    for (auto& subdir : system_folders_) {
      std::string path = base_path + "/" + subdir;
      if (!boost::filesystem::exists(path)) {
        return false;
      }
    }
    return true;
  }

Here is the caller graph for this function:

void EmbeddedDatabase::DBEngineImpl::cleanCatalog ( const std::string &  base_path )

inlineprotected

Definition at line 373 of file DBEngine.cpp.

References system_folders_.

                                                {
    if (boost::filesystem::exists(base_path)) {
      for (auto& subdir : system_folders_) {
        std::string path = base_path + "/" + subdir;
        if (boost::filesystem::exists(path)) {
          boost::filesystem::remove_all(path);
        }
      }
    }
  }

std::string EmbeddedDatabase::DBEngineImpl::createCatalog ( const std::string &  base_path )

inlineprotected

Definition at line 384 of file DBEngine.cpp.

References is_temp_db_, and system_folders_.

Referenced by init().

                                                      {
    std::string root_dir = base_path;
    if (base_path.empty()) {
      boost::system::error_code error;
      auto tmp_path = boost::filesystem::temp_directory_path(error);
      if (boost::system::errc::success != error.value()) {
        std::cerr << error.message() << std::endl;
        return "";
      }
      tmp_path /= "omnidbe_%%%%-%%%%-%%%%";
      auto uniq_path = boost::filesystem::unique_path(tmp_path, error);
      if (boost::system::errc::success != error.value()) {
        std::cerr << error.message() << std::endl;
        return "";
      }
      root_dir = uniq_path.string();
      is_temp_db_ = true;
    }
    if (!boost::filesystem::exists(root_dir)) {
      if (!boost::filesystem::create_directory(root_dir)) {
        std::cerr << "Cannot create database directory: " << root_dir << std::endl;
        return "";
      }
    }
    size_t absent_count = 0;
    for (auto& sub_dir : system_folders_) {
      std::string path = root_dir + "/" + sub_dir;
      if (!boost::filesystem::exists(path)) {
        if (!boost::filesystem::create_directory(path)) {
          std::cerr << "Cannot create database subdirectory: " << path << std::endl;
          return "";
        }
        ++absent_count;
      }
    }
    if ((absent_count > 0) && (absent_count < system_folders_.size())) {
      std::cerr << "Database directory structure is broken: " << root_dir << std::endl;
      return "";
    }
    return root_dir;
  }

Here is the caller graph for this function:

void EmbeddedDatabase::DBEngineImpl::executeDDL ( const std::string &  query )

inline

Definition at line 215 of file DBEngine.cpp.

References run_benchmark_import::res, session_id_, and sql_execute_dbe().

Referenced by EmbeddedDatabase::DBEngine::executeDDL().

                                          {
    auto res = sql_execute_dbe(session_id_, query, false, -1, -1);
  }

Here is the call graph for this function:

Here is the caller graph for this function:

std::shared_ptr<CursorImpl> EmbeddedDatabase::DBEngineImpl::executeDML ( const std::string &  query )

inline

Definition at line 253 of file DBEngine.cpp.

References session_id_, and sql_execute_dbe().

Referenced by EmbeddedDatabase::DBEngine::executeDML().

                                                               {
    return sql_execute_dbe(session_id_, query, false, -1, -1);
  }

Here is the call graph for this function:

Here is the caller graph for this function:

std::shared_ptr<CursorImpl> EmbeddedDatabase::DBEngineImpl::executeRA ( const std::string &  query )

inline

Definition at line 257 of file DBEngine.cpp.

References session_id_, and sql_execute_dbe().

Referenced by EmbeddedDatabase::DBEngine::executeRA().

                                                              {
    return sql_execute_dbe(session_id_, query, false, -1, -1);
  }

Here is the call graph for this function:

Here is the caller graph for this function:

std::vector<ColumnDetails> EmbeddedDatabase::DBEngineImpl::getTableDetails ( const std::string &  table_name )

inline

Definition at line 279 of file DBEngine.cpp.

References EmbeddedDatabase::ColumnDetails::col_name, EmbeddedDatabase::ColumnDetails::col_type, EmbeddedDatabase::ColumnDetails::comp_param, db_handler_, EmbeddedDatabase::DICT, EmbeddedDatabase::ColumnDetails::encoding, EmbeddedDatabase::ColumnDetails::is_array, IS_GEO, kARRAY, EmbeddedDatabase::ColumnDetails::nullable, EmbeddedDatabase::ColumnDetails::precision, run_benchmark_import::result, EmbeddedDatabase::ColumnDetails::scale, session_id_, EmbeddedDatabase::sqlToColumnEncoding(), and EmbeddedDatabase::sqlToColumnType().

Referenced by EmbeddedDatabase::DBEngine::getTableDetails().

                                                                        {
    std::vector<ColumnDetails> result;
    auto catalog = db_handler_->get_session_copy(session_id_).get_catalog_ptr();
    if (catalog) {
      auto metadata = catalog->getMetadataForTable(table_name, false);
      if (metadata) {
        const auto col_descriptors =
            catalog->getAllColumnMetadataForTable(metadata->tableId, false, true, false);
        const auto deleted_cd = catalog->getDeletedColumn(metadata);
        for (const auto cd : col_descriptors) {
          if (cd == deleted_cd) {
            continue;
          }
          ColumnDetails col_details;
          col_details.col_name = cd->columnName;
          auto ct = cd->columnType;
          SQLTypes sql_type = ct.get_type();
          EncodingType sql_enc = ct.get_compression();
          col_details.col_type = sqlToColumnType(sql_type);
          col_details.encoding = sqlToColumnEncoding(sql_enc);
          col_details.nullable = !ct.get_notnull();
          col_details.is_array = (sql_type == kARRAY);
          if (IS_GEO(sql_type)) {
            col_details.precision = static_cast<int>(ct.get_subtype());
            col_details.scale = ct.get_output_srid();
          } else {
            col_details.precision = ct.get_precision();
            col_details.scale = ct.get_scale();
          }
          if (col_details.encoding == ColumnEncoding::DICT) {
            // have to get the actual size of the encoding from the dictionary
            // definition
            const int dict_id = ct.get_comp_param();
            auto dd = catalog->getMetadataForDict(dict_id, false);
            if (dd) {
              col_details.comp_param = dd->dictNBits;
            } else {
              throw std::runtime_error("Dictionary definition for column doesn't exist");
            }
          } else {
            col_details.comp_param = ct.get_comp_param();
            if (ct.is_date_in_days() && col_details.comp_param == 0) {
              col_details.comp_param = 32;
            }
          }
          result.push_back(col_details);
        }
      }
    }
    return result;
  }

Here is the call graph for this function:

Here is the caller graph for this function:

std::vector<std::string> EmbeddedDatabase::DBEngineImpl::getTables ( )

inline

Definition at line 261 of file DBEngine.cpp.

References db_handler_, session_id_, and run_benchmark_import::tables.

Referenced by EmbeddedDatabase::DBEngine::getTables().

                                   {
    std::vector<std::string> table_names;
    auto catalog = db_handler_->get_session_copy(session_id_).get_catalog_ptr();
    if (catalog) {
      const auto tables = catalog->getAllTableMetadata();
      for (const auto td : tables) {
        if (td->shard >= 0) {
          // skip shards, they're not standalone tables
          continue;
        }
        table_names.push_back(td->tableName);
      }
    } else {
      throw std::runtime_error("System catalog uninitialized");
    }
    return table_names;
  }

Here is the caller graph for this function:

void EmbeddedDatabase::DBEngineImpl::importArrowTable ( const std::string &  name, std::shared_ptr< arrow::Table > &  table, uint64_t  fragment_size  )

inline

Definition at line 219 of file DBEngine.cpp.

References Data_Namespace::CPU_LEVEL, Catalog_Namespace::SysCatalog::createDBObject(), db_handler_, DEFAULT_FRAGMENT_ROWS, DEFAULT_MAX_CHUNK_SIZE, DEFAULT_MAX_ROWS, DEFAULT_PAGE_SIZE, TableDescriptor::fragmenter, TableDescriptor::fragPageSize, TableDescriptor::fragType, Fragmenter_Namespace::INSERT_ORDER, Catalog_Namespace::SysCatalog::instance(), TableDescriptor::isView, TableDescriptor::keyMetainfo, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRows, setup::name, TableDescriptor::persistenceLevel, releaseArrowTable(), session_id_, setArrowTable(), TableDescriptor::storageType, TableDBObjectType, TableDescriptor::tableName, and TableDescriptor::userId.

Referenced by EmbeddedDatabase::DBEngine::importArrowTable().

                                                {
    setArrowTable(name, table);
    try {
      auto session = db_handler_->get_session_copy(session_id_);
      TableDescriptor td;
      td.tableName = name;
      td.userId = session.get_currentUser().userId;
      td.storageType = "ARROW:" + name;
      td.persistenceLevel = Data_Namespace::MemoryLevel::CPU_LEVEL;
      td.isView = false;
      td.fragmenter = nullptr;
      td.fragType = Fragmenter_Namespace::FragmenterType::INSERT_ORDER;
      td.maxFragRows = fragment_size > 0 ? fragment_size : DEFAULT_FRAGMENT_ROWS;
      td.maxChunkSize = DEFAULT_MAX_CHUNK_SIZE;
      td.fragPageSize = DEFAULT_PAGE_SIZE;
      td.maxRows = DEFAULT_MAX_ROWS;
      td.keyMetainfo = "[]";

      std::list<ColumnDescriptor> cols;
      std::vector<Parser::SharedDictionaryDef> dictionaries;
      auto catalog = session.get_catalog_ptr();
      // nColumns
      catalog->createTable(td, cols, dictionaries, false);
      Catalog_Namespace::SysCatalog::instance().createDBObject(
          session.get_currentUser(), td.tableName, TableDBObjectType, *catalog);
    } catch (...) {
      releaseArrowTable(name);
      throw;
    }
    releaseArrowTable(name);
  }

Here is the call graph for this function:

Here is the caller graph for this function:

bool EmbeddedDatabase::DBEngineImpl::init ( const std::string &  cmd_line )

inline

Definition at line 104 of file DBEngine.cpp.

References CommandLineOptions::allow_loop_joins, CommandLineOptions::allow_multifrag, CommandLineOptions::authMetadata, CommandLineOptions::base_path, base_path_, SystemParameters::calcite_keepalive, catalogExists(), createCatalog(), db_handler_, CommandLineOptions::db_leaves, CommandLineOptions::disk_cache_config, CommandLineOptions::enable_auto_clear_render_mem, CommandLineOptions::enable_legacy_syntax, CommandLineOptions::enable_rendering, CommandLineOptions::enable_runtime_udf, logger::FATAL, g_serialize_temp_tables, CommandLineOptions::idle_session_duration, CommandLineOptions::init_logging(), CommandLineOptions::intel_jit_profile, CommandLineOptions::jit_debug, LOG, CommandLineOptions::max_concurrent_render_sessions, CommandLineOptions::max_session_duration, CommandLineOptions::num_reader_threads, OMNISCI_DEFAULT_DB, OMNISCI_ROOT_PASSWD_DEFAULT, OMNISCI_ROOT_USER, SystemParameters::omnisci_server_port, CommandLineOptions::parse_command_line(), CommandLineOptions::read_only, CommandLineOptions::render_compositor_use_last_gpu, CommandLineOptions::render_mem_bytes, CommandLineOptions::render_oom_retry_threshold, CommandLineOptions::renderer_use_vulkan_driver, CommandLineOptions::reserved_gpu_mem, session_id_, CommandLineOptions::string_leaves, CommandLineOptions::system_parameters, CommandLineOptions::udf_compiler_options, CommandLineOptions::udf_compiler_path, and CommandLineOptions::udf_file_name.

                                       {
    static bool initialized{false};
    if (initialized) {
      throw std::runtime_error("Database engine already initialized");
    }

    g_serialize_temp_tables = true;

    // Split the command line into parameters
    std::vector<std::string> parameters;
    if (!cmd_line.empty()) {
      parameters = boost::program_options::split_unix(cmd_line);
    }

    // Generate command line to initialize CommandLineOptions for DBHandler
    const char* log_option = "omnisci_dbe";
    std::vector<const char*> cstrings;
    cstrings.push_back(log_option);
    for (auto& param : parameters) {
      cstrings.push_back(param.c_str());
    }
    int argc = cstrings.size();
    const char** argv = cstrings.data();

    CommandLineOptions prog_config_opts(log_option);
    if (prog_config_opts.parse_command_line(argc, argv, false)) {
      throw std::runtime_error("DBE paramerameters parsing failed");
    }

    auto base_path = prog_config_opts.base_path;

    // Check path to the database
    bool is_new_db = base_path.empty() || !catalogExists(base_path);
    if (is_new_db) {
      base_path = createCatalog(base_path);
      if (base_path.empty()) {
        throw std::runtime_error("Database directory could not be created");
      }
    }
    prog_config_opts.base_path = base_path;
    prog_config_opts.init_logging();

    prog_config_opts.system_parameters.omnisci_server_port = -1;
    prog_config_opts.system_parameters.calcite_keepalive = true;

    try {
      db_handler_ =
          mapd::make_shared<DBHandler>(prog_config_opts.db_leaves,
                                       prog_config_opts.string_leaves,
                                       prog_config_opts.base_path,
                                       prog_config_opts.allow_multifrag,
                                       prog_config_opts.jit_debug,
                                       prog_config_opts.intel_jit_profile,
                                       prog_config_opts.read_only,
                                       prog_config_opts.allow_loop_joins,
                                       prog_config_opts.enable_rendering,
                                       prog_config_opts.renderer_use_vulkan_driver,
                                       prog_config_opts.enable_auto_clear_render_mem,
                                       prog_config_opts.render_oom_retry_threshold,
                                       prog_config_opts.render_mem_bytes,
                                       prog_config_opts.max_concurrent_render_sessions,
                                       prog_config_opts.reserved_gpu_mem,
                                       prog_config_opts.render_compositor_use_last_gpu,
                                       prog_config_opts.num_reader_threads,
                                       prog_config_opts.authMetadata,
                                       prog_config_opts.system_parameters,
                                       prog_config_opts.enable_legacy_syntax,
                                       prog_config_opts.idle_session_duration,
                                       prog_config_opts.max_session_duration,
                                       prog_config_opts.enable_runtime_udf,
                                       prog_config_opts.udf_file_name,
                                       prog_config_opts.udf_compiler_path,
                                       prog_config_opts.udf_compiler_options,
#ifdef ENABLE_GEOS
                                       prog_config_opts.libgeos_so_filename,
#endif
                                       prog_config_opts.disk_cache_config,
                                       is_new_db);
    } catch (const std::exception& e) {
      LOG(FATAL) << "Failed to initialize database handler: " << e.what();
    }
    db_handler_->connect(
        session_id_, OMNISCI_ROOT_USER, OMNISCI_ROOT_PASSWD_DEFAULT, OMNISCI_DEFAULT_DB);
    base_path_ = base_path;
    initialized = true;
    return true;
  }

Here is the call graph for this function:

bool EmbeddedDatabase::DBEngineImpl::login ( std::string &  db_name, std::string &  user_name, const std::string &  password  )

inline

Definition at line 338 of file DBEngine.cpp.

References db_handler_, and session_id_.

Referenced by EmbeddedDatabase::DBEngine::login().

                                                                                  {
    db_handler_->disconnect(session_id_);
    db_handler_->connect(session_id_, user_name, password, db_name);
    return true;
  }

Here is the caller graph for this function:

void EmbeddedDatabase::DBEngineImpl::reset ( )

inlineprotected

Definition at line 345 of file DBEngine.cpp.

References base_path_, db_handler_, Catalog_Namespace::SysCatalog::destroy(), is_temp_db_, session_id_, and logger::shutdown().

Referenced by ~DBEngineImpl().

               {
    if (db_handler_) {
      db_handler_->disconnect(session_id_);
      db_handler_->shutdown();
    }
    Catalog_Namespace::SysCatalog::destroy();
    db_handler_.reset();

    logger::shutdown();
    if (is_temp_db_) {
      boost::filesystem::remove_all(base_path_);
    }
    base_path_.clear();
  }

Here is the call graph for this function:

Here is the caller graph for this function:

bool EmbeddedDatabase::DBEngineImpl::setDatabase ( std::string &  db_name )

inline

Definition at line 331 of file DBEngine.cpp.

References db_handler_, Catalog_Namespace::SysCatalog::instance(), and session_id_.

Referenced by EmbeddedDatabase::DBEngine::setDatabase().

                                       {
    auto& sys_cat = Catalog_Namespace::SysCatalog::instance();
    auto& user = db_handler_->get_session_copy(session_id_).get_currentUser();
    sys_cat.switchDatabase(db_name, user.userName);
    return true;
  }

Here is the call graph for this function:

Here is the caller graph for this function:

std::shared_ptr<CursorImpl> EmbeddedDatabase::DBEngineImpl::sql_execute_dbe ( const TSessionId &  session_id, const std::string &  query_str, const bool  column_format, const int32_t  first_n, const int32_t  at_most_n  )

inline

Definition at line 192 of file DBEngine.cpp.

References CPU, db_handler_, and run_benchmark_import::result.

Referenced by executeDDL(), executeDML(), and executeRA().

                                                                       {
    ExecutionResult result{std::make_shared<ResultSet>(std::vector<TargetInfo>{},
                                                       ExecutorDeviceType::CPU,
                                                       QueryMemoryDescriptor(),
                                                       nullptr,
                                                       nullptr,
                                                       0,
                                                       0),
                           {}};
    db_handler_->sql_execute(
        result, session_id, query_str, column_format, first_n, at_most_n);
    auto& targets = result.getTargetsMeta();
    std::vector<std::string> col_names;
    for (const auto target : targets) {
      col_names.push_back(target.get_resname());
    }
    return std::make_shared<CursorImpl>(result.getRows(), col_names);
  }

Here is the caller graph for this function:

Member Data Documentation

std::string EmbeddedDatabase::DBEngineImpl::base_path_

private

Definition at line 427 of file DBEngine.cpp.

Referenced by init(), and reset().

mapd::shared_ptr<DBHandler> EmbeddedDatabase::DBEngineImpl::db_handler_

private

Definition at line 429 of file DBEngine.cpp.

Referenced by getTableDetails(), getTables(), importArrowTable(), init(), login(), reset(), setDatabase(), and sql_execute_dbe().

bool EmbeddedDatabase::DBEngineImpl::is_temp_db_

private

Definition at line 430 of file DBEngine.cpp.

Referenced by createCatalog(), and reset().

std::string EmbeddedDatabase::DBEngineImpl::session_id_

private

Definition at line 428 of file DBEngine.cpp.

Referenced by executeDDL(), executeDML(), executeRA(), getTableDetails(), getTables(), importArrowTable(), init(), login(), reset(), and setDatabase().

std::vector<std::string> EmbeddedDatabase::DBEngineImpl::system_folders_

private

Initial value:

= {"mapd_catalogs",
                                              "mapd_data",
                                              "mapd_export"}

Definition at line 433 of file DBEngine.cpp.

Referenced by catalogExists(), cleanCatalog(), and createCatalog().

std::string EmbeddedDatabase::DBEngineImpl::udf_filename_

private

Definition at line 431 of file DBEngine.cpp.

---

The documentation for this class was generated from the following file:

• /home/jenkins-slave/workspace/core-os-doxygen/Embedded/DBEngine.cpp