18 #include <boost/filesystem.hpp>
36 namespace EmbeddedDatabase {
45 CursorImpl(std::shared_ptr<ResultSet> result_set, std::vector<std::string> col_names)
61 return row.empty() ?
Row() :
Row(row);
106 bool init(
const std::string& cmd_line) {
107 static bool initialized{
false};
109 throw std::runtime_error(
"Database engine already initialized");
115 std::vector<std::string> parameters;
116 if (!cmd_line.empty()) {
117 parameters = boost::program_options::split_unix(cmd_line);
121 const char* log_option =
"omnisci_dbe";
122 std::vector<const char*> cstrings;
123 cstrings.push_back(log_option);
124 for (
auto& param : parameters) {
125 cstrings.push_back(param.c_str());
127 int argc = cstrings.size();
128 const char** argv = cstrings.data();
132 throw std::runtime_error(
"DBE paramerameters parsing failed");
136 apache::thrift::GlobalOutput.setOutputFunction([](
const char* msg) {});
139 auto base_path = prog_config_opts.
base_path;
142 bool is_new_db = base_path.empty() || !
catalogExists(base_path);
145 if (base_path.empty()) {
146 throw std::runtime_error(
"Database directory could not be created");
157 std::make_shared<DBHandler>(prog_config_opts.
db_leaves,
185 prog_config_opts.libgeos_so_filename,
189 }
catch (
const std::exception& e) {
190 LOG(
FATAL) <<
"Failed to initialize database handler: " << e.what();
202 const std::string& query_str,
203 const bool column_format,
204 const int32_t first_n,
205 const int32_t at_most_n) {
215 result, session_id, query_str, column_format, first_n, at_most_n, locks);
216 auto& targets =
result.getTargetsMeta();
217 std::vector<std::string> col_names;
218 for (
const auto target : targets) {
219 col_names.push_back(target.get_resname());
221 return std::make_shared<CursorImpl>(
result.getRows(), col_names);
229 std::shared_ptr<arrow::Table>& table,
230 uint64_t fragment_size) {
236 td.
userId = session.get_currentUser().userId;
248 std::list<ColumnDescriptor> cols;
249 std::vector<Parser::SharedDictionaryDef> dictionaries;
250 auto catalog = session.get_catalog_ptr();
252 catalog->createTable(td, cols, dictionaries,
false);
262 std::shared_ptr<CursorImpl>
executeDML(
const std::string& query) {
266 std::shared_ptr<CursorImpl>
executeRA(
const std::string& query) {
271 std::vector<std::string> table_names;
274 const auto tables = catalog->getAllTableMetadata();
275 for (
const auto td :
tables) {
276 if (td->shard >= 0) {
280 table_names.push_back(td->tableName);
283 throw std::runtime_error(
"System catalog uninitialized");
289 std::vector<ColumnDetails>
result;
292 auto metadata = catalog->getMetadataForTable(table_name,
false);
294 const auto col_descriptors =
295 catalog->getAllColumnMetadataForTable(metadata->tableId,
false,
true,
false);
296 const auto deleted_cd = catalog->getDeletedColumn(metadata);
297 for (
const auto cd : col_descriptors) {
298 if (cd == deleted_cd) {
302 col_details.
col_name = cd->columnName;
303 auto ct = cd->columnType;
308 col_details.
nullable = !ct.get_notnull();
311 col_details.
precision =
static_cast<int>(ct.get_subtype());
312 col_details.
scale = ct.get_output_srid();
314 col_details.
precision = ct.get_precision();
315 col_details.
scale = ct.get_scale();
320 const int dict_id = ct.get_comp_param();
321 auto dd = catalog->getMetadataForDict(dict_id,
false);
325 throw std::runtime_error(
"Dictionary definition for column doesn't exist");
329 if (ct.is_date_in_days() && col_details.
comp_param == 0) {
333 result.push_back(col_details);
343 sys_cat.switchDatabase(db_name, user.userName);
347 bool login(std::string& db_name, std::string& user_name,
const std::string& password) {
370 if (!boost::filesystem::exists(base_path)) {
374 std::string path = base_path +
"/" + subdir;
375 if (!boost::filesystem::exists(path)) {
383 if (boost::filesystem::exists(base_path)) {
385 std::string path = base_path +
"/" + subdir;
386 if (boost::filesystem::exists(path)) {
387 boost::filesystem::remove_all(path);
394 std::string root_dir = base_path;
395 if (base_path.empty()) {
396 boost::system::error_code error;
397 auto tmp_path = boost::filesystem::temp_directory_path(error);
398 if (boost::system::errc::success != error.value()) {
399 std::cerr << error.message() << std::endl;
402 tmp_path /=
"omnidbe_%%%%-%%%%-%%%%";
403 auto uniq_path = boost::filesystem::unique_path(tmp_path, error);
404 if (boost::system::errc::success != error.value()) {
405 std::cerr << error.message() << std::endl;
408 root_dir = uniq_path.string();
411 if (!boost::filesystem::exists(root_dir)) {
412 if (!boost::filesystem::create_directory(root_dir)) {
413 std::cerr <<
"Cannot create database directory: " << root_dir << std::endl;
417 size_t absent_count = 0;
419 std::string path = root_dir +
"/" + sub_dir;
420 if (!boost::filesystem::exists(path)) {
421 if (!boost::filesystem::create_directory(path)) {
422 std::cerr <<
"Cannot create database subdirectory: " << path << std::endl;
428 if ((absent_count > 0) && (absent_count < system_folders_.size())) {
429 std::cerr <<
"Database directory structure is broken: " << root_dir << std::endl;
453 auto engine = std::make_shared<DBEngineImpl>();
454 if (!
engine->init(cmd_line)) {
455 throw std::runtime_error(
"DBE initialization failed");
487 std::shared_ptr<arrow::Table>& table,
488 uint64_t fragment_size) {
509 std::string& user_name,
510 const std::string& password) {
512 return engine->
login(db_name, user_name, password);
Classes used to wrap parser calls for calcite redirection.
std::vector< std::unique_ptr< lockmgr::AbstractLockContainer< const TableDescriptor * >>> LockedTableDescriptors
const std::string kDataDirectoryName
bool setDatabase(std::string &db_name)
std::shared_ptr< DBHandler > db_handler_
std::shared_ptr< CursorImpl > sql_execute_dbe(const TSessionId &session_id, const std::string &query_str, const bool column_format, const int32_t first_n, const int32_t at_most_n)
int idle_session_duration
unsigned renderer_vulkan_timeout_ms
std::vector< LeafHostInfo > string_leaves
std::string udf_compiler_path
void importArrowTable(const std::string &name, std::shared_ptr< arrow::Table > &table, uint64_t fragment_size=0)
std::string udf_filename_
std::shared_ptr< arrow::RecordBatch > getArrowRecordBatch()
bool renderer_use_ppll_polys
std::string udf_file_name
#define DEFAULT_MAX_CHUNK_SIZE
bool setDatabase(std::string &db_name)
std::vector< ColumnDetails > getTableDetails(const std::string &table_name)
void setArrowTable(std::string name, std::shared_ptr< arrow::Table > table)
void executeDDL(const std::string &query)
bool init(const std::string &cmd_line)
bool catalogExists(const std::string &base_path)
boost::optional< int > parse_command_line(int argc, char const *const *argv, const bool should_init_logging=false)
const std::string kDefaultExportDirName
bool render_compositor_use_last_gpu
bool renderer_prefer_igpu
HOST DEVICE SQLTypes get_type() const
void createDBObject(const UserMetadata &user, const std::string &objectName, DBObjectType type, const Catalog_Namespace::Catalog &catalog, int32_t objectId=-1)
void releaseArrowTable(std::string name)
ColumnType getColType(uint32_t col_num)
std::vector< ColumnDetails > getTableDetails(const std::string &table_name)
ColumnType sqlToColumnType(const SQLTypes &type)
bool login(std::string &db_name, std::string &user_name, const std::string &password)
size_t max_concurrent_render_sessions
Supported runtime functions management and retrieval.
static SysCatalog & instance()
std::shared_ptr< Cursor > executeDML(const std::string &query)
size_t num_reader_threads
std::shared_ptr< ResultSet > result_set_
std::vector< LeafHostInfo > db_leaves
const std::string kDefaultDbName
CursorImpl(std::shared_ptr< ResultSet > result_set, std::vector< std::string > col_names)
bool enable_auto_clear_render_mem
void importArrowTable(const std::string &name, std::shared_ptr< arrow::Table > &table, uint64_t fragment_size)
int render_oom_retry_threshold
std::shared_ptr< arrow::RecordBatch > record_batch_
std::shared_ptr< CursorImpl > executeRA(const std::string &query)
std::shared_ptr< arrow::RecordBatch > getArrowRecordBatch()
std::vector< std::string > col_names_
std::string createCatalog(const std::string &base_path)
AuthMetadata authMetadata
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
bool g_serialize_temp_tables
#define DEFAULT_PAGE_SIZE
const std::string kRootUsername
std::mutex engine_create_mutex
std::vector< std::string > getTables()
static std::shared_ptr< DBEngine > create(const std::string &cmd_line)
const std::string kDefaultRootPasswd
#define DEFAULT_FRAGMENT_ROWS
Fragmenter_Namespace::FragmenterType fragType
Data_Namespace::MemoryLevel persistenceLevel
std::vector< std::string > udf_compiler_options
ColumnEncoding sqlToColumnEncoding(const EncodingType &type)
const std::string kCatalogDirectoryName
bool login(std::string &db_name, std::string &user_name, const std::string &password)
bool g_enable_watchdog false
ColumnType getColType(uint32_t col_num)
DBEngineImpl * getImpl(DBEngine *ptr)
File_Namespace::DiskCacheConfig disk_cache_config
void executeDDL(const std::string &query)
bool g_enable_thrift_logs
std::vector< std::string > system_folders_
std::shared_ptr< CursorImpl > executeDML(const std::string &query)
bool enable_legacy_syntax
std::shared_ptr< Cursor > executeRA(const std::string &query)
void cleanCatalog(const std::string &base_path)
SystemParameters system_parameters
std::vector< std::string > getTables()