26 #include <boost/algorithm/string/predicate.hpp>
27 #include <boost/filesystem.hpp>
28 #include <boost/range/adaptor/map.hpp>
29 #include <boost/version.hpp>
42 #if BOOST_VERSION >= 106600
43 #include <boost/uuid/detail/sha1.hpp>
45 #include <boost/uuid/sha1.hpp>
47 #include <rapidjson/document.h>
48 #include <rapidjson/istreamwrapper.h>
49 #include <rapidjson/ostreamwrapper.h>
50 #include <rapidjson/writer.h>
80 #include "MapDRelease.h"
92 using std::runtime_error;
108 namespace Catalog_Namespace {
134 "SELECT name FROM sqlite_master WHERE type='table' AND name='mapd_dashboards'");
141 "CREATE TABLE mapd_dashboards (id integer primary key autoincrement, name text , "
142 "userid integer references mapd_users, state text, image_hash text, update_time "
144 "metadata text, UNIQUE(userid, name) )");
147 "insert into mapd_dashboards (id, name , "
148 "userid, state, image_hash, update_time , "
150 "SELECT viewid , name , userid, view_state, image_hash, update_time, "
152 "from mapd_frontend_views");
153 }
catch (
const std::exception& e) {
163 const std::string& db_name) {
165 db_name +
"_temp_tables.json");
175 std::shared_ptr<Data_Namespace::DataMgr> dataMgr,
176 const std::vector<LeafHostInfo>& string_dict_hosts,
177 std::shared_ptr<Calcite> calcite,
179 : basePath_(basePath)
183 , string_dict_hosts_(string_dict_hosts)
184 , calciteMgr_(calcite)
187 , dcatalogMutex_(std::make_unique<heavyai::DistributedSharedMutex>(
199 ,
dsqliteMutex_(std::make_unique<heavyai::DistributedSharedMutex>(
212 CheckAndExecuteMigrations();
218 createDefaultServersIfNotExists();
221 CheckAndExecuteMigrationsPostBuildMaps();
226 conditionallyInitializeSystemObjects();
238 tableDescIt->second->fragmenter =
nullptr;
239 delete tableDescIt->second;
247 delete columnDescIt->second;
270 std::vector<std::string> cols;
274 if (std::find(cols.begin(), cols.end(), std::string(
"max_chunk_size")) ==
276 string queryString(
"ALTER TABLE mapd_tables ADD max_chunk_size BIGINT DEFAULT " +
280 if (std::find(cols.begin(), cols.end(), std::string(
"shard_column_id")) ==
282 string queryString(
"ALTER TABLE mapd_tables ADD shard_column_id BIGINT DEFAULT " +
286 if (std::find(cols.begin(), cols.end(), std::string(
"shard")) == cols.end()) {
287 string queryString(
"ALTER TABLE mapd_tables ADD shard BIGINT DEFAULT " +
291 if (std::find(cols.begin(), cols.end(), std::string(
"num_shards")) == cols.end()) {
292 string queryString(
"ALTER TABLE mapd_tables ADD num_shards BIGINT DEFAULT " +
296 if (std::find(cols.begin(), cols.end(), std::string(
"key_metainfo")) == cols.end()) {
297 string queryString(
"ALTER TABLE mapd_tables ADD key_metainfo TEXT DEFAULT '[]'");
300 if (std::find(cols.begin(), cols.end(), std::string(
"userid")) == cols.end()) {
301 string queryString(
"ALTER TABLE mapd_tables ADD userid integer DEFAULT " +
305 if (std::find(cols.begin(), cols.end(), std::string(
"sort_column_id")) ==
308 "ALTER TABLE mapd_tables ADD sort_column_id INTEGER DEFAULT 0");
310 if (std::find(cols.begin(), cols.end(), std::string(
"storage_type")) == cols.end()) {
311 string queryString(
"ALTER TABLE mapd_tables ADD storage_type TEXT DEFAULT ''");
314 if (std::find(cols.begin(), cols.end(), std::string(
"max_rollback_epochs")) ==
316 string queryString(
"ALTER TABLE mapd_tables ADD max_rollback_epochs INT DEFAULT " +
320 if (std::find(cols.begin(), cols.end(), std::string(
"is_system_table")) ==
322 string queryString(
"ALTER TABLE mapd_tables ADD is_system_table BOOLEAN DEFAULT 0");
325 }
catch (std::exception& e) {
337 "select name from sqlite_master WHERE type='table' AND "
338 "name='mapd_version_history'");
341 "CREATE TABLE mapd_version_history(version integer, migration_history text "
345 "select * from mapd_version_history where migration_history = "
346 "'notnull_fixlen_arrays'");
356 "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
358 LOG(
INFO) <<
"Updating mapd_columns, legacy fixlen arrays";
360 string queryString(
"UPDATE mapd_columns SET is_notnull=1 WHERE coltype=" +
363 }
catch (std::exception& e) {
375 "select name from sqlite_master WHERE type='table' AND "
376 "name='mapd_version_history'");
379 "CREATE TABLE mapd_version_history(version integer, migration_history text "
383 "select * from mapd_version_history where migration_history = "
384 "'notnull_geo_columns'");
394 "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
396 LOG(
INFO) <<
"Updating mapd_columns, legacy geo columns";
405 }
catch (std::exception& e) {
418 "SELECT name FROM sqlite_master WHERE type='table' AND "
419 "name='mapd_frontend_views'");
427 std::vector<std::string> cols;
431 if (std::find(cols.begin(), cols.end(), std::string(
"image_hash")) == cols.end()) {
434 if (std::find(cols.begin(), cols.end(), std::string(
"update_time")) == cols.end()) {
437 if (std::find(cols.begin(), cols.end(), std::string(
"view_metadata")) == cols.end()) {
440 }
catch (std::exception& e) {
452 "CREATE TABLE IF NOT EXISTS mapd_links (linkid integer primary key, userid "
453 "integer references mapd_users, "
454 "link text unique, view_state text, update_time timestamp, view_metadata text)");
456 std::vector<std::string> cols;
460 if (std::find(cols.begin(), cols.end(), std::string(
"view_metadata")) == cols.end()) {
463 }
catch (
const std::exception& e) {
477 "SELECT name FROM sqlite_master WHERE type='table' AND "
478 "name='mapd_frontend_views'");
486 "UPDATE mapd_frontend_views SET userid = 0 WHERE userid IS NULL");
487 }
catch (
const std::exception& e) {
507 std::vector<std::string> cols;
511 if (std::find(cols.begin(), cols.end(), std::string(
"version_num")) == cols.end()) {
512 LOG(
INFO) <<
"Updating mapd_tables updatePageSize";
517 string queryString(
"ALTER TABLE mapd_tables ADD version_num BIGINT DEFAULT " +
521 }
catch (std::exception& e) {
533 std::vector<std::string> cols;
537 if (std::find(cols.begin(), cols.end(), std::string(
"version_num")) == cols.end()) {
538 LOG(
INFO) <<
"Updating mapd_columns updateDeletedColumnIndicator";
540 string queryString(
"ALTER TABLE mapd_columns ADD version_num BIGINT DEFAULT " +
546 "ALTER TABLE mapd_columns ADD is_deletedcol boolean default 0 ");
548 }
catch (std::exception& e) {
560 std::vector<std::string> cols;
564 if (std::find(cols.begin(), cols.end(), std::string(
"default_value")) == cols.end()) {
565 LOG(
INFO) <<
"Adding support for default values to mapd_columns";
568 }
catch (std::exception& e) {
569 LOG(
ERROR) <<
"Failed to make metadata update for default values` support";
588 std::vector<std::string> cols;
592 if (std::find(cols.begin(), cols.end(), std::string(
"version_num")) == cols.end()) {
595 string dictQuery(
"SELECT dictid, name from mapd_dictionaries");
598 for (
size_t r = 0; r < numRows; ++r) {
608 int result = rename(oldName.c_str(), newName.c_str());
611 LOG(
INFO) <<
"Dictionary upgrade: successfully renamed " << oldName <<
" to "
614 LOG(
ERROR) <<
"Failed to rename old dictionary directory " << oldName <<
" to "
620 string queryString(
"ALTER TABLE mapd_dictionaries ADD version_num BIGINT DEFAULT " +
624 }
catch (std::exception& e) {
636 "CREATE TABLE IF NOT EXISTS mapd_logical_to_physical("
637 "logical_table_id integer, physical_table_id integer)");
638 }
catch (
const std::exception& e) {
655 const auto physicalTables = physicalTableIt->second;
656 CHECK(!physicalTables.empty());
657 for (
size_t i = 0; i < physicalTables.size(); i++) {
658 int32_t physical_tb_id = physicalTables[i];
660 "INSERT OR REPLACE INTO mapd_logical_to_physical (logical_table_id, "
661 "physical_table_id) VALUES (?1, ?2)",
666 }
catch (std::exception& e) {
678 std::vector<std::string> cols;
682 if (std::find(cols.begin(), cols.end(), std::string(
"refcount")) == cols.end()) {
685 }
catch (std::exception& e) {
698 }
catch (std::exception& e) {
711 "select name from sqlite_master WHERE type='table' AND "
712 "name='mapd_version_history'");
713 static const std::string migration_name{
"rename_legacy_data_wrappers"};
716 "CREATE TABLE mapd_version_history(version integer, migration_history text "
720 "select * from mapd_version_history where migration_history = "
722 migration_name +
"'");
729 LOG(
INFO) <<
"Executing " << migration_name <<
" migration.";
734 std::map<std::string, std::string> old_to_new_wrapper_names{
735 {
"OMNISCI_CSV", DataWrapperType::CSV},
736 {
"OMNISCI_PARQUET", DataWrapperType::PARQUET},
737 {
"OMNISCI_REGEX_PARSER", DataWrapperType::REGEX_PARSER},
738 {
"OMNISCI_INTERNAL_CATALOG", DataWrapperType::INTERNAL_CATALOG},
739 {
"INTERNAL_OMNISCI_MEMORY_STATS", DataWrapperType::INTERNAL_MEMORY_STATS},
740 {
"INTERNAL_OMNISCI_STORAGE_STATS", DataWrapperType::INTERNAL_STORAGE_STATS}
744 for (
const auto& [old_wrapper_name, new_wrapper_name] : old_to_new_wrapper_names) {
746 "UPDATE omnisci_foreign_servers SET data_wrapper_type = ? WHERE "
747 "data_wrapper_type = ?",
748 std::vector<std::string>{new_wrapper_name, old_wrapper_name});
753 "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
755 LOG(
INFO) << migration_name <<
" migration completed.";
756 }
catch (std::exception& e) {
768 }
catch (
const std::exception& e) {
776 return "CREATE TABLE " + (if_not_exists ? std::string{
"IF NOT EXISTS "} :
"") +
777 "omnisci_foreign_servers(id integer primary key, name text unique, " +
778 "data_wrapper_type text, owner_user_id integer, creation_time integer, " +
783 return "CREATE TABLE " + (if_not_exists ? std::string{
"IF NOT EXISTS "} :
"") +
784 "omnisci_foreign_tables(table_id integer unique, server_id integer, " +
785 "options text, last_refresh_time integer, next_refresh_time integer, " +
786 "FOREIGN KEY(table_id) REFERENCES mapd_tables(tableid), " +
787 "FOREIGN KEY(server_id) REFERENCES omnisci_foreign_servers(id))";
791 return "CREATE TABLE " + (if_not_exists ? std::string{
"IF NOT EXISTS "} :
"") +
792 "omnisci_custom_expressions(id integer primary key, name text, " +
793 "expression_json text, data_source_type text, " +
794 "data_source_id integer, is_deleted boolean)";
800 std::vector<DBObject> objects;
803 "SELECT name FROM sqlite_master WHERE type='table' AND "
804 "name='mapd_record_ownership_marker'");
831 "INSERT INTO mapd_record_ownership_marker (dummy) VALUES (?1)",
834 static const std::map<const DBObjectType, const AccessPrivileges>
835 object_level_all_privs_lookup{
845 auto _key_place = [&key](
auto type) {
849 for (
auto& it : object_level_all_privs_lookup) {
850 objects.emplace_back(_key_place(it.first), it.second, db.
dbOwner);
857 "SELECT tableid, name, userid, isview FROM mapd_tables WHERE userid > 0");
860 for (
size_t r = 0; r < numRows; ++r) {
879 objects.push_back(obj);
885 string tableQuery(
"SELECT id, name, userid FROM mapd_dashboards WHERE userid > 0");
888 for (
size_t r = 0; r < numRows; ++r) {
904 objects.push_back(obj);
907 }
catch (
const std::exception& e) {
917 }
catch (
const std::exception& e) {
918 LOG(
ERROR) <<
" Issue during migration of DB " <<
name() <<
" issue was " << e.what();
919 throw std::runtime_error(
" Issue during migration of DB " +
name() +
" issue was " +
932 std::unordered_map<std::string, std::pair<int, std::string>> dashboards;
933 std::vector<std::string> dashboard_ids;
934 static const std::string migration_name{
"dashboard_roles_migration"};
942 "select * from mapd_version_history where migration_history = '" +
943 migration_name +
"'");
949 LOG(
INFO) <<
"Performing dashboard internal roles Migration.";
963 }
catch (
const std::exception& e) {
970 const auto active_grantees =
975 for (
auto dash : dashboards) {
980 auto result = active_grantees.find(dash.first);
981 if (
result != active_grantees.end()) {
991 "select * from mapd_version_history where migration_history = '" +
992 migration_name +
"'");
997 "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
999 }
catch (
const std::exception& e) {
1000 LOG(
ERROR) <<
"Failed to create dashboard system roles during migration: "
1004 LOG(
INFO) <<
"Successfully created dashboard system roles during migration.";
1039 std::map<int32_t, std::string> user_name_by_user_id;
1040 for (
const auto& user : users) {
1041 user_name_by_user_id[user.userId] = user.userName;
1043 return user_name_by_user_id;
1048 const std::map<int32_t, std::string>& user_name_by_user_id) {
1049 auto entry = user_name_by_user_id.find(
id);
1050 if (entry != user_name_by_user_id.end()) {
1051 return entry->second;
1060 if (column_type.is_dict_encoded_string() ||
1061 column_type.is_subtype_dict_encoded_string()) {
1068 std::string dictQuery(
1069 "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
1072 for (
size_t r = 0; r < numRows; ++r) {
1082 dict_ref, dictName, dictNBits, is_shared, refcount, fname,
false);
1099 list<DictDescriptor> dicts;
1100 std::string dictQuery(
1101 "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
1104 for (
size_t r = 0; r < numRows; ++r) {
1113 DictDescriptor dd(dict_ref, dictName, dictNBits, is_shared, refcount, fname,
false);
1124 std::list<ColumnDescriptor*> original_cds;
1127 original_td = it1->second;
1131 CHECK_EQ(original_td, it2->second);
1138 for (
int column_id = 0; column_id < original_td->
nColumns; ++column_id) {
1142 original_cds.push_back(original_cd);
1149 std::string tableQuery(
1150 "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
1151 "max_chunk_size, frag_page_size, max_rows, partitions, shard_column_id, shard, "
1152 "num_shards, key_metainfo, userid, sort_column_id, storage_type, "
1153 "max_rollback_epochs, is_system_table from mapd_tables WHERE tableid = " +
1155 sqliteConnector_.query(tableQuery);
1156 numRows = sqliteConnector_.getNumRows();
1162 const auto& storage_type = sqliteConnector_.getData<
string>(0, 17);
1164 const auto& table_name = sqliteConnector_.getData<
string>(0, 1);
1165 LOG(
FATAL) <<
"Unable to read Catalog metadata: storage type is currently not a "
1166 "supported table option (table "
1167 << table_name <<
" [" << table_id <<
"] in database " << currentDB_.dbName
1177 td->storageType = storage_type;
1178 td->tableId = sqliteConnector_.getData<
int>(0, 0);
1179 td->tableName = sqliteConnector_.getData<
string>(0, 1);
1180 td->nColumns = sqliteConnector_.getData<
int>(0, 2);
1181 td->isView = sqliteConnector_.getData<
bool>(0, 3);
1182 td->fragments = sqliteConnector_.getData<
string>(0, 4);
1185 td->maxFragRows = sqliteConnector_.getData<
int>(0, 6);
1186 td->maxChunkSize = sqliteConnector_.getData<int64_t>(0, 7);
1187 td->fragPageSize = sqliteConnector_.getData<
int>(0, 8);
1188 td->maxRows = sqliteConnector_.getData<int64_t>(0, 9);
1189 td->partitions = sqliteConnector_.getData<
string>(0, 10);
1190 td->shardedColumnId = sqliteConnector_.getData<
int>(0, 11);
1191 td->shard = sqliteConnector_.getData<
int>(0, 12);
1192 td->nShards = sqliteConnector_.getData<
int>(0, 13);
1193 td->keyMetainfo = sqliteConnector_.getData<
string>(0, 14);
1194 td->userId = sqliteConnector_.getData<
int>(0, 15);
1195 td->sortedColumnId =
1196 sqliteConnector_.isNull(0, 16) ? 0 : sqliteConnector_.getData<
int>(0, 16);
1198 td->fragmenter =
nullptr;
1200 td->maxRollbackEpochs = sqliteConnector_.getData<
int>(0, 18);
1201 td->is_system_table = sqliteConnector_.getData<
bool>(0, 19);
1202 td->hasDeletedCol =
false;
1207 updateViewUnlocked(*td);
1210 if (
auto tableDescIt = tableDescriptorMapById_.find(table_id);
1211 tableDescIt != tableDescriptorMapById_.end()) {
1212 tableDescIt->second->fragmenter =
nullptr;
1213 delete tableDescIt->second;
1217 std::list<ColumnDescriptor*> cds;
1218 std::string columnQuery(
1219 "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
1220 "is_notnull, compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
1221 "virtual_expr, is_deletedcol, default_value from mapd_columns WHERE tableid = " +
1223 sqliteConnector_.query(columnQuery);
1224 numRows = sqliteConnector_.getNumRows();
1225 int32_t skip_physical_cols = 0;
1226 for (
size_t r = 0; r < numRows; ++r) {
1228 cd->
tableId = sqliteConnector_.getData<
int>(r, 0);
1229 cd->
columnId = sqliteConnector_.getData<
int>(r, 1);
1230 cd->
columnName = sqliteConnector_.getData<
string>(r, 2);
1239 cd->
chunks = sqliteConnector_.getData<
string>(r, 11);
1240 cd->
isSystemCol = sqliteConnector_.getData<
bool>(r, 12);
1241 cd->
isVirtualCol = sqliteConnector_.getData<
bool>(r, 13);
1242 cd->
virtualExpr = sqliteConnector_.getData<
string>(r, 14);
1243 cd->
isDeletedCol = sqliteConnector_.getData<
bool>(r, 15);
1244 if (sqliteConnector_.isNull(r, 16)) {
1247 cd->
default_value = std::make_optional(sqliteConnector_.getData<
string>(r, 16));
1250 cd->
db_id = getDatabaseId();
1257 td->mutex_ = original_td->mutex_;
1259 original_td =
nullptr;
1264 original_cds.clear();
1265 tableDescriptorMap_[
to_upper(td->tableName)] = td;
1266 tableDescriptorMapById_[td->tableId] = td;
1267 skip_physical_cols = 0;
1271 if (skip_physical_cols <= 0) {
1276 td->hasDeletedCol =
true;
1277 setDeletedColumnUnlocked(td, cd);
1279 td->columnIdBySpi_.push_back(cd->
columnId);
1284 calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
1288 void Catalog::reloadCatalogMetadata(
1289 const std::map<int32_t, std::string>& user_name_by_user_id) {
1296 void Catalog::reloadCatalogMetadataUnlocked(
1297 const std::map<int32_t, std::string>& user_name_by_user_id) {
1304 std::set<int> cluster_table_ids;
1305 std::string tableQuery(
"SELECT tableid from mapd_tables");
1306 sqliteConnector_.query(tableQuery);
1307 auto numRows = sqliteConnector_.getNumRows();
1308 for (
size_t r = 0; r < numRows; ++r) {
1309 const auto table_id = sqliteConnector_.getData<
int>(r, 0);
1310 cluster_table_ids.insert(table_id);
1315 std::set<int> ignored_table_ids;
1328 std::set<int> reload_table_ids;
1329 for (
auto const& cluster_table_id : cluster_table_ids) {
1330 if (ignored_table_ids.find(cluster_table_id) == ignored_table_ids.end()) {
1331 reload_table_ids.insert(cluster_table_id);
1334 for (
auto const& [cached_table_id, td] : tableDescriptorMapById_) {
1335 if (cluster_table_ids.find(cached_table_id) == cluster_table_ids.end()) {
1336 reload_table_ids.insert(cached_table_id);
1341 for (
auto const& reload_table_id : reload_table_ids) {
1342 reloadTableMetadataUnlocked(reload_table_id);
1347 dashboardDescriptorMap_.clear();
1348 linkDescriptorMap_.clear();
1349 linkDescriptorMapById_.clear();
1350 foreignServerMap_.clear();
1351 foreignServerMapById_.clear();
1352 custom_expr_map_by_id_.clear();
1355 buildForeignServerMapUnlocked();
1358 updateViewsInMapUnlocked();
1359 buildDashboardsMapUnlocked(user_name_by_user_id);
1360 buildLinksMapUnlocked();
1361 buildCustomExpressionsMapUnlocked();
1365 calciteMgr_->updateMetadata(currentDB_.dbName, {});
1369 void Catalog::buildTablesMapUnlocked() {
1370 std::string tableQuery(
1371 "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
1372 "max_chunk_size, frag_page_size, "
1373 "max_rows, partitions, shard_column_id, shard, num_shards, key_metainfo, userid, "
1374 "sort_column_id, storage_type, max_rollback_epochs, is_system_table "
1375 "from mapd_tables");
1376 sqliteConnector_.query(tableQuery);
1377 auto numRows = sqliteConnector_.getNumRows();
1378 for (
size_t r = 0; r < numRows; ++r) {
1380 const auto& storage_type = sqliteConnector_.getData<
string>(r, 17);
1382 const auto table_id = sqliteConnector_.getData<
int>(r, 0);
1383 const auto& table_name = sqliteConnector_.getData<
string>(r, 1);
1384 LOG(
FATAL) <<
"Unable to read Catalog metadata: storage type is currently not a "
1385 "supported table option (table "
1386 << table_name <<
" [" << table_id <<
"] in database "
1387 << currentDB_.dbName <<
").";
1397 td->
tableId = sqliteConnector_.getData<
int>(r, 0);
1398 td->
tableName = sqliteConnector_.getData<
string>(r, 1);
1399 td->
nColumns = sqliteConnector_.getData<
int>(r, 2);
1400 td->
isView = sqliteConnector_.getData<
bool>(r, 3);
1401 td->
fragments = sqliteConnector_.getData<
string>(r, 4);
1404 td->
maxFragRows = sqliteConnector_.getData<
int>(r, 6);
1405 td->
maxChunkSize = sqliteConnector_.getData<int64_t>(r, 7);
1406 td->
fragPageSize = sqliteConnector_.getData<
int>(r, 8);
1407 td->
maxRows = sqliteConnector_.getData<int64_t>(r, 9);
1408 td->
partitions = sqliteConnector_.getData<
string>(r, 10);
1410 td->
shard = sqliteConnector_.getData<
int>(r, 12);
1411 td->
nShards = sqliteConnector_.getData<
int>(r, 13);
1412 td->
keyMetainfo = sqliteConnector_.getData<
string>(r, 14);
1413 td->
userId = sqliteConnector_.getData<
int>(r, 15);
1415 sqliteConnector_.isNull(r, 16) ? 0 : sqliteConnector_.getData<
int>(r, 16);
1424 tableDescriptorMapById_[td->
tableId] = td;
1428 void Catalog::buildColumnsMapUnlocked() {
1429 std::string columnQuery(
1430 "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
1431 "is_notnull, compression, comp_param, "
1432 "size, chunks, is_systemcol, is_virtualcol, virtual_expr, is_deletedcol, "
1433 "default_value from "
1434 "mapd_columns ORDER BY tableid, "
1436 sqliteConnector_.query(columnQuery);
1437 auto numRows = sqliteConnector_.getNumRows();
1438 int32_t skip_physical_cols = 0;
1439 for (
size_t r = 0; r < numRows; ++r) {
1441 cd->
tableId = sqliteConnector_.getData<
int>(r, 0);
1442 cd->
columnId = sqliteConnector_.getData<
int>(r, 1);
1443 cd->
columnName = sqliteConnector_.getData<
string>(r, 2);
1452 cd->
chunks = sqliteConnector_.getData<
string>(r, 11);
1453 cd->
isSystemCol = sqliteConnector_.getData<
bool>(r, 12);
1454 cd->
isVirtualCol = sqliteConnector_.getData<
bool>(r, 13);
1455 cd->
virtualExpr = sqliteConnector_.getData<
string>(r, 14);
1456 cd->
isDeletedCol = sqliteConnector_.getData<
bool>(r, 15);
1457 if (sqliteConnector_.isNull(r, 16)) {
1460 cd->
default_value = std::make_optional(sqliteConnector_.getData<
string>(r, 16));
1463 cd->
db_id = getDatabaseId();
1467 if (skip_physical_cols <= 0) {
1471 auto td_itr = tableDescriptorMapById_.find(cd->
tableId);
1472 CHECK(td_itr != tableDescriptorMapById_.end());
1475 td_itr->second->hasDeletedCol =
true;
1476 setDeletedColumnUnlocked(td_itr->second, cd);
1478 tableDescriptorMapById_[cd->
tableId]->columnIdBySpi_.push_back(cd->
columnId);
1483 for (
auto& tit : tableDescriptorMapById_) {
1484 std::sort(tit.second->columnIdBySpi_.begin(),
1485 tit.second->columnIdBySpi_.end(),
1486 [](
const size_t a,
const size_t b) ->
bool {
return a < b; });
1491 std::string viewQuery(
"SELECT sql FROM mapd_views where tableid = " +
1493 sqliteConnector_.query(viewQuery);
1494 auto num_rows = sqliteConnector_.getNumRows();
1495 CHECK_EQ(num_rows, 1U) <<
"Expected single entry in mapd_views for view '"
1496 << td.
tableName <<
"', instead got " << num_rows;
1497 td.
viewSQL = sqliteConnector_.getData<
string>(0, 0);
1500 void Catalog::updateViewsInMapUnlocked() {
1501 std::string viewQuery(
"SELECT tableid, sql FROM mapd_views");
1502 sqliteConnector_.query(viewQuery);
1503 auto numRows = sqliteConnector_.getNumRows();
1504 for (
size_t r = 0; r < numRows; ++r) {
1505 auto tableId = sqliteConnector_.getData<
int>(r, 0);
1506 auto td = tableDescriptorMapById_[tableId];
1507 td->viewSQL = sqliteConnector_.getData<
string>(r, 1);
1508 td->fragmenter =
nullptr;
1512 void Catalog::buildDashboardsMapUnlocked(
1513 const std::map<int32_t, std::string>& user_name_by_user_id) {
1514 std::string frontendViewQuery(
1515 "SELECT id, state, name, image_hash, strftime('%Y-%m-%dT%H:%M:%SZ', update_time), "
1518 "FROM mapd_dashboards");
1519 sqliteConnector_.query(frontendViewQuery);
1520 auto numRows = sqliteConnector_.getNumRows();
1521 for (
size_t r = 0; r < numRows; ++r) {
1522 auto vd = std::make_shared<DashboardDescriptor>();
1523 vd->dashboardId = sqliteConnector_.getData<
int>(r, 0);
1524 vd->dashboardState = sqliteConnector_.getData<
string>(r, 1);
1525 vd->dashboardName = sqliteConnector_.getData<
string>(r, 2);
1526 vd->imageHash = sqliteConnector_.getData<
string>(r, 3);
1527 vd->updateTime = sqliteConnector_.getData<
string>(r, 4);
1528 vd->userId = sqliteConnector_.getData<
int>(r, 5);
1529 vd->dashboardMetadata = sqliteConnector_.getData<
string>(r, 6);
1532 std::to_string(currentDB_.dbId), sqliteConnector_.getData<
string>(r, 0));
1533 dashboardDescriptorMap_[
std::to_string(vd->userId) +
":" + vd->dashboardName] = vd;
1537 void Catalog::buildLinksMapUnlocked() {
1538 std::string linkQuery(
1539 "SELECT linkid, userid, link, view_state, strftime('%Y-%m-%dT%H:%M:%SZ', "
1540 "update_time), view_metadata "
1542 sqliteConnector_.query(linkQuery);
1543 auto numRows = sqliteConnector_.getNumRows();
1544 for (
size_t r = 0; r < numRows; ++r) {
1546 ld->linkId = sqliteConnector_.getData<
int>(r, 0);
1547 ld->userId = sqliteConnector_.getData<
int>(r, 1);
1548 ld->link = sqliteConnector_.getData<
string>(r, 2);
1549 ld->viewState = sqliteConnector_.getData<
string>(r, 3);
1550 ld->updateTime = sqliteConnector_.getData<
string>(r, 4);
1551 ld->viewMetadata = sqliteConnector_.getData<
string>(r, 5);
1552 linkDescriptorMap_[
std::to_string(currentDB_.dbId) + ld->link] = ld;
1553 linkDescriptorMapById_[ld->linkId] = ld;
1557 void Catalog::buildLogicalToPhysicalMapUnlocked() {
1559 std::string logicalToPhysicalTableMapQuery(
1560 "SELECT logical_table_id, physical_table_id "
1561 "FROM mapd_logical_to_physical");
1562 sqliteConnector_.query(logicalToPhysicalTableMapQuery);
1563 auto numRows = sqliteConnector_.getNumRows();
1564 for (
size_t r = 0; r < numRows; ++r) {
1565 auto logical_tb_id = sqliteConnector_.getData<
int>(r, 0);
1566 auto physical_tb_id = sqliteConnector_.getData<
int>(r, 1);
1567 const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
1568 if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
1570 std::vector<int32_t> physicalTables{physical_tb_id};
1572 logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
1573 CHECK(it_ok.second);
1576 physicalTableIt->second.push_back(physical_tb_id);
1584 void Catalog::buildMaps() {
1593 buildDictionaryMapUnlocked();
1594 buildTablesMapUnlocked();
1597 buildForeignServerMapUnlocked();
1598 updateForeignTablesInMapUnlocked();
1601 buildColumnsMapUnlocked();
1602 updateViewsInMapUnlocked();
1603 buildDashboardsMapUnlocked(user_name_by_user_id);
1604 buildLinksMapUnlocked();
1605 buildLogicalToPhysicalMapUnlocked();
1606 buildCustomExpressionsMapUnlocked();
1609 void Catalog::buildCustomExpressionsMapUnlocked() {
1610 sqliteConnector_.query(
1611 "SELECT id, name, expression_json, data_source_type, data_source_id, "
1613 "FROM omnisci_custom_expressions");
1614 auto num_rows = sqliteConnector_.getNumRows();
1615 for (
size_t row = 0; row < num_rows; row++) {
1616 auto custom_expr = getCustomExpressionFromConnector(row);
1617 custom_expr_map_by_id_[custom_expr->id] = std::move(custom_expr);
1621 std::unique_ptr<CustomExpression> Catalog::getCustomExpressionFromConnector(
size_t row) {
1622 auto id = sqliteConnector_.getData<
int>(row, 0);
1623 auto name = sqliteConnector_.getData<
string>(row, 1);
1624 auto expression_json = sqliteConnector_.getData<
string>(row, 2);
1625 auto data_source_type_str = sqliteConnector_.getData<
string>(row, 3);
1626 auto data_source_id = sqliteConnector_.getData<
int>(row, 4);
1627 auto is_deleted = sqliteConnector_.getData<
bool>(row, 5);
1628 return std::make_unique<CustomExpression>(
1632 CustomExpression::dataSourceTypeFromString(data_source_type_str),
1638 const list<ColumnDescriptor>& columns,
1639 const list<DictDescriptor>& dicts) {
1644 if (foreign_table) {
1646 *new_foreign_table = *foreign_table;
1647 new_td = new_foreign_table;
1653 new_td->
mutex_ = std::make_shared<std::mutex>();
1655 tableDescriptorMapById_[td->
tableId] = new_td;
1656 for (
auto cd : columns) {
1659 addToColumnMap(new_cd);
1664 setDeletedColumnUnlocked(new_td, new_cd);
1670 [](
const size_t a,
const size_t b) ->
bool {
return a < b; });
1674 std::unique_ptr<StringDictionaryClient> client;
1675 DictRef dict_ref(currentDB_.dbId, -1);
1676 if (!string_dict_hosts_.empty()) {
1679 for (
auto dd : dicts) {
1680 if (!dd.dictRef.dictId) {
1684 dict_ref.
dictId = dd.dictRef.dictId;
1686 client->create(dict_ref, dd.dictIsTemp);
1689 dictDescriptorMapByRef_[dict_ref].reset(new_dd);
1690 if (!dd.dictIsTemp) {
1696 void Catalog::removeTableFromMap(
const string& tableName,
1698 const bool is_on_error) {
1700 TableDescriptorMapById::iterator tableDescIt = tableDescriptorMapById_.find(tableId);
1701 if (tableDescIt == tableDescriptorMapById_.end()) {
1702 throw runtime_error(
"Table " + tableName +
" does not exist.");
1708 const auto ret = deletedColumnPerTable_.erase(td);
1712 tableDescriptorMapById_.erase(tableDescIt);
1713 tableDescriptorMap_.erase(
to_upper(tableName));
1715 dict_columns_by_table_id_.erase(tableId);
1720 std::unique_ptr<StringDictionaryClient> client;
1721 if (SysCatalog::instance().isAggregator()) {
1722 CHECK(!string_dict_hosts_.empty());
1723 DictRef dict_ref(currentDB_.dbId, -1);
1729 for (
auto cit = columnDescriptorMapById_.begin();
1730 cit != columnDescriptorMapById_.end();) {
1731 if (tableId != std::get<0>(cit->first)) {
1734 int i = std::get<1>(cit++->first);
1736 ColumnDescriptorMapById::iterator colDescIt = columnDescriptorMapById_.find(cidKey);
1738 columnDescriptorMapById_.erase(colDescIt);
1740 columnDescriptorMap_.erase(cnameKey);
1746 DictRef dict_ref(currentDB_.dbId, dictId);
1747 const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
1752 CHECK(dictIt != dictDescriptorMapByRef_.end());
1754 if (dictIt == dictDescriptorMapByRef_.end()) {
1758 const auto& dd = dictIt->second;
1761 if (!dd->refcount) {
1762 dd->stringDict.reset();
1767 client->drop(dict_ref);
1769 dictDescriptorMapByRef_.erase(dictIt);
1780 addFrontendViewToMapNoLock(vd);
1786 std::make_shared<DashboardDescriptor>(vd);
1789 std::vector<DBObject> Catalog::parseDashboardObjects(
const std::string& view_meta,
1790 const int& user_id) {
1791 std::vector<DBObject> objects;
1793 key.
dbId = currentDB_.dbId;
1794 auto _key_place = [&key](
auto type,
auto id) {
1800 auto td = getMetadataForTable(object_name,
false);
1804 LOG(
INFO) <<
"Ignoring dashboard source Table/View: " << object_name
1805 <<
" no longer exists in current DB.";
1812 objects.emplace_back(_key_place(object_type, td->tableId), priv, user_id);
1814 objects.back().setName(td->tableName);
1819 void Catalog::createOrUpdateDashboardSystemRole(
const std::string& view_meta,
1820 const int32_t& user_id,
1821 const std::string& dash_role_name) {
1822 auto objects = parseDashboardObjects(view_meta, user_id);
1823 Role* rl = SysCatalog::instance().getRoleGrantee(dash_role_name);
1828 SysCatalog::instance().createRole(
1829 dash_role_name,
false,
false);
1830 SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *
this);
1834 std::set<DBObjectKey> revoke_keys;
1836 for (
auto key : *ex_objects | boost::adaptors::map_keys) {
1842 for (
auto obj : objects) {
1843 found = key == obj.getObjectKey() ?
true :
false;
1849 revoke_keys.insert(key);
1852 for (
auto& key : revoke_keys) {
1856 SysCatalog::instance().revokeDBObjectPrivileges(
1861 SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *
this);
1870 linkDescriptorMapById_[ld.
linkId] = new_ld;
1878 vector<Chunk> chunkVec;
1879 auto columnDescs = getAllColumnMetadataForTable(td->
tableId,
true,
false,
true);
1880 Chunk::translateColumnDescriptorsToChunkVec(columnDescs, chunkVec);
1883 td->
fragmenter = std::make_shared<SortedOrderFragmenter>(chunkKeyPrefix,
1895 td->
fragmenter = std::make_shared<InsertOrderFragmenter>(chunkKeyPrefix,
1909 LOG(
INFO) <<
"Instantiating Fragmenter for table " << td->
tableName <<
" took "
1914 const std::string& tableName)
const {
1915 auto tableDescIt = tableDescriptorMap_.find(
to_upper(tableName));
1916 if (tableDescIt == tableDescriptorMap_.end()) {
1923 const std::string& tableName)
const {
1925 return getForeignTableUnlocked(tableName);
1928 const TableDescriptor* Catalog::getMetadataForTable(
const string& tableName,
1929 const bool populateFragmenter)
const {
1933 auto td = getMutableMetadataForTableUnlocked(tableName);
1938 if (populateFragmenter) {
1939 std::unique_lock<std::mutex> td_lock(*td->mutex_.get());
1940 if (td->fragmenter ==
nullptr && !td->isView) {
1941 instantiateFragmenter(td);
1948 bool populateFragmenter)
const {
1950 auto td = getMutableMetadataForTableUnlocked(table_id);
1955 if (populateFragmenter) {
1956 std::unique_lock<std::mutex> td_lock(*td->mutex_.get());
1957 if (td->fragmenter ==
nullptr && !td->isView) {
1958 instantiateFragmenter(td);
1964 std::optional<std::string> Catalog::getTableName(int32_t table_id)
const {
1966 auto td = getMutableMetadataForTableUnlocked(table_id);
1970 return td->tableName;
1973 std::optional<int32_t> Catalog::getTableId(
const std::string& table_name)
const {
1975 auto td = getMutableMetadataForTableUnlocked(table_name);
1983 const std::string& table_name)
const {
1984 auto it = tableDescriptorMap_.find(
to_upper(table_name));
1985 if (it == tableDescriptorMap_.end()) {
1992 auto tableDescIt = tableDescriptorMapById_.find(table_id);
1993 if (tableDescIt == tableDescriptorMapById_.end()) {
1996 return tableDescIt->second;
2000 const bool load_dict)
const {
2002 const DictRef dictRef(currentDB_.dbId, dict_id);
2003 auto dictDescIt = dictDescriptorMapByRef_.find(dictRef);
2005 dictDescriptorMapByRef_.end()) {
2008 auto& dd = dictDescIt->second;
2012 if (!dd->stringDict) {
2014 if (string_dict_hosts_.empty()) {
2015 if (dd->dictIsTemp) {
2016 dd->stringDict = std::make_shared<StringDictionary>(
2019 dd->stringDict = std::make_shared<StringDictionary>(
2024 std::make_shared<StringDictionary>(string_dict_hosts_.front(), dd->dictRef);
2027 LOG(
INFO) <<
"Time to load Dictionary " << dd->dictRef.dbId <<
"_"
2028 << dd->dictRef.dictId <<
" was " << time_ms <<
"ms";
2035 const std::vector<LeafHostInfo>& Catalog::getStringDictionaryHosts()
const {
2036 return string_dict_hosts_;
2040 const string& columnName)
const {
2044 auto colDescIt = columnDescriptorMap_.find(columnKey);
2046 columnDescriptorMap_.end()) {
2049 return colDescIt->second;
2055 auto colDescIt = columnDescriptorMapById_.find(columnIdKey);
2056 if (colDescIt == columnDescriptorMapById_
2060 return colDescIt->second;
2063 const std::optional<std::string> Catalog::getColumnName(
int table_id,
2064 int column_id)
const {
2066 auto it = columnDescriptorMapById_.find(
ColumnIdKey{table_id, column_id});
2067 if (it == columnDescriptorMapById_.end()) {
2070 return it->second->columnName;
2073 const int Catalog::getColumnIdBySpiUnlocked(
const int table_id,
const size_t spi)
const {
2074 const auto tabDescIt = tableDescriptorMapById_.find(table_id);
2075 CHECK(tableDescriptorMapById_.end() != tabDescIt);
2076 const auto& columnIdBySpi = tabDescIt->second->columnIdBySpi_;
2086 CHECK(0 < spx && spx <= columnIdBySpi.size())
2087 <<
"spx = " << spx <<
", size = " << columnIdBySpi.size();
2088 return columnIdBySpi[spx - 1] + phi;
2091 const int Catalog::getColumnIdBySpi(
const int table_id,
const size_t spi)
const {
2093 return getColumnIdBySpiUnlocked(table_id, spi);
2097 const size_t spi)
const {
2100 const auto columnId = getColumnIdBySpiUnlocked(tableId, spi);
2102 const auto colDescIt = columnDescriptorMapById_.find(columnIdKey);
2103 return columnDescriptorMapById_.end() == colDescIt ?
nullptr : colDescIt->second;
2106 void Catalog::deleteMetadataForDashboards(
const std::vector<int32_t> dashboard_ids,
2108 std::stringstream invalid_ids, restricted_ids;
2110 for (int32_t dashboard_id : dashboard_ids) {
2111 if (!getMetadataForDashboard(dashboard_id)) {
2112 invalid_ids << (!invalid_ids.str().empty() ?
", " :
"") << dashboard_id;
2116 object.loadKey(*
this);
2118 std::vector<DBObject> privs = {
object};
2119 if (!SysCatalog::instance().checkPrivileges(user, privs)) {
2120 restricted_ids << (!restricted_ids.str().empty() ?
", " :
"") << dashboard_id;
2124 if (invalid_ids.str().size() > 0 || restricted_ids.str().size() > 0) {
2125 std::stringstream error_message;
2126 error_message <<
"Delete dashboard(s) failed with error(s):";
2127 if (invalid_ids.str().size() > 0) {
2128 error_message <<
"\nDashboard id: " << invalid_ids.str()
2129 <<
" - Dashboard id does not exist";
2131 if (restricted_ids.str().size() > 0) {
2133 <<
"\nDashboard id: " << restricted_ids.str()
2134 <<
" - User should be either owner of dashboard or super user to delete it";
2136 throw std::runtime_error(error_message.str());
2138 std::vector<DBObject> dash_objs;
2140 for (int32_t dashboard_id : dashboard_ids) {
2144 SysCatalog::instance().revokeDBObjectPrivilegesFromAllBatch(dash_objs,
this);
2149 sqliteConnector_.query(
"BEGIN TRANSACTION");
2151 for (int32_t dashboard_id : dashboard_ids) {
2152 auto dash = getMetadataForDashboard(dashboard_id);
2156 throw std::runtime_error(
2157 std::string(
"Delete dashboard(s) failed with error(s):\nDashboard id: ") +
2158 std::to_string(dashboard_id) +
" - Dashboard id does not exist ");
2161 std::string dash_name = dash->dashboardName;
2162 auto viewDescIt = dashboardDescriptorMap_.find(user_id +
":" + dash_name);
2163 dashboardDescriptorMap_.erase(viewDescIt);
2164 sqliteConnector_.query_with_text_params(
2165 "DELETE FROM mapd_dashboards WHERE name = ? and userid = ?",
2166 std::vector<std::string>{dash_name, user_id});
2168 }
catch (std::exception& e) {
2169 sqliteConnector_.query(
"ROLLBACK TRANSACTION");
2172 sqliteConnector_.query(
"END TRANSACTION");
2177 const string& userId,
2178 const string& dashName)
const {
2181 auto viewDescIt = dashboardDescriptorMap_.find(userId +
":" + dashName);
2182 if (viewDescIt == dashboardDescriptorMap_.end()) {
2185 return viewDescIt->second.get();
2194 for (
auto descp : dashboardDescriptorMap_) {
2195 auto dash = descp.second.get();
2196 if (dash->dashboardId ==
id) {
2198 name = dash->dashboardName;
2205 return getMetadataForDashboard(userId, name);
2210 const LinkDescriptor* Catalog::getMetadataForLink(
const string& link)
const {
2212 auto linkDescIt = linkDescriptorMap_.find(link);
2213 if (linkDescIt == linkDescriptorMap_.end()) {
2216 return linkDescIt->second;
2221 auto linkDescIt = linkDescriptorMapById_.find(linkId);
2222 if (linkDescIt == linkDescriptorMapById_.end()) {
2225 return linkDescIt->second;
2230 const auto table = getMutableMetadataForTableUnlocked(table_id);
2233 CHECK(foreign_table);
2234 return foreign_table;
2237 void Catalog::getAllColumnMetadataForTableImpl(
2239 list<const ColumnDescriptor*>& columnDescriptors,
2240 const bool fetchSystemColumns,
2241 const bool fetchVirtualColumns,
2242 const bool fetchPhysicalColumns)
const {
2243 int32_t skip_physical_cols = 0;
2244 for (
const auto& columnDescriptor : columnDescriptorMapById_) {
2245 if (!fetchPhysicalColumns && skip_physical_cols > 0) {
2246 --skip_physical_cols;
2249 auto cd = columnDescriptor.second;
2259 if (!fetchPhysicalColumns) {
2263 columnDescriptors.push_back(cd);
2267 std::list<const ColumnDescriptor*> Catalog::getAllColumnMetadataForTable(
2269 const bool fetchSystemColumns,
2270 const bool fetchVirtualColumns,
2271 const bool fetchPhysicalColumns)
const {
2273 std::list<const ColumnDescriptor*> columnDescriptors;
2274 const TableDescriptor* td = getMutableMetadataForTableUnlocked(tableId);
2275 getAllColumnMetadataForTableImpl(td,
2278 fetchVirtualColumns,
2279 fetchPhysicalColumns);
2280 return columnDescriptors;
2283 list<const TableDescriptor*> Catalog::getAllTableMetadata()
const {
2285 list<const TableDescriptor*> table_list;
2286 for (
auto p : tableDescriptorMapById_) {
2287 table_list.push_back(p.second);
2292 std::vector<TableDescriptor> Catalog::getAllTableMetadataCopy()
const {
2294 std::vector<TableDescriptor>
tables;
2295 tables.reserve(tableDescriptorMapById_.size());
2296 for (
auto table_entry : tableDescriptorMapById_) {
2297 tables.emplace_back(*table_entry.second);
2298 tables.back().fragmenter =
nullptr;
2303 list<const DashboardDescriptor*> Catalog::getAllDashboardsMetadata()
const {
2305 list<const DashboardDescriptor*> dashboards;
2306 for (
auto dashboard_entry : dashboardDescriptorMap_) {
2307 dashboards.push_back(dashboard_entry.second.get());
2312 std::vector<DashboardDescriptor> Catalog::getAllDashboardsMetadataCopy()
const {
2314 std::vector<DashboardDescriptor> dashboards;
2315 dashboards.reserve(dashboardDescriptorMap_.size());
2316 for (
auto dashboard_entry : dashboardDescriptorMap_) {
2317 dashboards.emplace_back(*dashboard_entry.second);
2324 const auto& td = *tableDescriptorMapById_[cd.
tableId];
2325 list<DictDescriptor> dds;
2326 setColumnDictionary(cd, dds, td,
true);
2327 auto& dd = dds.back();
2328 CHECK(dd.dictRef.dictId);
2330 std::unique_ptr<StringDictionaryClient> client;
2331 if (!string_dict_hosts_.empty()) {
2333 string_dict_hosts_.front(),
DictRef(currentDB_.dbId, -1),
true));
2336 client->create(dd.dictRef, dd.dictIsTemp);
2340 dictDescriptorMapByRef_[dd.dictRef].reset(new_dd);
2341 if (!dd.dictIsTemp) {
2359 const auto td = getMetadataForTable(cd.
tableId,
false);
2361 sqliteConnector_.query_with_text_param(
2362 "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
2364 sqliteConnector_.query_with_text_param(
2365 "SELECT refcount FROM mapd_dictionaries WHERE dictid = ?",
std::to_string(dictId));
2366 const auto refcount = sqliteConnector_.getData<
int>(0, 0);
2367 VLOG(3) <<
"Dictionary " << dictId <<
"from dropped table has reference count "
2372 const DictRef dictRef(currentDB_.dbId, dictId);
2373 sqliteConnector_.query_with_text_param(
"DELETE FROM mapd_dictionaries WHERE dictid = ?",
2379 std::unique_ptr<StringDictionaryClient> client;
2380 if (!string_dict_hosts_.empty()) {
2384 client->drop(dictRef);
2387 dictDescriptorMapByRef_.erase(dictRef);
2391 std::map<int, StringDictionary*>& stringDicts) {
2394 CHECK(cit != columnDescriptorMap_.end());
2395 auto& ccd = *cit->second;
2397 if (!(ccd.columnType.is_string() || ccd.columnType.is_string_array())) {
2403 if (!(ccd.columnType.get_comp_param() > 0)) {
2407 auto dictId = ccd.columnType.get_comp_param();
2408 getMetadataForDict(dictId);
2410 const DictRef dictRef(currentDB_.dbId, dictId);
2411 auto dit = dictDescriptorMapByRef_.find(dictRef);
2412 CHECK(dit != dictDescriptorMapByRef_.end());
2414 CHECK(dit->second.get()->stringDict);
2415 stringDicts[ccd.columnId] = dit->second.get()->stringDict.get();
2421 cd.
db_id = getDatabaseId();
2423 for (
const auto shard : getPhysicalTablesDescriptors(&td)) {
2425 addColumn(*shard, shard_cd);
2433 std::vector<BindType> types(17, BindType::TEXT);
2435 types[16] = BindType::NULL_TYPE;
2437 sqliteConnector_.query_with_text_params(
2438 "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
2439 "colscale, is_notnull, "
2440 "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
2441 "is_deletedcol, default_value) "
2443 "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
2446 "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2466 sqliteConnector_.query_with_text_params(
2467 "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
2470 sqliteConnector_.query_with_text_params(
2471 "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
2473 cd.
columnId = sqliteConnector_.getData<
int>(0, 0);
2475 ++tableDescriptorMapById_[td.
tableId]->nColumns;
2477 addToColumnMap(ncd);
2478 columnDescriptorsForRoll.emplace_back(
nullptr, ncd);
2486 sqliteConnector_.query_with_text_params(
2487 "DELETE FROM mapd_columns where tableid = ? and columnid = ?",
2491 sqliteConnector_.query_with_text_params(
2492 "UPDATE mapd_tables SET ncolumns = ncolumns - 1 WHERE tableid = ?",
2495 ColumnDescriptorMap::iterator columnDescIt =
2497 CHECK(columnDescIt != columnDescriptorMap_.end());
2499 columnDescriptorsForRoll.emplace_back(columnDescIt->second,
nullptr);
2500 removeFromColumnMap(columnDescIt->second);
2501 --tableDescriptorMapById_[td.
tableId]->nColumns;
2506 for (
const auto shard : getPhysicalTablesDescriptors(&td)) {
2507 const auto shard_cd = getMetadataForColumn(shard->tableId, cd.
columnId);
2509 dropColumn(*shard, *shard_cd);
2514 void Catalog::roll(
const bool forward) {
2516 std::set<const TableDescriptor*> tds;
2518 for (
const auto& cdr : columnDescriptorsForRoll) {
2519 auto ocd = cdr.first;
2520 auto ncd = cdr.second;
2522 auto tabDescIt = tableDescriptorMapById_.find((ncd ? ncd : ocd)->tableId);
2523 CHECK(tableDescriptorMapById_.end() != tabDescIt);
2524 auto td = tabDescIt->second;
2525 auto& vc = td->columnIdBySpi_;
2528 if (
nullptr == ncd ||
2529 ncd->columnType.get_comp_param() != ocd->columnType.get_comp_param()) {
2530 delDictionary(*ocd);
2533 vc.erase(std::remove(vc.begin(), vc.end(), ocd->columnId), vc.end());
2539 if (vc.end() == std::find(vc.begin(), vc.end(), ncd->columnId)) {
2540 if (!ncd->isGeoPhyCol) {
2541 vc.push_back(ncd->columnId);
2548 addToColumnMap(ocd);
2552 removeFromColumnMap(ncd);
2553 if (
nullptr == ocd ||
2554 ocd->columnType.get_comp_param() != ncd->columnType.get_comp_param()) {
2555 delDictionary(*ncd);
2561 columnDescriptorsForRoll.clear();
2564 for (
const auto td : tds) {
2565 calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
2571 list<ColumnDescriptor>& columns) {
2573 if (
IS_GEO(col_ti.get_type())) {
2574 switch (col_ti.get_type()) {
2583 col_ti.get_comp_param() == 32) {
2584 unit_size = 4 *
sizeof(int8_t);
2587 unit_size = 8 *
sizeof(int8_t);
2591 columns.push_back(physical_cd_coords);
2605 columns.push_back(physical_cd_coords);
2611 bounds_ti.
set_size(4 *
sizeof(
double));
2613 columns.push_back(physical_cd_bounds);
2626 columns.push_back(physical_cd_coords);
2632 physical_cd_linestring_sizes.
columnType = linestring_sizes_ti;
2633 columns.push_back(physical_cd_linestring_sizes);
2639 bounds_ti.
set_size(4 *
sizeof(
double));
2641 columns.push_back(physical_cd_bounds);
2654 columns.push_back(physical_cd_coords);
2660 physical_cd_ring_sizes.
columnType = ring_sizes_ti;
2661 columns.push_back(physical_cd_ring_sizes);
2667 bounds_ti.
set_size(4 *
sizeof(
double));
2669 columns.push_back(physical_cd_bounds);
2674 physical_cd_render_group.
columnType = render_group_ti;
2675 columns.push_back(physical_cd_render_group);
2688 columns.push_back(physical_cd_coords);
2694 physical_cd_ring_sizes.
columnType = ring_sizes_ti;
2695 columns.push_back(physical_cd_ring_sizes);
2701 physical_cd_poly_rings.
columnType = poly_rings_ti;
2702 columns.push_back(physical_cd_poly_rings);
2708 bounds_ti.
set_size(4 *
sizeof(
double));
2710 columns.push_back(physical_cd_bounds);
2715 physical_cd_render_group.
columnType = render_group_ti;
2716 columns.push_back(physical_cd_render_group);
2723 throw runtime_error(
"Unrecognized geometry type.");
2731 auto timing_type_entry =
2733 CHECK(timing_type_entry != foreign_table.
options.end());
2734 if (timing_type_entry->second ==
2743 void Catalog::createTable(
2745 const list<ColumnDescriptor>& cols,
2746 const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs,
2747 bool isLogicalTable) {
2749 list<ColumnDescriptor> cds = cols;
2750 list<DictDescriptor> dds;
2751 std::set<std::string> toplevel_column_names;
2752 list<ColumnDescriptor> columns;
2757 throw std::runtime_error(
"Only temporary tables can be backed by foreign storage.");
2759 dataMgr_->getForeignStorageInterface()->prepareTable(getCurrentDB().dbId, td, cds);
2762 for (
auto cd : cds) {
2764 throw std::runtime_error(
2765 "Cannot create column with name rowid. rowid is a system defined column.");
2767 columns.push_back(cd);
2770 expandGeoColumn(cd, columns);
2780 #ifdef MATERIALIZED_ROWID
2784 cd.
virtualExpr =
"MAPD_FRAG_ID * MAPD_ROWS_PER_FRAG + MAPD_FRAG_ROW_ID";
2786 columns.push_back(cd);
2797 columns.push_back(cd_del);
2800 for (
auto& column : columns) {
2801 column.db_id = getDatabaseId();
2807 sqliteConnector_.query(
"BEGIN TRANSACTION");
2810 sqliteConnector_.query_with_text_params(
2811 R
"(INSERT INTO mapd_tables (name, userid, ncolumns, isview, fragments, frag_type, max_frag_rows, max_chunk_size, frag_page_size, max_rows, partitions, shard_column_id, shard, num_shards, sort_column_id, storage_type, max_rollback_epochs, is_system_table, key_metainfo) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?))",
2833 sqliteConnector_.query_with_text_param(
2834 "SELECT tableid FROM mapd_tables WHERE name = ?", td.
tableName);
2835 td.
tableId = sqliteConnector_.getData<
int>(0, 0);
2837 for (
auto cd : columns) {
2839 const bool is_foreign_col =
2840 setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2841 if (!is_foreign_col) {
2848 auto use_temp_dictionary =
false;
2849 setColumnDictionary(cd, dds, td, isLogicalTable, use_temp_dictionary);
2853 if (toplevel_column_names.count(cd.
columnName)) {
2860 std::vector<BindType> types(17, BindType::TEXT);
2862 types[16] = BindType::NULL_TYPE;
2864 sqliteConnector_.query_with_text_params(
2865 "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, "
2866 "coldim, colscale, is_notnull, "
2867 "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
2868 "virtual_expr, is_deletedcol, default_value) "
2869 "VALUES (?, ?, ?, ?, ?, "
2871 "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2895 sqliteConnector_.query_with_text_params(
2896 "INSERT INTO mapd_views (tableid, sql) VALUES (?,?)",
2902 sqliteConnector_.query_with_text_params(
2903 "INSERT INTO omnisci_foreign_tables (table_id, server_id, options, "
2904 "last_refresh_time, next_refresh_time) VALUES (?, ?, ?, ?, ?)",
2907 foreign_table.getOptionsAsJsonString(),
2911 }
catch (std::exception& e) {
2912 sqliteConnector_.query(
"ROLLBACK TRANSACTION");
2916 td.
tableId = nextTempTableId_++;
2918 for (
auto cd : columns) {
2920 const bool is_foreign_col =
2921 setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2923 if (!is_foreign_col) {
2925 std::string fileName(
"");
2926 std::string folderPath(
"");
2927 DictRef dict_ref(currentDB_.dbId, nextTempDictId_);
2944 if (toplevel_column_names.count(cd.
columnName)) {
2955 serializeTableJsonUnlocked(&td, cds);
2960 auto cache = dataMgr_->getPersistentStorageMgr()->getDiskCache();
2962 CHECK(!cache->hasCachedMetadataForKeyPrefix({getCurrentDB().dbId, td.
tableId}))
2963 <<
"Disk cache at " + cache->getCacheDirectory()
2964 <<
" contains preexisting data for new table. Please "
2965 "delete or clear cache before continuing";
2968 addTableToMap(&td, cds, dds);
2969 calciteMgr_->updateMetadata(currentDB_.dbName, td.
tableName);
2971 dataMgr_->getForeignStorageInterface()->registerTable(
this, td, cds);
2973 }
catch (std::exception& e) {
2974 sqliteConnector_.query(
"ROLLBACK TRANSACTION");
2975 removeTableFromMap(td.tableName, td.tableId,
true);
2978 sqliteConnector_.query(
"END TRANSACTION");
2982 sqlite_lock.unlock();
2983 getMetadataForTable(td.tableName,
2989 const std::list<ColumnDescriptor>& cds)
const {
2991 using namespace rapidjson;
2993 VLOG(1) <<
"Serializing temporary table " << td->
tableName <<
" to JSON for Calcite.";
2995 const auto db_name = currentDB_.dbName;
2999 if (boost::filesystem::exists(file_path)) {
3001 std::ifstream reader(file_path.string());
3002 CHECK(reader.is_open());
3003 IStreamWrapper json_read_wrapper(reader);
3004 d.ParseStream(json_read_wrapper);
3008 CHECK(d.IsObject());
3011 Value table(kObjectType);
3013 "name",
Value().SetString(StringRef(td->
tableName.c_str())), d.GetAllocator());
3014 table.AddMember(
"id",
Value().SetInt(td->
tableId), d.GetAllocator());
3015 table.AddMember(
"columns",
Value(kArrayType), d.GetAllocator());
3017 for (
const auto& cd : cds) {
3018 Value column(kObjectType);
3020 "name",
Value().SetString(StringRef(cd.
columnName)), d.GetAllocator());
3021 column.AddMember(
"coltype",
3024 column.AddMember(
"colsubtype",
3027 column.AddMember(
"compression",
3030 column.AddMember(
"comp_param",
3033 column.AddMember(
"size",
3042 column.AddMember(
"is_systemcol",
Value().SetBool(cd.
isSystemCol), d.GetAllocator());
3043 column.AddMember(
"is_virtualcol",
Value().SetBool(cd.
isVirtualCol), d.GetAllocator());
3044 column.AddMember(
"is_deletedcol",
Value().SetBool(cd.
isDeletedCol), d.GetAllocator());
3045 table[
"columns"].PushBack(column, d.GetAllocator());
3047 d.AddMember(StringRef(td->
tableName.c_str()), table, d.GetAllocator());
3050 std::ofstream writer(file_path.string(), std::ios::trunc | std::ios::out);
3051 CHECK(writer.is_open());
3052 OStreamWrapper json_wrapper(writer);
3054 Writer<OStreamWrapper> json_writer(json_wrapper);
3055 d.Accept(json_writer);
3059 void Catalog::dropTableFromJsonUnlocked(
const std::string& table_name)
const {
3061 using namespace rapidjson;
3063 VLOG(1) <<
"Dropping temporary table " << table_name <<
" to JSON for Calcite.";
3065 const auto db_name = currentDB_.dbName;
3068 CHECK(boost::filesystem::exists(file_path));
3071 std::ifstream reader(file_path.string());
3072 CHECK(reader.is_open());
3073 IStreamWrapper json_read_wrapper(reader);
3074 d.ParseStream(json_read_wrapper);
3076 CHECK(d.IsObject());
3077 auto table_name_ref = StringRef(table_name.c_str());
3078 CHECK(d.HasMember(table_name_ref));
3079 CHECK(d.RemoveMember(table_name_ref));
3082 std::ofstream writer(file_path.string(), std::ios::trunc | std::ios::out);
3083 CHECK(writer.is_open());
3084 OStreamWrapper json_wrapper(writer);
3086 Writer<OStreamWrapper> json_writer(json_wrapper);
3087 d.Accept(json_writer);
3091 void Catalog::createForeignServer(
3092 std::unique_ptr<foreign_storage::ForeignServer> foreign_server,
3093 bool if_not_exists) {
3096 createForeignServerNoLocks(std::move(foreign_server), if_not_exists);
3099 void Catalog::createForeignServerNoLocks(
3100 std::unique_ptr<foreign_storage::ForeignServer> foreign_server,
3101 bool if_not_exists) {
3102 const auto&
name = foreign_server->name;
3104 sqliteConnector_.query_with_text_params(
3105 "SELECT name from omnisci_foreign_servers where name = ?",
3106 std::vector<std::string>{
name});
3108 if (sqliteConnector_.getNumRows() == 0) {
3109 foreign_server->creation_time = std::time(
nullptr);
3110 sqliteConnector_.query_with_text_params(
3111 "INSERT INTO omnisci_foreign_servers (name, data_wrapper_type, owner_user_id, "
3114 "VALUES (?, ?, ?, ?, ?)",
3115 std::vector<std::string>{
name,
3116 foreign_server->data_wrapper_type,
3119 foreign_server->getOptionsAsJsonString()});
3120 sqliteConnector_.query_with_text_params(
3121 "SELECT id from omnisci_foreign_servers where name = ?",
3122 std::vector<std::string>{
name});
3123 CHECK_EQ(sqliteConnector_.getNumRows(), size_t(1));
3124 foreign_server->id = sqliteConnector_.getData<int32_t>(0, 0);
3125 std::shared_ptr<foreign_storage::ForeignServer> foreign_server_shared =
3126 std::move(foreign_server);
3127 CHECK(foreignServerMap_.find(
name) == foreignServerMap_.end())
3128 <<
"Attempting to insert a foreign server into foreign server map that already "
3130 foreignServerMap_[
name] = foreign_server_shared;
3131 foreignServerMapById_[foreign_server_shared->id] = foreign_server_shared;
3132 }
else if (!if_not_exists) {
3133 throw std::runtime_error{
"A foreign server with name \"" + foreign_server->name +
3134 "\" already exists."};
3137 const auto& server_it = foreignServerMap_.find(
name);
3138 CHECK(server_it != foreignServerMap_.end());
3139 CHECK(foreignServerMapById_.find(server_it->second->id) != foreignServerMapById_.end());
3143 const std::string& server_name)
const {
3147 if (foreignServerMap_.find(server_name) != foreignServerMap_.end()) {
3148 foreign_server = foreignServerMap_.find(server_name)->second.get();
3150 return foreign_server;
3153 const std::unique_ptr<const foreign_storage::ForeignServer>
3154 Catalog::getForeignServerFromStorage(
const std::string& server_name) {
3155 std::unique_ptr<foreign_storage::ForeignServer> foreign_server =
nullptr;
3157 sqliteConnector_.query_with_text_params(
3158 "SELECT id, name, data_wrapper_type, options, owner_user_id, creation_time "
3159 "FROM omnisci_foreign_servers WHERE name = ?",
3160 std::vector<std::string>{server_name});
3161 if (sqliteConnector_.getNumRows() > 0) {
3162 foreign_server = std::make_unique<foreign_storage::ForeignServer>(
3163 sqliteConnector_.getData<
int>(0, 0),
3164 sqliteConnector_.getData<std::string>(0, 1),
3165 sqliteConnector_.getData<std::string>(0, 2),
3166 sqliteConnector_.getData<std::string>(0, 3),
3167 sqliteConnector_.getData<std::int32_t>(0, 4),
3168 sqliteConnector_.getData<std::int32_t>(0, 5));
3170 return foreign_server;
3173 const std::unique_ptr<const foreign_storage::ForeignTable>
3174 Catalog::getForeignTableFromStorage(
int table_id) {
3175 std::unique_ptr<foreign_storage::ForeignTable> foreign_table =
nullptr;
3177 sqliteConnector_.query_with_text_params(
3178 "SELECT table_id, server_id, options, last_refresh_time, next_refresh_time from "
3179 "omnisci_foreign_tables WHERE table_id = ?",
3180 std::vector<std::string>{
to_string(table_id)});
3181 auto num_rows = sqliteConnector_.getNumRows();
3184 foreign_table = std::make_unique<foreign_storage::ForeignTable>(
3185 sqliteConnector_.getData<
int>(0, 0),
3186 foreignServerMapById_[sqliteConnector_.getData<int32_t>(0, 1)].get(),
3187 sqliteConnector_.getData<std::string>(0, 2),
3188 sqliteConnector_.getData<int64_t>(0, 3),
3189 sqliteConnector_.getData<int64_t>(0, 4));
3191 return foreign_table;
3194 void Catalog::changeForeignServerOwner(
const std::string& server_name,
3195 const int new_owner_id) {
3198 foreignServerMap_.find(server_name)->second.get();
3199 CHECK(foreign_server);
3200 setForeignServerProperty(server_name,
"owner_user_id",
std::to_string(new_owner_id));
3202 foreign_server->
user_id = new_owner_id;
3205 void Catalog::setForeignServerDataWrapper(
const std::string& server_name,
3206 const std::string& data_wrapper) {
3208 auto data_wrapper_type =
to_upper(data_wrapper);
3211 foreignServerMap_.find(server_name)->second.get();
3212 CHECK(foreign_server);
3217 }
catch (
const std::exception& e) {
3223 setForeignServerProperty(server_name,
"data_wrapper_type", data_wrapper_type);
3226 void Catalog::setForeignServerOptions(
const std::string& server_name,
3227 const std::string& options) {
3231 foreignServerMap_.find(server_name)->second.get();
3232 CHECK(foreign_server);
3233 auto saved_options = foreign_server->
options;
3237 }
catch (
const std::exception& e) {
3240 foreign_server->
options = saved_options;
3243 setForeignServerProperty(server_name,
"options", options);
3246 void Catalog::renameForeignServer(
const std::string& server_name,
3247 const std::string&
name) {
3249 auto foreign_server_it = foreignServerMap_.find(server_name);
3250 CHECK(foreign_server_it != foreignServerMap_.end());
3251 setForeignServerProperty(server_name,
"name", name);
3252 auto foreign_server_shared = foreign_server_it->second;
3253 foreign_server_shared->name =
name;
3254 foreignServerMap_[
name] = foreign_server_shared;
3255 foreignServerMap_.erase(foreign_server_it);
3258 void Catalog::dropForeignServer(
const std::string& server_name) {
3262 sqliteConnector_.query_with_text_params(
3263 "SELECT id from omnisci_foreign_servers where name = ?",
3264 std::vector<std::string>{server_name});
3265 auto num_rows = sqliteConnector_.getNumRows();
3268 auto server_id = sqliteConnector_.getData<int32_t>(0, 0);
3269 sqliteConnector_.query_with_text_param(
3270 "SELECT table_id from omnisci_foreign_tables where server_id = ?",
3272 if (sqliteConnector_.getNumRows() > 0) {
3273 throw std::runtime_error{
"Foreign server \"" + server_name +
3275 "by existing foreign tables and cannot be dropped."};
3277 sqliteConnector_.query(
"BEGIN TRANSACTION");
3279 sqliteConnector_.query_with_text_params(
3280 "DELETE FROM omnisci_foreign_servers WHERE name = ?",
3281 std::vector<std::string>{server_name});
3282 }
catch (
const std::exception& e) {
3283 sqliteConnector_.query(
"ROLLBACK TRANSACTION");
3286 sqliteConnector_.query(
"END TRANSACTION");
3287 foreignServerMap_.erase(server_name);
3288 foreignServerMapById_.erase(server_id);
3292 void Catalog::getForeignServersForUser(
3293 const rapidjson::Value* filters,
3295 std::vector<const foreign_storage::ForeignServer*>& results) {
3300 std::map<std::string, std::string> col_names{{
"server_name",
"name"},
3301 {
"data_wrapper",
"data_wrapper_type"},
3302 {
"created_at",
"creation_time"},
3303 {
"options",
"options"}};
3306 std::stringstream filter_string;
3307 std::vector<std::string> arguments;
3309 if (filters !=
nullptr) {
3311 int num_filters = 0;
3312 filter_string <<
" WHERE";
3313 for (
auto& filter_def : filters->GetArray()) {
3314 if (num_filters > 0) {
3315 filter_string <<
" " << std::string(filter_def[
"chain"].GetString());
3319 if (col_names.find(std::string(filter_def[
"attribute"].GetString())) ==
3321 throw std::runtime_error{
"Attribute with name \"" +
3322 std::string(filter_def[
"attribute"].GetString()) +
3323 "\" does not exist."};
3326 filter_string <<
" " << col_names[std::string(filter_def[
"attribute"].GetString())];
3328 bool equals_operator =
false;
3329 if (std::strcmp(filter_def[
"operation"].GetString(),
"EQUALS") == 0) {
3330 filter_string <<
" = ? ";
3331 equals_operator =
true;
3333 filter_string <<
" LIKE ? ";
3336 bool timestamp_column =
3337 (std::strcmp(filter_def[
"attribute"].GetString(),
"created_at") == 0);
3339 if (timestamp_column && !equals_operator) {
3340 throw std::runtime_error{
"LIKE operator is incompatible with TIMESTAMP data"};
3343 if (timestamp_column && equals_operator) {
3345 dateTimeParse<kTIMESTAMP>(filter_def[
"value"].GetString(), 0)));
3347 arguments.emplace_back(filter_def[
"value"].GetString());
3354 std::string query = std::string(
"SELECT name from omnisci_foreign_servers ");
3355 query += filter_string.str();
3357 sqliteConnector_.query_with_text_params(query, arguments);
3358 auto num_rows = sqliteConnector_.getNumRows();
3360 if (sqliteConnector_.getNumRows() == 0) {
3364 CHECK(sqliteConnector_.getNumCols() == 1);
3366 results.reserve(num_rows);
3367 for (
size_t row = 0; row < num_rows; ++row) {
3368 const auto& server_name = sqliteConnector_.getData<std::string>(row, 0);
3373 CHECK(foreign_server !=
nullptr);
3377 std::vector<DBObject> privObjects = {dbObject};
3378 if (!SysCatalog::instance().hasAnyPrivileges(user, privObjects)) {
3382 results.push_back(foreign_server);
3387 int32_t Catalog::getTableEpoch(
const int32_t db_id,
const int32_t table_id)
const {
3389 const auto td = getMetadataForTable(table_id,
false);
3391 std::stringstream table_not_found_error_message;
3392 table_not_found_error_message <<
"Table (" << db_id <<
"," << table_id
3394 throw std::runtime_error(table_not_found_error_message.str());
3396 const auto physicalTableIt = logicalToPhysicalTableMapById_.find(table_id);
3397 if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
3399 const auto physicalTables = physicalTableIt->second;
3400 CHECK(!physicalTables.empty());
3401 size_t curr_epoch{0}, first_epoch{0};
3402 int32_t first_table_id{0};
3403 bool are_epochs_inconsistent{
false};
3404 for (
size_t i = 0; i < physicalTables.size(); i++) {
3405 int32_t physical_tb_id = physicalTables[i];
3406 const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id,
false);
3409 curr_epoch = dataMgr_->getTableEpoch(db_id, physical_tb_id);
3410 LOG(
INFO) <<
"Got sharded table epoch for db id: " << db_id
3411 <<
", table id: " << physical_tb_id <<
", epoch: " << curr_epoch;
3413 first_epoch = curr_epoch;
3414 first_table_id = physical_tb_id;
3415 }
else if (first_epoch != curr_epoch) {
3416 are_epochs_inconsistent =
true;
3417 LOG(
ERROR) <<
"Epochs on shards do not all agree on table id: " << table_id
3418 <<
", db id: " << db_id
3419 <<
". First table (table id: " << first_table_id
3420 <<
") has epoch: " << first_epoch <<
". Table id: " << physical_tb_id
3421 <<
", has inconsistent epoch: " << curr_epoch
3422 <<
". See previous INFO logs for all epochs and their table ids.";
3425 if (are_epochs_inconsistent) {
3431 auto epoch = dataMgr_->getTableEpoch(db_id, table_id);
3432 LOG(
INFO) <<
"Got table epoch for db id: " << db_id <<
", table id: " << table_id
3433 <<
", epoch: " << epoch;
3438 std::vector<const foreign_storage::ForeignTable*>
3439 Catalog::getAllForeignTablesForForeignServer(
const int32_t foreign_server_id) {
3441 std::vector<const foreign_storage::ForeignTable*> foreign_tables;
3442 for (
auto entry : tableDescriptorMapById_) {
3443 auto table_descriptor = entry.second;
3446 CHECK(foreign_table);
3447 if (foreign_table->foreign_server->id == foreign_server_id) {
3448 foreign_tables.emplace_back(foreign_table);
3452 return foreign_tables;
3455 void Catalog::setTableEpoch(
const int db_id,
const int table_id,
int new_epoch) {
3456 LOG(
INFO) <<
"Set table epoch db:" << db_id <<
" Table ID " << table_id
3457 <<
" back to new epoch " << new_epoch;
3458 const auto td = getMetadataForTable(table_id,
false);
3460 std::stringstream table_not_found_error_message;
3461 table_not_found_error_message <<
"Table (" << db_id <<
"," << table_id
3463 throw std::runtime_error(table_not_found_error_message.str());
3466 std::stringstream is_temp_table_error_message;
3467 is_temp_table_error_message <<
"Cannot set epoch on temporary table";
3468 throw std::runtime_error(is_temp_table_error_message.str());
3472 file_mgr_params.
epoch = new_epoch;
3475 const auto physical_tables = getPhysicalTablesDescriptors(td,
false);
3476 CHECK(!physical_tables.empty());
3477 for (
const auto table : physical_tables) {
3478 auto table_id = table->tableId;
3479 LOG(
INFO) <<
"Set sharded table epoch db:" << db_id <<
" Table ID " << table_id
3480 <<
" back to new epoch " << new_epoch;
3483 removeChunks(table_id);
3484 dataMgr_->getGlobalFileMgr()->setFileMgrParams(db_id, table_id, file_mgr_params);
3488 void Catalog::alterPhysicalTableMetadata(
3499 sqliteConnector_.query_with_text_params(
3500 "UPDATE mapd_tables SET max_rollback_epochs = ? WHERE tableid = ?",
3507 sqliteConnector_.query_with_text_params(
3508 "UPDATE mapd_tables SET max_rows = ? WHERE tableid = ?",
3519 sqliteConnector_.query(
"BEGIN TRANSACTION");
3521 const auto physical_table_it = logicalToPhysicalTableMapById_.find(td->
tableId);
3522 if (physical_table_it != logicalToPhysicalTableMapById_.end()) {
3523 const auto physical_tables = physical_table_it->second;
3524 CHECK(!physical_tables.empty());
3525 for (
size_t i = 0; i < physical_tables.size(); i++) {
3526 int32_t physical_tb_id = physical_tables[i];
3527 const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id,
false);
3529 alterPhysicalTableMetadata(phys_td, table_update_params);
3532 alterPhysicalTableMetadata(td, table_update_params);
3533 }
catch (std::exception& e) {
3534 sqliteConnector_.query(
"ROLLBACK TRANSACTION");
3537 sqliteConnector_.query(
"END TRANSACTION");
3540 void Catalog::setMaxRollbackEpochs(
const int32_t table_id,
3541 const int32_t max_rollback_epochs) {
3544 if (max_rollback_epochs <= -1) {
3545 throw std::runtime_error(
"Cannot set max_rollback_epochs < 0.");
3547 const auto td = getMetadataForTable(
3553 if (table_update_params == td) {
3554 LOG(
INFO) <<
"Setting max_rollback_epochs for table " << table_id
3555 <<
" to existing value, skipping operation";
3559 file_mgr_params.
epoch = -1;
3561 setTableFileMgrParams(table_id, file_mgr_params);
3562 alterTableMetadata(td, table_update_params);
3565 void Catalog::setMaxRows(
const int32_t table_id,
const int64_t max_rows) {
3567 throw std::runtime_error(
"Max rows cannot be a negative number.");
3569 const auto td = getMetadataForTable(table_id);
3572 table_update_params.
max_rows = max_rows;
3573 if (table_update_params == td) {
3574 LOG(
INFO) <<
"Max rows value of " << max_rows
3575 <<
" is the same as the existing value. Skipping update.";
3578 alterTableMetadata(td, table_update_params);
3579 CHECK(td->fragmenter);
3580 td->fragmenter->dropFragmentsToSize(max_rows);
3584 void Catalog::setUncappedTableEpoch(
const std::string& table_name) {
3586 auto td_entry = tableDescriptorMap_.find(
to_upper(table_name));
3587 CHECK(td_entry != tableDescriptorMap_.end());
3588 auto td = td_entry->second;
3590 std::vector<int> table_key{getCurrentDB().dbId, td->tableId};
3597 alterTableMetadata(td, table_update_params);
3600 setTableFileMgrParams(td->tableId, file_mgr_params);
3603 void Catalog::setTableFileMgrParams(
3607 const auto td = getMetadataForTable(table_id,
false);
3608 const auto db_id = this->getDatabaseId();
3610 std::stringstream table_not_found_error_message;
3611 table_not_found_error_message <<
"Table (" << db_id <<
"," << table_id
3613 throw std::runtime_error(table_not_found_error_message.str());
3616 std::stringstream is_temp_table_error_message;
3617 is_temp_table_error_message <<
"Cannot set storage params on temporary table";
3618 throw std::runtime_error(is_temp_table_error_message.str());
3621 const auto physical_tables = getPhysicalTablesDescriptors(td,
false);
3622 CHECK(!physical_tables.empty());
3623 for (
const auto table : physical_tables) {
3624 auto table_id = table->tableId;
3625 removeChunks(table_id);
3626 dataMgr_->getGlobalFileMgr()->setFileMgrParams(db_id, table_id, file_mgr_params);
3630 std::vector<TableEpochInfo> Catalog::getTableEpochs(
const int32_t db_id,
3631 const int32_t table_id)
const {
3633 std::vector<TableEpochInfo> table_epochs;
3634 const auto physical_table_it = logicalToPhysicalTableMapById_.find(table_id);
3635 if (physical_table_it != logicalToPhysicalTableMapById_.end()) {
3636 const auto physical_tables = physical_table_it->second;
3637 CHECK(!physical_tables.empty());
3639 for (
const auto physical_tb_id : physical_tables) {
3640 const auto phys_td = getMutableMetadataForTableUnlocked(physical_tb_id);
3643 auto table_id = phys_td->tableId;
3644 auto epoch = dataMgr_->getTableEpoch(db_id, phys_td->tableId);
3645 table_epochs.emplace_back(table_id, epoch);
3646 LOG(
INFO) <<
"Got sharded table epoch for db id: " << db_id
3647 <<
", table id: " << table_id <<
", epoch: " << epoch;
3650 auto epoch = dataMgr_->getTableEpoch(db_id, table_id);
3651 LOG(
INFO) <<
"Got table epoch for db id: " << db_id <<
", table id: " << table_id
3652 <<
", epoch: " << epoch;
3653 table_epochs.emplace_back(table_id, epoch);
3655 return table_epochs;
3658 void Catalog::setTableEpochs(
const int32_t db_id,
3659 const std::vector<TableEpochInfo>& table_epochs)
const {
3660 const auto td = getMetadataForTable(table_epochs[0].table_id,
false);
3665 for (
const auto& table_epoch_info : table_epochs) {
3666 removeChunks(table_epoch_info.table_id);
3667 file_mgr_params.
epoch = table_epoch_info.table_epoch;
3668 dataMgr_->getGlobalFileMgr()->setFileMgrParams(
3669 db_id, table_epoch_info.table_id, file_mgr_params);
3670 LOG(
INFO) <<
"Set table epoch for db id: " << db_id
3671 <<
", table id: " << table_epoch_info.table_id
3672 <<
", back to epoch: " << table_epoch_info.table_epoch;
3678 std::string table_epochs_str{
"["};
3679 bool first_entry{
true};
3680 for (
const auto& table_epoch : table_epochs) {
3682 first_entry =
false;
3684 table_epochs_str +=
", ";
3686 table_epochs_str +=
"(table_id: " +
std::to_string(table_epoch.table_id) +
3689 table_epochs_str +=
"]";
3690 return table_epochs_str;
3694 void Catalog::setTableEpochsLogExceptions(
3695 const int32_t db_id,
3696 const std::vector<TableEpochInfo>& table_epochs)
const {
3698 setTableEpochs(db_id, table_epochs);
3699 }
catch (std::exception& e) {
3700 LOG(
ERROR) <<
"An error occurred when attempting to set table epochs. DB id: "
3702 <<
", Error: " << e.what();
3708 const auto it = deletedColumnPerTable_.find(td);
3709 return it != deletedColumnPerTable_.end() ? it->second :
nullptr;
3713 int delete_column_id)
const {
3718 return fragmenter->hasDeletedRows(delete_column_id);
3726 std::vector<const TableDescriptor*> tds;
3731 const auto it = deletedColumnPerTable_.find(td);
3733 if (it == deletedColumnPerTable_.end()) {
3737 tds = getPhysicalTablesDescriptors(td,
false);
3740 for (
auto tdd : tds) {
3741 if (checkMetadataForDeletedRecs(tdd, cd->
columnId)) {
3751 setDeletedColumnUnlocked(td, cd);
3756 const auto it_ok = deletedColumnPerTable_.emplace(td, cd);
3757 CHECK(it_ok.second);
3776 const bool persist_reference) {
3779 CHECK(foreign_ref_col);
3780 referencing_column.
columnType = foreign_ref_col->columnType;
3782 const DictRef dict_ref(currentDB_.dbId, dict_id);
3783 const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
3784 CHECK(dictIt != dictDescriptorMapByRef_.end());
3785 const auto& dd = dictIt->second;
3788 if (persist_reference) {
3790 sqliteConnector_.query_with_text_params(
3791 "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
3796 bool Catalog::setColumnSharedDictionary(
3798 std::list<ColumnDescriptor>& cdd,
3799 std::list<DictDescriptor>& dds,
3801 const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs) {
3805 if (shared_dict_defs.empty()) {
3808 for (
const auto& shared_dict_def : shared_dict_defs) {
3810 const auto& column = shared_dict_def.get_column();
3812 if (!shared_dict_def.get_foreign_table().compare(td.
tableName)) {
3814 const auto& ref_column = shared_dict_def.get_foreign_column();
3816 std::find_if(cdd.begin(), cdd.end(), [ref_column](
const ColumnDescriptor it) {
3817 return !ref_column.compare(it.columnName);
3819 CHECK(colIt != cdd.end());
3824 auto dictIt = std::find_if(
3825 dds.begin(), dds.end(), [
this, dict_id](
const DictDescriptor it) {
3826 return it.dictRef.dbId == this->currentDB_.dbId &&
3827 it.dictRef.dictId == dict_id;
3829 if (dictIt != dds.end()) {
3835 sqliteConnector_.query_with_text_params(
3836 "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
3846 const auto& foreign_table_name = shared_dict_def.get_foreign_table();
3847 const auto foreign_td = getMetadataForTable(foreign_table_name,
false);
3850 throw std::runtime_error(
3851 "Only temporary tables can share dictionaries with other temporary "
3854 addReferenceToForeignDict(cd, shared_dict_def,
false);
3866 std::list<DictDescriptor>& dds,
3868 bool is_logical_table,
3869 bool use_temp_dictionary) {
3872 std::string dictName{
"Initial_key"};
3874 std::string folderPath;
3875 if (is_logical_table) {
3878 sqliteConnector_.query_with_text_params(
3879 "INSERT INTO mapd_dictionaries (name, nbits, is_shared, refcount) VALUES (?, ?, "
3881 std::vector<std::string>{
3883 sqliteConnector_.query_with_text_param(
3884 "SELECT dictid FROM mapd_dictionaries WHERE name = ?", dictName);
3885 dictId = sqliteConnector_.getData<
int>(0, 0);
3887 sqliteConnector_.query_with_text_param(
3888 "UPDATE mapd_dictionaries SET name = ? WHERE name = 'Initial_key'", dictName);
3899 use_temp_dictionary);
3908 void Catalog::createShardedTable(
3910 const list<ColumnDescriptor>& cols,
3911 const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs) {
3914 createTable(*tdl, cols, shared_dict_defs,
true);
3915 int32_t logical_tb_id = tdl->
tableId;
3916 std::string logical_table_name = tdl->
tableName;
3919 std::vector<int32_t> physicalTables;
3920 for (int32_t i = 1; i <= td.
nShards; i++) {
3922 tdp->
tableName = generatePhysicalTableName(logical_table_name, i);
3924 createTable(*tdp, cols, shared_dict_defs,
false);
3925 int32_t physical_tb_id = tdp->
tableId;
3928 physicalTables.push_back(physical_tb_id);
3931 if (!physicalTables.empty()) {
3935 logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
3936 CHECK(it_ok.second);
3939 updateLogicalToPhysicalTableMap(logical_tb_id);
3946 const auto physical_tables = getPhysicalTablesDescriptors(td);
3947 for (
const auto table : physical_tables) {
3948 doTruncateTable(table);
3954 removeFragmenterForTable(td->
tableId);
3956 const int tableId = td->
tableId;
3957 ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
3962 dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
3965 std::unique_ptr<StringDictionaryClient> client;
3966 if (SysCatalog::instance().isAggregator()) {
3967 CHECK(!string_dict_hosts_.empty());
3968 DictRef dict_ref(currentDB_.dbId, -1);
3973 for (
const auto& columnDescriptor : columnDescriptorMapById_) {
3974 auto cd = columnDescriptor.second;
3981 const DictRef dict_ref(currentDB_.dbId, dict_id);
3982 const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
3983 CHECK(dictIt != dictDescriptorMapByRef_.end());
3984 const auto& dd = dictIt->second;
3987 if (dd->refcount == 1) {
3989 dd->stringDict.reset();
3992 client->drop(dd->dictRef);
3994 if (!dd->dictIsTemp) {
3995 boost::filesystem::create_directory(dd->dictFolderPath);
4006 dictDescriptorMapByRef_.erase(dictIt);
4011 dictDescriptorMapByRef_[new_dd->
dictRef].reset(new_dd);
4019 for (
auto col_id = 0; col_id < td.
nColumns; ++col_id) {
4020 if (
auto it = columnDescriptorMapById_.find({td.
tableId, col_id});
4021 it != columnDescriptorMapById_.end()) {
4022 auto cd = it->second;
4025 DictRef dict_ref(currentDB_.dbId, dict_id);
4026 if (
auto dict_it = dictDescriptorMapByRef_.find(dict_ref);
4027 dict_it != dictDescriptorMapByRef_.end()) {
4029 dict_it->second->stringDict =
nullptr;
4031 getMetadataForDict(dict_id,
true);
4038 void Catalog::invalidateCachesForTable(
const int table_id) {
4041 ChunkKey const table_key{getDatabaseId(), table_id};
4042 auto td = getMutableMetadataForTableUnlocked(table_id);
4048 refreshDictionaryCachesForTableUnlocked(*td);
4052 if (td->fragmenter !=
nullptr) {
4053 auto tableDescIt = tableDescriptorMapById_.find(table_id);
4054 CHECK(tableDescIt != tableDescriptorMapById_.end());
4055 tableDescIt->second->fragmenter =
nullptr;
4056 CHECK(td->fragmenter ==
nullptr);
4061 instantiateFragmenter(td);
4064 void Catalog::removeFragmenterForTable(
const int table_id)
const {
4066 auto td = getMetadataForTable(table_id,
false);
4067 if (td->fragmenter !=
nullptr) {
4068 auto tableDescIt = tableDescriptorMapById_.find(table_id);
4069 CHECK(tableDescIt != tableDescriptorMapById_.end());
4070 tableDescIt->second->fragmenter =
nullptr;
4071 CHECK(td->fragmenter ==
nullptr);
4076 void Catalog::removeChunks(
const int table_id)
const {
4077 removeFragmenterForTable(table_id);
4080 ChunkKey chunkKey = {currentDB_.dbId, table_id};
4087 SysCatalog::instance().revokeDBObjectPrivilegesFromAll(
4089 std::vector<const TableDescriptor*> tables_to_drop;
4092 const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->
tableId);
4093 if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
4095 const auto physicalTables = physicalTableIt->second;
4096 CHECK(!physicalTables.empty());
4097 for (
size_t i = 0; i < physicalTables.size(); i++) {
4098 int32_t physical_tb_id = physicalTables[i];
4100 getMutableMetadataForTableUnlocked(physical_tb_id);
4102 tables_to_drop.emplace_back(phys_td);
4105 tables_to_drop.emplace_back(td);
4108 for (
auto table : tables_to_drop) {
4109 eraseTablePhysicalData(table);
4111 deleteTableCatalogMetadata(td, tables_to_drop);
4114 void Catalog::deleteTableCatalogMetadata(
4116 const std::vector<const TableDescriptor*>& physical_tables) {
4119 sqliteConnector_.query(
"BEGIN TRANSACTION");
4122 sqliteConnector_.query_with_text_param(
4123 "DELETE FROM mapd_logical_to_physical WHERE logical_table_id = ?",
4125 logicalToPhysicalTableMapById_.erase(logical_table->
tableId);
4126 for (
auto table : physical_tables) {
4127 eraseTableMetadata(table);
4129 }
catch (std::exception& e) {
4130 sqliteConnector_.query(
"ROLLBACK TRANSACTION");
4133 sqliteConnector_.query(
"END TRANSACTION");
4137 executeDropTableSqliteQueries(td);
4139 dropTableFromJsonUnlocked(td->
tableName);
4141 calciteMgr_->updateMetadata(currentDB_.dbName, td->
tableName);
4149 const int tableId = td->
tableId;
4150 sqliteConnector_.query_with_text_param(
"DELETE FROM mapd_tables WHERE tableid = ?",
4152 sqliteConnector_.query_with_text_params(
4153 "select comp_param from mapd_columns where compression = ? and tableid = ?",
4155 int numRows = sqliteConnector_.getNumRows();
4156 std::vector<int> dict_id_list;
4157 for (
int r = 0; r < numRows; ++r) {
4158 dict_id_list.push_back(sqliteConnector_.getData<
int>(r, 0));
4160 for (
auto dict_id : dict_id_list) {
4161 sqliteConnector_.query_with_text_params(
4162 "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
4165 sqliteConnector_.query_with_text_params(
4166 "DELETE FROM mapd_dictionaries WHERE dictid in (select comp_param from "
4167 "mapd_columns where compression = ? "
4168 "and tableid = ?) and refcount = 0",
4170 sqliteConnector_.query_with_text_param(
"DELETE FROM mapd_columns WHERE tableid = ?",
4173 sqliteConnector_.query_with_text_param(
"DELETE FROM mapd_views WHERE tableid = ?",
4177 sqliteConnector_.query_with_text_param(
4178 "DELETE FROM omnisci_foreign_tables WHERE table_id = ?",
std::to_string(tableId));
4182 void Catalog::renamePhysicalTable(
const TableDescriptor* td,
const string& newTableName) {
4186 sqliteConnector_.query(
"BEGIN TRANSACTION");
4188 sqliteConnector_.query_with_text_params(
4189 "UPDATE mapd_tables SET name = ? WHERE tableid = ?",
4191 }
catch (std::exception& e) {
4192 sqliteConnector_.query(
"ROLLBACK TRANSACTION");
4195 sqliteConnector_.query(
"END TRANSACTION");
4196 TableDescriptorMap::iterator tableDescIt =
4198 CHECK(tableDescIt != tableDescriptorMap_.end());
4199 calciteMgr_->updateMetadata(currentDB_.dbName, td->
tableName);
4203 tableDescriptorMap_.erase(tableDescIt);
4204 tableDescriptorMap_[
to_upper(newTableName)] = changeTd;
4205 calciteMgr_->updateMetadata(currentDB_.dbName, td->
tableName);
4208 void Catalog::renameTable(
const TableDescriptor* td,
const string& newTableName) {
4213 const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->
tableId);
4214 if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
4215 const auto physicalTables = physicalTableIt->second;
4216 CHECK(!physicalTables.empty());
4217 for (
size_t i = 0; i < physicalTables.size(); i++) {
4218 int32_t physical_tb_id = physicalTables[i];
4221 std::string newPhysTableName =
4222 generatePhysicalTableName(newTableName, static_cast<int32_t>(i + 1));
4223 renamePhysicalTable(phys_td, newPhysTableName);
4226 renamePhysicalTable(td, newTableName);
4232 key.
dbId = currentDB_.dbId;
4235 object.setObjectKey(key);
4236 auto objdescs = SysCatalog::instance().getMetadataForObject(
4238 for (
auto obj : objdescs) {
4239 Grantee* grnt = SysCatalog::instance().getGrantee(obj->roleName);
4244 SysCatalog::instance().renameObjectsInDescriptorMap(
object, *
this);
4248 void Catalog::renamePhysicalTable(std::vector<std::pair<std::string, std::string>>& names,
4249 std::vector<int>& tableIds) {
4255 for (
size_t i = 0; i < names.size(); i++) {
4256 int tableId = tableIds[i];
4257 std::string& newTableName = names[i].second;
4259 sqliteConnector_.query_with_text_params(
4260 "UPDATE mapd_tables SET name = ? WHERE tableid = ?",
4261 std::vector<std::string>{newTableName,
std::to_string(tableId)});
4263 }
catch (std::exception& e) {
4264 sqliteConnector_.query(
"ROLLBACK TRANSACTION");
4269 for (
size_t i = 0; i < names.size(); i++) {
4270 std::string& curTableName = names[i].first;
4271 std::string& newTableName = names[i].second;
4273 TableDescriptorMap::iterator tableDescIt =
4274 tableDescriptorMap_.find(
to_upper(curTableName));
4275 CHECK(tableDescIt != tableDescriptorMap_.end());
4276 calciteMgr_->updateMetadata(currentDB_.dbName, curTableName);
4281 tableDescriptorMap_.erase(tableDescIt);
4282 tableDescriptorMap_[
to_upper(newTableName)] = changeTd;
4283 calciteMgr_->updateMetadata(currentDB_.dbName, curTableName);
4292 std::map<std::string, int>& cachedTableMap,
4293 std::string& curTableName) {
4294 auto iter = cachedTableMap.find(curTableName);
4295 if ((iter != cachedTableMap.end())) {
4298 int tableId = (*iter).second;
4299 if (tableId == -1) {
4311 std::string& curTableName,
4312 std::string& newTableName,
4315 cachedTableMap[curTableName] = -1;
4318 cachedTableMap[newTableName] = tableId;
4321 void Catalog::renameTable(std::vector<std::pair<std::string, std::string>>& names) {
4324 std::vector<int> tableIds;
4331 std::map<int, size_t> uniqueOrderedTableIds;
4334 std::map<std::string, int> cachedTableMap;
4339 for (
size_t i = 0; i < names.size(); i++) {
4340 std::string& curTableName = names[i].first;
4341 std::string& newTableName = names[i].second;
4348 tableIds.push_back(td->
tableId);
4349 if (uniqueOrderedTableIds.find(td->
tableId) == uniqueOrderedTableIds.end()) {
4351 uniqueOrderedTableIds[td->
tableId] = i;
4356 CHECK_EQ(tableIds.size(), names.size());
4370 for (
auto& idPair : uniqueOrderedTableIds) {
4371 std::string& tableName = names[idPair.second].first;
4372 tableLocks.emplace_back(
4375 *
this, tableName,
false)));
4384 sqliteConnector_.query(
"BEGIN TRANSACTION");
4387 std::vector<std::pair<std::string, std::string>> allNames;
4388 std::vector<int> allTableIds;
4390 for (
size_t i = 0; i < names.size(); i++) {
4391 int tableId = tableIds[i];
4392 std::string& curTableName = names[i].first;
4393 std::string& newTableName = names[i].second;
4396 const auto physicalTableIt = logicalToPhysicalTableMapById_.find(tableId);
4397 if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
4398 const auto physicalTables = physicalTableIt->second;
4399 CHECK(!physicalTables.empty());
4400 for (
size_t k = 0; k < physicalTables.size(); k++) {
4401 int32_t physical_tb_id = physicalTables[k];
4404 std::string newPhysTableName =
4405 generatePhysicalTableName(newTableName, static_cast<int32_t>(k + 1));
4406 allNames.emplace_back(phys_td->
tableName, newPhysTableName);
4407 allTableIds.push_back(phys_td->
tableId);
4410 allNames.emplace_back(curTableName, newTableName);
4411 allTableIds.push_back(tableId);
4414 renamePhysicalTable(allNames, allTableIds);
4416 sqliteConnector_.query(
"END TRANSACTION");
4421 for (
size_t i = 0; i < names.size(); i++) {
4422 int tableId = tableIds[i];
4423 std::string& newTableName = names[i].second;
4427 key.
dbId = currentDB_.dbId;
4432 object.setObjectKey(key);
4434 auto objdescs = SysCatalog::instance().getMetadataForObject(
4436 for (
auto obj : objdescs) {
4437 Grantee* grnt = SysCatalog::instance().getGrantee(obj->roleName);
4442 SysCatalog::instance().renameObjectsInDescriptorMap(
object, *
this);
4454 const string& newColumnName) {
4457 sqliteConnector_.query(
"BEGIN TRANSACTION");
4462 std::string new_column_name = cdx->columnName;
4463 new_column_name.replace(0, cd->
columnName.size(), newColumnName);
4464 sqliteConnector_.query_with_text_params(
4465 "UPDATE mapd_columns SET name = ? WHERE tableid = ? AND columnid = ?",
4466 std::vector<std::string>{new_column_name,
4470 }
catch (std::exception& e) {
4471 sqliteConnector_.query(
"ROLLBACK TRANSACTION");
4474 sqliteConnector_.query(
"END TRANSACTION");
4475 calciteMgr_->updateMetadata(currentDB_.dbName, td->
tableName);
4479 ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.find(
4481 CHECK(columnDescIt != columnDescriptorMap_.end());
4484 columnDescriptorMap_.erase(columnDescIt);
4485 columnDescriptorMap_[std::make_tuple(td->
tableId,
to_upper(changeCd->columnName))] =
4488 calciteMgr_->updateMetadata(currentDB_.dbName, td->
tableName);
4494 sqliteConnector_.query(
"BEGIN TRANSACTION");
4497 sqliteConnector_.query_with_text_params(
4498 "SELECT id FROM mapd_dashboards WHERE name = ? and userid = ?",
4500 if (sqliteConnector_.getNumRows() > 0) {
4501 sqliteConnector_.query_with_text_params(
4502 "UPDATE mapd_dashboards SET state = ?, image_hash = ?, metadata = ?, "
4504 "datetime('now') where name = ? "
4512 sqliteConnector_.query_with_text_params(
4513 "INSERT INTO mapd_dashboards (name, state, image_hash, metadata, "
4518 "datetime('now'), ?)",
4525 }
catch (std::exception& e) {
4526 sqliteConnector_.query(
"ROLLBACK TRANSACTION");
4529 sqliteConnector_.query(
"END TRANSACTION");
4533 sqliteConnector_.query_with_text_params(
4534 "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_dashboards "
4535 "WHERE name = ? and userid = ?",
4537 vd.
dashboardId = sqliteConnector_.getData<
int>(0, 0);
4538 vd.
updateTime = sqliteConnector_.getData<std::string>(0, 1);
4539 }
catch (std::exception& e) {
4544 addFrontendViewToMap(vd);
4547 if (!isInfoSchemaDb()) {
4549 createOrUpdateDashboardSystemRole(
4559 CHECK(sqliteConnector_.getSqlitePtr());
4560 sqliteConnector_.query(
"BEGIN TRANSACTION");
4562 sqliteConnector_.query_with_text_params(
4563 "SELECT id FROM mapd_dashboards WHERE id = ?",
4565 if (sqliteConnector_.getNumRows() > 0) {
4566 sqliteConnector_.query_with_text_params(
4567 "UPDATE mapd_dashboards SET name = ?, state = ?, image_hash = ?, metadata = "
4568 "?, userid = ?, update_time = datetime('now') where id = ? ",
4577 <<
" does not exist in db";
4578 throw runtime_error(
"Error replacing dashboard id " +
4581 }
catch (std::exception& e) {
4582 sqliteConnector_.query(
"ROLLBACK TRANSACTION");
4585 sqliteConnector_.query(
"END TRANSACTION");
4588 for (
auto descp : dashboardDescriptorMap_) {
4589 auto dash = descp.second.get();
4592 auto viewDescIt = dashboardDescriptorMap_.find(
std::to_string(dash->userId) +
":" +
4593 dash->dashboardName);
4595 dashboardDescriptorMap_.end()) {
4596 LOG(
ERROR) <<
"No metadata for dashboard for user " << dash->userId
4597 <<
" dashboard " << dash->dashboardName <<
" does not exist in map";
4598 throw runtime_error(
"No metadata for dashboard for user " +
4600 dash->dashboardName +
" does not exist in map");
4602 dashboardDescriptorMap_.erase(viewDescIt);
4608 <<
" does not exist in map";
4610 " does not exist in map");
4614 sqliteConnector_.query_with_text_params(
4615 "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM "
4619 vd.
updateTime = sqliteConnector_.getData<
string>(0, 1);
4622 addFrontendViewToMapNoLock(vd);
4625 if (!isInfoSchemaDb()) {
4627 createOrUpdateDashboardSystemRole(
4632 std::string Catalog::calculateSHA1(
const std::string& data) {
4633 boost::uuids::detail::sha1 sha1;
4634 unsigned int digest[5];
4635 sha1.process_bytes(data.c_str(), data.length());
4636 sha1.get_digest(digest);
4637 std::stringstream ss;
4638 for (
size_t i = 0; i < 5; i++) {
4639 ss << std::hex << digest[i];
4647 sqliteConnector_.query(
"BEGIN TRANSACTION");
4651 sqliteConnector_.query_with_text_params(
4652 "SELECT linkid FROM mapd_links WHERE link = ? and userid = ?",
4654 if (sqliteConnector_.getNumRows() > 0) {
4655 sqliteConnector_.query_with_text_params(
4656 "UPDATE mapd_links SET update_time = datetime('now') WHERE userid = ? AND "
4660 sqliteConnector_.query_with_text_params(
4661 "INSERT INTO mapd_links (userid, link, view_state, view_metadata, "
4662 "update_time) VALUES (?,?,?,?, datetime('now'))",
4663 std::vector<std::string>{
4667 sqliteConnector_.query_with_text_param(
4668 "SELECT linkid, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_links "
4671 ld.
linkId = sqliteConnector_.getData<
int>(0, 0);
4672 ld.
updateTime = sqliteConnector_.getData<std::string>(0, 1);
4673 }
catch (std::exception& e) {
4674 sqliteConnector_.query(
"ROLLBACK TRANSACTION");
4677 sqliteConnector_.query(
"END TRANSACTION");
4686 const auto column_descriptors =
4687 getAllColumnMetadataForTable(td->
tableId,
false,
true,
true);
4691 for (
auto cd_itr = column_descriptors.begin(); cd_itr != column_descriptors.end();
4700 std::vector<const TableDescriptor*> Catalog::getPhysicalTablesDescriptors(
4702 bool populate_fragmenter)
const {
4704 const auto physicalTableIt =
4705 logicalToPhysicalTableMapById_.find(logical_table_desc->
tableId);
4706 if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
4707 return {logical_table_desc};
4709 const auto physicalTablesIds = physicalTableIt->second;
4710 CHECK(!physicalTablesIds.empty());
4712 std::vector<const TableDescriptor*> physicalTables;
4713 for (
size_t i = 0; i < physicalTablesIds.size(); i++) {
4714 physicalTables.push_back(
4715 getMetadataForTable(physicalTablesIds[i], populate_fragmenter));
4717 return physicalTables;
4720 std::vector<std::pair<int32_t, int32_t>> Catalog::getAllPersistedTableAndShardIds()
4723 std::vector<std::pair<int32_t, int32_t>> table_and_shard_ids;
4724 table_and_shard_ids.reserve(tableDescriptorMapById_.size());
4725 for (
const auto [table_id, td] : tableDescriptorMapById_) {
4728 logicalToPhysicalTableMapById_.find(table_id) ==
4729 logicalToPhysicalTableMapById_.end()) {
4730 table_and_shard_ids.emplace_back(table_id, td->
shard);
4733 return table_and_shard_ids;
4736 const std::map<int, const ColumnDescriptor*> Catalog::getDictionaryToColumnMapping() {
4739 std::map<int, const ColumnDescriptor*> mapping;
4741 const auto tables = getAllTableMetadata();
4742 for (
const auto td :
tables) {
4743 if (td->
shard >= 0) {
4748 for (
auto& cd : getAllColumnMetadataForTable(td->
tableId,
false,
false,
true)) {
4750 if (ti.is_string()) {
4756 if (dict_id > 0 && mapping.end() == mapping.find(dict_id)) {
4757 mapping[dict_id] = cd;
4770 if (td->
shard >= 0) {
4774 switch (get_tables_type) {
4792 std::vector<DBObject> privObjects = {dbObject};
4793 if (!SysCatalog::instance().hasAnyPrivileges(user_metadata, privObjects)) {
4800 std::vector<std::string> Catalog::getTableNamesForUser(
4805 std::vector<std::string> table_names;
4806 const auto tables = getAllTableMetadata();
4807 for (
const auto td :
tables) {
4808 if (filterTableByTypeAndUser(td, user_metadata, get_tables_type)) {
4815 std::vector<TableMetadata> Catalog::getTablesMetadataForUser(
4818 const std::string& filter_table_name)
const {
4822 std::vector<TableMetadata> tables_metadata;
4823 const auto tables = getAllTableMetadata();
4824 for (
const auto td :
tables) {
4825 if (filterTableByTypeAndUser(td, user_metadata, get_tables_type)) {
4826 if (!filter_table_name.empty()) {
4827 if (td->
tableName != filter_table_name) {
4833 tables_metadata.emplace_back(table_metadata);
4836 return tables_metadata;
4839 int Catalog::getLogicalTableId(
const int physicalTableId)
const {
4841 for (
const auto& l : logicalToPhysicalTableMapById_) {
4842 if (l.second.end() != std::find_if(l.second.begin(),
4844 [&](decltype(*l.second.begin()) tid) ->
bool {
4845 return physicalTableId == tid;
4850 return physicalTableId;
4853 void Catalog::checkpoint(
const int logicalTableId)
const {
4854 const auto td = getMetadataForTable(logicalTableId);
4855 const auto shards = getPhysicalTablesDescriptors(td);
4856 for (
const auto shard : shards) {
4857 getDataMgr().checkpoint(getCurrentDB().dbId, shard->tableId);
4861 void Catalog::checkpointWithAutoRollback(
const int logical_table_id)
const {
4862 auto table_epochs = getTableEpochs(getDatabaseId(), logical_table_id);
4864 checkpoint(logical_table_id);
4866 setTableEpochsLogExceptions(getDatabaseId(), table_epochs);
4871 void Catalog::resetTableEpochFloor(
const int logicalTableId)
const {
4873 const auto td = getMetadataForTable(logicalTableId,
false);
4874 const auto shards = getPhysicalTablesDescriptors(td,
false);
4875 for (
const auto shard : shards) {
4876 getDataMgr().resetTableEpochFloor(getCurrentDB().dbId, shard->tableId);
4880 void Catalog::eraseDbMetadata() {
4881 const auto tables = getAllTableMetadata();
4882 for (
const auto table :
tables) {
4883 eraseTableMetadata(table);
4888 calciteMgr_->updateMetadata(currentDB_.dbName,
"");
4891 void Catalog::eraseDbPhysicalData() {
4892 const auto tables = getAllTableMetadata();
4893 for (
const auto table :
tables) {
4894 eraseTablePhysicalData(table);
4899 const int tableId = td->
tableId;
4901 removeFragmenterForTable(tableId);
4903 ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
4912 dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
4916 std::string Catalog::generatePhysicalTableName(
const std::string& logicalTableName,
4917 const int32_t& shardNumber) {
4918 std::string physicalTableName =
4919 logicalTableName + physicalTableNameTag_ +
std::to_string(shardNumber);
4920 return (physicalTableName);
4923 void Catalog::buildForeignServerMapUnlocked() {
4925 sqliteConnector_.query(
4926 "SELECT id, name, data_wrapper_type, options, owner_user_id, creation_time FROM "
4927 "omnisci_foreign_servers");
4928 auto num_rows = sqliteConnector_.getNumRows();
4930 for (
size_t row = 0; row < num_rows; row++) {
4931 auto foreign_server = std::make_shared<foreign_storage::ForeignServer>(
4932 sqliteConnector_.getData<
int>(row, 0),
4933 sqliteConnector_.getData<std::string>(row, 1),
4934 sqliteConnector_.getData<std::string>(row, 2),
4935 sqliteConnector_.getData<std::string>(row, 3),
4936 sqliteConnector_.getData<std::int32_t>(row, 4),
4937 sqliteConnector_.getData<std::int32_t>(row, 5));
4938 foreignServerMap_[foreign_server->name] = foreign_server;
4939 foreignServerMapById_[foreign_server->id] = foreign_server;
4943 void Catalog::updateForeignTablesInMapUnlocked() {
4945 sqliteConnector_.query(
4946 "SELECT table_id, server_id, options, last_refresh_time, next_refresh_time from "
4947 "omnisci_foreign_tables");
4948 auto num_rows = sqliteConnector_.getNumRows();
4949 for (
size_t r = 0; r < num_rows; r++) {
4950 const auto table_id = sqliteConnector_.getData<int32_t>(r, 0);
4951 const auto server_id = sqliteConnector_.getData<int32_t>(r, 1);
4952 const auto& options = sqliteConnector_.getData<std::string>(r, 2);
4953 const auto last_refresh_time = sqliteConnector_.getData<int64_t>(r, 3);
4954 const auto next_refresh_time = sqliteConnector_.getData<int64_t>(r, 4);
4956 CHECK(tableDescriptorMapById_.find(table_id) != tableDescriptorMapById_.end());
4957 auto foreign_table =
4959 CHECK(foreign_table);
4960 foreign_table->foreign_server = foreignServerMapById_[server_id].get();
4961 CHECK(foreign_table->foreign_server);
4962 foreign_table->populateOptionsMap(options);
4963 foreign_table->last_refresh_time = last_refresh_time;
4964 foreign_table->next_refresh_time = next_refresh_time;
4965 if (foreign_table->is_system_table) {
4966 foreign_table->is_in_memory_system_table =
4968 foreign_table->foreign_server->data_wrapper_type);
4973 void Catalog::setForeignServerProperty(
const std::string& server_name,
4974 const std::string& property,
4975 const std::string& value) {
4977 sqliteConnector_.query_with_text_params(
4978 "SELECT id from omnisci_foreign_servers where name = ?",
4979 std::vector<std::string>{server_name});
4980 auto num_rows = sqliteConnector_.getNumRows();
4983 auto server_id = sqliteConnector_.getData<int32_t>(0, 0);
4984 sqliteConnector_.query_with_text_params(
4985 "UPDATE omnisci_foreign_servers SET " + property +
" = ? WHERE id = ?",
4988 throw std::runtime_error{
"Can not change property \"" +
property +
4989 "\" for foreign server." +
" Foreign server \"" +
4990 server_name +
"\" is not found."};
4994 void Catalog::createDefaultServersIfNotExists() {
5000 auto local_csv_server = std::make_unique<foreign_storage::ForeignServer>(
5001 "default_local_delimited",
5005 local_csv_server->validate();
5006 createForeignServerNoLocks(std::move(local_csv_server),
true);
5008 #ifdef ENABLE_IMPORT_PARQUET
5009 auto local_parquet_server = std::make_unique<foreign_storage::ForeignServer>(
5010 "default_local_parquet",
5014 local_parquet_server->validate();
5015 createForeignServerNoLocks(std::move(local_parquet_server),
true);
5018 auto local_regex_parser_server = std::make_unique<foreign_storage::ForeignServer>(
5019 "default_local_regex_parsed",
5023 local_regex_parser_server->validate();
5024 createForeignServerNoLocks(std::move(local_regex_parser_server),
true);
5028 void Catalog::setForReload(
const int32_t tableId) {
5029 const auto td = getMetadataForTable(tableId);
5030 for (
const auto shard : getPhysicalTablesDescriptors(td)) {
5031 const auto tableEpoch = getTableEpoch(currentDB_.dbId, shard->tableId);
5032 setTableEpoch(currentDB_.dbId, shard->tableId, tableEpoch);
5037 std::vector<std::string> Catalog::getTableDataDirectories(
5039 const auto global_file_mgr = getDataMgr().getGlobalFileMgr();
5040 std::vector<std::string> file_paths;
5041 for (
auto shard : getPhysicalTablesDescriptors(td)) {
5043 global_file_mgr->getFileMgr(currentDB_.dbId, shard->tableId));
5044 boost::filesystem::path file_path(file_mgr->getFileMgrBasePath());
5045 file_paths.push_back(file_path.filename().string());
5052 bool file_name_only)
const {
5057 const DictRef dictRef(currentDB_.dbId, dictId);
5058 const auto dit = dictDescriptorMapByRef_.find(dictRef);
5059 CHECK(dit != dictDescriptorMapByRef_.end());
5061 if (file_name_only) {
5062 boost::filesystem::path file_path(dit->second->dictFolderPath);
5063 return file_path.filename().string();
5065 return dit->second->dictFolderPath;
5068 return std::string();
5072 std::vector<std::string> Catalog::getTableDictDirectories(
5074 std::vector<std::string> file_paths;
5075 for (
auto cd : getAllColumnMetadataForTable(td->
tableId,
false,
false,
true)) {
5076 auto file_base = getColumnDictDirectory(cd);
5077 if (!file_base.empty() &&
5078 file_paths.end() == std::find(file_paths.begin(), file_paths.end(), file_base)) {
5079 file_paths.push_back(file_base);
5085 std::set<std::string> Catalog::getTableDictDirectoryPaths(int32_t table_id)
const {
5087 std::set<std::string> directory_paths;
5088 auto it = dict_columns_by_table_id_.find(table_id);
5089 if (it != dict_columns_by_table_id_.end()) {
5090 for (
auto cd : it->second) {
5091 auto directory_path = getColumnDictDirectory(cd,
false);
5092 if (!directory_path.empty()) {
5093 directory_paths.emplace(directory_path);
5097 return directory_paths;
5107 std::ostringstream os;
5108 os <<
"CREATE TABLE @T (";
5110 const auto cds = getAllColumnMetadataForTable(td->
tableId,
false,
false,
false);
5112 std::vector<std::string> shared_dicts;
5113 std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
5114 for (
const auto cd : cds) {
5126 os <<
" " << ti.get_type_name();
5128 os << (ti.get_notnull() ?
" NOT NULL" :
"");
5132 if (ti.is_string() || (ti.is_array() && ti.get_subtype() ==
kTEXT)) {
5133 auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
5136 const auto dict_id = ti.get_comp_param();
5137 const DictRef dict_ref(currentDB_.dbId, dict_id);
5138 const auto dict_it = dictDescriptorMapByRef_.find(dict_ref);
5139 CHECK(dict_it != dictDescriptorMapByRef_.end());
5140 const auto dict_name = dict_it->second->dictName;
5143 if (dict_root_cds.end() == dict_root_cds.find(dict_name)) {
5144 dict_root_cds[dict_name] = cd;
5145 os <<
" ENCODING " << ti.get_compression_name() <<
"(" << (size * 8) <<
")";
5147 const auto dict_root_cd = dict_root_cds[dict_name];
5148 shared_dicts.push_back(
"SHARED DICTIONARY (" + cd->
columnName +
5149 ") REFERENCES @T(" + dict_root_cd->columnName +
")");
5154 os <<
" ENCODING NONE";
5156 }
else if (ti.is_date_in_days() ||
5157 (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
5158 const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
5159 os <<
" ENCODING " << ti.get_compression_name() <<
"(" << comp_param <<
")";
5160 }
else if (ti.is_geometry()) {
5162 os <<
" ENCODING " << ti.get_compression_name() <<
"(" << ti.get_comp_param()
5165 os <<
" ENCODING NONE";
5172 if (shared_dicts.size()) {
5176 std::vector<std::string> with_options;
5181 with_options.emplace_back(td->
hasDeletedCol ?
"VACUUM='DELAYED'"
5182 :
"VACUUM='IMMEDIATE'");
5184 with_options.push_back(
"PARTITIONS='" + td->
partitions +
"'");
5189 os <<
", SHARD KEY(" << shard_cd->columnName <<
")";
5190 with_options.push_back(
5197 with_options.push_back(
"SORT_COLUMN='" + sort_cd->columnName +
"'");
5201 with_options.push_back(
"MAX_ROLLBACK_EPOCHS=" +
5212 return std::find_if(str.begin(), str.end(), [](
const unsigned char& ch) {
5213 return std::isspace(ch);
5219 std::string_view str,
5220 std::string_view chars =
"`~!@#$%^&*()-=+[{]}\\|;:'\",<.>/?") {
5221 return str.find_first_of(chars) != std::string_view::npos;
5231 bool multiline_formatting,
5232 bool dump_defaults)
const {
5234 return dumpCreateTableUnlocked(td, multiline_formatting, dump_defaults);
5237 std::optional<std::string> Catalog::dumpCreateTable(int32_t table_id,
5238 bool multiline_formatting,
5239 bool dump_defaults)
const {
5241 const auto td = getMutableMetadataForTableUnlocked(table_id);
5245 return dumpCreateTableUnlocked(td, multiline_formatting, dump_defaults);
5249 bool multiline_formatting,
5250 bool dump_defaults)
const {
5252 std::ostringstream os;
5255 os <<
"CREATE FOREIGN TABLE " << td->
tableName <<
" (";
5256 }
else if (!td->
isView) {
5267 std::vector<std::string> additional_info;
5268 std::set<std::string> shared_dict_column_names;
5270 gatherAdditionalInfo(additional_info, shared_dict_column_names, td);
5273 const auto cds = getAllColumnMetadataForTable(td->
tableId,
false,
false,
false);
5274 std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
5276 for (
const auto cd : cds) {
5281 if (!multiline_formatting) {
5287 if (multiline_formatting) {
5300 os <<
" " << ti.get_type_name();
5302 os << (ti.get_notnull() ?
" NOT NULL" :
"");
5306 if (shared_dict_column_names.find(cd->
columnName) ==
5307 shared_dict_column_names.end()) {
5310 if (ti.is_string() || (ti.is_array() && ti.get_subtype() ==
kTEXT)) {
5311 auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
5313 os <<
" ENCODING " << ti.get_compression_name() <<
"(" << (size * 8) <<
")";
5315 os <<
" ENCODING NONE";
5317 }
else if (ti.is_date_in_days() ||
5318 (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
5319 const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
5320 os <<
" ENCODING " << ti.get_compression_name() <<
"(" << comp_param <<
")";
5321 }
else if (ti.is_geometry()) {
5323 os <<
" ENCODING " << ti.get_compression_name() <<
"(" << ti.get_comp_param()
5326 os <<
" ENCODING NONE";
5333 if (additional_info.size()) {
5335 if (!multiline_formatting) {
5345 std::vector<std::string> with_options;
5347 if (multiline_formatting) {
5352 os <<
"SERVER " << foreign_table->foreign_server->name;
5355 for (
const auto& [option, value] : foreign_table->options) {
5356 with_options.emplace_back(option +
"='" + value +
"'");