OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Catalog.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
23 #include "Catalog/Catalog.h"
24 
25 #include <algorithm>
26 #include <boost/algorithm/string/predicate.hpp>
27 #include <boost/filesystem.hpp>
28 #include <boost/range/adaptor/map.hpp>
29 #include <boost/version.hpp>
30 #include <cassert>
31 #include <cerrno>
32 #include <cstdio>
33 #include <cstring>
34 #include <exception>
35 #include <fstream>
36 #include <list>
37 #include <memory>
38 #include <random>
39 #include <regex>
40 #include <sstream>
41 
42 #if BOOST_VERSION >= 106600
43 #include <boost/uuid/detail/sha1.hpp>
44 #else
45 #include <boost/uuid/sha1.hpp>
46 #endif
47 #include <rapidjson/document.h>
48 #include <rapidjson/istreamwrapper.h>
49 #include <rapidjson/ostreamwrapper.h>
50 #include <rapidjson/writer.h>
51 
52 #include "Catalog/SysCatalog.h"
53 
54 #include "QueryEngine/Execute.h"
56 
63 #include "Fragmenter/Fragmenter.h"
65 #include "LockMgr/LockMgr.h"
68 #include "Parser/ParserNode.h"
69 #include "QueryEngine/Execute.h"
71 #include "RefreshTimeCalculator.h"
72 #include "Shared/DateTimeParser.h"
73 #include "Shared/File.h"
74 #include "Shared/StringTransform.h"
75 #include "Shared/SysDefinitions.h"
76 #include "Shared/measure.h"
77 #include "Shared/misc.h"
79 
80 #include "MapDRelease.h"
81 #include "RWLocks.h"
83 
84 #include "Shared/distributed.h"
85 
86 using Chunk_NS::Chunk;
89 using std::list;
90 using std::map;
91 using std::pair;
92 using std::runtime_error;
93 using std::string;
94 using std::vector;
95 
96 bool g_enable_fsi{true};
97 bool g_enable_s3_fsi{false};
101 extern bool g_cache_string_hash;
102 extern bool g_enable_system_tables;
103 
104 // Serialize temp tables to a json file in the Catalogs directory for Calcite parsing
105 // under unit testing.
107 
108 namespace Catalog_Namespace {
109 
110 const int DEFAULT_INITIAL_VERSION = 1; // start at version 1
112  1073741824; // 2^30, give room for over a billion non-temp tables
114  1073741824; // 2^30, give room for over a billion non-temp dictionaries
115 
116 const std::string Catalog::physicalTableNameTag_("_shard_#");
117 
118 thread_local bool Catalog::thread_holds_read_lock = false;
119 
124 
125 // migration will be done as two step process this release
126 // will create and use new table
127 // next release will remove old table, doing this to have fall back path
128 // incase of migration failure
131  sqliteConnector_.query("BEGIN TRANSACTION");
132  try {
134  "SELECT name FROM sqlite_master WHERE type='table' AND name='mapd_dashboards'");
135  if (sqliteConnector_.getNumRows() != 0) {
136  // already done
137  sqliteConnector_.query("END TRANSACTION");
138  return;
139  }
141  "CREATE TABLE mapd_dashboards (id integer primary key autoincrement, name text , "
142  "userid integer references mapd_users, state text, image_hash text, update_time "
143  "timestamp, "
144  "metadata text, UNIQUE(userid, name) )");
145  // now copy content from old table to new table
147  "insert into mapd_dashboards (id, name , "
148  "userid, state, image_hash, update_time , "
149  "metadata) "
150  "SELECT viewid , name , userid, view_state, image_hash, update_time, "
151  "view_metadata "
152  "from mapd_frontend_views");
153  } catch (const std::exception& e) {
154  sqliteConnector_.query("ROLLBACK TRANSACTION");
155  throw;
156  }
157  sqliteConnector_.query("END TRANSACTION");
158 }
159 
160 namespace {
161 
162 inline auto table_json_filepath(const std::string& base_path,
163  const std::string& db_name) {
164  return boost::filesystem::path(base_path + "/" + shared::kCatalogDirectoryName + "/" +
165  db_name + "_temp_tables.json");
166 }
167 
168 std::map<int32_t, std::string> get_user_id_to_user_name_map();
169 } // namespace
170 
172 
173 Catalog::Catalog(const string& basePath,
174  const DBMetadata& curDB,
175  std::shared_ptr<Data_Namespace::DataMgr> dataMgr,
176  const std::vector<LeafHostInfo>& string_dict_hosts,
177  std::shared_ptr<Calcite> calcite,
178  bool is_new_db)
179  : basePath_(basePath)
180  , sqliteConnector_(curDB.dbName, basePath + "/" + shared::kCatalogDirectoryName + "/")
181  , currentDB_(curDB)
182  , dataMgr_(dataMgr)
183  , string_dict_hosts_(string_dict_hosts)
184  , calciteMgr_(calcite)
185  , nextTempTableId_(MAPD_TEMP_TABLE_START_ID)
186  , nextTempDictId_(MAPD_TEMP_DICT_START_ID)
187  , dcatalogMutex_(std::make_unique<heavyai::DistributedSharedMutex>(
188  std::filesystem::path(basePath_) / shared::kLockfilesDirectoryName /
189  shared::kCatalogDirectoryName / (currentDB_.dbName + ".lockfile"),
190  [this](size_t) {
191  if (!initialized_) {
192  return;
193  }
194  const auto user_name_by_user_id = get_user_id_to_user_name_map();
196  *dsqliteMutex_);
197  reloadCatalogMetadataUnlocked(user_name_by_user_id);
198  }))
199  , dsqliteMutex_(std::make_unique<heavyai::DistributedSharedMutex>(
200  std::filesystem::path(basePath_) / shared::kLockfilesDirectoryName /
201  shared::kCatalogDirectoryName / (currentDB_.dbName + ".sqlite.lockfile")))
202  , sqliteMutex_()
203  , sharedMutex_()
206  if (!g_enable_fsi) {
207  CHECK(!g_enable_system_tables) << "System tables require FSI to be enabled";
208  CHECK(!g_enable_s3_fsi) << "S3 FSI requires FSI to be enabled";
209  }
210 
211  if (!is_new_db) {
212  CheckAndExecuteMigrations();
213  }
214 
215  buildMaps();
216 
217  if (g_enable_fsi) {
218  createDefaultServersIfNotExists();
219  }
220  if (!is_new_db) {
221  CheckAndExecuteMigrationsPostBuildMaps();
222  }
224  boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
225  }
226  conditionallyInitializeSystemObjects();
227  // once all initialized use real object
228  initialized_ = true;
229 }
230 
232  // cat_write_lock write_lock(this);
233 
234  // must clean up heap-allocated TableDescriptor and ColumnDescriptor structs
235  for (TableDescriptorMap::iterator tableDescIt = tableDescriptorMap_.begin();
236  tableDescIt != tableDescriptorMap_.end();
237  ++tableDescIt) {
238  tableDescIt->second->fragmenter = nullptr;
239  delete tableDescIt->second;
240  }
241 
242  // TableDescriptorMapById points to the same descriptors. No need to delete
243 
244  for (ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.begin();
245  columnDescIt != columnDescriptorMap_.end();
246  ++columnDescIt) {
247  delete columnDescIt->second;
248  }
249 
250  // ColumnDescriptorMapById points to the same descriptors. No need to delete
251 
253  boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
254  }
255 }
256 
258  if (initialized_) {
259  return this;
260  } else {
261  return SysCatalog::instance().getDummyCatalog().get();
262  }
263 }
264 
267  sqliteConnector_.query("BEGIN TRANSACTION");
268  try {
269  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_tables)");
270  std::vector<std::string> cols;
271  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
272  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
273  }
274  if (std::find(cols.begin(), cols.end(), std::string("max_chunk_size")) ==
275  cols.end()) {
276  string queryString("ALTER TABLE mapd_tables ADD max_chunk_size BIGINT DEFAULT " +
278  sqliteConnector_.query(queryString);
279  }
280  if (std::find(cols.begin(), cols.end(), std::string("shard_column_id")) ==
281  cols.end()) {
282  string queryString("ALTER TABLE mapd_tables ADD shard_column_id BIGINT DEFAULT " +
283  std::to_string(0));
284  sqliteConnector_.query(queryString);
285  }
286  if (std::find(cols.begin(), cols.end(), std::string("shard")) == cols.end()) {
287  string queryString("ALTER TABLE mapd_tables ADD shard BIGINT DEFAULT " +
288  std::to_string(-1));
289  sqliteConnector_.query(queryString);
290  }
291  if (std::find(cols.begin(), cols.end(), std::string("num_shards")) == cols.end()) {
292  string queryString("ALTER TABLE mapd_tables ADD num_shards BIGINT DEFAULT " +
293  std::to_string(0));
294  sqliteConnector_.query(queryString);
295  }
296  if (std::find(cols.begin(), cols.end(), std::string("key_metainfo")) == cols.end()) {
297  string queryString("ALTER TABLE mapd_tables ADD key_metainfo TEXT DEFAULT '[]'");
298  sqliteConnector_.query(queryString);
299  }
300  if (std::find(cols.begin(), cols.end(), std::string("userid")) == cols.end()) {
301  string queryString("ALTER TABLE mapd_tables ADD userid integer DEFAULT " +
303  sqliteConnector_.query(queryString);
304  }
305  if (std::find(cols.begin(), cols.end(), std::string("sort_column_id")) ==
306  cols.end()) {
308  "ALTER TABLE mapd_tables ADD sort_column_id INTEGER DEFAULT 0");
309  }
310  if (std::find(cols.begin(), cols.end(), std::string("storage_type")) == cols.end()) {
311  string queryString("ALTER TABLE mapd_tables ADD storage_type TEXT DEFAULT ''");
312  sqliteConnector_.query(queryString);
313  }
314  if (std::find(cols.begin(), cols.end(), std::string("max_rollback_epochs")) ==
315  cols.end()) {
316  string queryString("ALTER TABLE mapd_tables ADD max_rollback_epochs INT DEFAULT " +
317  std::to_string(-1));
318  sqliteConnector_.query(queryString);
319  }
320  if (std::find(cols.begin(), cols.end(), std::string("is_system_table")) ==
321  cols.end()) {
322  string queryString("ALTER TABLE mapd_tables ADD is_system_table BOOLEAN DEFAULT 0");
323  sqliteConnector_.query(queryString);
324  }
325  } catch (std::exception& e) {
326  sqliteConnector_.query("ROLLBACK TRANSACTION");
327  throw;
328  }
329  sqliteConnector_.query("END TRANSACTION");
330 }
331 
334  sqliteConnector_.query("BEGIN TRANSACTION");
335  try {
337  "select name from sqlite_master WHERE type='table' AND "
338  "name='mapd_version_history'");
339  if (sqliteConnector_.getNumRows() == 0) {
341  "CREATE TABLE mapd_version_history(version integer, migration_history text "
342  "unique)");
343  } else {
345  "select * from mapd_version_history where migration_history = "
346  "'notnull_fixlen_arrays'");
347  if (sqliteConnector_.getNumRows() != 0) {
348  // legacy fixlen arrays had migrated
349  // no need for further execution
350  sqliteConnector_.query("END TRANSACTION");
351  return;
352  }
353  }
354  // Insert check for migration
356  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
357  std::vector<std::string>{std::to_string(MAPD_VERSION), "notnull_fixlen_arrays"});
358  LOG(INFO) << "Updating mapd_columns, legacy fixlen arrays";
359  // Upating all fixlen array columns
360  string queryString("UPDATE mapd_columns SET is_notnull=1 WHERE coltype=" +
361  std::to_string(kARRAY) + " AND size>0;");
362  sqliteConnector_.query(queryString);
363  } catch (std::exception& e) {
364  sqliteConnector_.query("ROLLBACK TRANSACTION");
365  throw;
366  }
367  sqliteConnector_.query("END TRANSACTION");
368 }
369 
372  sqliteConnector_.query("BEGIN TRANSACTION");
373  try {
375  "select name from sqlite_master WHERE type='table' AND "
376  "name='mapd_version_history'");
377  if (sqliteConnector_.getNumRows() == 0) {
379  "CREATE TABLE mapd_version_history(version integer, migration_history text "
380  "unique)");
381  } else {
383  "select * from mapd_version_history where migration_history = "
384  "'notnull_geo_columns'");
385  if (sqliteConnector_.getNumRows() != 0) {
386  // legacy geo columns had migrated
387  // no need for further execution
388  sqliteConnector_.query("END TRANSACTION");
389  return;
390  }
391  }
392  // Insert check for migration
394  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
395  std::vector<std::string>{std::to_string(MAPD_VERSION), "notnull_geo_columns"});
396  LOG(INFO) << "Updating mapd_columns, legacy geo columns";
397  // Upating all geo columns
398  string queryString(
399  "UPDATE mapd_columns SET is_notnull=1 WHERE coltype=" + std::to_string(kPOINT) +
400  " OR coltype=" + std::to_string(kMULTIPOINT) + " OR coltype=" +
402  " OR coltype=" + std::to_string(kPOLYGON) +
403  " OR coltype=" + std::to_string(kMULTIPOLYGON) + ";");
404  sqliteConnector_.query(queryString);
405  } catch (std::exception& e) {
406  sqliteConnector_.query("ROLLBACK TRANSACTION");
407  throw;
408  }
409  sqliteConnector_.query("END TRANSACTION");
410 }
411 
414  sqliteConnector_.query("BEGIN TRANSACTION");
415  try {
416  // check table still exists
418  "SELECT name FROM sqlite_master WHERE type='table' AND "
419  "name='mapd_frontend_views'");
420  if (sqliteConnector_.getNumRows() == 0) {
421  // table does not exists
422  // no need to migrate
423  sqliteConnector_.query("END TRANSACTION");
424  return;
425  }
426  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_frontend_views)");
427  std::vector<std::string> cols;
428  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
429  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
430  }
431  if (std::find(cols.begin(), cols.end(), std::string("image_hash")) == cols.end()) {
432  sqliteConnector_.query("ALTER TABLE mapd_frontend_views ADD image_hash text");
433  }
434  if (std::find(cols.begin(), cols.end(), std::string("update_time")) == cols.end()) {
435  sqliteConnector_.query("ALTER TABLE mapd_frontend_views ADD update_time timestamp");
436  }
437  if (std::find(cols.begin(), cols.end(), std::string("view_metadata")) == cols.end()) {
438  sqliteConnector_.query("ALTER TABLE mapd_frontend_views ADD view_metadata text");
439  }
440  } catch (std::exception& e) {
441  sqliteConnector_.query("ROLLBACK TRANSACTION");
442  throw;
443  }
444  sqliteConnector_.query("END TRANSACTION");
445 }
446 
449  sqliteConnector_.query("BEGIN TRANSACTION");
450  try {
452  "CREATE TABLE IF NOT EXISTS mapd_links (linkid integer primary key, userid "
453  "integer references mapd_users, "
454  "link text unique, view_state text, update_time timestamp, view_metadata text)");
455  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_links)");
456  std::vector<std::string> cols;
457  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
458  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
459  }
460  if (std::find(cols.begin(), cols.end(), std::string("view_metadata")) == cols.end()) {
461  sqliteConnector_.query("ALTER TABLE mapd_links ADD view_metadata text");
462  }
463  } catch (const std::exception& e) {
464  sqliteConnector_.query("ROLLBACK TRANSACTION");
465  throw;
466  }
467  sqliteConnector_.query("END TRANSACTION");
468 }
469 
472  sqliteConnector_.query("BEGIN TRANSACTION");
473  try {
474  sqliteConnector_.query("UPDATE mapd_links SET userid = 0 WHERE userid IS NULL");
475  // check table still exists
477  "SELECT name FROM sqlite_master WHERE type='table' AND "
478  "name='mapd_frontend_views'");
479  if (sqliteConnector_.getNumRows() == 0) {
480  // table does not exists
481  // no need to migrate
482  sqliteConnector_.query("END TRANSACTION");
483  return;
484  }
486  "UPDATE mapd_frontend_views SET userid = 0 WHERE userid IS NULL");
487  } catch (const std::exception& e) {
488  sqliteConnector_.query("ROLLBACK TRANSACTION");
489  throw;
490  }
491  sqliteConnector_.query("END TRANSACTION");
492 }
493 
494 // introduce DB version into the tables table
495 // if the DB does not have a version reset all pagesizes to 2097152 to be compatible with
496 // old value
497 
500  if (currentDB_.dbName.length() == 0) {
501  // updateDictionaryNames dbName length is zero nothing to do here
502  return;
503  }
504  sqliteConnector_.query("BEGIN TRANSACTION");
505  try {
506  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_tables)");
507  std::vector<std::string> cols;
508  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
509  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
510  }
511  if (std::find(cols.begin(), cols.end(), std::string("version_num")) == cols.end()) {
512  LOG(INFO) << "Updating mapd_tables updatePageSize";
513  // No version number
514  // need to update the defaul tpagesize to old correct value
515  sqliteConnector_.query("UPDATE mapd_tables SET frag_page_size = 2097152 ");
516  // need to add new version info
517  string queryString("ALTER TABLE mapd_tables ADD version_num BIGINT DEFAULT " +
519  sqliteConnector_.query(queryString);
520  }
521  } catch (std::exception& e) {
522  sqliteConnector_.query("ROLLBACK TRANSACTION");
523  throw;
524  }
525  sqliteConnector_.query("END TRANSACTION");
526 }
527 
530  sqliteConnector_.query("BEGIN TRANSACTION");
531  try {
532  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_columns)");
533  std::vector<std::string> cols;
534  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
535  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
536  }
537  if (std::find(cols.begin(), cols.end(), std::string("version_num")) == cols.end()) {
538  LOG(INFO) << "Updating mapd_columns updateDeletedColumnIndicator";
539  // need to add new version info
540  string queryString("ALTER TABLE mapd_columns ADD version_num BIGINT DEFAULT " +
542  sqliteConnector_.query(queryString);
543  // need to add new column to table defintion to indicate deleted column, column used
544  // as bitmap for deleted rows.
546  "ALTER TABLE mapd_columns ADD is_deletedcol boolean default 0 ");
547  }
548  } catch (std::exception& e) {
549  sqliteConnector_.query("ROLLBACK TRANSACTION");
550  throw;
551  }
552  sqliteConnector_.query("END TRANSACTION");
553 }
554 
557  sqliteConnector_.query("BEGIN TRANSACTION");
558  try {
559  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_columns)");
560  std::vector<std::string> cols;
561  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
562  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
563  }
564  if (std::find(cols.begin(), cols.end(), std::string("default_value")) == cols.end()) {
565  LOG(INFO) << "Adding support for default values to mapd_columns";
566  sqliteConnector_.query("ALTER TABLE mapd_columns ADD default_value TEXT");
567  }
568  } catch (std::exception& e) {
569  sqliteConnector_.query("ROLLBACK TRANSACTION");
570  LOG(ERROR) << "Failed to make metadata update for default values` support";
571  throw;
572  }
573  sqliteConnector_.query("END TRANSACTION");
574 }
575 
576 // introduce DB version into the dictionary tables
577 // if the DB does not have a version rename all dictionary tables
578 
581  if (currentDB_.dbName.length() == 0) {
582  // updateDictionaryNames dbName length is zero nothing to do here
583  return;
584  }
585  sqliteConnector_.query("BEGIN TRANSACTION");
586  try {
587  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_dictionaries)");
588  std::vector<std::string> cols;
589  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
590  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
591  }
592  if (std::find(cols.begin(), cols.end(), std::string("version_num")) == cols.end()) {
593  // No version number
594  // need to rename dictionaries
595  string dictQuery("SELECT dictid, name from mapd_dictionaries");
596  sqliteConnector_.query(dictQuery);
597  size_t numRows = sqliteConnector_.getNumRows();
598  for (size_t r = 0; r < numRows; ++r) {
599  int dictId = sqliteConnector_.getData<int>(r, 0);
600  std::string dictName = sqliteConnector_.getData<string>(r, 1);
601 
602  std::string oldName = g_base_path + "/" + shared::kDataDirectoryName + "/" +
603  currentDB_.dbName + "_" + dictName;
604  std::string newName = g_base_path + "/" + shared::kDataDirectoryName + "/DB_" +
605  std::to_string(currentDB_.dbId) + "_DICT_" +
606  std::to_string(dictId);
607 
608  int result = rename(oldName.c_str(), newName.c_str());
609 
610  if (result == 0) {
611  LOG(INFO) << "Dictionary upgrade: successfully renamed " << oldName << " to "
612  << newName;
613  } else {
614  LOG(ERROR) << "Failed to rename old dictionary directory " << oldName << " to "
615  << newName + " dbname '" << currentDB_.dbName << "' error code "
616  << std::to_string(result);
617  }
618  }
619  // need to add new version info
620  string queryString("ALTER TABLE mapd_dictionaries ADD version_num BIGINT DEFAULT " +
622  sqliteConnector_.query(queryString);
623  }
624  } catch (std::exception& e) {
625  sqliteConnector_.query("ROLLBACK TRANSACTION");
626  throw;
627  }
628  sqliteConnector_.query("END TRANSACTION");
629 }
630 
633  sqliteConnector_.query("BEGIN TRANSACTION");
634  try {
636  "CREATE TABLE IF NOT EXISTS mapd_logical_to_physical("
637  "logical_table_id integer, physical_table_id integer)");
638  } catch (const std::exception& e) {
639  sqliteConnector_.query("ROLLBACK TRANSACTION");
640  throw;
641  }
642  sqliteConnector_.query("END TRANSACTION");
643 }
644 
645 void Catalog::updateLogicalToPhysicalTableMap(const int32_t logical_tb_id) {
646  /* this proc inserts/updates all pairs of (logical_tb_id, physical_tb_id) in
647  * sqlite mapd_logical_to_physical table for given logical_tb_id as needed
648  */
649 
651  sqliteConnector_.query("BEGIN TRANSACTION");
652  try {
653  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
654  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
655  const auto physicalTables = physicalTableIt->second;
656  CHECK(!physicalTables.empty());
657  for (size_t i = 0; i < physicalTables.size(); i++) {
658  int32_t physical_tb_id = physicalTables[i];
660  "INSERT OR REPLACE INTO mapd_logical_to_physical (logical_table_id, "
661  "physical_table_id) VALUES (?1, ?2)",
662  std::vector<std::string>{std::to_string(logical_tb_id),
663  std::to_string(physical_tb_id)});
664  }
665  }
666  } catch (std::exception& e) {
667  sqliteConnector_.query("ROLLBACK TRANSACTION");
668  throw;
669  }
670  sqliteConnector_.query("END TRANSACTION");
671 }
672 
675  sqliteConnector_.query("BEGIN TRANSACTION");
676  try {
677  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_dictionaries)");
678  std::vector<std::string> cols;
679  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
680  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
681  }
682  if (std::find(cols.begin(), cols.end(), std::string("refcount")) == cols.end()) {
683  sqliteConnector_.query("ALTER TABLE mapd_dictionaries ADD refcount DEFAULT 1");
684  }
685  } catch (std::exception& e) {
686  sqliteConnector_.query("ROLLBACK TRANSACTION");
687  throw;
688  }
689  sqliteConnector_.query("END TRANSACTION");
690 }
691 
694  sqliteConnector_.query("BEGIN TRANSACTION");
695  try {
698  } catch (std::exception& e) {
699  sqliteConnector_.query("ROLLBACK TRANSACTION");
700  throw;
701  }
702  sqliteConnector_.query("END TRANSACTION");
703 }
704 
706  // TODO: Move common migration logic to a shared function.
708  sqliteConnector_.query("BEGIN TRANSACTION");
709  try {
711  "select name from sqlite_master WHERE type='table' AND "
712  "name='mapd_version_history'");
713  static const std::string migration_name{"rename_legacy_data_wrappers"};
714  if (sqliteConnector_.getNumRows() == 0) {
716  "CREATE TABLE mapd_version_history(version integer, migration_history text "
717  "unique)");
718  } else {
720  "select * from mapd_version_history where migration_history = "
721  "'" +
722  migration_name + "'");
723  if (sqliteConnector_.getNumRows() != 0) {
724  // Migration already done.
725  sqliteConnector_.query("END TRANSACTION");
726  return;
727  }
728  }
729  LOG(INFO) << "Executing " << migration_name << " migration.";
730 
731  // Update legacy data wrapper names
733  // clang-format off
734  std::map<std::string, std::string> old_to_new_wrapper_names{
735  {"OMNISCI_CSV", DataWrapperType::CSV},
736  {"OMNISCI_PARQUET", DataWrapperType::PARQUET},
737  {"OMNISCI_REGEX_PARSER", DataWrapperType::REGEX_PARSER},
738  {"OMNISCI_INTERNAL_CATALOG", DataWrapperType::INTERNAL_CATALOG},
739  {"INTERNAL_OMNISCI_MEMORY_STATS", DataWrapperType::INTERNAL_MEMORY_STATS},
740  {"INTERNAL_OMNISCI_STORAGE_STATS", DataWrapperType::INTERNAL_STORAGE_STATS}
741  };
742  // clang-format on
743 
744  for (const auto& [old_wrapper_name, new_wrapper_name] : old_to_new_wrapper_names) {
746  "UPDATE omnisci_foreign_servers SET data_wrapper_type = ? WHERE "
747  "data_wrapper_type = ?",
748  std::vector<std::string>{new_wrapper_name, old_wrapper_name});
749  }
750 
751  // Record migration.
753  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
754  std::vector<std::string>{std::to_string(MAPD_VERSION), migration_name});
755  LOG(INFO) << migration_name << " migration completed.";
756  } catch (std::exception& e) {
757  sqliteConnector_.query("ROLLBACK TRANSACTION");
758  throw;
759  }
760  sqliteConnector_.query("END TRANSACTION");
761 }
762 
765  sqliteConnector_.query("BEGIN TRANSACTION");
766  try {
768  } catch (const std::exception& e) {
769  sqliteConnector_.query("ROLLBACK TRANSACTION");
770  throw;
771  }
772  sqliteConnector_.query("END TRANSACTION");
773 }
774 
775 const std::string Catalog::getForeignServerSchema(bool if_not_exists) {
776  return "CREATE TABLE " + (if_not_exists ? std::string{"IF NOT EXISTS "} : "") +
777  "omnisci_foreign_servers(id integer primary key, name text unique, " +
778  "data_wrapper_type text, owner_user_id integer, creation_time integer, " +
779  "options text)";
780 }
781 
782 const std::string Catalog::getForeignTableSchema(bool if_not_exists) {
783  return "CREATE TABLE " + (if_not_exists ? std::string{"IF NOT EXISTS "} : "") +
784  "omnisci_foreign_tables(table_id integer unique, server_id integer, " +
785  "options text, last_refresh_time integer, next_refresh_time integer, " +
786  "FOREIGN KEY(table_id) REFERENCES mapd_tables(tableid), " +
787  "FOREIGN KEY(server_id) REFERENCES omnisci_foreign_servers(id))";
788 }
789 
790 const std::string Catalog::getCustomExpressionsSchema(bool if_not_exists) {
791  return "CREATE TABLE " + (if_not_exists ? std::string{"IF NOT EXISTS "} : "") +
792  "omnisci_custom_expressions(id integer primary key, name text, " +
793  "expression_json text, data_source_type text, " +
794  "data_source_id integer, is_deleted boolean)";
795 }
796 
799  sqliteConnector_.query("BEGIN TRANSACTION");
800  std::vector<DBObject> objects;
801  try {
803  "SELECT name FROM sqlite_master WHERE type='table' AND "
804  "name='mapd_record_ownership_marker'");
805  // check if mapd catalog - marker exists
806  if (sqliteConnector_.getNumRows() != 0 && currentDB_.dbId == 1) {
807  // already done
808  sqliteConnector_.query("END TRANSACTION");
809  return;
810  }
811  // check if different catalog - marker exists
812  else if (sqliteConnector_.getNumRows() != 0 && currentDB_.dbId != 1) {
813  sqliteConnector_.query("SELECT dummy FROM mapd_record_ownership_marker");
814  // Check if migration is being performed on existing non mapd catalogs
815  // Older non mapd dbs will have table but no record in them
816  if (sqliteConnector_.getNumRows() != 0) {
817  // already done
818  sqliteConnector_.query("END TRANSACTION");
819  return;
820  }
821  }
822  // marker not exists - create one
823  else {
824  sqliteConnector_.query("CREATE TABLE mapd_record_ownership_marker (dummy integer)");
825  }
826 
827  DBMetadata db;
828  CHECK(SysCatalog::instance().getMetadataForDB(currentDB_.dbName, db));
829  // place dbId as a refernce for migration being performed
831  "INSERT INTO mapd_record_ownership_marker (dummy) VALUES (?1)",
832  std::vector<std::string>{std::to_string(db.dbOwner)});
833 
834  static const std::map<const DBObjectType, const AccessPrivileges>
835  object_level_all_privs_lookup{
841 
842  // grant owner all permissions on DB
843  DBObjectKey key;
844  key.dbId = currentDB_.dbId;
845  auto _key_place = [&key](auto type) {
846  key.permissionType = type;
847  return key;
848  };
849  for (auto& it : object_level_all_privs_lookup) {
850  objects.emplace_back(_key_place(it.first), it.second, db.dbOwner);
851  objects.back().setName(currentDB_.dbName);
852  }
853 
854  {
855  // other users tables and views
856  string tableQuery(
857  "SELECT tableid, name, userid, isview FROM mapd_tables WHERE userid > 0");
858  sqliteConnector_.query(tableQuery);
859  size_t numRows = sqliteConnector_.getNumRows();
860  for (size_t r = 0; r < numRows; ++r) {
861  int32_t tableid = sqliteConnector_.getData<int>(r, 0);
862  std::string tableName = sqliteConnector_.getData<string>(r, 1);
863  int32_t ownerid = sqliteConnector_.getData<int>(r, 2);
864  bool isview = sqliteConnector_.getData<bool>(r, 3);
865 
868  DBObjectKey key;
869  key.dbId = currentDB_.dbId;
870  key.objectId = tableid;
871  key.permissionType = type;
872 
873  DBObject obj(tableName, type);
874  obj.setObjectKey(key);
875  obj.setOwner(ownerid);
878 
879  objects.push_back(obj);
880  }
881  }
882 
883  {
884  // other users dashboards
885  string tableQuery("SELECT id, name, userid FROM mapd_dashboards WHERE userid > 0");
886  sqliteConnector_.query(tableQuery);
887  size_t numRows = sqliteConnector_.getNumRows();
888  for (size_t r = 0; r < numRows; ++r) {
889  int32_t dashId = sqliteConnector_.getData<int>(r, 0);
890  std::string dashName = sqliteConnector_.getData<string>(r, 1);
891  int32_t ownerid = sqliteConnector_.getData<int>(r, 2);
892 
894  DBObjectKey key;
895  key.dbId = currentDB_.dbId;
896  key.objectId = dashId;
897  key.permissionType = type;
898 
899  DBObject obj(dashName, type);
900  obj.setObjectKey(key);
901  obj.setOwner(ownerid);
903 
904  objects.push_back(obj);
905  }
906  }
907  } catch (const std::exception& e) {
908  sqliteConnector_.query("ROLLBACK TRANSACTION");
909  throw;
910  }
911  sqliteConnector_.query("END TRANSACTION");
912 
913  // now apply the objects to the syscat to track the permisisons
914  // moved outside transaction to avoid lock in sqlite
915  try {
917  } catch (const std::exception& e) {
918  LOG(ERROR) << " Issue during migration of DB " << name() << " issue was " << e.what();
919  throw std::runtime_error(" Issue during migration of DB " + name() + " issue was " +
920  e.what());
921  // will need to remove the mapd_record_ownership_marker table and retry
922  }
923 }
924 
929 }
930 
932  // do not take cat_sqlite_lock here as Catalog functions do that themselves
934 }
935 
937  std::unordered_map<std::string, std::pair<int, std::string>> dashboards;
938  std::vector<std::string> dashboard_ids;
939  static const std::string migration_name{"dashboard_roles_migration"};
940  {
942  sqliteConnector_.query("BEGIN TRANSACTION");
943  try {
944  // migration_history should be present in all catalogs by now
945  // if not then would be created before this migration
947  "select * from mapd_version_history where migration_history = '" +
948  migration_name + "'");
949  if (sqliteConnector_.getNumRows() != 0) {
950  // no need for further execution
951  sqliteConnector_.query("END TRANSACTION");
952  return;
953  }
954  LOG(INFO) << "Performing dashboard internal roles Migration.";
955  sqliteConnector_.query("select id, userid, metadata from mapd_dashboards");
956  for (size_t i = 0; i < sqliteConnector_.getNumRows(); ++i) {
959  sqliteConnector_.getData<string>(i, 0)))) {
960  // Successfully created roles during previous migration/crash
961  // No need to include them
962  continue;
963  }
964  dashboards[sqliteConnector_.getData<string>(i, 0)] = std::make_pair(
965  sqliteConnector_.getData<int>(i, 1), sqliteConnector_.getData<string>(i, 2));
966  dashboard_ids.push_back(sqliteConnector_.getData<string>(i, 0));
967  }
968  } catch (const std::exception& e) {
969  sqliteConnector_.query("ROLLBACK TRANSACTION");
970  throw;
971  }
972  sqliteConnector_.query("END TRANSACTION");
973  }
974  // All current grantees with shared dashboards.
975  const auto active_grantees =
977 
978  try {
979  // NOTE(wamsi): Transactionally unsafe
980  for (auto dash : dashboards) {
981  createOrUpdateDashboardSystemRole(dash.second.second,
982  dash.second.first,
984  std::to_string(currentDB_.dbId), dash.first));
985  auto result = active_grantees.find(dash.first);
986  if (result != active_grantees.end()) {
989  dash.first)},
990  result->second);
991  }
992  }
994  // check if this has already been completed
996  "select * from mapd_version_history where migration_history = '" +
997  migration_name + "'");
998  if (sqliteConnector_.getNumRows() != 0) {
999  return;
1000  }
1002  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
1003  std::vector<std::string>{std::to_string(MAPD_VERSION), migration_name});
1004  } catch (const std::exception& e) {
1005  LOG(ERROR) << "Failed to create dashboard system roles during migration: "
1006  << e.what();
1007  throw;
1008  }
1009  LOG(INFO) << "Successfully created dashboard system roles during migration.";
1010 }
1011 
1013  cat_write_lock write_lock(this);
1017  updateGeoColumns();
1020  updateLinkSchema();
1024  updatePageSize();
1028  if (g_enable_fsi) {
1029  updateFsiSchemas();
1031  }
1034 }
1035 
1039 }
1040 
1041 namespace {
1042 std::map<int32_t, std::string> get_user_id_to_user_name_map() {
1043  auto users = SysCatalog::instance().getAllUserMetadata();
1044  std::map<int32_t, std::string> user_name_by_user_id;
1045  for (const auto& user : users) {
1046  user_name_by_user_id[user.userId] = user.userName;
1047  }
1048  return user_name_by_user_id;
1049 }
1050 
1052  int32_t id,
1053  const std::map<int32_t, std::string>& user_name_by_user_id) {
1054  auto entry = user_name_by_user_id.find(id);
1055  if (entry != user_name_by_user_id.end()) {
1056  return entry->second;
1057  }
1058  // a user could be deleted and a dashboard still exist?
1059  return "Unknown";
1060 }
1061 
1063  CHECK_GT(cd.db_id, 0);
1064  auto& column_type = cd.columnType;
1065  if (column_type.is_dict_encoded_string() ||
1066  column_type.is_subtype_dict_encoded_string()) {
1067  column_type.setStringDictKey({cd.db_id, column_type.get_comp_param()});
1068  }
1069 }
1070 } // namespace
1071 
1073  std::string dictQuery(
1074  "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
1075  sqliteConnector_.query(dictQuery);
1076  auto numRows = sqliteConnector_.getNumRows();
1077  for (size_t r = 0; r < numRows; ++r) {
1078  auto dictId = sqliteConnector_.getData<int>(r, 0);
1079  auto dictName = sqliteConnector_.getData<string>(r, 1);
1080  auto dictNBits = sqliteConnector_.getData<int>(r, 2);
1081  auto is_shared = sqliteConnector_.getData<bool>(r, 3);
1082  auto refcount = sqliteConnector_.getData<int>(r, 4);
1083  auto fname = g_base_path + "/" + shared::kDataDirectoryName + "/DB_" +
1084  std::to_string(currentDB_.dbId) + "_DICT_" + std::to_string(dictId);
1085  DictRef dict_ref(currentDB_.dbId, dictId);
1086  auto dd = new DictDescriptor(
1087  dict_ref, dictName, dictNBits, is_shared, refcount, fname, false);
1088  dictDescriptorMapByRef_[dict_ref].reset(dd);
1089  }
1090 }
1091 
1092 // NOTE(sy): Only used by --multi-instance clusters.
1093 void Catalog::reloadTableMetadata(int table_id) {
1094  cat_write_lock write_lock(this);
1096  reloadTableMetadataUnlocked(table_id);
1097 }
1098 
1099 // NOTE(sy): Only used by --multi-instance clusters.
1101  // Reload dictionaries first.
1102  // TODO(sy): Does dictionary reloading really belong here?
1103  // We don't have dictionary locks in the system but maybe we need them.
1105 
1106  // Delete this table's metadata from the in-memory cache before reloading.
1107  TableDescriptor* original_td = nullptr;
1108  std::list<ColumnDescriptor*> original_cds;
1109  if (auto it1 = tableDescriptorMapById_.find(table_id);
1110  it1 != tableDescriptorMapById_.end()) {
1111  original_td = it1->second;
1112  if (dynamic_cast<foreign_storage::ForeignTable*>(original_td)) {
1113  // If have a foreign table then we need to destroy the local data wrapper or it will
1114  // contain (potentially invalid) cached data. This needs to be done before we
1115  // remove the table descriptors.
1116  dataMgr_->removeTableRelatedDS(currentDB_.dbId, table_id);
1117  } else {
1118  dataMgr_->removeMutableTableDiskCacheData(currentDB_.dbId, table_id);
1119  }
1120  tableDescriptorMapById_.erase(it1);
1121  if (auto it2 = tableDescriptorMap_.find(to_upper(original_td->tableName));
1122  it2 != tableDescriptorMap_.end()) {
1123  CHECK_EQ(original_td, it2->second);
1124  tableDescriptorMap_.erase(it2);
1125  }
1126  if (original_td->hasDeletedCol) {
1127  const auto ret = deletedColumnPerTable_.erase(original_td);
1128  CHECK_EQ(ret, size_t(1));
1129  }
1130  for (int column_id = 0; column_id < original_td->nColumns; ++column_id) {
1131  if (auto it3 = columnDescriptorMapById_.find({table_id, column_id});
1132  it3 != columnDescriptorMapById_.end()) {
1133  ColumnDescriptor* original_cd = it3->second;
1134  original_cds.push_back(original_cd);
1135  removeFromColumnMap(original_cd);
1136  }
1137  }
1138  }
1139 
1140  TableDescriptor* td;
1141  try {
1142  td = createTableFromDiskUnlocked(table_id);
1143  } catch (const NoTableFoundException& e) {
1144  // No entry found for table on disk. Another node may have deleted it.
1145  return;
1146  }
1147 
1148  if (auto tableDescIt = tableDescriptorMapById_.find(table_id);
1149  tableDescIt != tableDescriptorMapById_.end()) {
1150  tableDescIt->second->fragmenter = nullptr;
1151  delete tableDescIt->second;
1152  }
1153 
1154  // Reload the column descriptors.
1155  auto cds = sqliteGetColumnsForTableUnlocked(table_id);
1156 
1157  // Store the descriptors into the cache.
1158  if (original_td) {
1159  td->mutex_ = original_td->mutex_; // TODO(sy): Unnecessary?
1160  delete original_td;
1161  original_td = nullptr;
1162  }
1163  for (ColumnDescriptor* original_cd : original_cds) {
1164  delete original_cd;
1165  }
1166  original_cds.clear();
1167  tableDescriptorMap_[to_upper(td->tableName)] = td;
1168  tableDescriptorMapById_[td->tableId] = td;
1169  int32_t skip_physical_cols = 0;
1170  for (ColumnDescriptor* cd : cds) {
1171  addToColumnMap(cd);
1172 
1173  if (skip_physical_cols <= 0) {
1174  skip_physical_cols = cd->columnType.get_physical_cols();
1175  }
1176 
1177  if (cd->isDeletedCol) {
1178  td->hasDeletedCol = true;
1179  setDeletedColumnUnlocked(td, cd);
1180  } else if (cd->columnType.is_geometry() || skip_physical_cols-- <= 0) {
1181  td->columnIdBySpi_.push_back(cd->columnId);
1182  }
1183  }
1184 
1185  // Notify Calcite about the reloaded table.
1186  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
1187 }
1188 
1189 // NOTE(sy): Only used by --multi-instance clusters.
1190 void Catalog::reloadCatalogMetadata(
1191  const std::map<int32_t, std::string>& user_name_by_user_id) {
1192  cat_write_lock write_lock(this);
1193  cat_sqlite_lock sqlite_lock(getObjForLock());
1194  reloadCatalogMetadataUnlocked(get_user_id_to_user_name_map());
1195 }
1196 
1197 // NOTE(sy): Only used by --multi-instance clusters.
1198 void Catalog::reloadCatalogMetadataUnlocked(
1199  const std::map<int32_t, std::string>& user_name_by_user_id) {
1201 
1202  sqliteConnector_.reset(basePath_ + "/" + shared::kCatalogDirectoryName + "/" +
1203  currentDB_.dbName);
1204 
1205  // Notice when tables or columns have been created, dropped, or changed by other nodes.
1206  // Needed so that users will see reasonably-correct lists of what objects exist.
1207 
1208  // Load the list of table ID's that exist on disk storage.
1209  std::set<int> cluster_table_ids;
1210  std::string tableQuery("SELECT tableid from mapd_tables");
1211  sqliteConnector_.query(tableQuery);
1212  auto numRows = sqliteConnector_.getNumRows();
1213  for (size_t r = 0; r < numRows; ++r) {
1214  const auto table_id = sqliteConnector_.getData<int>(r, 0);
1215  cluster_table_ids.insert(table_id);
1216  }
1217 
1218  // Ignore any table ID's locked by other threads on this node.
1219  // Those other threads are already handling any necessary reloading for those tables.
1220  std::set<int> ignored_table_ids;
1221  for (ChunkKey const& k : lockmgr::TableSchemaLockMgr::instance().getLockedTables()) {
1222  CHECK_EQ(k.size(), 2U);
1223  if (k[CHUNK_KEY_DB_IDX] != currentDB_.dbId) {
1224  continue;
1225  }
1226  ignored_table_ids.insert(k[CHUNK_KEY_TABLE_IDX]);
1227  }
1228 
1229  // For this node's Catalog cache:
1230  // Newly created table schemas created by other nodes need to be loaded.
1231  // Unlocked table schemas might have been renamed by other nodes; just reload them all.
1232  // Deleted table schemas still in this node's cache need to be flushed.
1233  std::set<int> reload_table_ids;
1234  for (auto const& cluster_table_id : cluster_table_ids) {
1235  if (ignored_table_ids.find(cluster_table_id) == ignored_table_ids.end()) {
1236  reload_table_ids.insert(cluster_table_id);
1237  }
1238  }
1239  for (auto const& [cached_table_id, td] : tableDescriptorMapById_) {
1240  if (cluster_table_ids.find(cached_table_id) == cluster_table_ids.end()) {
1241  reload_table_ids.insert(cached_table_id);
1242  }
1243  }
1244 
1245  // Reload tables.
1246  for (auto const& reload_table_id : reload_table_ids) {
1247  reloadTableMetadataUnlocked(reload_table_id);
1248  }
1249 
1251 
1252  dashboardDescriptorMap_.clear();
1253  linkDescriptorMap_.clear();
1254  linkDescriptorMapById_.clear();
1255  foreignServerMap_.clear();
1256  foreignServerMapById_.clear();
1257  custom_expr_map_by_id_.clear();
1258 
1259  if (g_enable_fsi) {
1260  buildForeignServerMapUnlocked();
1261  }
1262 
1263  updateViewsInMapUnlocked();
1264  buildDashboardsMapUnlocked(user_name_by_user_id);
1265  buildLinksMapUnlocked();
1266  buildCustomExpressionsMapUnlocked();
1267 
1268  // Notify Calcite about the reloaded database.
1269  if (calciteMgr_) {
1270  calciteMgr_->updateMetadata(currentDB_.dbName, {});
1271  }
1272 }
1273 
1274 void Catalog::buildTablesMapUnlocked() {
1275  std::string tableQuery(
1276  "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
1277  "max_chunk_size, frag_page_size, "
1278  "max_rows, partitions, shard_column_id, shard, num_shards, key_metainfo, userid, "
1279  "sort_column_id, storage_type, max_rollback_epochs, is_system_table "
1280  "from mapd_tables");
1281  sqliteConnector_.query(tableQuery);
1282  auto numRows = sqliteConnector_.getNumRows();
1283  for (size_t r = 0; r < numRows; ++r) {
1284  TableDescriptor* td;
1285  const auto& storage_type = sqliteConnector_.getData<string>(r, 17);
1286  if (!storage_type.empty() && storage_type != StorageType::FOREIGN_TABLE) {
1287  const auto table_id = sqliteConnector_.getData<int>(r, 0);
1288  const auto& table_name = sqliteConnector_.getData<string>(r, 1);
1289  LOG(FATAL) << "Unable to read Catalog metadata: storage type is currently not a "
1290  "supported table option (table "
1291  << table_name << " [" << table_id << "] in database "
1292  << currentDB_.dbName << ").";
1293  }
1294 
1295  if (storage_type == StorageType::FOREIGN_TABLE) {
1296  td = new foreign_storage::ForeignTable();
1297  } else {
1298  td = new TableDescriptor();
1299  }
1300 
1301  td->storageType = storage_type;
1302  td->tableId = sqliteConnector_.getData<int>(r, 0);
1303  td->tableName = sqliteConnector_.getData<string>(r, 1);
1304  td->nColumns = sqliteConnector_.getData<int>(r, 2);
1305  td->isView = sqliteConnector_.getData<bool>(r, 3);
1306  td->fragments = sqliteConnector_.getData<string>(r, 4);
1307  td->fragType =
1308  (Fragmenter_Namespace::FragmenterType)sqliteConnector_.getData<int>(r, 5);
1309  td->maxFragRows = sqliteConnector_.getData<int>(r, 6);
1310  td->maxChunkSize = sqliteConnector_.getData<int64_t>(r, 7);
1311  td->fragPageSize = sqliteConnector_.getData<int>(r, 8);
1312  td->maxRows = sqliteConnector_.getData<int64_t>(r, 9);
1313  td->partitions = sqliteConnector_.getData<string>(r, 10);
1314  td->shardedColumnId = sqliteConnector_.getData<int>(r, 11);
1315  td->shard = sqliteConnector_.getData<int>(r, 12);
1316  td->nShards = sqliteConnector_.getData<int>(r, 13);
1317  td->keyMetainfo = sqliteConnector_.getData<string>(r, 14);
1318  td->userId = sqliteConnector_.getData<int>(r, 15);
1319  td->sortedColumnId =
1320  sqliteConnector_.isNull(r, 16) ? 0 : sqliteConnector_.getData<int>(r, 16);
1321  if (!td->isView) {
1322  td->fragmenter = nullptr;
1323  }
1324  td->maxRollbackEpochs = sqliteConnector_.getData<int>(r, 18);
1325  td->is_system_table = sqliteConnector_.getData<bool>(r, 19);
1326  td->hasDeletedCol = false;
1327 
1328  tableDescriptorMap_[to_upper(td->tableName)] = td;
1329  tableDescriptorMapById_[td->tableId] = td;
1330  }
1331 }
1332 
1333 void Catalog::buildColumnsMapUnlocked() {
1334  std::string columnQuery(
1335  "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
1336  "is_notnull, compression, comp_param, "
1337  "size, chunks, is_systemcol, is_virtualcol, virtual_expr, is_deletedcol, "
1338  "default_value from "
1339  "mapd_columns ORDER BY tableid, "
1340  "columnid");
1341  sqliteConnector_.query(columnQuery);
1342  auto numRows = sqliteConnector_.getNumRows();
1343  int32_t skip_physical_cols = 0;
1344  for (size_t r = 0; r < numRows; ++r) {
1345  ColumnDescriptor* cd = new ColumnDescriptor();
1346  cd->tableId = sqliteConnector_.getData<int>(r, 0);
1347  cd->columnId = sqliteConnector_.getData<int>(r, 1);
1348  cd->columnName = sqliteConnector_.getData<string>(r, 2);
1349  cd->columnType.set_type((SQLTypes)sqliteConnector_.getData<int>(r, 3));
1350  cd->columnType.set_subtype((SQLTypes)sqliteConnector_.getData<int>(r, 4));
1351  cd->columnType.set_dimension(sqliteConnector_.getData<int>(r, 5));
1352  cd->columnType.set_scale(sqliteConnector_.getData<int>(r, 6));
1353  cd->columnType.set_notnull(sqliteConnector_.getData<bool>(r, 7));
1354  cd->columnType.set_compression((EncodingType)sqliteConnector_.getData<int>(r, 8));
1355  cd->columnType.set_comp_param(sqliteConnector_.getData<int>(r, 9));
1356  cd->columnType.set_size(sqliteConnector_.getData<int>(r, 10));
1357  cd->chunks = sqliteConnector_.getData<string>(r, 11);
1358  cd->isSystemCol = sqliteConnector_.getData<bool>(r, 12);
1359  cd->isVirtualCol = sqliteConnector_.getData<bool>(r, 13);
1360  cd->virtualExpr = sqliteConnector_.getData<string>(r, 14);
1361  cd->isDeletedCol = sqliteConnector_.getData<bool>(r, 15);
1362  if (sqliteConnector_.isNull(r, 16)) {
1363  cd->default_value = std::nullopt;
1364  } else {
1365  cd->default_value = std::make_optional(sqliteConnector_.getData<string>(r, 16));
1366  }
1367  cd->isGeoPhyCol = skip_physical_cols > 0;
1368  cd->db_id = getDatabaseId();
1369  set_dict_key(*cd);
1370  addToColumnMap(cd);
1371 
1372  if (skip_physical_cols <= 0) {
1373  skip_physical_cols = cd->columnType.get_physical_cols();
1374  }
1375 
1376  auto td_itr = tableDescriptorMapById_.find(cd->tableId);
1377  CHECK(td_itr != tableDescriptorMapById_.end());
1378 
1379  if (cd->isDeletedCol) {
1380  td_itr->second->hasDeletedCol = true;
1381  setDeletedColumnUnlocked(td_itr->second, cd);
1382  } else if (cd->columnType.is_geometry() || skip_physical_cols-- <= 0) {
1383  tableDescriptorMapById_[cd->tableId]->columnIdBySpi_.push_back(cd->columnId);
1384  }
1385  }
1386 
1387  // sort columnIdBySpi_ based on columnId
1388  for (auto& tit : tableDescriptorMapById_) {
1389  std::sort(tit.second->columnIdBySpi_.begin(),
1390  tit.second->columnIdBySpi_.end(),
1391  [](const size_t a, const size_t b) -> bool { return a < b; });
1392  }
1393 }
1394 
1395 void Catalog::updateViewUnlocked(TableDescriptor& td) {
1396  std::string viewQuery("SELECT sql FROM mapd_views where tableid = " +
1397  std::to_string(td.tableId));
1398  sqliteConnector_.query(viewQuery);
1399  auto num_rows = sqliteConnector_.getNumRows();
1400  CHECK_EQ(num_rows, 1U) << "Expected single entry in mapd_views for view '"
1401  << td.tableName << "', instead got " << num_rows;
1402  td.viewSQL = sqliteConnector_.getData<string>(0, 0);
1403 }
1404 
1405 void Catalog::updateViewsInMapUnlocked() {
1406  std::string viewQuery("SELECT tableid, sql FROM mapd_views");
1407  sqliteConnector_.query(viewQuery);
1408  auto numRows = sqliteConnector_.getNumRows();
1409  for (size_t r = 0; r < numRows; ++r) {
1410  auto tableId = sqliteConnector_.getData<int>(r, 0);
1411  auto td = tableDescriptorMapById_[tableId];
1412  td->viewSQL = sqliteConnector_.getData<string>(r, 1);
1413  td->fragmenter = nullptr;
1414  }
1415 }
1416 
1417 void Catalog::buildDashboardsMapUnlocked(
1418  const std::map<int32_t, std::string>& user_name_by_user_id) {
1419  std::string frontendViewQuery(
1420  "SELECT id, state, name, image_hash, strftime('%Y-%m-%dT%H:%M:%SZ', update_time), "
1421  "userid, "
1422  "metadata "
1423  "FROM mapd_dashboards");
1424  sqliteConnector_.query(frontendViewQuery);
1425  auto numRows = sqliteConnector_.getNumRows();
1426  for (size_t r = 0; r < numRows; ++r) {
1427  auto vd = std::make_shared<DashboardDescriptor>();
1428  vd->dashboardId = sqliteConnector_.getData<int>(r, 0);
1429  vd->dashboardState = sqliteConnector_.getData<string>(r, 1);
1430  vd->dashboardName = sqliteConnector_.getData<string>(r, 2);
1431  vd->imageHash = sqliteConnector_.getData<string>(r, 3);
1432  vd->updateTime = sqliteConnector_.getData<string>(r, 4);
1433  vd->userId = sqliteConnector_.getData<int>(r, 5);
1434  vd->dashboardMetadata = sqliteConnector_.getData<string>(r, 6);
1435  vd->user = get_user_name_from_id(vd->userId, user_name_by_user_id);
1436  vd->dashboardSystemRoleName = generate_dashboard_system_rolename(
1437  std::to_string(currentDB_.dbId), sqliteConnector_.getData<string>(r, 0));
1438  dashboardDescriptorMap_[std::to_string(vd->userId) + ":" + vd->dashboardName] = vd;
1439  }
1440 }
1441 
1442 void Catalog::buildLinksMapUnlocked() {
1443  std::string linkQuery(
1444  "SELECT linkid, userid, link, view_state, strftime('%Y-%m-%dT%H:%M:%SZ', "
1445  "update_time), view_metadata "
1446  "FROM mapd_links");
1447  sqliteConnector_.query(linkQuery);
1448  auto numRows = sqliteConnector_.getNumRows();
1449  for (size_t r = 0; r < numRows; ++r) {
1450  auto ld = new LinkDescriptor();
1451  ld->linkId = sqliteConnector_.getData<int>(r, 0);
1452  ld->userId = sqliteConnector_.getData<int>(r, 1);
1453  ld->link = sqliteConnector_.getData<string>(r, 2);
1454  ld->viewState = sqliteConnector_.getData<string>(r, 3);
1455  ld->updateTime = sqliteConnector_.getData<string>(r, 4);
1456  ld->viewMetadata = sqliteConnector_.getData<string>(r, 5);
1457  linkDescriptorMap_[std::to_string(currentDB_.dbId) + ld->link] = ld;
1458  linkDescriptorMapById_[ld->linkId] = ld;
1459  }
1460 }
1461 
1462 void Catalog::buildLogicalToPhysicalMapUnlocked() {
1463  /* rebuild map linking logical tables to corresponding physical ones */
1464  std::string logicalToPhysicalTableMapQuery(
1465  "SELECT logical_table_id, physical_table_id "
1466  "FROM mapd_logical_to_physical");
1467  sqliteConnector_.query(logicalToPhysicalTableMapQuery);
1468  auto numRows = sqliteConnector_.getNumRows();
1469  for (size_t r = 0; r < numRows; ++r) {
1470  auto logical_tb_id = sqliteConnector_.getData<int>(r, 0);
1471  auto physical_tb_id = sqliteConnector_.getData<int>(r, 1);
1472  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
1473  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
1474  /* add new entity to the map logicalToPhysicalTableMapById_ */
1475  std::vector<int32_t> physicalTables{physical_tb_id};
1476  const auto it_ok =
1477  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
1478  CHECK(it_ok.second);
1479  } else {
1480  /* update map logicalToPhysicalTableMapById_ */
1481  physicalTableIt->second.push_back(physical_tb_id);
1482  }
1483  }
1484 }
1485 
1486 // The catalog uses a series of maps to cache data that have been read from the sqlite
1487 // tables. Usually we update these maps whenever we write using sqlite, so this function
1488 // is responsible for initializing all of them based on the sqlite db state.
1489 void Catalog::buildMaps() {
1490  // Get all user id to username mapping here in order to avoid making a call to
1491  // SysCatalog (and attempting to acquire SysCatalog locks) while holding locks for this
1492  // catalog.
1493  const auto user_name_by_user_id = get_user_id_to_user_name_map();
1494 
1495  cat_write_lock write_lock(this);
1496  cat_sqlite_lock sqlite_lock(getObjForLock());
1497 
1498  buildDictionaryMapUnlocked();
1499  buildTablesMapUnlocked();
1500 
1501  if (g_enable_fsi) {
1502  buildForeignServerMapUnlocked();
1503  updateForeignTablesInMapUnlocked();
1504  }
1505 
1506  buildColumnsMapUnlocked();
1507  updateViewsInMapUnlocked();
1508  buildDashboardsMapUnlocked(user_name_by_user_id);
1509  buildLinksMapUnlocked();
1510  buildLogicalToPhysicalMapUnlocked();
1511  buildCustomExpressionsMapUnlocked();
1512 }
1513 
1514 void Catalog::buildCustomExpressionsMapUnlocked() {
1515  sqliteConnector_.query(
1516  "SELECT id, name, expression_json, data_source_type, data_source_id, "
1517  "is_deleted "
1518  "FROM omnisci_custom_expressions");
1519  auto num_rows = sqliteConnector_.getNumRows();
1520  for (size_t row = 0; row < num_rows; row++) {
1521  auto custom_expr = getCustomExpressionFromConnector(row);
1522  custom_expr_map_by_id_[custom_expr->id] = std::move(custom_expr);
1523  }
1524 }
1525 
1526 std::unique_ptr<CustomExpression> Catalog::getCustomExpressionFromConnector(size_t row) {
1527  auto id = sqliteConnector_.getData<int>(row, 0);
1528  auto name = sqliteConnector_.getData<string>(row, 1);
1529  auto expression_json = sqliteConnector_.getData<string>(row, 2);
1530  auto data_source_type_str = sqliteConnector_.getData<string>(row, 3);
1531  auto data_source_id = sqliteConnector_.getData<int>(row, 4);
1532  auto is_deleted = sqliteConnector_.getData<bool>(row, 5);
1533  return std::make_unique<CustomExpression>(
1534  id,
1535  name,
1536  expression_json,
1537  CustomExpression::dataSourceTypeFromString(data_source_type_str),
1538  data_source_id,
1539  is_deleted);
1540 }
1541 
1542 void Catalog::addTableToMap(const TableDescriptor* td,
1543  const list<ColumnDescriptor>& columns,
1544  const list<DictDescriptor>& dicts) {
1545  cat_write_lock write_lock(this);
1546  TableDescriptor* new_td;
1547 
1548  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
1549  if (foreign_table) {
1550  auto new_foreign_table = new foreign_storage::ForeignTable();
1551  *new_foreign_table = *foreign_table;
1552  new_td = new_foreign_table;
1553  } else {
1554  new_td = new TableDescriptor();
1555  *new_td = *td;
1556  }
1557 
1558  new_td->mutex_ = std::make_shared<std::mutex>();
1559  tableDescriptorMap_[to_upper(td->tableName)] = new_td;
1560  tableDescriptorMapById_[td->tableId] = new_td;
1561  for (auto cd : columns) {
1562  ColumnDescriptor* new_cd = new ColumnDescriptor();
1563  *new_cd = cd;
1564  addToColumnMap(new_cd);
1565 
1566  // Add deleted column to the map
1567  if (cd.isDeletedCol) {
1568  CHECK(new_td->hasDeletedCol);
1569  setDeletedColumnUnlocked(new_td, new_cd);
1570  }
1571  }
1572 
1573  std::sort(new_td->columnIdBySpi_.begin(),
1574  new_td->columnIdBySpi_.end(),
1575  [](const size_t a, const size_t b) -> bool { return a < b; });
1576  // TODO(sy): Why does addTableToMap() sort columnIdBySpi_ but not insert into it while
1577  // buildColumnsMapUnlocked() does both?
1578 
1579  std::unique_ptr<StringDictionaryClient> client;
1580  DictRef dict_ref(currentDB_.dbId, -1);
1581  if (!string_dict_hosts_.empty()) {
1582  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1583  }
1584  for (auto dd : dicts) {
1585  if (!dd.dictRef.dictId) {
1586  // Dummy entry created for a shard of a logical table, nothing to do.
1587  continue;
1588  }
1589  dict_ref.dictId = dd.dictRef.dictId;
1590  if (client) {
1591  client->create(dict_ref, dd.dictIsTemp);
1592  }
1593  DictDescriptor* new_dd = new DictDescriptor(dd);
1594  dictDescriptorMapByRef_[dict_ref].reset(new_dd);
1595  if (!dd.dictIsTemp) {
1596  boost::filesystem::create_directory(new_dd->dictFolderPath);
1597  }
1598  }
1599 }
1600 
1601 void Catalog::removeTableFromMap(const string& tableName,
1602  const int tableId,
1603  const bool is_on_error) {
1604  cat_write_lock write_lock(this);
1605  TableDescriptorMapById::iterator tableDescIt = tableDescriptorMapById_.find(tableId);
1606  if (tableDescIt == tableDescriptorMapById_.end()) {
1607  throw TableNotFoundException(tableName, currentDB_.dbName, " Cannot remove.");
1608  }
1609 
1610  TableDescriptor* td = tableDescIt->second;
1611 
1612  if (td->hasDeletedCol) {
1613  const auto ret = deletedColumnPerTable_.erase(td);
1614  CHECK_EQ(ret, size_t(1));
1615  }
1616 
1617  tableDescriptorMapById_.erase(tableDescIt);
1618  tableDescriptorMap_.erase(to_upper(tableName));
1619  td->fragmenter = nullptr;
1620  dict_columns_by_table_id_.erase(tableId);
1621 
1623  delete td;
1624 
1625  std::unique_ptr<StringDictionaryClient> client;
1626  if (SysCatalog::instance().isAggregator()) {
1627  CHECK(!string_dict_hosts_.empty());
1628  DictRef dict_ref(currentDB_.dbId, -1);
1629  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1630  }
1631 
1632  // delete all column descriptors for the table
1633  // no more link columnIds to sequential indexes!
1634  for (auto cit = columnDescriptorMapById_.begin();
1635  cit != columnDescriptorMapById_.end();) {
1636  if (tableId != std::get<0>(cit->first)) {
1637  ++cit;
1638  } else {
1639  int i = std::get<1>(cit++->first);
1640  ColumnIdKey cidKey(tableId, i);
1641  ColumnDescriptorMapById::iterator colDescIt = columnDescriptorMapById_.find(cidKey);
1642  ColumnDescriptor* cd = colDescIt->second;
1643  columnDescriptorMapById_.erase(colDescIt);
1644  ColumnKey cnameKey(tableId, to_upper(cd->columnName));
1645  columnDescriptorMap_.erase(cnameKey);
1646  const int dictId = cd->columnType.get_comp_param();
1647  // Dummy dictionaries created for a shard of a logical table have the id set to
1648  // zero.
1649  if (cd->columnType.get_compression() == kENCODING_DICT && dictId) {
1650  INJECT_TIMER(removingDicts);
1651  DictRef dict_ref(currentDB_.dbId, dictId);
1652  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
1653  // If we're removing this table due to an error, it is possible that the string
1654  // dictionary reference was never populated. Don't crash, just continue cleaning
1655  // up the TableDescriptor and ColumnDescriptors
1656  if (!is_on_error) {
1657  CHECK(dictIt != dictDescriptorMapByRef_.end());
1658  } else {
1659  if (dictIt == dictDescriptorMapByRef_.end()) {
1660  continue;
1661  }
1662  }
1663  const auto& dd = dictIt->second;
1664  CHECK_GE(dd->refcount, 1);
1665  --dd->refcount;
1666  if (!dd->refcount) {
1667  dd->stringDict.reset();
1668  if (!isTemp) {
1669  File_Namespace::renameForDelete(dd->dictFolderPath);
1670  }
1671  if (client) {
1672  client->drop(dict_ref);
1673  }
1674  dictDescriptorMapByRef_.erase(dictIt);
1675  }
1676  }
1677 
1678  delete cd;
1679  }
1680  }
1681 }
1682 
1683 void Catalog::addFrontendViewToMap(DashboardDescriptor& vd) {
1684  cat_write_lock write_lock(this);
1685  addFrontendViewToMapNoLock(vd);
1686 }
1687 
1688 void Catalog::addFrontendViewToMapNoLock(DashboardDescriptor& vd) {
1689  cat_write_lock write_lock(this);
1690  dashboardDescriptorMap_[std::to_string(vd.userId) + ":" + vd.dashboardName] =
1691  std::make_shared<DashboardDescriptor>(vd);
1692 }
1693 
1694 std::vector<DBObject> Catalog::parseDashboardObjects(const std::string& view_meta,
1695  const int& user_id) {
1696  std::vector<DBObject> objects;
1697  DBObjectKey key;
1698  key.dbId = currentDB_.dbId;
1699  auto _key_place = [&key](auto type, auto id) {
1700  key.permissionType = type;
1701  key.objectId = id;
1702  return key;
1703  };
1704  for (auto object_name : parse_underlying_dashboard_objects(view_meta)) {
1705  auto td = getMetadataForTable(object_name, false);
1706  if (!td) {
1707  // Parsed object source is not present in current database
1708  // LOG the info and ignore
1709  LOG(INFO) << "Ignoring dashboard source Table/View: " << object_name
1710  << " no longer exists in current DB.";
1711  continue;
1712  }
1713  // Dashboard source can be Table or View
1714  const auto object_type = td->isView ? ViewDBObjectType : TableDBObjectType;
1715  const auto priv = td->isView ? AccessPrivileges::SELECT_FROM_VIEW
1717  objects.emplace_back(_key_place(object_type, td->tableId), priv, user_id);
1718  objects.back().setObjectType(td->isView ? ViewDBObjectType : TableDBObjectType);
1719  objects.back().setName(td->tableName);
1720  }
1721  return objects;
1722 }
1723 
1724 void Catalog::createOrUpdateDashboardSystemRole(const std::string& view_meta,
1725  const int32_t& user_id,
1726  const std::string& dash_role_name) {
1727  auto objects = parseDashboardObjects(view_meta, user_id);
1728  Role* rl = SysCatalog::instance().getRoleGrantee(dash_role_name);
1729  if (!rl) {
1730  // Dashboard role does not exist
1731  // create role and grant privileges
1732  // NOTE(wamsi): Transactionally unsafe
1733  SysCatalog::instance().createRole(
1734  dash_role_name, /*user_private_role=*/false, /*is_temporary=*/false);
1735  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1736  } else {
1737  // Dashboard system role already exists
1738  // Add/remove privileges on objects
1739  std::set<DBObjectKey> revoke_keys;
1740  auto ex_objects = rl->getDbObjects(true);
1741  for (auto key : *ex_objects | boost::adaptors::map_keys) {
1742  if (key.permissionType != TableDBObjectType &&
1743  key.permissionType != ViewDBObjectType) {
1744  continue;
1745  }
1746  bool found = false;
1747  for (auto obj : objects) {
1748  found = key == obj.getObjectKey() ? true : false;
1749  if (found) {
1750  break;
1751  }
1752  }
1753  if (!found) {
1754  revoke_keys.insert(key);
1755  }
1756  }
1757  for (auto& key : revoke_keys) {
1758  // revoke privs on object since the object is no
1759  // longer used by the dashboard as source
1760  // NOTE(wamsi): Transactionally unsafe
1761  SysCatalog::instance().revokeDBObjectPrivileges(
1762  dash_role_name, *rl->findDbObject(key, true), *this);
1763  }
1764  // Update privileges on remaining objects
1765  // NOTE(wamsi): Transactionally unsafe
1766  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1767  }
1768 }
1769 
1770 void Catalog::addLinkToMap(LinkDescriptor& ld) {
1771  cat_write_lock write_lock(this);
1772  LinkDescriptor* new_ld = new LinkDescriptor();
1773  *new_ld = ld;
1774  linkDescriptorMap_[std::to_string(currentDB_.dbId) + ld.link] = new_ld;
1775  linkDescriptorMapById_[ld.linkId] = new_ld;
1776 }
1777 
1778 void Catalog::instantiateFragmenter(TableDescriptor* td) const {
1779  auto time_ms = measure<>::execution([&]() {
1780  // instanciate table fragmenter upon first use
1781  // assume only insert order fragmenter is supported
1783  vector<Chunk> chunkVec;
1784  auto columnDescs = getAllColumnMetadataForTable(td->tableId, true, false, true);
1785  Chunk::translateColumnDescriptorsToChunkVec(columnDescs, chunkVec);
1786  ChunkKey chunkKeyPrefix = {currentDB_.dbId, td->tableId};
1787  if (td->sortedColumnId > 0) {
1788  td->fragmenter = std::make_shared<SortedOrderFragmenter>(chunkKeyPrefix,
1789  chunkVec,
1790  dataMgr_.get(),
1791  const_cast<Catalog*>(this),
1792  td->tableId,
1793  td->shard,
1794  td->maxFragRows,
1795  td->maxChunkSize,
1796  td->fragPageSize,
1797  td->maxRows,
1798  td->persistenceLevel);
1799  } else {
1800  td->fragmenter = std::make_shared<InsertOrderFragmenter>(chunkKeyPrefix,
1801  chunkVec,
1802  dataMgr_.get(),
1803  const_cast<Catalog*>(this),
1804  td->tableId,
1805  td->shard,
1806  td->maxFragRows,
1807  td->maxChunkSize,
1808  td->fragPageSize,
1809  td->maxRows,
1810  td->persistenceLevel,
1811  !td->storageType.empty());
1812  }
1813  });
1814  LOG(INFO) << "Instantiating Fragmenter for table " << td->tableName << " took "
1815  << time_ms << "ms";
1816 }
1817 
1818 foreign_storage::ForeignTable* Catalog::getForeignTableUnlocked(
1819  const std::string& tableName) const {
1820  auto tableDescIt = tableDescriptorMap_.find(to_upper(tableName));
1821  if (tableDescIt == tableDescriptorMap_.end()) { // check to make sure table exists
1822  return nullptr;
1823  }
1824  return dynamic_cast<foreign_storage::ForeignTable*>(tableDescIt->second);
1825 }
1826 
1827 const foreign_storage::ForeignTable* Catalog::getForeignTable(
1828  const std::string& tableName) const {
1829  cat_read_lock read_lock(this);
1830  return getForeignTableUnlocked(tableName);
1831 }
1832 
1833 const TableDescriptor* Catalog::getMetadataForTable(const string& tableName,
1834  const bool populateFragmenter) const {
1835  // we give option not to populate fragmenter (default true/yes) as it can be heavy for
1836  // pure metadata calls
1837  cat_read_lock read_lock(this);
1838  auto td = getMutableMetadataForTableUnlocked(tableName);
1839  if (!td) {
1840  return nullptr;
1841  }
1842  read_lock.unlock();
1843  if (populateFragmenter) {
1844  std::unique_lock<std::mutex> td_lock(*td->mutex_.get());
1845  if (td->fragmenter == nullptr && !td->isView) {
1846  instantiateFragmenter(td);
1847  }
1848  }
1849  return td; // returns pointer to table descriptor
1850 }
1851 
1852 const TableDescriptor* Catalog::getMetadataForTable(int table_id,
1853  bool populateFragmenter) const {
1854  cat_read_lock read_lock(this);
1855  auto td = getMutableMetadataForTableUnlocked(table_id);
1856  if (!td) {
1857  return nullptr;
1858  }
1859  read_lock.unlock();
1860  if (populateFragmenter) {
1861  std::unique_lock<std::mutex> td_lock(*td->mutex_.get());
1862  if (td->fragmenter == nullptr && !td->isView) {
1863  instantiateFragmenter(td);
1864  }
1865  }
1866  return td;
1867 }
1868 
1869 std::optional<std::string> Catalog::getTableName(int32_t table_id) const {
1870  cat_read_lock read_lock(this);
1871  auto td = getMutableMetadataForTableUnlocked(table_id);
1872  if (!td) {
1873  return {};
1874  }
1875  return td->tableName;
1876 }
1877 
1878 std::optional<int32_t> Catalog::getTableId(const std::string& table_name) const {
1879  cat_read_lock read_lock(this);
1880  auto td = getMutableMetadataForTableUnlocked(table_name);
1881  if (!td) {
1882  return {};
1883  }
1884  return td->tableId;
1885 }
1886 
1887 TableDescriptor* Catalog::getMutableMetadataForTableUnlocked(
1888  const std::string& table_name) const {
1889  auto it = tableDescriptorMap_.find(to_upper(table_name));
1890  if (it == tableDescriptorMap_.end()) {
1891  return nullptr;
1892  }
1893  return it->second;
1894 }
1895 
1896 TableDescriptor* Catalog::getMutableMetadataForTableUnlocked(int table_id) const {
1897  auto tableDescIt = tableDescriptorMapById_.find(table_id);
1898  if (tableDescIt == tableDescriptorMapById_.end()) { // check to make sure table exists
1899  return nullptr;
1900  }
1901  return tableDescIt->second;
1902 }
1903 
1904 const DictDescriptor* Catalog::getMetadataForDict(const int dict_id,
1905  const bool load_dict) const {
1906  cat_read_lock read_lock(this);
1907  const DictRef dictRef(currentDB_.dbId, dict_id);
1908  auto dictDescIt = dictDescriptorMapByRef_.find(dictRef);
1909  if (dictDescIt ==
1910  dictDescriptorMapByRef_.end()) { // check to make sure dictionary exists
1911  return nullptr;
1912  }
1913  auto& dd = dictDescIt->second;
1914 
1915  if (load_dict) {
1916  std::lock_guard string_dict_lock(*dd->string_dict_mutex);
1917  if (!dd->stringDict) {
1918  auto time_ms = measure<>::execution([&]() {
1919  if (string_dict_hosts_.empty()) {
1920  if (dd->dictIsTemp) {
1921  dd->stringDict = std::make_shared<StringDictionary>(
1922  dd->dictRef, dd->dictFolderPath, true, true, g_cache_string_hash);
1923  } else {
1924  dd->stringDict = std::make_shared<StringDictionary>(
1925  dd->dictRef, dd->dictFolderPath, false, true, g_cache_string_hash);
1926  }
1927  } else {
1928  dd->stringDict =
1929  std::make_shared<StringDictionary>(string_dict_hosts_.front(), dd->dictRef);
1930  }
1931  });
1932  LOG(INFO) << "Time to load Dictionary " << dd->dictRef.dbId << "_"
1933  << dd->dictRef.dictId << " was " << time_ms << "ms";
1934  }
1935  }
1936 
1937  return dd.get();
1938 }
1939 
1940 const std::vector<LeafHostInfo>& Catalog::getStringDictionaryHosts() const {
1941  return string_dict_hosts_;
1942 }
1943 
1944 const ColumnDescriptor* Catalog::getMetadataForColumn(int tableId,
1945  const string& columnName) const {
1946  cat_read_lock read_lock(this);
1947 
1948  ColumnKey columnKey(tableId, to_upper(columnName));
1949  auto colDescIt = columnDescriptorMap_.find(columnKey);
1950  if (colDescIt ==
1951  columnDescriptorMap_.end()) { // need to check to make sure column exists for table
1952  return nullptr;
1953  }
1954  return colDescIt->second;
1955 }
1956 
1957 const ColumnDescriptor* Catalog::getMetadataForColumn(int table_id, int column_id) const {
1958  cat_read_lock read_lock(this);
1959  ColumnIdKey columnIdKey(table_id, column_id);
1960  auto colDescIt = columnDescriptorMapById_.find(columnIdKey);
1961  if (colDescIt == columnDescriptorMapById_
1962  .end()) { // need to check to make sure column exists for table
1963  return nullptr;
1964  }
1965  return colDescIt->second;
1966 }
1967 
1968 const std::optional<std::string> Catalog::getColumnName(int table_id,
1969  int column_id) const {
1970  cat_read_lock read_lock(this);
1971  auto it = columnDescriptorMapById_.find(ColumnIdKey{table_id, column_id});
1972  if (it == columnDescriptorMapById_.end()) {
1973  return {};
1974  }
1975  return it->second->columnName;
1976 }
1977 
1978 const int Catalog::getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const {
1979  const auto tabDescIt = tableDescriptorMapById_.find(table_id);
1980  CHECK(tableDescriptorMapById_.end() != tabDescIt);
1981  const auto& columnIdBySpi = tabDescIt->second->columnIdBySpi_;
1982 
1983  auto spx = spi;
1984  int phi = 0;
1985  if (spx >= SPIMAP_MAGIC1) // see Catalog.h
1986  {
1987  phi = (spx - SPIMAP_MAGIC1) % SPIMAP_MAGIC2;
1988  spx = (spx - SPIMAP_MAGIC1) / SPIMAP_MAGIC2;
1989  }
1990 
1991  CHECK(0 < spx && spx <= columnIdBySpi.size())
1992  << "spx = " << spx << ", size = " << columnIdBySpi.size();
1993  return columnIdBySpi[spx - 1] + phi;
1994 }
1995 
1996 const int Catalog::getColumnIdBySpi(const int table_id, const size_t spi) const {
1997  cat_read_lock read_lock(this);
1998  return getColumnIdBySpiUnlocked(table_id, spi);
1999 }
2000 
2001 const ColumnDescriptor* Catalog::getMetadataForColumnBySpi(const int tableId,
2002  const size_t spi) const {
2003  cat_read_lock read_lock(this);
2004 
2005  const auto columnId = getColumnIdBySpiUnlocked(tableId, spi);
2006  ColumnIdKey columnIdKey(tableId, columnId);
2007  const auto colDescIt = columnDescriptorMapById_.find(columnIdKey);
2008  return columnDescriptorMapById_.end() == colDescIt ? nullptr : colDescIt->second;
2009 }
2010 
2011 void Catalog::deleteMetadataForDashboards(const std::vector<int32_t> dashboard_ids,
2012  const UserMetadata& user) {
2013  std::stringstream invalid_ids, restricted_ids;
2014 
2015  for (int32_t dashboard_id : dashboard_ids) {
2016  if (!getMetadataForDashboard(dashboard_id)) {
2017  invalid_ids << (!invalid_ids.str().empty() ? ", " : "") << dashboard_id;
2018  continue;
2019  }
2020  DBObject object(dashboard_id, DashboardDBObjectType);
2021  object.loadKey(*this);
2022  object.setPrivileges(AccessPrivileges::DELETE_DASHBOARD);
2023  std::vector<DBObject> privs = {object};
2024  if (!SysCatalog::instance().checkPrivileges(user, privs)) {
2025  restricted_ids << (!restricted_ids.str().empty() ? ", " : "") << dashboard_id;
2026  }
2027  }
2028 
2029  if (invalid_ids.str().size() > 0 || restricted_ids.str().size() > 0) {
2030  std::stringstream error_message;
2031  error_message << "Delete dashboard(s) failed with error(s):";
2032  if (invalid_ids.str().size() > 0) {
2033  error_message << "\nDashboard id: " << invalid_ids.str()
2034  << " - Dashboard id does not exist";
2035  }
2036  if (restricted_ids.str().size() > 0) {
2037  error_message
2038  << "\nDashboard id: " << restricted_ids.str()
2039  << " - User should be either owner of dashboard or super user to delete it";
2040  }
2041  throw std::runtime_error(error_message.str());
2042  }
2043  std::vector<DBObject> dash_objs;
2044 
2045  for (int32_t dashboard_id : dashboard_ids) {
2046  dash_objs.emplace_back(dashboard_id, DashboardDBObjectType);
2047  }
2048  // BE-5245: Transactionally unsafe (like other combined Catalog/Syscatalog operations)
2049  SysCatalog::instance().revokeDBObjectPrivilegesFromAllBatch(dash_objs, this);
2050  {
2051  cat_write_lock write_lock(this);
2052  cat_sqlite_lock sqlite_lock(getObjForLock());
2053 
2054  sqliteConnector_.query("BEGIN TRANSACTION");
2055  try {
2056  for (int32_t dashboard_id : dashboard_ids) {
2057  auto dash = getMetadataForDashboard(dashboard_id);
2058  // Dash should still exist if revokeDBObjectPrivileges passed but throw and
2059  // rollback if already deleted
2060  if (!dash) {
2061  throw std::runtime_error(
2062  std::string("Delete dashboard(s) failed with error(s):\nDashboard id: ") +
2063  std::to_string(dashboard_id) + " - Dashboard id does not exist ");
2064  }
2065  std::string user_id = std::to_string(dash->userId);
2066  std::string dash_name = dash->dashboardName;
2067  auto viewDescIt = dashboardDescriptorMap_.find(user_id + ":" + dash_name);
2068  dashboardDescriptorMap_.erase(viewDescIt);
2069  sqliteConnector_.query_with_text_params(
2070  "DELETE FROM mapd_dashboards WHERE name = ? and userid = ?",
2071  std::vector<std::string>{dash_name, user_id});
2072  }
2073  } catch (std::exception& e) {
2074  sqliteConnector_.query("ROLLBACK TRANSACTION");
2075  throw;
2076  }
2077  sqliteConnector_.query("END TRANSACTION");
2078  }
2079 }
2080 
2081 const DashboardDescriptor* Catalog::getMetadataForDashboard(
2082  const string& userId,
2083  const string& dashName) const {
2084  cat_read_lock read_lock(this);
2085 
2086  auto viewDescIt = dashboardDescriptorMap_.find(userId + ":" + dashName);
2087  if (viewDescIt == dashboardDescriptorMap_.end()) { // check to make sure view exists
2088  return nullptr;
2089  }
2090  return viewDescIt->second.get(); // returns pointer to view descriptor
2091 }
2092 
2093 const DashboardDescriptor* Catalog::getMetadataForDashboard(const int32_t id) const {
2094  cat_read_lock read_lock(this);
2095  std::string userId;
2096  std::string name;
2097  bool found{false};
2098  {
2099  for (auto descp : dashboardDescriptorMap_) {
2100  auto dash = descp.second.get();
2101  if (dash->dashboardId == id) {
2102  userId = std::to_string(dash->userId);
2103  name = dash->dashboardName;
2104  found = true;
2105  break;
2106  }
2107  }
2108  }
2109  if (found) {
2110  return getMetadataForDashboard(userId, name);
2111  }
2112  return nullptr;
2113 }
2114 
2115 const LinkDescriptor* Catalog::getMetadataForLink(const string& link) const {
2116  cat_read_lock read_lock(this);
2117  auto linkDescIt = linkDescriptorMap_.find(link);
2118  if (linkDescIt == linkDescriptorMap_.end()) { // check to make sure view exists
2119  return nullptr;
2120  }
2121  return linkDescIt->second; // returns pointer to view descriptor
2122 }
2123 
2124 const LinkDescriptor* Catalog::getMetadataForLink(int linkId) const {
2125  cat_read_lock read_lock(this);
2126  auto linkDescIt = linkDescriptorMapById_.find(linkId);
2127  if (linkDescIt == linkDescriptorMapById_.end()) { // check to make sure view exists
2128  return nullptr;
2129  }
2130  return linkDescIt->second;
2131 }
2132 
2133 const foreign_storage::ForeignTable* Catalog::getForeignTable(int table_id) const {
2134  cat_read_lock read_lock(this);
2135  const auto table = getMutableMetadataForTableUnlocked(table_id);
2136  CHECK(table);
2137  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(table);
2138  CHECK(foreign_table);
2139  return foreign_table;
2140 }
2141 
2142 void Catalog::getAllColumnMetadataForTableImpl(
2143  const TableDescriptor* td,
2144  list<const ColumnDescriptor*>& columnDescriptors,
2145  const bool fetchSystemColumns,
2146  const bool fetchVirtualColumns,
2147  const bool fetchPhysicalColumns) const {
2148  int32_t skip_physical_cols = 0;
2149  for (const auto& columnDescriptor : columnDescriptorMapById_) {
2150  if (!fetchPhysicalColumns && skip_physical_cols > 0) {
2151  --skip_physical_cols;
2152  continue;
2153  }
2154  auto cd = columnDescriptor.second;
2155  if (cd->tableId != td->tableId) {
2156  continue;
2157  }
2158  if (!fetchSystemColumns && cd->isSystemCol) {
2159  continue;
2160  }
2161  if (!fetchVirtualColumns && cd->isVirtualCol) {
2162  continue;
2163  }
2164  if (!fetchPhysicalColumns) {
2165  const auto& col_ti = cd->columnType;
2166  skip_physical_cols = col_ti.get_physical_cols();
2167  }
2168  columnDescriptors.push_back(cd);
2169  }
2170 }
2171 
2172 std::list<const ColumnDescriptor*> Catalog::getAllColumnMetadataForTable(
2173  const int tableId,
2174  const bool fetchSystemColumns,
2175  const bool fetchVirtualColumns,
2176  const bool fetchPhysicalColumns) const {
2177  cat_read_lock read_lock(this);
2178  std::list<const ColumnDescriptor*> columnDescriptors;
2179  const TableDescriptor* td = getMutableMetadataForTableUnlocked(tableId);
2180  getAllColumnMetadataForTableImpl(td,
2181  columnDescriptors,
2182  fetchSystemColumns,
2183  fetchVirtualColumns,
2184  fetchPhysicalColumns);
2185  return columnDescriptors;
2186 }
2187 
2188 list<const TableDescriptor*> Catalog::getAllTableMetadata() const {
2189  cat_read_lock read_lock(this);
2190  list<const TableDescriptor*> table_list;
2191  for (auto p : tableDescriptorMapById_) {
2192  table_list.push_back(p.second);
2193  }
2194  return table_list;
2195 }
2196 
2197 std::vector<TableDescriptor> Catalog::getAllTableMetadataCopy() const {
2198  cat_read_lock read_lock(this);
2199  std::vector<TableDescriptor> tables;
2200  tables.reserve(tableDescriptorMapById_.size());
2201  for (auto table_entry : tableDescriptorMapById_) {
2202  tables.emplace_back(*table_entry.second);
2203  tables.back().fragmenter = nullptr;
2204  }
2205  return tables;
2206 }
2207 
2208 list<const DashboardDescriptor*> Catalog::getAllDashboardsMetadata() const {
2209  cat_read_lock read_lock(this);
2210  list<const DashboardDescriptor*> dashboards;
2211  for (auto dashboard_entry : dashboardDescriptorMap_) {
2212  dashboards.push_back(dashboard_entry.second.get());
2213  }
2214  return dashboards;
2215 }
2216 
2217 std::vector<DashboardDescriptor> Catalog::getAllDashboardsMetadataCopy() const {
2218  cat_read_lock read_lock(this);
2219  std::vector<DashboardDescriptor> dashboards;
2220  dashboards.reserve(dashboardDescriptorMap_.size());
2221  for (auto dashboard_entry : dashboardDescriptorMap_) {
2222  dashboards.emplace_back(*dashboard_entry.second);
2223  }
2224  return dashboards;
2225 }
2226 
2227 DictRef Catalog::addDictionaryTransactional(ColumnDescriptor& cd) {
2228  cat_write_lock write_lock(this);
2230  DictRef ref{};
2231  sqliteConnector_.query("BEGIN TRANSACTION");
2232  try {
2233  ref = addDictionaryNontransactional(cd);
2234  } catch (std::exception& e) {
2235  sqliteConnector_.query("ROLLBACK TRANSACTION");
2236  throw;
2237  }
2238  sqliteConnector_.query("END TRANSACTION");
2239  return ref;
2240 }
2241 
2242 DictRef Catalog::addDictionaryNontransactional(ColumnDescriptor& cd) {
2243  cat_write_lock write_lock(this);
2244  const auto& td = *tableDescriptorMapById_[cd.tableId];
2245  list<DictDescriptor> dds;
2246  setColumnDictionary(cd, dds, td, true);
2247  auto& dd = dds.back();
2248  CHECK(dd.dictRef.dictId);
2249 
2250  std::unique_ptr<StringDictionaryClient> client;
2251  if (!string_dict_hosts_.empty()) {
2252  client.reset(new StringDictionaryClient(
2253  string_dict_hosts_.front(), DictRef(currentDB_.dbId, -1), true));
2254  }
2255  if (client) {
2256  client->create(dd.dictRef, dd.dictIsTemp);
2257  }
2258 
2259  DictDescriptor* new_dd = new DictDescriptor(dd);
2260  dictDescriptorMapByRef_[dd.dictRef].reset(new_dd);
2261  if (!dd.dictIsTemp) {
2262  boost::filesystem::create_directory(new_dd->dictFolderPath);
2263  }
2264  return dd.dictRef;
2265 }
2266 
2267 void Catalog::delDictionaryTransactional(const ColumnDescriptor& cd) {
2268  cat_write_lock write_lock(this);
2270  sqliteConnector_.query("BEGIN TRANSACTION");
2271  try {
2272  delDictionaryNontransactional(cd);
2273  } catch (std::exception& e) {
2274  sqliteConnector_.query("ROLLBACK TRANSACTION");
2275  throw;
2276  }
2277  sqliteConnector_.query("END TRANSACTION");
2278 }
2279 
2280 void Catalog::delDictionaryNontransactional(const ColumnDescriptor& cd) {
2281  cat_write_lock write_lock(this);
2282  cat_sqlite_lock sqlite_lock(getObjForLock());
2283  if (!(cd.columnType.is_string() || cd.columnType.is_string_array())) {
2284  return;
2285  }
2286  if (!(cd.columnType.get_compression() == kENCODING_DICT)) {
2287  return;
2288  }
2289  const auto dictId = cd.columnType.get_comp_param();
2290  CHECK_GT(dictId, 0);
2291  // decrement and zero check dict ref count
2292  const auto td = getMetadataForTable(cd.tableId, false);
2293  CHECK(td);
2294  sqliteConnector_.query_with_text_param(
2295  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
2296  std::to_string(dictId));
2297  sqliteConnector_.query_with_text_param(
2298  "SELECT refcount FROM mapd_dictionaries WHERE dictid = ?", std::to_string(dictId));
2299  const auto refcount = sqliteConnector_.getData<int>(0, 0);
2300  VLOG(3) << "Dictionary " << dictId << "from dropped table has reference count "
2301  << refcount;
2302  if (refcount > 0) {
2303  return;
2304  }
2305  const DictRef dictRef(currentDB_.dbId, dictId);
2306  sqliteConnector_.query_with_text_param("DELETE FROM mapd_dictionaries WHERE dictid = ?",
2307  std::to_string(dictId));
2309  "/DB_" + std::to_string(currentDB_.dbId) + "_DICT_" +
2310  std::to_string(dictId));
2311 
2312  std::unique_ptr<StringDictionaryClient> client;
2313  if (!string_dict_hosts_.empty()) {
2314  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dictRef, true));
2315  }
2316  if (client) {
2317  client->drop(dictRef);
2318  }
2319 
2320  dictDescriptorMapByRef_.erase(dictRef);
2321 }
2322 
2323 std::list<const DictDescriptor*> Catalog::getAllDictionariesWithColumnInName(
2324  const ColumnDescriptor* cd) {
2325  cat_read_lock read_lock(this);
2326  std::list<const DictDescriptor*> dds;
2327 
2328  auto table_name_opt = getTableName(cd->tableId);
2329  CHECK(table_name_opt.has_value());
2330  auto table_name = table_name_opt.value();
2331 
2332  for (const auto& [dkey, dd] : dictDescriptorMapByRef_) {
2333  if (dd->dictName.find(table_name + "_" + cd->columnName + "_dict") !=
2334  std::string::npos) {
2335  dds.push_back(dd.get());
2336  }
2337  }
2338 
2339  return dds;
2340 }
2341 
2342 void Catalog::getDictionary(const ColumnDescriptor& cd,
2343  std::map<int, StringDictionary*>& stringDicts) {
2344  // learn 'committed' ColumnDescriptor of this column
2345  auto cit = columnDescriptorMap_.find(ColumnKey(cd.tableId, to_upper(cd.columnName)));
2346  CHECK(cit != columnDescriptorMap_.end());
2347  auto& ccd = *cit->second;
2348 
2349  if (!(ccd.columnType.is_string() || ccd.columnType.is_string_array())) {
2350  return;
2351  }
2352  if (!(ccd.columnType.get_compression() == kENCODING_DICT)) {
2353  return;
2354  }
2355  if (!(ccd.columnType.get_comp_param() > 0)) {
2356  return;
2357  }
2358 
2359  auto dictId = ccd.columnType.get_comp_param();
2360  getMetadataForDict(dictId);
2361 
2362  const DictRef dictRef(currentDB_.dbId, dictId);
2363  auto dit = dictDescriptorMapByRef_.find(dictRef);
2364  CHECK(dit != dictDescriptorMapByRef_.end());
2365  CHECK(dit->second);
2366  CHECK(dit->second.get()->stringDict);
2367  stringDicts[ccd.columnId] = dit->second.get()->stringDict.get();
2368 }
2369 
2370 size_t Catalog::getTotalMemorySizeForDictionariesForDatabase() const {
2371  size_t ret{0};
2372  for (auto const& kv : dictDescriptorMapByRef_) {
2373  if (kv.first.dbId == currentDB_.dbId) {
2374  auto dictionary = kv.second.get()->stringDict.get();
2375  if (dictionary) {
2376  ret += dictionary->computeCacheSize();
2377  }
2378  }
2379  }
2380  return ret;
2381 }
2382 
2383 void Catalog::alterColumnTypeTransactional(const ColumnDescriptor& cd) {
2384  cat_write_lock write_lock(this);
2386 
2387  sqliteConnector_.query("BEGIN TRANSACTION");
2388  try {
2389  const auto table_id = cd.tableId;
2390 
2391  auto catalog_cd = getMetadataForColumn(table_id, cd.columnId);
2392 
2393  CHECK(catalog_cd) << " can not alter non existing column";
2394 
2395  using BindType = SqliteConnector::BindType;
2396  std::vector<BindType> types(11, BindType::TEXT);
2397  if (!cd.default_value.has_value()) {
2398  types[8] = BindType::NULL_TYPE;
2399  }
2400  sqliteConnector_.query_with_text_params(
2401  "UPDATE mapd_columns SET "
2402  "coltype = ?,"
2403  "colsubtype = ?,"
2404  "coldim = ?,"
2405  "colscale = ?,"
2406  "is_notnull = ?,"
2407  "compression = ?,"
2408  "comp_param = ?,"
2409  "size = ?,"
2410  "default_value = ? "
2411  "WHERE tableid = ? and columnid = ?",
2412  std::vector<std::string>{std::to_string(cd.columnType.get_type()),
2420  cd.default_value.value_or("NULL"),
2421  std::to_string(table_id),
2422  std::to_string(cd.columnId)},
2423 
2424  types);
2425 
2426  auto ncd = new ColumnDescriptor(cd);
2427 
2428  ColumnDescriptorMap::iterator columnDescIt =
2429  columnDescriptorMap_.find(ColumnKey(cd.tableId, to_upper(cd.columnName)));
2430  CHECK(columnDescIt != columnDescriptorMap_.end());
2431  auto ocd = columnDescIt->second;
2432 
2433  updateInColumnMap(ncd, ocd);
2434  } catch (std::exception& e) {
2435  sqliteConnector_.query("ROLLBACK TRANSACTION");
2436  throw;
2437  }
2438  sqliteConnector_.query("END TRANSACTION");
2439 }
2440 
2441 int Catalog::getNextAddedColumnId(const TableDescriptor& td) {
2442  cat_read_lock read_lock(this);
2443  sqliteConnector_.query_with_text_params(
2444  "SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?",
2445  std::vector<std::string>{std::to_string(td.tableId)});
2446  return sqliteConnector_.getData<int>(0, 0);
2447 }
2448 
2449 void Catalog::addColumnTransactional(const TableDescriptor& td, ColumnDescriptor& cd) {
2450  cat_write_lock write_lock(this);
2452  sqliteConnector_.query("BEGIN TRANSACTION");
2453  try {
2454  addColumnNontransactional(td, cd);
2455  } catch (std::exception& e) {
2456  sqliteConnector_.query("ROLLBACK TRANSACTION");
2457  throw;
2458  }
2459  sqliteConnector_.query("END TRANSACTION");
2460 }
2461 
2462 void Catalog::addColumnNontransactional(const TableDescriptor& td, ColumnDescriptor& cd) {
2463  cat_write_lock write_lock(this);
2465  cd.tableId = td.tableId;
2466  cd.db_id = getDatabaseId();
2467  if (td.nShards > 0 && td.shard < 0) {
2468  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
2469  auto shard_cd = cd;
2470  addColumnNontransactional(*shard, shard_cd);
2471  }
2472  }
2474  addDictionaryNontransactional(cd);
2475  }
2476 
2477  using BindType = SqliteConnector::BindType;
2478  std::vector<BindType> types(17, BindType::TEXT);
2479  if (!cd.default_value.has_value()) {
2480  types[16] = BindType::NULL_TYPE;
2481  }
2482  sqliteConnector_.query_with_text_params(
2483  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
2484  "colscale, is_notnull, "
2485  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
2486  "is_deletedcol, default_value) "
2487  "VALUES (?, "
2488  "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
2489  "?, ?, ?, "
2490  "?, "
2491  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2492  std::vector<std::string>{std::to_string(td.tableId),
2493  std::to_string(td.tableId),
2494  cd.columnName,
2503  "",
2506  cd.virtualExpr,
2508  cd.default_value.value_or("NULL")},
2509  types);
2510 
2511  sqliteConnector_.query_with_text_params(
2512  "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
2513  std::vector<std::string>{std::to_string(td.tableId)});
2514 
2515  sqliteConnector_.query_with_text_params(
2516  "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
2517  std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
2518  cd.columnId = sqliteConnector_.getData<int>(0, 0);
2519 
2520  ++tableDescriptorMapById_[td.tableId]->nColumns;
2521  auto ncd = new ColumnDescriptor(cd);
2522  addToColumnMap(ncd);
2523  addColumnDescriptor(ncd);
2524  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
2525 }
2526 
2527 // NOTE: this function is deprecated
2528 void Catalog::addColumn(const TableDescriptor& td, ColumnDescriptor& cd) {
2529  // caller must handle sqlite/chunk transaction TOGETHER
2530  cd.tableId = td.tableId;
2531  cd.db_id = getDatabaseId();
2532  if (td.nShards > 0 && td.shard < 0) {
2533  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
2534  auto shard_cd = cd;
2535  addColumn(*shard, shard_cd);
2536  }
2537  }
2539  addDictionaryNontransactional(cd);
2540  }
2541 
2542  using BindType = SqliteConnector::BindType;
2543  std::vector<BindType> types(17, BindType::TEXT);
2544  if (!cd.default_value.has_value()) {
2545  types[16] = BindType::NULL_TYPE;
2546  }
2547  sqliteConnector_.query_with_text_params(
2548  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
2549  "colscale, is_notnull, "
2550  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
2551  "is_deletedcol, default_value) "
2552  "VALUES (?, "
2553  "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
2554  "?, ?, ?, "
2555  "?, "
2556  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2557  std::vector<std::string>{std::to_string(td.tableId),
2558  std::to_string(td.tableId),
2559  cd.columnName,
2568  "",
2571  cd.virtualExpr,
2573  cd.default_value.value_or("NULL")},
2574  types);
2575 
2576  sqliteConnector_.query_with_text_params(
2577  "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
2578  std::vector<std::string>{std::to_string(td.tableId)});
2579 
2580  sqliteConnector_.query_with_text_params(
2581  "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
2582  std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
2583  cd.columnId = sqliteConnector_.getData<int>(0, 0);
2584 
2585  ++tableDescriptorMapById_[td.tableId]->nColumns;
2586  auto ncd = new ColumnDescriptor(cd);
2587  addToColumnMap(ncd);
2588  columnDescriptorsForRoll.emplace_back(nullptr, ncd);
2589 }
2590 
2591 void Catalog::dropColumnTransactional(const TableDescriptor& td,
2592  const ColumnDescriptor& cd) {
2593  dropColumnPolicies(td, cd);
2594 
2595  cat_write_lock write_lock(this);
2597  sqliteConnector_.query("BEGIN TRANSACTION");
2598  try {
2599  dropColumnNontransactional(td, cd);
2600  } catch (std::exception& e) {
2601  sqliteConnector_.query("ROLLBACK TRANSACTION");
2602  throw;
2603  }
2604  sqliteConnector_.query("END TRANSACTION");
2605 }
2606 
2607 void Catalog::dropColumnNontransactional(const TableDescriptor& td,
2608  const ColumnDescriptor& cd) {
2609  cat_write_lock write_lock(this);
2611 
2612  sqliteConnector_.query_with_text_params(
2613  "DELETE FROM mapd_columns where tableid = ? and columnid = ?",
2614  std::vector<std::string>{std::to_string(td.tableId), std::to_string(cd.columnId)});
2615 
2616  sqliteConnector_.query_with_text_params(
2617  "UPDATE mapd_tables SET ncolumns = ncolumns - 1 WHERE tableid = ?",
2618  std::vector<std::string>{std::to_string(td.tableId)});
2619 
2620  ColumnDescriptorMap::iterator columnDescIt =
2621  columnDescriptorMap_.find(ColumnKey(cd.tableId, to_upper(cd.columnName)));
2622  CHECK(columnDescIt != columnDescriptorMap_.end());
2623 
2624  auto ocd = columnDescIt->second;
2625  removeFromColumnMap(ocd);
2626  --tableDescriptorMapById_[td.tableId]->nColumns;
2627  removeColumnDescriptor(ocd);
2628  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
2629 
2630  // for each shard
2631  if (td.nShards > 0 && td.shard < 0) {
2632  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
2633  const auto shard_cd = getMetadataForColumn(shard->tableId, cd.columnId);
2634  CHECK(shard_cd);
2635  dropColumnNontransactional(*shard, *shard_cd);
2636  }
2637  }
2638 }
2639 
2640 void Catalog::dropColumnPolicies(const TableDescriptor& td, const ColumnDescriptor& cd) {
2641 
2642  // for each shard
2643  if (td.nShards > 0 && td.shard < 0) {
2644  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
2645  const auto shard_cd = getMetadataForColumn(shard->tableId, cd.columnId);
2646  CHECK(shard_cd);
2647  dropColumnPolicies(*shard, *shard_cd);
2648  }
2649  }
2650 }
2651 
2652 // NOTE: this function is deprecated
2653 void Catalog::dropColumn(const TableDescriptor& td, const ColumnDescriptor& cd) {
2654  {
2655  cat_write_lock write_lock(this);
2656  cat_sqlite_lock sqlite_lock(getObjForLock());
2657  // caller must handle sqlite/chunk transaction TOGETHER
2658  sqliteConnector_.query_with_text_params(
2659  "DELETE FROM mapd_columns where tableid = ? and columnid = ?",
2660  std::vector<std::string>{std::to_string(td.tableId),
2661  std::to_string(cd.columnId)});
2662 
2663  sqliteConnector_.query_with_text_params(
2664  "UPDATE mapd_tables SET ncolumns = ncolumns - 1 WHERE tableid = ?",
2665  std::vector<std::string>{std::to_string(td.tableId)});
2666 
2667  ColumnDescriptorMap::iterator columnDescIt =
2668  columnDescriptorMap_.find(ColumnKey(cd.tableId, to_upper(cd.columnName)));
2669  CHECK(columnDescIt != columnDescriptorMap_.end());
2670 
2671  columnDescriptorsForRoll.emplace_back(columnDescIt->second, nullptr);
2672  removeFromColumnMap(columnDescIt->second);
2673  --tableDescriptorMapById_[td.tableId]->nColumns;
2674  }
2675 
2676  // for each shard
2677  if (td.nShards > 0 && td.shard < 0) {
2678  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
2679  const auto shard_cd = getMetadataForColumn(shard->tableId, cd.columnId);
2680  CHECK(shard_cd);
2681  dropColumn(*shard, *shard_cd);
2682  }
2683  }
2684 }
2685 
2686 void Catalog::removeColumnDescriptor(const ColumnDescriptor* cd) {
2687  if (!cd) {
2688  return;
2689  }
2690 
2691  auto tabDescIt = tableDescriptorMapById_.find(cd->tableId);
2692  CHECK(tableDescriptorMapById_.end() != tabDescIt);
2693  auto td = tabDescIt->second;
2694  auto& cd_by_spi = td->columnIdBySpi_;
2695  cd_by_spi.erase(std::remove(cd_by_spi.begin(), cd_by_spi.end(), cd->columnId),
2696  cd_by_spi.end());
2697  delete cd;
2698  std::sort(cd_by_spi.begin(), cd_by_spi.end());
2699 }
2700 
2701 void Catalog::addColumnDescriptor(const ColumnDescriptor* cd) {
2702  if (!cd || cd->isGeoPhyCol) {
2703  return;
2704  }
2705 
2706  auto tabDescIt = tableDescriptorMapById_.find(cd->tableId);
2707  CHECK(tableDescriptorMapById_.end() != tabDescIt);
2708  auto td = tabDescIt->second;
2709  auto& cd_by_spi = td->columnIdBySpi_;
2710 
2711  if (cd_by_spi.end() == std::find(cd_by_spi.begin(), cd_by_spi.end(), cd->columnId)) {
2712  cd_by_spi.push_back(cd->columnId);
2713  }
2714  std::sort(cd_by_spi.begin(), cd_by_spi.end());
2715 }
2716 
2717 // NOTE: this function is deprecated
2718 void Catalog::rollLegacy(const bool forward) {
2719  cat_write_lock write_lock(this);
2720  std::set<const TableDescriptor*> tds;
2721 
2722  for (const auto& cdr : columnDescriptorsForRoll) {
2723  auto ocd = cdr.first;
2724  auto ncd = cdr.second;
2725  CHECK(ocd || ncd);
2726  auto tabDescIt = tableDescriptorMapById_.find((ncd ? ncd : ocd)->tableId);
2727  CHECK(tableDescriptorMapById_.end() != tabDescIt);
2728  auto td = tabDescIt->second;
2729  auto& vc = td->columnIdBySpi_;
2730  if (forward) {
2731  if (ocd) {
2732  if (nullptr == ncd ||
2733  ncd->columnType.get_comp_param() != ocd->columnType.get_comp_param()) {
2734  delDictionaryNontransactional(*ocd);
2735  }
2736 
2737  vc.erase(std::remove(vc.begin(), vc.end(), ocd->columnId), vc.end());
2738 
2739  delete ocd;
2740  }
2741  if (ncd) {
2742  // append columnId if its new and not phy geo
2743  if (vc.end() == std::find(vc.begin(), vc.end(), ncd->columnId)) {
2744  if (!ncd->isGeoPhyCol) {
2745  vc.push_back(ncd->columnId);
2746  }
2747  }
2748  }
2749  tds.insert(td);
2750  } else {
2751  if (ocd) {
2752  addToColumnMap(ocd);
2753  }
2754  // roll back the dict of new column
2755  if (ncd) {
2756  removeFromColumnMap(ncd);
2757  if (nullptr == ocd ||
2758  ocd->columnType.get_comp_param() != ncd->columnType.get_comp_param()) {
2759  delDictionaryNontransactional(*ncd);
2760  }
2761  delete ncd;
2762  }
2763  }
2764  }
2765  columnDescriptorsForRoll.clear();
2766 
2767  if (forward) {
2768  for (const auto td : tds) {
2769  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
2770  }
2771  }
2772 }
2773 
2774 void Catalog::expandGeoColumn(const ColumnDescriptor& cd,
2775  list<ColumnDescriptor>& columns) {
2776  const auto& col_ti = cd.columnType;
2777  if (IS_GEO(col_ti.get_type())) {
2778  switch (col_ti.get_type()) {
2779  case kPOINT: {
2780  ColumnDescriptor physical_cd_coords(true);
2781  physical_cd_coords.columnName = cd.columnName + "_coords";
2782  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2783  // Raw data: compressed/uncompressed coords
2784  coords_ti.set_subtype(kTINYINT);
2785  size_t unit_size;
2786  if (col_ti.get_compression() == kENCODING_GEOINT &&
2787  col_ti.get_comp_param() == 32) {
2788  unit_size = 4 * sizeof(int8_t);
2789  } else {
2790  CHECK(col_ti.get_compression() == kENCODING_NONE);
2791  unit_size = 8 * sizeof(int8_t);
2792  }
2793  coords_ti.set_size(2 * unit_size);
2794  physical_cd_coords.columnType = coords_ti;
2795  columns.push_back(physical_cd_coords);
2796 
2797  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2798 
2799  break;
2800  }
2801  case kMULTIPOINT:
2802  case kLINESTRING: {
2803  ColumnDescriptor physical_cd_coords(true);
2804  physical_cd_coords.columnName = cd.columnName + "_coords";
2805  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2806  // Raw data: compressed/uncompressed coords
2807  coords_ti.set_subtype(kTINYINT);
2808  physical_cd_coords.columnType = coords_ti;
2809  columns.push_back(physical_cd_coords);
2810 
2811  ColumnDescriptor physical_cd_bounds(true);
2812  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2813  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2814  bounds_ti.set_subtype(kDOUBLE);
2815  bounds_ti.set_size(4 * sizeof(double));
2816  physical_cd_bounds.columnType = bounds_ti;
2817  columns.push_back(physical_cd_bounds);
2818 
2819  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2820 
2821  break;
2822  }
2823  case kMULTILINESTRING: {
2824  ColumnDescriptor physical_cd_coords(true);
2825  physical_cd_coords.columnName = cd.columnName + "_coords";
2826  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2827  // Raw data: compressed/uncompressed coords
2828  coords_ti.set_subtype(kTINYINT);
2829  physical_cd_coords.columnType = coords_ti;
2830  columns.push_back(physical_cd_coords);
2831 
2832  ColumnDescriptor physical_cd_linestring_sizes(true);
2833  physical_cd_linestring_sizes.columnName = cd.columnName + "_linestring_sizes";
2834  SQLTypeInfo linestring_sizes_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2835  linestring_sizes_ti.set_subtype(kINT);
2836  physical_cd_linestring_sizes.columnType = linestring_sizes_ti;
2837  columns.push_back(physical_cd_linestring_sizes);
2838 
2839  ColumnDescriptor physical_cd_bounds(true);
2840  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2841  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2842  bounds_ti.set_subtype(kDOUBLE);
2843  bounds_ti.set_size(4 * sizeof(double));
2844  physical_cd_bounds.columnType = bounds_ti;
2845  columns.push_back(physical_cd_bounds);
2846 
2847  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2848 
2849  break;
2850  }
2851  case kPOLYGON: {
2852  ColumnDescriptor physical_cd_coords(true);
2853  physical_cd_coords.columnName = cd.columnName + "_coords";
2854  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2855  // Raw data: compressed/uncompressed coords
2856  coords_ti.set_subtype(kTINYINT);
2857  physical_cd_coords.columnType = coords_ti;
2858  columns.push_back(physical_cd_coords);
2859 
2860  ColumnDescriptor physical_cd_ring_sizes(true);
2861  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
2862  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2863  ring_sizes_ti.set_subtype(kINT);
2864  physical_cd_ring_sizes.columnType = ring_sizes_ti;
2865  columns.push_back(physical_cd_ring_sizes);
2866 
2867  ColumnDescriptor physical_cd_bounds(true);
2868  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2869  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2870  bounds_ti.set_subtype(kDOUBLE);
2871  bounds_ti.set_size(4 * sizeof(double));
2872  physical_cd_bounds.columnType = bounds_ti;
2873  columns.push_back(physical_cd_bounds);
2874 
2875  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2876 
2877  break;
2878  }
2879  case kMULTIPOLYGON: {
2880  ColumnDescriptor physical_cd_coords(true);
2881  physical_cd_coords.columnName = cd.columnName + "_coords";
2882  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2883  // Raw data: compressed/uncompressed coords
2884  coords_ti.set_subtype(kTINYINT);
2885  physical_cd_coords.columnType = coords_ti;
2886  columns.push_back(physical_cd_coords);
2887 
2888  ColumnDescriptor physical_cd_ring_sizes(true);
2889  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
2890  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2891  ring_sizes_ti.set_subtype(kINT);
2892  physical_cd_ring_sizes.columnType = ring_sizes_ti;
2893  columns.push_back(physical_cd_ring_sizes);
2894 
2895  ColumnDescriptor physical_cd_poly_rings(true);
2896  physical_cd_poly_rings.columnName = cd.columnName + "_poly_rings";
2897  SQLTypeInfo poly_rings_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2898  poly_rings_ti.set_subtype(kINT);
2899  physical_cd_poly_rings.columnType = poly_rings_ti;
2900  columns.push_back(physical_cd_poly_rings);
2901 
2902  ColumnDescriptor physical_cd_bounds(true);
2903  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2904  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2905  bounds_ti.set_subtype(kDOUBLE);
2906  bounds_ti.set_size(4 * sizeof(double));
2907  physical_cd_bounds.columnType = bounds_ti;
2908  columns.push_back(physical_cd_bounds);
2909 
2910  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2911 
2912  break;
2913  }
2914  default:
2915  throw runtime_error("Unrecognized geometry type.");
2916  break;
2917  }
2918  }
2919 }
2920 
2921 namespace {
2923  auto timing_type_entry =
2925  CHECK(timing_type_entry != foreign_table.options.end());
2926  if (timing_type_entry->second ==
2929  foreign_table.options);
2930  }
2932 }
2933 } // namespace
2934 
2935 void Catalog::createTable(
2936  TableDescriptor& td,
2937  const list<ColumnDescriptor>& cols,
2938  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs,
2939  bool isLogicalTable) {
2940  cat_write_lock write_lock(this);
2941  list<ColumnDescriptor> cds = cols;
2942  list<DictDescriptor> dds;
2943  std::set<std::string> toplevel_column_names;
2944  list<ColumnDescriptor> columns;
2945 
2946  if (!td.storageType.empty() &&
2949  throw std::runtime_error("Only temporary tables can be backed by foreign storage.");
2950  }
2951  dataMgr_->getForeignStorageInterface()->prepareTable(getCurrentDB().dbId, td, cds);
2952  }
2953 
2954  for (auto cd : cds) {
2955  if (cd.columnName == "rowid") {
2956  throw std::runtime_error(
2957  "Cannot create column with name rowid. rowid is a system defined column.");
2958  }
2959  columns.push_back(cd);
2960  toplevel_column_names.insert(cd.columnName);
2961  if (cd.columnType.is_geometry()) {
2962  expandGeoColumn(cd, columns);
2963  }
2964  }
2965  cds.clear();
2966 
2967  ColumnDescriptor cd;
2968  // add row_id column -- Must be last column in the table
2969  cd.columnName = "rowid";
2970  cd.isSystemCol = true;
2971  cd.columnType = SQLTypeInfo(kBIGINT, true);
2972 #ifdef MATERIALIZED_ROWID
2973  cd.isVirtualCol = false;
2974 #else
2975  cd.isVirtualCol = true;
2976  cd.virtualExpr = "MAPD_FRAG_ID * MAPD_ROWS_PER_FRAG + MAPD_FRAG_ROW_ID";
2977 #endif
2978  columns.push_back(cd);
2979  toplevel_column_names.insert(cd.columnName);
2980 
2981  if (td.hasDeletedCol) {
2982  ColumnDescriptor cd_del;
2983  cd_del.columnName = "$deleted$";
2984  cd_del.isSystemCol = true;
2985  cd_del.isVirtualCol = false;
2986  cd_del.columnType = SQLTypeInfo(kBOOLEAN, true);
2987  cd_del.isDeletedCol = true;
2988 
2989  columns.push_back(cd_del);
2990  }
2991 
2992  for (auto& column : columns) {
2993  column.db_id = getDatabaseId();
2994  }
2995 
2996  td.nColumns = columns.size();
2997  // TODO(sy): don't take disk locks or touch sqlite connector for temporary tables
2998  cat_sqlite_lock sqlite_lock(getObjForLock());
2999  sqliteConnector_.query("BEGIN TRANSACTION");
3001  try {
3002  sqliteConnector_.query_with_text_params(
3003  R"(INSERT INTO mapd_tables (name, userid, ncolumns, isview, fragments, frag_type, max_frag_rows, max_chunk_size, frag_page_size, max_rows, partitions, shard_column_id, shard, num_shards, sort_column_id, storage_type, max_rollback_epochs, is_system_table, key_metainfo) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?))",
3004  std::vector<std::string>{td.tableName,
3005  std::to_string(td.userId),
3007  std::to_string(td.isView),
3008  "",
3013  std::to_string(td.maxRows),
3014  td.partitions,
3016  std::to_string(td.shard),
3017  std::to_string(td.nShards),
3019  td.storageType,
3022  td.keyMetainfo});
3023 
3024  // now get the auto generated tableid
3025  sqliteConnector_.query_with_text_param(
3026  "SELECT tableid FROM mapd_tables WHERE name = ?", td.tableName);
3027  td.tableId = sqliteConnector_.getData<int>(0, 0);
3028  int colId = 1;
3029  for (auto cd : columns) {
3031  const bool is_foreign_col =
3032  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
3033  if (!is_foreign_col) {
3034  // Ideally we would like to not persist string dictionaries for system tables,
3035  // since system table content can be highly dynamic and string dictionaries
3036  // are not currently vacuumed. However, in distributed this causes issues
3037  // when the cluster is out of sync (when the agg resets but leaves persist) so
3038  // for the sake of testing we need to leave this as normal dictionaries until
3039  // we solve the distributed issue.
3040  auto use_temp_dictionary = false; // td.is_system_table;
3041  setColumnDictionary(cd, dds, td, isLogicalTable, use_temp_dictionary);
3042  }
3043  }
3044 
3045  if (toplevel_column_names.count(cd.columnName)) {
3046  if (!cd.isGeoPhyCol) {
3047  td.columnIdBySpi_.push_back(colId);
3048  }
3049  }
3050 
3051  using BindType = SqliteConnector::BindType;
3052  std::vector<BindType> types(17, BindType::TEXT);
3053  if (!cd.default_value.has_value()) {
3054  types[16] = BindType::NULL_TYPE;
3055  }
3056  sqliteConnector_.query_with_text_params(
3057  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, "
3058  "coldim, colscale, is_notnull, "
3059  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
3060  "virtual_expr, is_deletedcol, default_value) "
3061  "VALUES (?, ?, ?, ?, ?, "
3062  "?, "
3063  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
3064  std::vector<std::string>{std::to_string(td.tableId),
3065  std::to_string(colId),
3066  cd.columnName,
3075  "",
3078  cd.virtualExpr,
3080  cd.default_value.value_or("NULL")},
3081  types);
3082  cd.tableId = td.tableId;
3083  cd.columnId = colId++;
3084  cds.push_back(cd);
3085  }
3086  if (td.isView) {
3087  sqliteConnector_.query_with_text_params(
3088  "INSERT INTO mapd_views (tableid, sql) VALUES (?,?)",
3089  std::vector<std::string>{std::to_string(td.tableId), td.viewSQL});
3090  }
3092  auto& foreign_table = dynamic_cast<foreign_storage::ForeignTable&>(td);
3093  foreign_table.next_refresh_time = get_next_refresh_time(foreign_table);
3094  sqliteConnector_.query_with_text_params(
3095  "INSERT INTO omnisci_foreign_tables (table_id, server_id, options, "
3096  "last_refresh_time, next_refresh_time) VALUES (?, ?, ?, ?, ?)",
3097  std::vector<std::string>{std::to_string(foreign_table.tableId),
3098  std::to_string(foreign_table.foreign_server->id),
3099  foreign_table.getOptionsAsJsonString(),
3100  std::to_string(foreign_table.last_refresh_time),
3101  std::to_string(foreign_table.next_refresh_time)});
3102  }
3103  } catch (std::exception& e) {
3104  sqliteConnector_.query("ROLLBACK TRANSACTION");
3105  throw;
3106  }
3107  } else { // Temporary table
3108  td.tableId = nextTempTableId_++;
3109  int colId = 1;
3110  for (auto cd : columns) {
3112  const bool is_foreign_col =
3113  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
3114 
3115  if (!is_foreign_col) {
3116  // Create a new temporary dictionary
3117  std::string fileName("");
3118  std::string folderPath("");
3119  DictRef dict_ref(currentDB_.dbId, nextTempDictId_);
3120  nextTempDictId_++;
3121  DictDescriptor dd(dict_ref,
3122  fileName,
3124  false,
3125  1,
3126  folderPath,
3127  true); // Is dictName (2nd argument) used?
3128  dds.push_back(dd);
3129  if (!cd.columnType.is_array()) {
3131  }
3132  cd.columnType.set_comp_param(dict_ref.dictId);
3133  set_dict_key(cd);
3134  }
3135  }
3136  if (toplevel_column_names.count(cd.columnName)) {
3137  if (!cd.isGeoPhyCol) {
3138  td.columnIdBySpi_.push_back(colId);
3139  }
3140  }
3141  cd.tableId = td.tableId;
3142  cd.columnId = colId++;
3143  cds.push_back(cd);
3144  }
3145 
3147  serializeTableJsonUnlocked(&td, cds);
3148  }
3149  }
3150 
3151  try {
3152  auto cache = dataMgr_->getPersistentStorageMgr()->getDiskCache();
3153  if (cache) {
3154  CHECK(!cache->hasCachedMetadataForKeyPrefix({getCurrentDB().dbId, td.tableId}))
3155  << "Disk cache at " + cache->getCacheDirectory()
3156  << " contains preexisting data for new table. Please "
3157  "delete or clear cache before continuing";
3158  }
3159 
3160  addTableToMap(&td, cds, dds);
3161  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
3162  if (!td.storageType.empty() && td.storageType != StorageType::FOREIGN_TABLE) {
3163  dataMgr_->getForeignStorageInterface()->registerTable(this, td, cds);
3164  }
3165  } catch (std::exception& e) {
3166  sqliteConnector_.query("ROLLBACK TRANSACTION");
3167  removeTableFromMap(td.tableName, td.tableId, true);
3168  throw;
3169  }
3170  sqliteConnector_.query("END TRANSACTION");
3171 
3172  if (td.storageType != StorageType::FOREIGN_TABLE) {
3173  write_lock.unlock();
3174  sqlite_lock.unlock();
3175  getMetadataForTable(td.tableName,
3176  true); // cause instantiateFragmenter() to be called
3177  }
3178 }
3179 
3180 void Catalog::serializeTableJsonUnlocked(const TableDescriptor* td,
3181  const std::list<ColumnDescriptor>& cds) const {
3182  // relies on the catalog write lock
3183  using namespace rapidjson;
3184 
3185  VLOG(1) << "Serializing temporary table " << td->tableName << " to JSON for Calcite.";
3186 
3187  const auto db_name = currentDB_.dbName;
3188  const auto file_path = table_json_filepath(basePath_, db_name);
3189 
3190  Document d;
3191  if (boost::filesystem::exists(file_path)) {
3192  // look for an existing file for this database
3193  std::ifstream reader(file_path.string());
3194  CHECK(reader.is_open());
3195  IStreamWrapper json_read_wrapper(reader);
3196  d.ParseStream(json_read_wrapper);
3197  } else {
3198  d.SetObject();
3199  }
3200  CHECK(d.IsObject());
3201  CHECK(!d.HasMember(StringRef(td->tableName.c_str())));
3202 
3203  Value table(kObjectType);
3204  table.AddMember(
3205  "name", Value().SetString(StringRef(td->tableName.c_str())), d.GetAllocator());
3206  table.AddMember("id", Value().SetInt(td->tableId), d.GetAllocator());
3207  table.AddMember("columns", Value(kArrayType), d.GetAllocator());
3208 
3209  for (const auto& cd : cds) {
3210  Value column(kObjectType);
3211  column.AddMember(
3212  "name", Value().SetString(StringRef(cd.columnName)), d.GetAllocator());
3213  column.AddMember("coltype",
3214  Value().SetInt(static_cast<int>(cd.columnType.get_type())),
3215  d.GetAllocator());
3216  column.AddMember("colsubtype",
3217  Value().SetInt(static_cast<int>(cd.columnType.get_subtype())),
3218  d.GetAllocator());
3219  column.AddMember("compression",
3220  Value().SetInt(static_cast<int>(cd.columnType.get_compression())),
3221  d.GetAllocator());
3222  column.AddMember("comp_param",
3223  Value().SetInt(static_cast<int>(cd.columnType.get_comp_param())),
3224  d.GetAllocator());
3225  column.AddMember("size",
3226  Value().SetInt(static_cast<int>(cd.columnType.get_size())),
3227  d.GetAllocator());
3228  column.AddMember(
3229  "coldim", Value().SetInt(cd.columnType.get_dimension()), d.GetAllocator());
3230  column.AddMember(
3231  "colscale", Value().SetInt(cd.columnType.get_scale()), d.GetAllocator());
3232  column.AddMember(
3233  "is_notnull", Value().SetBool(cd.columnType.get_notnull()), d.GetAllocator());
3234  column.AddMember("is_systemcol", Value().SetBool(cd.isSystemCol), d.GetAllocator());
3235  column.AddMember("is_virtualcol", Value().SetBool(cd.isVirtualCol), d.GetAllocator());
3236  column.AddMember("is_deletedcol", Value().SetBool(cd.isDeletedCol), d.GetAllocator());
3237  table["columns"].PushBack(column, d.GetAllocator());
3238  }
3239  d.AddMember(StringRef(td->tableName.c_str()), table, d.GetAllocator());
3240 
3241  // Overwrite the existing file
3242  std::ofstream writer(file_path.string(), std::ios::trunc | std::ios::out);
3243  CHECK(writer.is_open());
3244  OStreamWrapper json_wrapper(writer);
3245 
3246  Writer<OStreamWrapper> json_writer(json_wrapper);
3247  d.Accept(json_writer);
3248  writer.close();
3249 }
3250 
3251 void Catalog::dropTableFromJsonUnlocked(const std::string& table_name) const {
3252  // relies on the catalog write lock
3253  using namespace rapidjson;
3254 
3255  VLOG(1) << "Dropping temporary table " << table_name << " to JSON for Calcite.";
3256 
3257  const auto db_name = currentDB_.dbName;
3258  const auto file_path = table_json_filepath(basePath_, db_name);
3259 
3260  CHECK(boost::filesystem::exists(file_path));
3261  Document d;
3262 
3263  std::ifstream reader(file_path.string());
3264  CHECK(reader.is_open());
3265  IStreamWrapper json_read_wrapper(reader);
3266  d.ParseStream(json_read_wrapper);
3267 
3268  CHECK(d.IsObject());
3269  auto table_name_ref = StringRef(table_name.c_str());
3270  CHECK(d.HasMember(table_name_ref));
3271  CHECK(d.RemoveMember(table_name_ref));
3272 
3273  // Overwrite the existing file
3274  std::ofstream writer(file_path.string(), std::ios::trunc | std::ios::out);
3275  CHECK(writer.is_open());
3276  OStreamWrapper json_wrapper(writer);
3277 
3278  Writer<OStreamWrapper> json_writer(json_wrapper);
3279  d.Accept(json_writer);
3280  writer.close();
3281 }
3282 
3283 void Catalog::createForeignServer(
3284  std::unique_ptr<foreign_storage::ForeignServer> foreign_server,
3285  bool if_not_exists) {
3286  cat_write_lock write_lock(this);
3287  cat_sqlite_lock sqlite_lock(getObjForLock());
3288  createForeignServerNoLocks(std::move(foreign_server), if_not_exists);
3289 }
3290 
3291 void Catalog::createForeignServerNoLocks(
3292  std::unique_ptr<foreign_storage::ForeignServer> foreign_server,
3293  bool if_not_exists) {
3294  const auto& name = foreign_server->name;
3295 
3296  sqliteConnector_.query_with_text_params(
3297  "SELECT name from omnisci_foreign_servers where name = ?",
3298  std::vector<std::string>{name});
3299 
3300  if (sqliteConnector_.getNumRows() == 0) {
3301  foreign_server->creation_time = std::time(nullptr);
3302  sqliteConnector_.query_with_text_params(
3303  "INSERT INTO omnisci_foreign_servers (name, data_wrapper_type, owner_user_id, "
3304  "creation_time, "
3305  "options) "
3306  "VALUES (?, ?, ?, ?, ?)",
3307  std::vector<std::string>{name,
3308  foreign_server->data_wrapper_type,
3309  std::to_string(foreign_server->user_id),
3310  std::to_string(foreign_server->creation_time),
3311  foreign_server->getOptionsAsJsonString()});
3312  sqliteConnector_.query_with_text_params(
3313  "SELECT id from omnisci_foreign_servers where name = ?",
3314  std::vector<std::string>{name});
3315  CHECK_EQ(sqliteConnector_.getNumRows(), size_t(1));
3316  foreign_server->id = sqliteConnector_.getData<int32_t>(0, 0);
3317  std::shared_ptr<foreign_storage::ForeignServer> foreign_server_shared =
3318  std::move(foreign_server);
3319  CHECK(foreignServerMap_.find(name) == foreignServerMap_.end())
3320  << "Attempting to insert a foreign server into foreign server map that already "
3321  "exists.";
3322  foreignServerMap_[name] = foreign_server_shared;
3323  foreignServerMapById_[foreign_server_shared->id] = foreign_server_shared;
3324  } else if (!if_not_exists) {
3325  throw std::runtime_error{"A foreign server with name \"" + foreign_server->name +
3326  "\" already exists."};
3327  }
3328 
3329  const auto& server_it = foreignServerMap_.find(name);
3330  CHECK(server_it != foreignServerMap_.end());
3331  CHECK(foreignServerMapById_.find(server_it->second->id) != foreignServerMapById_.end());
3332 }
3333 
3334 const foreign_storage::ForeignServer* Catalog::getForeignServer(
3335  const std::string& server_name) const {
3336  foreign_storage::ForeignServer* foreign_server = nullptr;
3337  cat_read_lock read_lock(this);
3338 
3339  if (foreignServerMap_.find(server_name) != foreignServerMap_.end()) {
3340  foreign_server = foreignServerMap_.find(server_name)->second.get();
3341  }
3342  return foreign_server;
3343 }
3344 
3345 const std::unique_ptr<const foreign_storage::ForeignServer>
3346 Catalog::getForeignServerFromStorage(const std::string& server_name) {
3347  std::unique_ptr<foreign_storage::ForeignServer> foreign_server = nullptr;
3348  cat_sqlite_lock sqlite_lock(getObjForLock());
3349  sqliteConnector_.query_with_text_params(
3350  "SELECT id, name, data_wrapper_type, options, owner_user_id, creation_time "
3351  "FROM omnisci_foreign_servers WHERE name = ?",
3352  std::vector<std::string>{server_name});
3353  if (sqliteConnector_.getNumRows() > 0) {
3354  foreign_server = std::make_unique<foreign_storage::ForeignServer>(
3355  sqliteConnector_.getData<int>(0, 0),
3356  sqliteConnector_.getData<std::string>(0, 1),
3357  sqliteConnector_.getData<std::string>(0, 2),
3358  sqliteConnector_.getData<std::string>(0, 3),
3359  sqliteConnector_.getData<std::int32_t>(0, 4),
3360  sqliteConnector_.getData<std::int32_t>(0, 5));
3361  }
3362  return foreign_server;
3363 }
3364 
3365 const std::unique_ptr<const foreign_storage::ForeignTable>
3366 Catalog::getForeignTableFromStorage(int table_id) {
3367  std::unique_ptr<foreign_storage::ForeignTable> foreign_table = nullptr;
3368  cat_sqlite_lock sqlite_lock(getObjForLock());
3369  sqliteConnector_.query_with_text_params(
3370  "SELECT table_id, server_id, options, last_refresh_time, next_refresh_time from "
3371  "omnisci_foreign_tables WHERE table_id = ?",
3372  std::vector<std::string>{to_string(table_id)});
3373  auto num_rows = sqliteConnector_.getNumRows();
3374  if (num_rows > 0) {
3375  CHECK_EQ(size_t(1), num_rows);
3376  foreign_table = std::make_unique<foreign_storage::ForeignTable>(
3377  sqliteConnector_.getData<int>(0, 0),
3378  foreignServerMapById_[sqliteConnector_.getData<int32_t>(0, 1)].get(),
3379  sqliteConnector_.getData<std::string>(0, 2),
3380  sqliteConnector_.getData<int64_t>(0, 3),
3381  sqliteConnector_.getData<int64_t>(0, 4));
3382  }
3383  return foreign_table;
3384 }
3385 
3386 void Catalog::changeForeignServerOwner(const std::string& server_name,
3387  const int new_owner_id) {
3388  cat_write_lock write_lock(this);
3389  foreign_storage::ForeignServer* foreign_server =
3390  foreignServerMap_.find(server_name)->second.get();
3391  CHECK(foreign_server);
3392  setForeignServerProperty(server_name, "owner_user_id", std::to_string(new_owner_id));
3393  // update in-memory server
3394  foreign_server->user_id = new_owner_id;
3395 }
3396 
3397 void Catalog::setForeignServerDataWrapper(const std::string& server_name,
3398  const std::string& data_wrapper) {
3399  cat_write_lock write_lock(this);
3400  auto data_wrapper_type = to_upper(data_wrapper);
3401  // update in-memory server
3402  foreign_storage::ForeignServer* foreign_server =
3403  foreignServerMap_.find(server_name)->second.get();
3404  CHECK(foreign_server);
3405  std::string saved_data_wrapper_type = foreign_server->data_wrapper_type;
3406  foreign_server->data_wrapper_type = data_wrapper_type;
3407  try {
3408  foreign_server->validate();
3409  } catch (const std::exception& e) {
3410  // validation did not succeed:
3411  // revert to saved data_wrapper_type & throw exception
3412  foreign_server->data_wrapper_type = saved_data_wrapper_type;
3413  throw;
3414  }
3415  setForeignServerProperty(server_name, "data_wrapper_type", data_wrapper_type);
3416 }
3417 
3418 void Catalog::setForeignServerOptions(const std::string& server_name,
3419  const std::string& options) {
3420  cat_write_lock write_lock(this);
3421  // update in-memory server
3422  foreign_storage::ForeignServer* foreign_server =
3423  foreignServerMap_.find(server_name)->second.get();
3424  CHECK(foreign_server);
3425  auto saved_options = foreign_server->options;
3426  foreign_server->populateOptionsMap(options, true);
3427  try {
3428  foreign_server->validate();
3429  } catch (const std::exception& e) {
3430  // validation did not succeed:
3431  // revert to saved options & throw exception
3432  foreign_server->options = saved_options;
3433  throw;
3434  }
3435  setForeignServerProperty(server_name, "options", options);
3436 }
3437 
3438 void Catalog::renameForeignServer(const std::string& server_name,
3439  const std::string& name) {
3440  cat_write_lock write_lock(this);
3441  auto foreign_server_it = foreignServerMap_.find(server_name);
3442  CHECK(foreign_server_it != foreignServerMap_.end());
3443  setForeignServerProperty(server_name, "name", name);
3444  auto foreign_server_shared = foreign_server_it->second;
3445  foreign_server_shared->name = name;
3446  foreignServerMap_[name] = foreign_server_shared;
3447  foreignServerMap_.erase(foreign_server_it);
3448 }
3449 
3450 void Catalog::dropForeignServer(const std::string& server_name) {
3451  cat_write_lock write_lock(this);
3452  cat_sqlite_lock sqlite_lock(getObjForLock());
3453 
3454  sqliteConnector_.query_with_text_params(
3455  "SELECT id from omnisci_foreign_servers where name = ?",
3456  std::vector<std::string>{server_name});
3457  auto num_rows = sqliteConnector_.getNumRows();
3458  if (num_rows > 0) {
3459  CHECK_EQ(size_t(1), num_rows);
3460  auto server_id = sqliteConnector_.getData<int32_t>(0, 0);
3461  sqliteConnector_.query_with_text_param(
3462  "SELECT table_id from omnisci_foreign_tables where server_id = ?",
3463  std::to_string(server_id));
3464  if (sqliteConnector_.getNumRows() > 0) {
3465  throw std::runtime_error{"Foreign server \"" + server_name +
3466  "\" is referenced "
3467  "by existing foreign tables and cannot be dropped."};
3468  }
3469  sqliteConnector_.query("BEGIN TRANSACTION");
3470  try {
3471  sqliteConnector_.query_with_text_params(
3472  "DELETE FROM omnisci_foreign_servers WHERE name = ?",
3473  std::vector<std::string>{server_name});
3474  } catch (const std::exception& e) {
3475  sqliteConnector_.query("ROLLBACK TRANSACTION");
3476  throw;
3477  }
3478  sqliteConnector_.query("END TRANSACTION");
3479  foreignServerMap_.erase(server_name);
3480  foreignServerMapById_.erase(server_id);
3481  }
3482 }
3483 
3484 void Catalog::getForeignServersForUser(
3485  const rapidjson::Value* filters,
3486  const UserMetadata& user,
3487  std::vector<const foreign_storage::ForeignServer*>& results) {
3488  sys_read_lock syscat_read_lock(&SysCatalog::instance());
3489  cat_read_lock read_lock(this);
3490  cat_sqlite_lock sqlite_lock(getObjForLock());
3491  // Customer facing and internal SQlite names
3492  std::map<std::string, std::string> col_names{{"server_name", "name"},
3493  {"data_wrapper", "data_wrapper_type"},
3494  {"created_at", "creation_time"},
3495  {"options", "options"}};
3496 
3497  // TODO add "owner" when FSI privilege is implemented
3498  std::stringstream filter_string;
3499  std::vector<std::string> arguments;
3500 
3501  if (filters != nullptr) {
3502  // Create SQL WHERE clause for SQLite query
3503  int num_filters = 0;
3504  filter_string << " WHERE";
3505  for (auto& filter_def : filters->GetArray()) {
3506  if (num_filters > 0) {
3507  filter_string << " " << std::string(filter_def["chain"].GetString());
3508  ;
3509  }
3510 
3511  if (col_names.find(std::string(filter_def["attribute"].GetString())) ==
3512  col_names.end()) {
3513  throw std::runtime_error{"Attribute with name \"" +
3514  std::string(filter_def["attribute"].GetString()) +
3515  "\" does not exist."};
3516  }
3517 
3518  filter_string << " " << col_names[std::string(filter_def["attribute"].GetString())];
3519 
3520  bool equals_operator = false;
3521  if (std::strcmp(filter_def["operation"].GetString(), "EQUALS") == 0) {
3522  filter_string << " = ? ";
3523  equals_operator = true;
3524  } else {
3525  filter_string << " LIKE ? ";
3526  }
3527 
3528  bool timestamp_column =
3529  (std::strcmp(filter_def["attribute"].GetString(), "created_at") == 0);
3530 
3531  if (timestamp_column && !equals_operator) {
3532  throw std::runtime_error{"LIKE operator is incompatible with TIMESTAMP data"};
3533  }
3534 
3535  if (timestamp_column && equals_operator) {
3536  arguments.push_back(std::to_string(
3537  dateTimeParse<kTIMESTAMP>(filter_def["value"].GetString(), 0)));
3538  } else {
3539  arguments.emplace_back(filter_def["value"].GetString());
3540  }
3541 
3542  num_filters++;
3543  }
3544  }
3545  // Create select query for the omnisci_foreign_servers table
3546  std::string query = std::string("SELECT name from omnisci_foreign_servers ");
3547  query += filter_string.str();
3548 
3549  sqliteConnector_.query_with_text_params(query, arguments);
3550  auto num_rows = sqliteConnector_.getNumRows();
3551 
3552  if (sqliteConnector_.getNumRows() == 0) {
3553  return;
3554  }
3555 
3556  CHECK(sqliteConnector_.getNumCols() == 1);
3557  // Return pointers to objects
3558  results.reserve(num_rows);
3559  for (size_t row = 0; row < num_rows; ++row) {
3560  const auto& server_name = sqliteConnector_.getData<std::string>(row, 0);
3561  if (shared::contains(INTERNAL_SERVERS, server_name)) {
3562  continue;
3563  }
3564  const foreign_storage::ForeignServer* foreign_server = getForeignServer(server_name);
3565  CHECK(foreign_server != nullptr);
3566 
3567  DBObject dbObject(foreign_server->name, ServerDBObjectType);
3568  dbObject.loadKey(*this);
3569  std::vector<DBObject> privObjects = {dbObject};
3570  if (!SysCatalog::instance().hasAnyPrivileges(user, privObjects)) {
3571  // skip server, as there are no privileges to access it
3572  continue;
3573  }
3574  results.push_back(foreign_server);
3575  }
3576 }
3577 
3578 // returns the table epoch or -1 if there is something wrong with the shared epoch
3579 int32_t Catalog::getTableEpoch(const int32_t db_id, const int32_t table_id) const {
3580  cat_read_lock read_lock(this);
3581  const auto td = getMetadataForTable(table_id, false);
3582  if (!td) {
3583  std::stringstream table_not_found_error_message;
3584  table_not_found_error_message << "Table (" << db_id << "," << table_id
3585  << ") not found";
3586  throw std::runtime_error(table_not_found_error_message.str());
3587  }
3588  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(table_id);
3589  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
3590  // check all shards have same checkpoint
3591  const auto physicalTables = physicalTableIt->second;
3592  CHECK(!physicalTables.empty());
3593  size_t curr_epoch{0}, first_epoch{0};
3594  int32_t first_table_id{0};
3595  bool are_epochs_inconsistent{false};
3596  for (size_t i = 0; i < physicalTables.size(); i++) {
3597  int32_t physical_tb_id = physicalTables[i];
3598  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id, false);
3599  CHECK(phys_td);
3600 
3601  curr_epoch = dataMgr_->getTableEpoch(db_id, physical_tb_id);
3602  LOG(INFO) << "Got sharded table epoch for db id: " << db_id
3603  << ", table id: " << physical_tb_id << ", epoch: " << curr_epoch;
3604  if (i == 0) {
3605  first_epoch = curr_epoch;
3606  first_table_id = physical_tb_id;
3607  } else if (first_epoch != curr_epoch) {
3608  are_epochs_inconsistent = true;
3609  LOG(ERROR) << "Epochs on shards do not all agree on table id: " << table_id
3610  << ", db id: " << db_id
3611  << ". First table (table id: " << first_table_id
3612  << ") has epoch: " << first_epoch << ". Table id: " << physical_tb_id
3613  << ", has inconsistent epoch: " << curr_epoch
3614  << ". See previous INFO logs for all epochs and their table ids.";
3615  }
3616  }
3617  if (are_epochs_inconsistent) {
3618  // oh dear the shards do not agree on the epoch for this table
3619  return -1;
3620  }
3621  return curr_epoch;
3622  } else {
3623  auto epoch = dataMgr_->getTableEpoch(db_id, table_id);
3624  LOG(INFO) << "Got table epoch for db id: " << db_id << ", table id: " << table_id
3625  << ", epoch: " << epoch;
3626  return epoch;
3627  }
3628 }
3629 
3630 std::vector<const foreign_storage::ForeignTable*>
3631 Catalog::getAllForeignTablesForForeignServer(const int32_t foreign_server_id) {
3632  cat_read_lock read_lock(this);
3633  std::vector<const foreign_storage::ForeignTable*> foreign_tables;
3634  for (auto entry : tableDescriptorMapById_) {
3635  auto table_descriptor = entry.second;
3636  if (table_descriptor->storageType == StorageType::FOREIGN_TABLE) {
3637  auto foreign_table = dynamic_cast<foreign_storage::ForeignTable*>(table_descriptor);
3638  CHECK(foreign_table);
3639  if (foreign_table->foreign_server->id == foreign_server_id) {
3640  foreign_tables.emplace_back(foreign_table);
3641  }
3642  }
3643  }
3644  return foreign_tables;
3645 }
3646 
3647 void Catalog::setTableEpoch(const int db_id, const int table_id, int new_epoch) {
3648  LOG(INFO) << "Set table epoch db:" << db_id << " Table ID " << table_id
3649  << " back to new epoch " << new_epoch;
3650  const auto td = getMetadataForTable(table_id, false);
3651  if (!td) {
3652  std::stringstream table_not_found_error_message;
3653  table_not_found_error_message << "Table (" << db_id << "," << table_id
3654  << ") not found";
3655  throw std::runtime_error(table_not_found_error_message.str());
3656  }
3657  if (td->persistenceLevel != Data_Namespace::MemoryLevel::DISK_LEVEL) {
3658  std::stringstream is_temp_table_error_message;
3659  is_temp_table_error_message << "Cannot set epoch on temporary table";
3660  throw std::runtime_error(is_temp_table_error_message.str());
3661  }
3662 
3663  File_Namespace::FileMgrParams file_mgr_params;
3664  file_mgr_params.epoch = new_epoch;
3665  file_mgr_params.max_rollback_epochs = td->maxRollbackEpochs;
3666 
3667  const auto physical_tables = getPhysicalTablesDescriptors(td, false);
3668  CHECK(!physical_tables.empty());
3669  for (const auto table : physical_tables) {
3670  auto table_id = table->tableId;
3671  LOG(INFO) << "Set sharded table epoch db:" << db_id << " Table ID " << table_id
3672  << " back to new epoch " << new_epoch;
3673  // Should have table lock from caller so safe to do this after, avoids
3674  // having to repopulate data on error
3675  removeChunks(table_id);
3676  dataMgr_->getGlobalFileMgr()->setFileMgrParams(db_id, table_id, file_mgr_params);
3677  }
3678 }
3679 
3680 void Catalog::alterPhysicalTableMetadata(
3681  const TableDescriptor* td,
3682  const TableDescriptorUpdateParams& table_update_params) {
3683  // Only called from parent alterTableParamMetadata, expect already to have catalog and
3684  // sqlite write locks
3685 
3686  // Sqlite transaction should have already been begun in parent alterTableCatalogMetadata
3687 
3688  TableDescriptor* mutable_td = getMutableMetadataForTableUnlocked(td->tableId);
3689  CHECK(mutable_td);
3690  if (td->maxRollbackEpochs != table_update_params.max_rollback_epochs) {
3691  sqliteConnector_.query_with_text_params(
3692  "UPDATE mapd_tables SET max_rollback_epochs = ? WHERE tableid = ?",
3693  std::vector<std::string>{std::to_string(table_update_params.max_rollback_epochs),
3694  std::to_string(td->tableId)});
3695  mutable_td->maxRollbackEpochs = table_update_params.max_rollback_epochs;
3696  }
3697 
3698  if (td->maxRows != table_update_params.max_rows) {
3699  sqliteConnector_.query_with_text_params(
3700  "UPDATE mapd_tables SET max_rows = ? WHERE tableid = ?",
3701  std::vector<std::string>{std::to_string(table_update_params.max_rows),
3702  std::to_string(td->tableId)});
3703  mutable_td->maxRows = table_update_params.max_rows;
3704  }
3705 }
3706 
3707 void Catalog::alterTableMetadata(const TableDescriptor* td,
3708  const TableDescriptorUpdateParams& table_update_params) {
3709  cat_write_lock write_lock(this);
3710  cat_sqlite_lock sqlite_lock(getObjForLock());
3711  sqliteConnector_.query("BEGIN TRANSACTION");
3712  try {
3713  const auto physical_table_it = logicalToPhysicalTableMapById_.find(td->tableId);
3714  if (physical_table_it != logicalToPhysicalTableMapById_.end()) {
3715  const auto physical_tables = physical_table_it->second;
3716  CHECK(!physical_tables.empty());
3717  for (size_t i = 0; i < physical_tables.size(); i++) {
3718  int32_t physical_tb_id = physical_tables[i];
3719  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id, false);
3720  CHECK(phys_td);
3721  alterPhysicalTableMetadata(phys_td, table_update_params);
3722  }
3723  }
3724  alterPhysicalTableMetadata(td, table_update_params);
3725  } catch (std::exception& e) {
3726  sqliteConnector_.query("ROLLBACK TRANSACTION");
3727  LOG(FATAL) << "Table '" << td->tableName << "' catalog update failed";
3728  }
3729  sqliteConnector_.query("END TRANSACTION");
3730 }
3731 
3732 void Catalog::setMaxRollbackEpochs(const int32_t table_id,
3733  const int32_t max_rollback_epochs) {
3734  // Must be called from AlterTableParamStmt or other method that takes executor and
3735  // TableSchema locks
3736  if (max_rollback_epochs <= -1) {
3737  throw std::runtime_error("Cannot set max_rollback_epochs < 0.");
3738  }
3739  const auto td = getMetadataForTable(
3740  table_id, false); // Deep copy as there will be gap between read and write locks
3741  CHECK(td); // Existence should have already been checked in
3742  // ParserNode::AlterTableParmStmt
3743  TableDescriptorUpdateParams table_update_params(td);
3744  table_update_params.max_rollback_epochs = max_rollback_epochs;
3745  if (table_update_params == td) { // Operator is overloaded to test for equality
3746  LOG(INFO) << "Setting max_rollback_epochs for table " << table_id
3747  << " to existing value, skipping operation";
3748  return;
3749  }
3750  File_Namespace::FileMgrParams file_mgr_params;
3751  file_mgr_params.epoch = -1; // Use existing epoch
3752  file_mgr_params.max_rollback_epochs = max_rollback_epochs;
3753  setTableFileMgrParams(table_id, file_mgr_params);
3754  alterTableMetadata(td, table_update_params);
3755 }
3756 
3757 void Catalog::setMaxRows(const int32_t table_id, const int64_t max_rows) {
3758  if (max_rows < 0) {
3759  throw std::runtime_error("Max rows cannot be a negative number.");
3760  }
3761  const auto td = getMetadataForTable(table_id);
3762  CHECK(td);
3763  TableDescriptorUpdateParams table_update_params(td);
3764  table_update_params.max_rows = max_rows;
3765  if (table_update_params == td) {
3766  LOG(INFO) << "Max rows value of " << max_rows
3767  << " is the same as the existing value. Skipping update.";
3768  return;
3769  }
3770  alterTableMetadata(td, table_update_params);
3771  CHECK(td->fragmenter);
3772  td->fragmenter->dropFragmentsToSize(max_rows);
3773 }
3774 
3775 // For testing purposes only
3776 void Catalog::setUncappedTableEpoch(const std::string& table_name) {
3777  cat_write_lock write_lock(this);
3778  auto td_entry = tableDescriptorMap_.find(to_upper(table_name));
3779  CHECK(td_entry != tableDescriptorMap_.end());
3780  auto td = td_entry->second;
3781 
3782  Executor::clearExternalCaches(true, td, getDatabaseId());
3783 
3784  TableDescriptorUpdateParams table_update_params(td);
3785  table_update_params.max_rollback_epochs = -1;
3786  write_lock.unlock();
3787 
3788  alterTableMetadata(td, table_update_params);
3789  File_Namespace::FileMgrParams file_mgr_params;
3790  file_mgr_params.max_rollback_epochs = -1;
3791  setTableFileMgrParams(td->tableId, file_mgr_params);
3792 }
3793 
3794 void Catalog::setTableFileMgrParams(
3795  const int table_id,
3796  const File_Namespace::FileMgrParams& file_mgr_params) {
3797  // Expects parent to have write lock
3798  const auto td = getMetadataForTable(table_id, false);
3799  const auto db_id = this->getDatabaseId();
3800  if (!td) {
3801  std::stringstream table_not_found_error_message;
3802  table_not_found_error_message << "Table (" << db_id << "," << table_id
3803  << ") not found";
3804  throw std::runtime_error(table_not_found_error_message.str());
3805  }
3806  if (td->persistenceLevel != Data_Namespace::MemoryLevel::DISK_LEVEL) {
3807  std::stringstream is_temp_table_error_message;
3808  is_temp_table_error_message << "Cannot set storage params on temporary table";
3809  throw std::runtime_error(is_temp_table_error_message.str());
3810  }
3811 
3812  const auto physical_tables = getPhysicalTablesDescriptors(td, false);
3813  CHECK(!physical_tables.empty());
3814  for (const auto table : physical_tables) {
3815  auto table_id = table->tableId;
3816  removeChunks(table_id);
3817  dataMgr_->getGlobalFileMgr()->setFileMgrParams(db_id, table_id, file_mgr_params);
3818  }
3819 }
3820 
3821 std::vector<TableEpochInfo> Catalog::getTableEpochs(const int32_t db_id,
3822  const int32_t table_id) const {
3823  cat_read_lock read_lock(this);
3824  std::vector<TableEpochInfo> table_epochs;
3825  const auto physical_table_it = logicalToPhysicalTableMapById_.find(table_id);
3826  if (physical_table_it != logicalToPhysicalTableMapById_.end()) {
3827  const auto physical_tables = physical_table_it->second;
3828  CHECK(!physical_tables.empty());
3829 
3830  for (const auto physical_tb_id : physical_tables) {
3831  const auto phys_td = getMutableMetadataForTableUnlocked(physical_tb_id);
3832  CHECK(phys_td);
3833 
3834  auto table_id = phys_td->tableId;
3835  auto epoch = dataMgr_->getTableEpoch(db_id, phys_td->tableId);
3836  table_epochs.emplace_back(table_id, epoch);
3837  LOG(INFO) << "Got sharded table epoch for db id: " << db_id
3838  << ", table id: " << table_id << ", epoch: " << epoch;
3839  }
3840  } else {
3841  auto epoch = dataMgr_->getTableEpoch(db_id, table_id);
3842  LOG(INFO) << "Got table epoch for db id: " << db_id << ", table id: " << table_id
3843  << ", epoch: " << epoch;
3844  table_epochs.emplace_back(table_id, epoch);
3845  }
3846  return table_epochs;
3847 }
3848 
3849 void Catalog::setTableEpochs(const int32_t db_id,
3850  const std::vector<TableEpochInfo>& table_epochs) const {
3851  const auto td = getMetadataForTable(table_epochs[0].table_id, false);
3852  CHECK(td);
3853  File_Namespace::FileMgrParams file_mgr_params;
3854  file_mgr_params.max_rollback_epochs = td->maxRollbackEpochs;
3855 
3856  for (const auto& table_epoch_info : table_epochs) {
3857  removeChunks(table_epoch_info.table_id);
3858  file_mgr_params.epoch = table_epoch_info.table_epoch;
3859  dataMgr_->getGlobalFileMgr()->setFileMgrParams(
3860  db_id, table_epoch_info.table_id, file_mgr_params);
3861  LOG(INFO) << "Set table epoch for db id: " << db_id
3862  << ", table id: " << table_epoch_info.table_id
3863  << ", back to epoch: " << table_epoch_info.table_epoch;
3864  }
3865 }
3866 
3867 namespace {
3868 std::string table_epochs_to_string(const std::vector<TableEpochInfo>& table_epochs) {
3869  std::string table_epochs_str{"["};
3870  bool first_entry{true};
3871  for (const auto& table_epoch : table_epochs) {
3872  if (first_entry) {
3873  first_entry = false;
3874  } else {
3875  table_epochs_str += ", ";
3876  }
3877  table_epochs_str += "(table_id: " + std::to_string(table_epoch.table_id) +
3878  ", epoch: " + std::to_string(table_epoch.table_epoch) + ")";
3879  }
3880  table_epochs_str += "]";
3881  return table_epochs_str;
3882 }
3883 } // namespace
3884 
3885 void Catalog::setTableEpochsLogExceptions(
3886  const int32_t db_id,
3887  const std::vector<TableEpochInfo>& table_epochs) const {
3888  try {
3889  setTableEpochs(db_id, table_epochs);
3890  } catch (std::exception& e) {
3891  LOG(ERROR) << "An error occurred when attempting to set table epochs. DB id: "
3892  << db_id << ", Table epochs: " << table_epochs_to_string(table_epochs)
3893  << ", Error: " << e.what();
3894  }
3895 }
3896 
3897 const ColumnDescriptor* Catalog::getDeletedColumn(const TableDescriptor* td) const {
3898  cat_read_lock read_lock(this);
3899  const auto it = deletedColumnPerTable_.find(td);
3900  return it != deletedColumnPerTable_.end() ? it->second : nullptr;
3901 }
3902 
3903 const bool Catalog::checkMetadataForDeletedRecs(const TableDescriptor* td,
3904  int delete_column_id) const {
3905  // check if there are rows deleted by examining the deletedColumn metadata
3906  CHECK(td);
3907  auto fragmenter = td->fragmenter;
3908  if (fragmenter) {
3909  return fragmenter->hasDeletedRows(delete_column_id);
3910  } else {
3911  return false;
3912  }
3913 }
3914 
3915 const ColumnDescriptor* Catalog::getDeletedColumnIfRowsDeleted(
3916  const TableDescriptor* td) const {
3917  std::vector<const TableDescriptor*> tds;
3918  const ColumnDescriptor* cd;
3919  {
3920  cat_read_lock read_lock(this);
3921 
3922  const auto it = deletedColumnPerTable_.find(td);
3923  // if not a table that supports delete return nullptr, nothing more to do
3924  if (it == deletedColumnPerTable_.end()) {
3925  return nullptr;
3926  }
3927  cd = it->second;
3928  tds = getPhysicalTablesDescriptors(td, false);
3929  }
3930  // individual tables are still protected by higher level locks
3931  for (auto tdd : tds) {
3932  if (checkMetadataForDeletedRecs(tdd, cd->columnId)) {
3933  return cd;
3934  }
3935  }
3936  // no deletes so far recorded in metadata
3937  return nullptr;
3938 }
3939 
3940 void Catalog::setDeletedColumn(const TableDescriptor* td, const ColumnDescriptor* cd) {
3941  cat_write_lock write_lock(this);
3942  setDeletedColumnUnlocked(td, cd);
3943 }
3944 
3945 void Catalog::setDeletedColumnUnlocked(const TableDescriptor* td,
3946  const ColumnDescriptor* cd) {
3947  const auto it_ok = deletedColumnPerTable_.emplace(td, cd);
3948  CHECK(it_ok.second);
3949 }
3950 
3951 namespace {
3952 
3954  const Catalog& cat,
3955  const Parser::SharedDictionaryDef& shared_dict_def) {
3956  const auto& table_name = shared_dict_def.get_foreign_table();
3957  const auto td = cat.getMetadataForTable(table_name, false);
3958  CHECK(td);
3959  const auto& foreign_col_name = shared_dict_def.get_foreign_column();
3960  return cat.getMetadataForColumn(td->tableId, foreign_col_name);
3961 }
3962 
3963 } // namespace
3964 
3965 void Catalog::addReferenceToForeignDict(ColumnDescriptor& referencing_column,
3966  Parser::SharedDictionaryDef shared_dict_def,
3967  const bool persist_reference) {
3968  cat_write_lock write_lock(this);
3969  const auto foreign_ref_col = get_foreign_col(*this, shared_dict_def);
3970  CHECK(foreign_ref_col);
3971  referencing_column.columnType = foreign_ref_col->columnType;
3972  const int dict_id = referencing_column.columnType.get_comp_param();
3973  const DictRef dict_ref(currentDB_.dbId, dict_id);
3974  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
3975  CHECK(dictIt != dictDescriptorMapByRef_.end());
3976  const auto& dd = dictIt->second;
3977  CHECK_GE(dd->refcount, 1);
3978  ++dd->refcount;
3979  if (persist_reference) {
3980  cat_sqlite_lock sqlite_lock(getObjForLock());
3981  sqliteConnector_.query_with_text_params(
3982  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
3983  {std::to_string(dict_id)});
3984  }
3985 }
3986 
3987 bool Catalog::setColumnSharedDictionary(
3988  ColumnDescriptor& cd,
3989  std::list<ColumnDescriptor>& cdd,
3990  std::list<DictDescriptor>& dds,
3991  const TableDescriptor td,
3992  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs) {
3993  cat_write_lock write_lock(this);
3994  cat_sqlite_lock sqlite_lock(getObjForLock());
3995 
3996  if (shared_dict_defs.empty()) {
3997  return false;
3998  }
3999  for (const auto& shared_dict_def : shared_dict_defs) {
4000  // check if the current column is a referencing column
4001  const auto& column = shared_dict_def.get_column();
4002  if (cd.columnName == column) {
4003  if (!shared_dict_def.get_foreign_table().compare(td.tableName)) {
4004  // Dictionaries are being shared in table to be created
4005  const auto& ref_column = shared_dict_def.get_foreign_column();
4006  auto colIt =
4007  std::find_if(cdd.begin(), cdd.end(), [ref_column](const ColumnDescriptor it) {
4008  return !ref_column.compare(it.columnName);
4009  });
4010  CHECK(colIt != cdd.end());
4011  cd.columnType = colIt->columnType;
4012 
4013  const int dict_id = colIt->columnType.get_comp_param();
4014  CHECK_GE(dict_id, 1);
4015  auto dictIt = std::find_if(
4016  dds.begin(), dds.end(), [this, dict_id](const DictDescriptor it) {
4017  return it.dictRef.dbId == this->currentDB_.dbId &&
4018  it.dictRef.dictId == dict_id;
4019  });
4020  if (dictIt != dds.end()) {
4021  // There exists dictionary definition of a dictionary column
4022  CHECK_GE(dictIt->refcount, 1);
4023  ++dictIt->refcount;
4024  if (!table_is_temporary(&td)) {
4025  // Persist reference count
4026  sqliteConnector_.query_with_text_params(
4027  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
4028  {std::to_string(dict_id)});
4029  }
4030  } else {
4031  // The dictionary is referencing a column which is referencing a column in
4032  // diffrent table
4033  auto root_dict_def = compress_reference_path(shared_dict_def, shared_dict_defs);
4034  addReferenceToForeignDict(cd, root_dict_def, !table_is_temporary(&td));
4035  }
4036  } else {
4037  const auto& foreign_table_name = shared_dict_def.get_foreign_table();
4038  const auto foreign_td = getMetadataForTable(foreign_table_name, false);
4039  if (table_is_temporary(foreign_td)) {
4040  if (!table_is_temporary(&td)) {
4041  throw std::runtime_error(
4042  "Only temporary tables can share dictionaries with other temporary "
4043  "tables.");
4044  }
4045  addReferenceToForeignDict(cd, shared_dict_def, false);
4046  } else {
4047  addReferenceToForeignDict(cd, shared_dict_def, !table_is_temporary(&td));
4048  }
4049  }
4050  return true;
4051  }
4052  }
4053  return false;
4054 }
4055 
4056 void Catalog::setColumnDictionary(ColumnDescriptor& cd,
4057  std::list<DictDescriptor>& dds,
4058  const TableDescriptor& td,
4059  bool is_logical_table,
4060  bool use_temp_dictionary) {
4061  cat_write_lock write_lock(this);
4062 
4063  std::string dictName{"Initial_key"};
4064  int dictId{0};
4065  std::string folderPath;
4066  if (is_logical_table) {
4067  cat_sqlite_lock sqlite_lock(getObjForLock());
4068 
4069  sqliteConnector_.query_with_text_params(
4070  "INSERT INTO mapd_dictionaries (name, nbits, is_shared, refcount) VALUES (?, ?, "
4071  "?, 1)",
4072  std::vector<std::string>{
4073  dictName, std::to_string(cd.columnType.get_comp_param()), "0"});
4074  sqliteConnector_.query_with_text_param(
4075  "SELECT dictid FROM mapd_dictionaries WHERE name = ?", dictName);
4076  dictId = sqliteConnector_.getData<int>(0, 0);
4077  dictName = td.tableName + "_" + cd.columnName + "_dict" + std::to_string(dictId);
4078  sqliteConnector_.query_with_text_param(
4079  "UPDATE mapd_dictionaries SET name = ? WHERE name = 'Initial_key'", dictName);
4080  folderPath = g_base_path + "/" + shared::kDataDirectoryName + "/DB_" +
4081  std::to_string(currentDB_.dbId) + "_DICT_" + std::to_string(dictId);
4082  }
4083  DictDescriptor dd(currentDB_.dbId,
4084  dictId,
4085  dictName,
4087  false,
4088  1,
4089  folderPath,
4090  use_temp_dictionary);
4091  dds.push_back(dd);
4092  if (!cd.columnType.is_array()) {
4094  }
4095  cd.columnType.set_comp_param(dictId);
4096  set_dict_key(cd);
4097 }
4098 
4099 void Catalog::createShardedTable(
4100  TableDescriptor& td,
4101  const list<ColumnDescriptor>& cols,
4102  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs) {
4103  /* create logical table */
4104  TableDescriptor* tdl = &td;
4105  createTable(*tdl, cols, shared_dict_defs, true); // create logical table
4106  int32_t logical_tb_id = tdl->tableId;
4107  std::string logical_table_name = tdl->tableName;
4108 
4109  /* create physical tables and link them to the logical table */
4110  std::vector<int32_t> physicalTables;
4111  for (int32_t i = 1; i <= td.nShards; i++) {
4112  TableDescriptor* tdp = &td;
4113  tdp->tableName = generatePhysicalTableName(logical_table_name, i);
4114  tdp->shard = i - 1;
4115  createTable(*tdp, cols, shared_dict_defs, false); // create physical table
4116  int32_t physical_tb_id = tdp->tableId;
4117 
4118  /* add physical table to the vector of physical tables */
4119  physicalTables.push_back(physical_tb_id);
4120  }
4121 
4122  if (!physicalTables.empty()) {
4123  cat_write_lock write_lock(this);
4124  /* add logical to physical tables correspondence to the map */
4125  const auto it_ok =
4126  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
4127  CHECK(it_ok.second);
4128  /* update sqlite mapd_logical_to_physical in sqlite database */
4129  if (!table_is_temporary(&td)) {
4130  updateLogicalToPhysicalTableMap(logical_tb_id);
4131  }
4132  }
4133 }
4134 
4135 void Catalog::truncateTable(const TableDescriptor* td) {
4136  // truncate all corresponding physical tables
4137  const auto physical_tables = getPhysicalTablesDescriptors(td);
4138  for (const auto table : physical_tables) {
4139  doTruncateTable(table);
4140  }
4141 }
4142 
4143 void Catalog::doTruncateTable(const TableDescriptor* td) {
4144  // must destroy fragmenter before deleteChunks is called.
4145  removeFragmenterForTable(td->tableId);
4146 
4147  const int tableId = td->tableId;
4148  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
4149  // assuming deleteChunksWithPrefix is atomic
4150  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
4151  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
4152 
4153  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
4154 
4155  cat_write_lock write_lock(this);
4156  std::unique_ptr<StringDictionaryClient> client;
4157  if (SysCatalog::instance().isAggregator()) {
4158  CHECK(!string_dict_hosts_.empty());
4159  DictRef dict_ref(currentDB_.dbId, -1);
4160  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
4161  }
4162  // clean up any dictionaries
4163  // delete all column descriptors for the table
4164  for (const auto& columnDescriptor : columnDescriptorMapById_) {
4165  auto cd = columnDescriptor.second;
4166  if (cd->tableId != td->tableId) {
4167  continue;
4168  }
4169  const int dict_id = cd->columnType.get_comp_param();
4170  // Dummy dictionaries created for a shard of a logical table have the id set to zero.
4171  if (cd->columnType.get_compression() == kENCODING_DICT && dict_id) {
4172  const DictRef dict_ref(currentDB_.dbId, dict_id);
4173  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
4174  CHECK(dictIt != dictDescriptorMapByRef_.end());
4175  const auto& dd = dictIt->second;
4176  CHECK_GE(dd->refcount, 1);
4177  // if this is the only table using this dict reset the dict
4178  if (dd->refcount == 1) {
4179  // close the dictionary
4180  dd->stringDict.reset();
4181  File_Namespace::renameForDelete(dd->dictFolderPath);
4182  if (client) {
4183  client->drop(dd->dictRef);
4184  }
4185  if (!dd->dictIsTemp) {
4186  boost::filesystem::create_directory(dd->dictFolderPath);
4187  }
4188  }
4189 
4190  DictDescriptor* new_dd = new DictDescriptor(dd->dictRef,
4191  dd->dictName,
4192  dd->dictNBits,
4193  dd->dictIsShared,
4194  dd->refcount,
4195  dd->dictFolderPath,
4196  dd->dictIsTemp);
4197  dictDescriptorMapByRef_.erase(dictIt);
4198  // now create new Dict -- need to figure out what to do here for temp tables
4199  if (client) {
4200  client->create(new_dd->dictRef, new_dd->dictIsTemp);
4201  }
4202  dictDescriptorMapByRef_[new_dd->dictRef].reset(new_dd);
4203  getMetadataForDict(new_dd->dictRef.dictId);
4204  }
4205  }
4206 }
4207 
4208 // NOTE(Misiu): Only used by --multi-instance clusters.
4209 void Catalog::refreshDictionaryCachesForTableUnlocked(const TableDescriptor& td) {
4210  for (auto col_id = 0; col_id < td.nColumns; ++col_id) {
4211  if (auto it = columnDescriptorMapById_.find({td.tableId, col_id});
4212  it != columnDescriptorMapById_.end()) {
4213  auto cd = it->second;
4214  auto dict_id = cd->columnType.get_comp_param();
4215  if (cd->columnType.get_compression() == kENCODING_DICT && dict_id) {
4216  DictRef dict_ref(currentDB_.dbId, dict_id);
4217  if (auto dict_it = dictDescriptorMapByRef_.find(dict_ref);
4218  dict_it != dictDescriptorMapByRef_.end()) {
4219  // getMetadataForDict() will only reload if the stringDict is null.
4220  dict_it->second->stringDict = nullptr;
4221  }
4222  getMetadataForDict(dict_id, true);
4223  }
4224  }
4225  }
4226 }
4227 
4228 // NOTE(sy): Only used by --multi-instance clusters.
4229 void Catalog::invalidateCachesForTable(const int table_id) {
4230  // When called, exactly one thread has a LockMgr data or insert lock for the table.
4231  cat_read_lock read_lock(this);
4232  ChunkKey const table_key{getDatabaseId(), table_id};
4233  auto td = getMutableMetadataForTableUnlocked(table_id);
4234  CHECK(td);
4235  getDataMgr().deleteChunksWithPrefix(table_key, MemoryLevel::GPU_LEVEL);
4236  getDataMgr().deleteChunksWithPrefix(table_key, MemoryLevel::CPU_LEVEL);
4237  Executor::clearExternalCaches(false, td, getDatabaseId());
4238 
4239  refreshDictionaryCachesForTableUnlocked(*td);
4240 
4241  // TODO(sy): doTruncateTable() says "destroy fragmenter before deleteChunks is called"
4242  // removeFragmenterForTable(table_key[CHUNK_KEY_TABLE_IDX]);
4243  if (td->fragmenter != nullptr) {
4244  auto tableDescIt = tableDescriptorMapById_.find(table_id);
4245  CHECK(tableDescIt != tableDescriptorMapById_.end());
4246  tableDescIt->second->fragmenter = nullptr;
4247  CHECK(td->fragmenter == nullptr);
4248  }
4249  getDataMgr().getGlobalFileMgr()->closeFileMgr(table_key[CHUNK_KEY_DB_IDX],
4250  table_key[CHUNK_KEY_TABLE_IDX]);
4251  if (dynamic_cast<foreign_storage::ForeignTable*>(td)) {
4252  dataMgr_->getPersistentStorageMgr()->getForeignStorageMgr()->refreshTable(table_key,
4253  true);
4254  } else {
4255  dataMgr_->removeMutableTableDiskCacheData(currentDB_.dbId, table_id);
4256  }
4257  instantiateFragmenter(td);
4258 }
4259 
4260 void Catalog::removeFragmenterForTable(const int table_id) const {
4261  cat_write_lock write_lock(this);
4262  auto td = getMetadataForTable(table_id, false);
4263  if (td->fragmenter != nullptr) {
4264  auto tableDescIt = tableDescriptorMapById_.find(table_id);
4265  CHECK(tableDescIt != tableDescriptorMapById_.end());
4266  tableDescIt->second->fragmenter = nullptr;
4267  CHECK(td->fragmenter == nullptr);
4268  }
4269 }
4270 
4271 // used by rollback_table_epoch to clean up in memory artifacts after a rollback
4272 void Catalog::removeChunks(const int table_id) const {
4273  removeFragmenterForTable(table_id);
4274 
4275  // remove the chunks from in memory structures
4276  ChunkKey chunkKey = {currentDB_.dbId, table_id};
4277 
4278  dataMgr_->deleteChunksWithPrefix(chunkKey, MemoryLevel::CPU_LEVEL);
4279  dataMgr_->deleteChunksWithPrefix(chunkKey, MemoryLevel::GPU_LEVEL);
4280 }
4281 
4282 void Catalog::dropTable(const TableDescriptor* td) {
4283  SysCatalog::instance().revokeDBObjectPrivilegesFromAll(
4285  std::vector<const TableDescriptor*> tables_to_drop;
4286  {
4287  cat_read_lock read_lock(this);
4288  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
4289  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
4290  // remove all corresponding physical tables if this is a logical table
4291  const auto physicalTables = physicalTableIt->second;
4292  CHECK(!physicalTables.empty());
4293  for (size_t i = 0; i < physicalTables.size(); i++) {
4294  int32_t physical_tb_id = physicalTables[i];
4295  const TableDescriptor* phys_td =
4296  getMutableMetadataForTableUnlocked(physical_tb_id);
4297  CHECK(phys_td);
4298  tables_to_drop.emplace_back(phys_td);
4299  }
4300  }
4301  tables_to_drop.emplace_back(td);
4302  }
4303 
4304  for (auto table : tables_to_drop) {
4305  eraseTablePhysicalData(table);
4306  }
4307  deleteTableCatalogMetadata(td, tables_to_drop);
4308 }
4309 
4310 void Catalog::deleteTableCatalogMetadata(
4311  const TableDescriptor* logical_table,
4312  const std::vector<const TableDescriptor*>& physical_tables) {
4313  cat_write_lock write_lock(this);
4314  cat_sqlite_lock sqlite_lock(getObjForLock());
4315  sqliteConnector_.query("BEGIN TRANSACTION");
4316  try {
4317  // remove corresponding record from the logicalToPhysicalTableMap in sqlite database
4318  sqliteConnector_.query_with_text_param(
4319  "DELETE FROM mapd_logical_to_physical WHERE logical_table_id = ?",
4320  std::to_string(logical_table->tableId));
4321  logicalToPhysicalTableMapById_.erase(logical_table->tableId);
4322  for (auto table : physical_tables) {
4323  eraseTableMetadata(table);
4324  }
4325  } catch (std::exception& e) {
4326  sqliteConnector_.query("ROLLBACK TRANSACTION");
4327  throw;
4328  }
4329  sqliteConnector_.query("END TRANSACTION");
4330 }
4331 
4332 void Catalog::eraseTableMetadata(const TableDescriptor* td) {
4333  executeDropTableSqliteQueries(td);
4335  dropTableFromJsonUnlocked(td->tableName);
4336  }
4337  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
4338  {
4339  INJECT_TIMER(removeTableFromMap_);
4340  removeTableFromMap(td->tableName, td->tableId);
4341  }
4342 }
4343 
4344 void Catalog::executeDropTableSqliteQueries(const TableDescriptor* td) {
4345  const int tableId = td->tableId;
4346  sqliteConnector_.query_with_text_param("DELETE FROM mapd_tables WHERE tableid = ?",
4347  std::to_string(tableId));
4348  sqliteConnector_.query_with_text_params(
4349  "select comp_param from mapd_columns where compression = ? and tableid = ?",
4350  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
4351  int numRows = sqliteConnector_.getNumRows();
4352  std::vector<int> dict_id_list;
4353  for (int r = 0; r < numRows; ++r) {
4354  dict_id_list.push_back(sqliteConnector_.getData<int>(r, 0));
4355  }
4356  for (auto dict_id : dict_id_list) {
4357  sqliteConnector_.query_with_text_params(
4358  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
4359  std::vector<std::string>{std::to_string(dict_id)});
4360  }
4361  sqliteConnector_.query_with_text_params(
4362  "DELETE FROM mapd_dictionaries WHERE dictid in (select comp_param from "
4363  "mapd_columns where compression = ? "
4364  "and tableid = ?) and refcount = 0",
4365  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
4366  sqliteConnector_.query_with_text_param("DELETE FROM mapd_columns WHERE tableid = ?",
4367  std::to_string(tableId));
4368  if (td->isView) {
4369  sqliteConnector_.query_with_text_param("DELETE FROM mapd_views WHERE tableid = ?",
4370  std::to_string(tableId));
4371  }
4373  sqliteConnector_.query_with_text_param(
4374  "DELETE FROM omnisci_foreign_tables WHERE table_id = ?", std::to_string(tableId));
4375  }
4376 }
4377 
4378 void Catalog::renamePhysicalTable(const TableDescriptor* td, const string& newTableName) {
4379  cat_write_lock write_lock(this);
4380  cat_sqlite_lock sqlite_lock(getObjForLock());
4381 
4382  sqliteConnector_.query("BEGIN TRANSACTION");
4383  try {
4384  sqliteConnector_.query_with_text_params(
4385  "UPDATE mapd_tables SET name = ? WHERE tableid = ?",
4386  std::vector<std::string>{newTableName, std::to_string(td->tableId)});
4387  } catch (std::exception& e) {
4388  sqliteConnector_.query("ROLLBACK TRANSACTION");
4389  throw;
4390  }
4391  sqliteConnector_.query("END TRANSACTION");
4392  TableDescriptorMap::iterator tableDescIt =
4393  tableDescriptorMap_.find(to_upper(td->tableName));
4394  CHECK(tableDescIt != tableDescriptorMap_.end());
4395  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
4396  // Get table descriptor to change it
4397  TableDescriptor* changeTd = tableDescIt->second;
4398  changeTd->tableName = newTableName;
4399  tableDescriptorMap_.erase(tableDescIt); // erase entry under old name
4400  tableDescriptorMap_[to_upper(newTableName)] = changeTd;
4401  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
4402 }
4403 
4404 void Catalog::renameTable(const TableDescriptor* td, const string& newTableName) {
4405  {
4406  cat_write_lock write_lock(this);
4407  cat_sqlite_lock sqlite_lock(getObjForLock());
4408  // rename all corresponding physical tables if this is a logical table
4409  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
4410  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
4411  const auto physicalTables = physicalTableIt->second;
4412  CHECK(!physicalTables.empty());
4413  for (size_t i = 0; i < physicalTables.size(); i++) {
4414  int32_t physical_tb_id = physicalTables[i];
4415  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
4416  CHECK(phys_td);
4417  std::string newPhysTableName =
4418  generatePhysicalTableName(newTableName, static_cast<int32_t>(i + 1));
4419  renamePhysicalTable(phys_td, newPhysTableName);
4420  }
4421  }
4422  renamePhysicalTable(td, newTableName);
4423  }
4424  {
4425  DBObject object(newTableName, TableDBObjectType);
4426  // update table name in direct and effective priv map
4427  DBObjectKey key;
4428  key.dbId = currentDB_.dbId;
4429  key.objectId = td->tableId;
4430  key.permissionType = static_cast<int>(DBObjectType::TableDBObjectType);
4431  object.setObjectKey(key);
4432  auto objdescs = SysCatalog::instance().getMetadataForObject(
4433  currentDB_.dbId, static_cast<int>(DBObjectType::TableDBObjectType), td->tableId);
4434  for (auto obj : objdescs) {
4435  Grantee* grnt = SysCatalog::instance().getGrantee(obj->roleName);
4436  if (grnt) {
4437  grnt->renameDbObject(object);
4438  }
4439  }
4440  SysCatalog::instance().renameObjectsInDescriptorMap(object, *this);
4441  }
4442 }
4443 
4444 void Catalog::renamePhysicalTables(
4445  std::vector<std::pair<std::string, std::string>>& names,
4446  std::vector<int>& tableIds) {
4447  cat_write_lock write_lock(this);
4448  cat_sqlite_lock sqlite_lock(getObjForLock());
4449 
4450  // execute the SQL query
4451  for (size_t i = 0; i < names.size(); i++) {
4452  int tableId = tableIds[i];
4453  const std::string& newTableName = names[i].second;
4454  sqliteConnector_.query_with_text_params(
4455  "UPDATE mapd_tables SET name = ? WHERE tableid = ?",
4456  std::vector<std::string>{newTableName, std::to_string(tableId)});
4457  }
4458 
4459  // reset the table descriptors, give Calcite a kick
4460  for (size_t i = 0; i < names.size(); i++) {
4461  const auto& [curTableName, newTableName] = names[i];
4462 
4463  TableDescriptorMap::iterator tableDescIt =
4464  tableDescriptorMap_.find(to_upper(curTableName));
4465  CHECK(tableDescIt != tableDescriptorMap_.end());
4466  calciteMgr_->updateMetadata(currentDB_.dbName, curTableName);
4467 
4468  // Get table descriptor to change it
4469  TableDescriptor* changeTd = tableDescIt->second;
4470  changeTd->tableName = newTableName;
4471  tableDescriptorMap_.erase(tableDescIt); // erase entry under old name
4472  tableDescriptorMap_[to_upper(newTableName)] = changeTd;
4473  calciteMgr_->updateMetadata(currentDB_.dbName, curTableName);
4474  }
4475 }
4476 
4477 // Collect an 'overlay' mapping of the tableNames->tableId
4478 // to account for possible chained renames
4479 // (for swap: a->b, b->c, c->d, d->a)
4480 const TableDescriptor* Catalog::getCachedTableDescriptor(
4481  const std::map<std::string, int>& cached_table_map,
4482  const std::string& cur_table_name) {
4483  if (auto it = cached_table_map.find(cur_table_name); it != cached_table_map.end()) {
4484  auto table_id = it->second;
4485  return (table_id == -1) ? NULL : getMetadataForTable(table_id);
4486  }
4487  return getMetadataForTable(cur_table_name);
4488 }
4489 
4490 namespace {
4491 void replace_cached_table_name(std::map<std::string, int>& cachedTableMap,
4492  const std::string& curTableName,
4493  const std::string& newTableName,
4494  int tableId) {
4495  // mark old/cur name as deleted
4496  cachedTableMap[curTableName] = -1;
4497 
4498  // insert the 'new' name
4499  cachedTableMap[newTableName] = tableId;
4500 }
4501 } // namespace
4502 
4503 void Catalog::renameTables(
4504  const std::vector<std::pair<std::string, std::string>>& names) {
4505  // tableId of all tables being renamed
4506  // ... in matching order to 'names'
4507  std::vector<int> tableIds;
4508 
4509  // (sorted & unique) list of tables ids for locking
4510  // (with names index of src in case of error)
4511  // <tableId, strIndex>
4512  // std::map is by definition/implementation sorted
4513  // std::map current usage below tests to avoid over-write
4514  std::map<int, size_t> uniqueOrderedTableIds;
4515 
4516  // mapping of modified tables names -> tableId
4517  std::map<std::string, int> cachedTableMap;
4518 
4519  // -------- Setup --------
4520 
4521  // gather tableIds pre-execute; build maps
4522  for (size_t i = 0; i < names.size(); i++) {
4523  const auto& [curTableName, newTableName] = names[i];
4524 
4525  // make sure the table being renamed exists,
4526  // or will exist when executed in 'name' order
4527  auto td = getCachedTableDescriptor(cachedTableMap, curTableName);
4528  CHECK(td);
4529 
4530  tableIds.push_back(td->tableId);
4531  if (uniqueOrderedTableIds.find(td->tableId) == uniqueOrderedTableIds.end()) {
4532  // don't overwrite as it should map to the first names index 'i'
4533  uniqueOrderedTableIds[td->tableId] = i;
4534  }
4535  replace_cached_table_name(cachedTableMap, curTableName, newTableName, td->tableId);
4536  }
4537 
4538  CHECK_EQ(tableIds.size(), names.size());
4539 
4540  // The outer Stmt created a write lock before calling the catalog rename table
4541  // -> TODO: might want to sort out which really should set the lock :
4542  // the comment in the outer scope indicates it should be in here
4543  // but it's not clear if the access done there *requires* it out there
4544  //
4545  // Lock tables pre-execute (may/will be in different order than rename occurs)
4546  // const auto execute_write_lock = heavyai::unique_lock<heavyai::shared_mutex>(
4547  // *legacylockmgr::LockMgr<heavyai::shared_mutex, bool>::getMutex(
4548  // legacylockmgr::ExecutorOuterLock, true));
4549 
4550  // acquire the locks for all tables being renamed
4552  for (auto& idPair : uniqueOrderedTableIds) {
4553  const std::string& tableName = names[idPair.second].first;
4554  tableLocks.emplace_back(
4557  *this, tableName, false)));
4558  }
4559 
4560  // -------- Rename --------
4561 
4562  {
4563  cat_write_lock write_lock(this);
4564  // collect all (tables + physical tables) into a single list
4565  std::vector<std::pair<std::string, std::string>> allNames;
4566  std::vector<int> allTableIds;
4567 
4568  for (size_t i = 0; i < names.size(); i++) {
4569  int tableId = tableIds[i];
4570  const auto& [curTableName, newTableName] = names[i];
4571 
4572  // rename all corresponding physical tables if this is a logical table
4573  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(tableId);
4574  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
4575  const auto physicalTables = physicalTableIt->second;
4576  CHECK(!physicalTables.empty());
4577  for (size_t k = 0; k < physicalTables.size(); k++) {
4578  int32_t physical_tb_id = physicalTables[k];
4579  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
4580  CHECK(phys_td);
4581  std::string newPhysTableName = generatePhysicalTableName(newTableName, (k + 1));
4582  allNames.emplace_back(phys_td->tableName, newPhysTableName);
4583  allTableIds.push_back(phys_td->tableId);
4584  }
4585  }
4586  allNames.emplace_back(curTableName, newTableName);
4587  allTableIds.push_back(tableId);
4588  }
4589 
4590  // rename all tables in one transaction
4591  execInTransaction(&Catalog::renamePhysicalTables, allNames, allTableIds);
4592  }
4593 
4594  // now update the SysCatalog
4595  for (size_t i = 0; i < names.size(); i++) {
4596  int tableId = tableIds[i];
4597  const std::string& newTableName = names[i].second;
4598  {
4599  // update table name in direct and effective priv map
4600  DBObjectKey key;
4601  key.dbId = currentDB_.dbId;
4602  key.objectId = tableId;
4603  key.permissionType = static_cast<int>(DBObjectType::TableDBObjectType);
4604 
4605  DBObject object(newTableName, TableDBObjectType);
4606  object.setObjectKey(key);
4607 
4608  auto objdescs = SysCatalog::instance().getMetadataForObject(
4609  currentDB_.dbId, static_cast<int>(DBObjectType::TableDBObjectType), tableId);
4610  for (auto obj : objdescs) {
4611  Grantee* grnt = SysCatalog::instance().getGrantee(obj->roleName);
4612  if (grnt) {
4613  grnt->renameDbObject(object);
4614  }
4615  }
4616  SysCatalog::instance().renameObjectsInDescriptorMap(object, *this);
4617  }
4618  }
4619 }
4620 
4621 void Catalog::renameColumn(const TableDescriptor* td,
4622  const ColumnDescriptor* cd,
4623  const string& newColumnName) {
4624  cat_write_lock write_lock(this);
4625  cat_sqlite_lock sqlite_lock(getObjForLock());
4626  sqliteConnector_.query("BEGIN TRANSACTION");
4627  try {
4628  for (int i = 0; i <= cd->columnType.get_physical_cols(); ++i) {
4629  auto cdx = getMetadataForColumn(td->tableId, cd->columnId + i);
4630  CHECK(cdx);
4631  std::string new_column_name = cdx->columnName;
4632  new_column_name.replace(0, cd->columnName.size(), newColumnName);
4633  sqliteConnector_.query_with_text_params(
4634  "UPDATE mapd_columns SET name = ? WHERE tableid = ? AND columnid = ?",
4635  std::vector<std::string>{new_column_name,
4636  std::to_string(td->tableId),
4637  std::to_string(cdx->columnId)});
4638  }
4639  } catch (std::exception& e) {
4640  sqliteConnector_.query("ROLLBACK TRANSACTION");
4641  throw;
4642  }
4643  sqliteConnector_.query("END TRANSACTION");
4644  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
4645  for (int i = 0; i <= cd->columnType.get_physical_cols(); ++i) {
4646  auto cdx = getMetadataForColumn(td->tableId, cd->columnId + i);
4647  CHECK(cdx);
4648  ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.find(
4649  std::make_tuple(td->tableId, to_upper(cdx->columnName)));
4650  CHECK(columnDescIt != columnDescriptorMap_.end());
4651  ColumnDescriptor* changeCd = columnDescIt->second;
4652  changeCd->columnName.replace(0, cd->columnName.size(), newColumnName);
4653  columnDescriptorMap_.erase(columnDescIt); // erase entry under old name
4654  columnDescriptorMap_[std::make_tuple(td->tableId, to_upper(changeCd->columnName))] =
4655  changeCd;
4656  }
4657  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
4658 }
4659 
4660 int32_t Catalog::createDashboard(DashboardDescriptor& vd) {
4661  cat_write_lock write_lock(this);
4662  cat_sqlite_lock sqlite_lock(getObjForLock());
4663  sqliteConnector_.query("BEGIN TRANSACTION");
4664  try {
4665  // TODO(andrew): this should be an upsert
4666  sqliteConnector_.query_with_text_params(
4667  "SELECT id FROM mapd_dashboards WHERE name = ? and userid = ?",
4668  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
4669  if (sqliteConnector_.getNumRows() > 0) {
4670  sqliteConnector_.query_with_text_params(
4671  "UPDATE mapd_dashboards SET state = ?, image_hash = ?, metadata = ?, "
4672  "update_time = "
4673  "datetime('now') where name = ? "
4674  "and userid = ?",
4675  std::vector<std::string>{vd.dashboardState,
4676  vd.imageHash,
4677  vd.dashboardMetadata,
4678  vd.dashboardName,
4679  std::to_string(vd.userId)});
4680  } else {
4681  sqliteConnector_.query_with_text_params(
4682  "INSERT INTO mapd_dashboards (name, state, image_hash, metadata, "
4683  "update_time, "
4684  "userid) "
4685  "VALUES "
4686  "(?,?,?,?, "
4687  "datetime('now'), ?)",
4688  std::vector<std::string>{vd.dashboardName,
4689  vd.dashboardState,
4690  vd.imageHash,
4691  vd.dashboardMetadata,
4692  std::to_string(vd.userId)});
4693  }
4694  } catch (std::exception& e) {
4695  sqliteConnector_.query("ROLLBACK TRANSACTION");
4696  throw;
4697  }
4698  sqliteConnector_.query("END TRANSACTION");
4699 
4700  // now get the auto generated dashboardId
4701  try {
4702  sqliteConnector_.query_with_text_params(
4703  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_dashboards "
4704  "WHERE name = ? and userid = ?",
4705  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
4706  vd.dashboardId = sqliteConnector_.getData<int>(0, 0);
4707  vd.updateTime = sqliteConnector_.getData<std::string>(0, 1);
4708  } catch (std::exception& e) {
4709  throw;
4710  }
4712  std::to_string(currentDB_.dbId), std::to_string(vd.dashboardId));
4713  addFrontendViewToMap(vd);
4714  sqlite_lock.unlock();
4715  write_lock.unlock();
4716  if (!isInfoSchemaDb()) {
4717  // NOTE(wamsi): Transactionally unsafe
4718  createOrUpdateDashboardSystemRole(
4720  }
4721  return vd.dashboardId;
4722 }
4723 
4724 void Catalog::replaceDashboard(DashboardDescriptor& vd) {
4725  cat_write_lock write_lock(this);
4726  cat_sqlite_lock sqlite_lock(getObjForLock());
4727 
4728  CHECK(sqliteConnector_.getSqlitePtr());
4729  sqliteConnector_.query("BEGIN TRANSACTION");
4730  try {
4731  sqliteConnector_.query_with_text_params(
4732  "SELECT id FROM mapd_dashboards WHERE id = ?",
4733  std::vector<std::string>{std::to_string(vd.dashboardId)});
4734  if (sqliteConnector_.getNumRows() > 0) {
4735  sqliteConnector_.query_with_text_params(
4736  "UPDATE mapd_dashboards SET name = ?, state = ?, image_hash = ?, metadata = "
4737  "?, userid = ?, update_time = datetime('now') where id = ? ",
4738  std::vector<std::string>{vd.dashboardName,
4739  vd.dashboardState,
4740  vd.imageHash,
4741  vd.dashboardMetadata,
4742  std::to_string(vd.userId),
4744  } else {
4745  LOG(ERROR) << "Error replacing dashboard id " << vd.dashboardId
4746  << " does not exist in db";
4747  throw runtime_error("Error replacing dashboard id " +
4748  std::to_string(vd.dashboardId) + " does not exist in db");
4749  }
4750  } catch (std::exception& e) {
4751  sqliteConnector_.query("ROLLBACK TRANSACTION");
4752  throw;
4753  }
4754  sqliteConnector_.query("END TRANSACTION");
4755 
4756  bool found{false};
4757  for (auto descp : dashboardDescriptorMap_) {
4758  auto dash = descp.second.get();
4759  if (dash->dashboardId == vd.dashboardId) {
4760  found = true;
4761  auto viewDescIt = dashboardDescriptorMap_.find(std::to_string(dash->userId) + ":" +
4762  dash->dashboardName);
4763  if (viewDescIt ==
4764  dashboardDescriptorMap_.end()) { // check to make sure view exists
4765  LOG(ERROR) << "No metadata for dashboard for user " << dash->userId
4766  << " dashboard " << dash->dashboardName << " does not exist in map";
4767  throw runtime_error("No metadata for dashboard for user " +
4768  std::to_string(dash->userId) + " dashboard " +
4769  dash->dashboardName + " does not exist in map");
4770  }
4771  dashboardDescriptorMap_.erase(viewDescIt);
4772  break;
4773  }
4774  }
4775  if (!found) {
4776  LOG(ERROR) << "Error replacing dashboard id " << vd.dashboardId
4777  << " does not exist in map";
4778  throw runtime_error("Error replacing dashboard id " + std::to_string(vd.dashboardId) +
4779  " does not exist in map");
4780  }
4781 
4782  // now reload the object
4783  sqliteConnector_.query_with_text_params(
4784  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM "
4785  "mapd_dashboards "
4786  "WHERE id = ?",
4787  std::vector<std::string>{std::to_string(vd.dashboardId)});
4788  vd.updateTime = sqliteConnector_.getData<string>(0, 1);
4790  std::to_string(currentDB_.dbId), std::to_string(vd.dashboardId));
4791  addFrontendViewToMapNoLock(vd);
4792  sqlite_lock.unlock();
4793  write_lock.unlock();
4794  if (!isInfoSchemaDb()) {
4795  // NOTE(wamsi): Transactionally unsafe
4796  createOrUpdateDashboardSystemRole(
4798  }
4799 }
4800 
4801 std::string Catalog::calculateSHA1(const std::string& data) {
4802  boost::uuids::detail::sha1 sha1;
4803  unsigned int digest[5];
4804  sha1.process_bytes(data.c_str(), data.length());
4805  sha1.get_digest(digest);
4806  std::stringstream ss;
4807  for (size_t i = 0; i < 5; i++) {
4808  ss << std::hex << digest[i];
4809  }
4810  return ss.str();
4811 }
4812 
4813 std::string Catalog::createLink(LinkDescriptor& ld, size_t min_length) {
4814  cat_write_lock write_lock(this);
4815  cat_sqlite_lock sqlite_lock(getObjForLock());
4816  sqliteConnector_.query("BEGIN TRANSACTION");
4817  try {
4818  ld.link = calculateSHA1(ld.viewState + ld.viewMetadata + std::to_string(ld.userId))
4819  .substr(0, 8);
4820  sqliteConnector_.query_with_text_params(
4821  "SELECT linkid FROM mapd_links WHERE link = ? and userid = ?",
4822  std::vector<std::string>{ld.link, std::to_string(ld.userId)});
4823  if (sqliteConnector_.getNumRows() > 0) {
4824  sqliteConnector_.query_with_text_params(
4825  "UPDATE mapd_links SET update_time = datetime('now') WHERE userid = ? AND "
4826  "link = ?",
4827  std::vector<std::string>{std::to_string(ld.userId), ld.link});
4828  } else {
4829  sqliteConnector_.query_with_text_params(
4830  "INSERT INTO mapd_links (userid, link, view_state, view_metadata, "
4831  "update_time) VALUES (?,?,?,?, datetime('now'))",
4832  std::vector<std::string>{
4833  std::to_string(ld.userId), ld.link, ld.viewState, ld.viewMetadata});
4834  }
4835  // now get the auto generated dashid
4836  sqliteConnector_.query_with_text_param(
4837  "SELECT linkid, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_links "
4838  "WHERE link = ?",
4839  ld.link);
4840  ld.linkId = sqliteConnector_.getData<int>(0, 0);
4841  ld.updateTime = sqliteConnector_.getData<std::string>(0, 1);
4842  } catch (std::exception& e) {
4843  sqliteConnector_.query("ROLLBACK TRANSACTION");
4844  throw;
4845  }
4846  sqliteConnector_.query("END TRANSACTION");
4847  addLinkToMap(ld);
4848  return ld.link;
4849 }
4850 
4851 const ColumnDescriptor* Catalog::getShardColumnMetadataForTable(
4852  const TableDescriptor* td) const {
4853  cat_read_lock read_lock(this);
4854 
4855  const auto column_descriptors =
4856  getAllColumnMetadataForTable(td->tableId, false, true, true);
4857 
4858  const ColumnDescriptor* shard_cd{nullptr};
4859  int i = 1;
4860  for (auto cd_itr = column_descriptors.begin(); cd_itr != column_descriptors.end();
4861  ++cd_itr, ++i) {
4862  if (i == td->shardedColumnId) {
4863  shard_cd = *cd_itr;
4864  }
4865  }
4866  return shard_cd;
4867 }
4868 
4869 std::vector<const TableDescriptor*> Catalog::getPhysicalTablesDescriptors(
4870  const TableDescriptor* logical_table_desc,
4871  bool populate_fragmenter) const {
4872  cat_read_lock read_lock(this);
4873  const auto physicalTableIt =
4874  logicalToPhysicalTableMapById_.find(logical_table_desc->tableId);
4875  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
4876  return {logical_table_desc};
4877  }
4878  const auto physicalTablesIds = physicalTableIt->second;
4879  CHECK(!physicalTablesIds.empty());
4880  read_lock.unlock();
4881  std::vector<const TableDescriptor*> physicalTables;
4882  for (size_t i = 0; i < physicalTablesIds.size(); i++) {
4883  physicalTables.push_back(
4884  getMetadataForTable(physicalTablesIds[i], populate_fragmenter));
4885  }
4886  return physicalTables;
4887 }
4888 
4889 std::vector<std::pair<int32_t, int32_t>> Catalog::getAllPersistedTableAndShardIds()
4890  const {
4891  cat_read_lock read_lock(this);
4892  std::vector<std::pair<int32_t, int32_t>> table_and_shard_ids;
4893  table_and_shard_ids.reserve(tableDescriptorMapById_.size());
4894  for (const auto [table_id, td] : tableDescriptorMapById_) {
4895  // Only include ids for physical persisted tables
4896  if (!td->isView && !td->isTemporaryTable() && !td->isForeignTable() &&
4897  logicalToPhysicalTableMapById_.find(table_id) ==
4898  logicalToPhysicalTableMapById_.end()) {
4899  table_and_shard_ids.emplace_back(table_id, td->shard);
4900  }
4901  }
4902  return table_and_shard_ids;
4903 }
4904 
4905 const std::map<int, const ColumnDescriptor*> Catalog::getDictionaryToColumnMapping() {
4906  cat_read_lock read_lock(this);
4907 
4908  std::map<int, const ColumnDescriptor*> mapping;
4909 
4910  const auto tables = getAllTableMetadata();
4911  for (const auto td : tables) {
4912  if (td->shard >= 0) {
4913  // skip shards, they're not standalone tables
4914  continue;
4915  }
4916 
4917  for (auto& cd : getAllColumnMetadataForTable(td->tableId, false, false, true)) {
4918  const auto& ti = cd->columnType;
4919  if (ti.is_string()) {
4920  if (ti.get_compression() == kENCODING_DICT) {
4921  // if foreign reference, get referenced tab.col
4922  const auto dict_id = ti.get_comp_param();
4923 
4924  // ignore temp (negative) dictionaries
4925  if (dict_id > 0 && mapping.end() == mapping.find(dict_id)) {
4926  mapping[dict_id] = cd;
4927  }
4928  }
4929  }
4930  }
4931  }
4932 
4933  return mapping;
4934 }
4935 
4936 bool Catalog::filterTableByTypeAndUser(const TableDescriptor* td,
4937  const UserMetadata& user_metadata,
4938  const GetTablesType get_tables_type) const {
4939  if (td->shard >= 0) {
4940  // skip shards, they're not standalone tables
4941  return false;
4942  }
4943  switch (get_tables_type) {
4944  case GET_PHYSICAL_TABLES: {
4945  if (td->isView) {
4946  return false;
4947  }
4948  break;
4949  }
4950  case GET_VIEWS: {
4951  if (!td->isView) {
4952  return false;
4953  }
4954  break;
4955  }
4956  default:
4957  break;
4958  }
4960  dbObject.loadKey(*this);
4961  std::vector<DBObject> privObjects = {dbObject};
4962  if (!SysCatalog::instance().hasAnyPrivileges(user_metadata, privObjects)) {
4963  // skip table, as there are no privileges to access it
4964  return false;
4965  }
4966  return true;
4967 }
4968 
4969 std::vector<std::string> Catalog::getTableNamesForUser(
4970  const UserMetadata& user_metadata,
4971  const GetTablesType get_tables_type) const {
4972  sys_read_lock syscat_read_lock(&SysCatalog::instance());
4973  cat_read_lock read_lock(this);
4974  std::vector<std::string> table_names;
4975  const auto tables = getAllTableMetadata();
4976  for (const auto td : tables) {
4977  if (filterTableByTypeAndUser(td, user_metadata, get_tables_type)) {
4978  table_names.push_back(td->tableName);
4979  }
4980  }
4981  return table_names;
4982 }
4983 
4984 std::vector<TableMetadata> Catalog::getTablesMetadataForUser(
4985  const UserMetadata& user_metadata,
4986  const GetTablesType get_tables_type,
4987  const std::string& filter_table_name) const {
4988  sys_read_lock syscat_read_lock(&SysCatalog::instance());
4989  cat_read_lock read_lock(this);
4990 
4991  std::vector<TableMetadata> tables_metadata;
4992  const auto tables = getAllTableMetadata();
4993  for (const auto td : tables) {
4994  if (filterTableByTypeAndUser(td, user_metadata, get_tables_type)) {
4995  if (!filter_table_name.empty()) {
4996  if (td->tableName != filter_table_name) {
4997  continue;
4998  }
4999  }
5000  TableMetadata table_metadata(td); // Makes a copy, not safe to access raw table
5001  // descriptor outside catalog lock
5002  tables_metadata.emplace_back(table_metadata);
5003  }
5004  }
5005  return tables_metadata;
5006 }
5007 
5008 int Catalog::getLogicalTableId(const int physicalTableId) const {
5009  cat_read_lock read_lock(this);
5010  for (const auto& l : logicalToPhysicalTableMapById_) {
5011  if (l.second.end() != std::find_if(l.second.begin(),
5012  l.second.end(),
5013  [&](decltype(*l.second.begin()) tid) -> bool {
5014  return physicalTableId == tid;
5015  })) {
5016  return l.first;
5017  }
5018  }
5019  return physicalTableId;
5020 }
5021 
5022 void Catalog::checkpoint(const int logicalTableId) const {
5023  const auto td = getMetadataForTable(logicalTableId);
5024  const auto shards = getPhysicalTablesDescriptors(td);
5025  for (const auto shard : shards) {
5026  getDataMgr().checkpoint(getCurrentDB().dbId, shard->tableId);
5027  }
5028 }
5029 
5030 void Catalog::checkpointWithAutoRollback(const int logical_table_id) const {
5031  auto table_epochs = getTableEpochs(getDatabaseId(), logical_table_id);
5032  try {
5033  checkpoint(logical_table_id);
5034  } catch (...) {
5035  setTableEpochsLogExceptions(getDatabaseId(), table_epochs);
5036  throw;
5037  }
5038 }
5039 
5040 void Catalog::resetTableEpochFloor(const int logicalTableId) const {
5041  cat_read_lock read_lock(this);
5042  const auto td = getMetadataForTable(logicalTableId, false);
5043  const auto shards = getPhysicalTablesDescriptors(td, false);
5044  for (const auto shard : shards) {
5045  getDataMgr().resetTableEpochFloor(getCurrentDB().dbId, shard->tableId);
5046  }
5047 }
5048 
5049 void Catalog::eraseDbMetadata() {
5050  const auto tables = getAllTableMetadata();
5051  for (const auto table : tables) {
5052  eraseTableMetadata(table);
5053  }
5054  // Physically erase database metadata
5055  boost::filesystem::remove(basePath_ + "/" + shared::kCatalogDirectoryName + "/" +
5056  currentDB_.dbName);
5057  calciteMgr_->updateMetadata(currentDB_.dbName, "");
5058 }
5059 
5060 void Catalog::eraseDbPhysicalData() {
5061  const auto tables = getAllTableMetadata();
5062  for (const auto table : tables) {
5063  eraseTablePhysicalData(table);
5064  }
5065 }
5066 
5067 void Catalog::eraseTablePhysicalData(const TableDescriptor* td) {
5068  const int tableId = td->tableId;
5069  // must destroy fragmenter before deleteChunks is called.
5070  removeFragmenterForTable(tableId);
5071 
5072  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
5073  {
5074  INJECT_TIMER(deleteChunksWithPrefix);
5075  // assuming deleteChunksWithPrefix is atomic
5076  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
5077  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
5078  }
5079  if (!td->isView) {
5080  INJECT_TIMER(Remove_Table);
5081  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
5082  }
5083 }
5084 
5085 std::string Catalog::generatePhysicalTableName(const std::string& logicalTableName,
5086  const size_t shardNumber) {
5087  std::string physicalTableName =
5088  logicalTableName + physicalTableNameTag_ + std::to_string(shardNumber);
5089  return (physicalTableName);
5090 }
5091 
5092 void Catalog::buildForeignServerMapUnlocked() {
5094  sqliteConnector_.query(
5095  "SELECT id, name, data_wrapper_type, options, owner_user_id, creation_time FROM "
5096  "omnisci_foreign_servers");
5097  auto num_rows = sqliteConnector_.getNumRows();
5098 
5099  for (size_t row = 0; row < num_rows; row++) {
5100  auto foreign_server = std::make_shared<foreign_storage::ForeignServer>(
5101  sqliteConnector_.getData<int>(row, 0),
5102  sqliteConnector_.getData<std::string>(row, 1),
5103  sqliteConnector_.getData<std::string>(row, 2),
5104  sqliteConnector_.getData<std::string>(row, 3),
5105  sqliteConnector_.getData<std::int32_t>(row, 4),
5106  sqliteConnector_.getData<std::int32_t>(row, 5));
5107  foreignServerMap_[foreign_server->name] = foreign_server;
5108  foreignServerMapById_[foreign_server->id] = foreign_server;
5109  }
5110 }
5111 
5112 void Catalog::updateForeignTablesInMapUnlocked() {
5114  sqliteConnector_.query(
5115  "SELECT table_id, server_id, options, last_refresh_time, next_refresh_time from "
5116  "omnisci_foreign_tables");
5117  auto num_rows = sqliteConnector_.getNumRows();
5118  for (size_t r = 0; r < num_rows; r++) {
5119  const auto table_id = sqliteConnector_.getData<int32_t>(r, 0);
5120  const auto server_id = sqliteConnector_.getData<int32_t>(r, 1);
5121  const auto& options = sqliteConnector_.getData<std::string>(r, 2);
5122  const auto last_refresh_time = sqliteConnector_.getData<int64_t>(r, 3);
5123  const auto next_refresh_time = sqliteConnector_.getData<int64_t>(r, 4);
5124 
5125  CHECK(tableDescriptorMapById_.find(table_id) != tableDescriptorMapById_.end());
5126  auto foreign_table =
5127  dynamic_cast<foreign_storage::ForeignTable*>(tableDescriptorMapById_[table_id]);
5128  CHECK(foreign_table);
5129  foreign_table->foreign_server = foreignServerMapById_[server_id].get();
5130  CHECK(foreign_table->foreign_server);
5131  foreign_table->populateOptionsMap(options);
5132  foreign_table->last_refresh_time = last_refresh_time;
5133  foreign_table->next_refresh_time = next_refresh_time;
5134  if (foreign_table->is_system_table) {
5135  foreign_table->is_in_memory_system_table =
5137  foreign_table->foreign_server->data_wrapper_type);
5138  }
5139  }
5140 }
5141 
5142 void Catalog::reloadForeignTableUnlocked(foreign_storage::ForeignTable& foreign_table) {
5144  CHECK_NE(foreign_table.tableId, 0)
5145  << "reloadForeignTable expects a table with valid id";
5146  sqliteConnector_.query(
5147  "SELECT server_id, options, last_refresh_time, next_refresh_time from "
5148  "omnisci_foreign_tables WHERE table_id == " +
5149  std::to_string(foreign_table.tableId));
5150  auto num_rows = sqliteConnector_.getNumRows();
5151  CHECK_EQ(num_rows, 1U) << "Expected single entry in omnisci_foreign_tables for table'"
5152  << foreign_table.tableName << "', instead got " << num_rows;
5153  const auto server_id = sqliteConnector_.getData<int32_t>(0, 0);
5154  const auto& options = sqliteConnector_.getData<std::string>(0, 1);
5155  const auto last_refresh_time = sqliteConnector_.getData<int64_t>(0, 2);
5156  const auto next_refresh_time = sqliteConnector_.getData<int64_t>(0, 3);
5157 
5158  foreign_table.foreign_server = foreignServerMapById_[server_id].get();
5159  CHECK(foreign_table.foreign_server);
5160  foreign_table.populateOptionsMap(options);
5161  foreign_table.last_refresh_time = last_refresh_time;
5162  foreign_table.next_refresh_time = next_refresh_time;
5163  if (foreign_table.is_system_table) {
5164  foreign_table.is_in_memory_system_table =
5166  foreign_table.foreign_server->data_wrapper_type);
5167  }
5168 }
5169 
5170 void Catalog::reloadDictionariesFromDiskUnlocked() {
5171  std::string dictQuery(
5172  "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
5173  sqliteConnector_.query(dictQuery);
5174  auto numRows = sqliteConnector_.getNumRows();
5175  for (size_t r = 0; r < numRows; ++r) {
5176  auto dictId = sqliteConnector_.getData<int>(r, 0);
5177  auto dictName = sqliteConnector_.getData<string>(r, 1);
5178  auto dictNBits = sqliteConnector_.getData<int>(r, 2);
5179  auto is_shared = sqliteConnector_.getData<bool>(r, 3);
5180  auto refcount = sqliteConnector_.getData<int>(r, 4);
5181  auto fname = g_base_path + "/" + shared::kDataDirectoryName + "/DB_" +
5182  std::to_string(currentDB_.dbId) + "_DICT_" + std::to_string(dictId);
5183  DictRef dict_ref(currentDB_.dbId, dictId);
5184  DictDescriptor dd(dict_ref, dictName, dictNBits, is_shared, refcount, fname, false);
5185  if (auto it = dictDescriptorMapByRef_.find(dict_ref);
5186  it == dictDescriptorMapByRef_.end()) {
5187  dictDescriptorMapByRef_[dict_ref] = std::make_unique<DictDescriptor>(dd);
5188  } else {
5189  *it->second = dd;
5190  }
5191  }
5192 }
5193 
5194 std::list<ColumnDescriptor*> Catalog::sqliteGetColumnsForTableUnlocked(int32_t table_id) {
5195  std::list<ColumnDescriptor*> cds;
5196  // TODO(Misiu): Change ColumnDescriptorMap_ to use smartpointers. Right now we use
5197  // temporary smartpointers that release their memory once initialized.
5198  std::list<std::unique_ptr<ColumnDescriptor>> smart_cds;
5199  std::string columnQuery(
5200  "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
5201  "is_notnull, compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
5202  "virtual_expr, is_deletedcol, default_value from mapd_columns WHERE tableid = " +
5203  std::to_string(table_id) + " ORDER BY tableid, columnid");
5204  sqliteConnector_.query(columnQuery);
5205  auto numRows = sqliteConnector_.getNumRows();
5206  int32_t skip_physical_cols = 0;
5207  for (size_t r = 0; r < numRows; ++r) {
5208  std::unique_ptr<ColumnDescriptor> cd = std::make_unique<ColumnDescriptor>();
5209  cd->tableId = sqliteConnector_.getData<int>(r, 0);
5210  cd->columnId = sqliteConnector_.getData<int>(r, 1);
5211  cd->columnName = sqliteConnector_.getData<string>(r, 2);
5212  cd->columnType.set_type((SQLTypes)sqliteConnector_.getData<int>(r, 3));
5213  cd->columnType.set_subtype((SQLTypes)sqliteConnector_.getData<int>(r, 4));
5214  cd->columnType.set_dimension(sqliteConnector_.getData<int>(r, 5));
5215  cd->columnType.set_scale(sqliteConnector_.getData<int>(r, 6));
5216  cd->columnType.set_notnull(sqliteConnector_.getData<bool>(r, 7));
5217  cd->columnType.set_compression((EncodingType)sqliteConnector_.getData<int>(r, 8));
5218  cd->columnType.set_comp_param(sqliteConnector_.getData<int>(r, 9));
5219  cd->columnType.set_size(sqliteConnector_.getData<int>(r, 10));
5220  cd->chunks = sqliteConnector_.getData<string>(r, 11);
5221  cd->isSystemCol = sqliteConnector_.getData<bool>(r, 12);
5222  cd->isVirtualCol = sqliteConnector_.getData<bool>(r, 13);
5223  cd->virtualExpr = sqliteConnector_.getData<string>(r, 14);
5224  cd->isDeletedCol = sqliteConnector_.getData<bool>(r, 15);
5225  if (sqliteConnector_.isNull(r, 16)) {
5226  cd->default_value = std::nullopt;
5227  } else {
5228  cd->default_value = std::make_optional(sqliteConnector_.getData<string>(r, 16));
5229  }
5230  cd->isGeoPhyCol = skip_physical_cols-- > 0;
5231  cd->db_id = getDatabaseId();
5232  set_dict_key(*cd);
5233  smart_cds.emplace_back(std::move(cd));
5234  }
5235  // Once we have correctly initialized all columns, release their ownership as we
5236  // currently handle them as free pointers.
5237  for (auto& cd : smart_cds) {
5238  cds.emplace_back(cd.release());
5239  }
5240  return cds;
5241 }
5242 
5243 TableDescriptor* Catalog::createTableFromDiskUnlocked(int32_t table_id) {
5244  std::string query(
5245  "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
5246  "max_chunk_size, frag_page_size, max_rows, partitions, shard_column_id, shard, "
5247  "num_shards, key_metainfo, userid, sort_column_id, storage_type, "
5248  "max_rollback_epochs, is_system_table from mapd_tables WHERE tableid = " +
5249  std::to_string(table_id));
5250  sqliteConnector_.query(query);
5251  auto numRows = sqliteConnector_.getNumRows();
5252  if (!numRows) {
5253  throw NoTableFoundException(table_id);
5254  }
5255 
5256  const auto& storage_type = sqliteConnector_.getData<string>(0, 17);
5257  if (!storage_type.empty() && storage_type != StorageType::FOREIGN_TABLE) {
5258  const auto& table_name = sqliteConnector_.getData<string>(0, 1);
5259  LOG(FATAL) << "Unable to read Catalog metadata: storage type is currently not a "
5260  "supported table option (table "
5261  << table_name << " [" << table_id << "] in database " << currentDB_.dbName
5262  << ").";
5263  }
5264 
5265  // TODO(Misiu): Get rid of manual memory allocation and use smart pointers for
5266  // TableDecriptorMap_. Currently we use a smartpointer to cleanup if we catch
5267  // exceptions during initialization and then release ownership into the existing system.
5268  std::unique_ptr<TableDescriptor> td;
5269  td = (storage_type == StorageType::FOREIGN_TABLE)
5270  ? std::make_unique<foreign_storage::ForeignTable>()
5271  : std::make_unique<TableDescriptor>();
5272 
5273  td->tableId = sqliteConnector_.getData<int>(0, 0);
5274  td->tableName = sqliteConnector_.getData<string>(0, 1);
5275  td->nColumns = sqliteConnector_.getData<int>(0, 2);
5276  td->isView = sqliteConnector_.getData<bool>(0, 3);
5277  td->fragments = sqliteConnector_.getData<string>(0, 4);
5278  td->fragType = static_cast<Fragmenter_Namespace::FragmenterType>(
5279  sqliteConnector_.getData<int>(0, 5));
5280  td->maxFragRows = sqliteConnector_.getData<int>(0, 6);
5281  td->maxChunkSize = sqliteConnector_.getData<int64_t>(0, 7);
5282  td->fragPageSize = sqliteConnector_.getData<int>(0, 8);
5283  td->maxRows = sqliteConnector_.getData<int64_t>(0, 9);
5284  td->partitions = sqliteConnector_.getData<string>(0, 10);
5285  td->shardedColumnId = sqliteConnector_.getData<int>(0, 11);
5286  td->shard = sqliteConnector_.getData<int>(0, 12);
5287  td->nShards = sqliteConnector_.getData<int>(0, 13);
5288  td->keyMetainfo = sqliteConnector_.getData<string>(0, 14);
5289  td->userId = sqliteConnector_.getData<int>(0, 15);
5290  td->sortedColumnId =
5291  sqliteConnector_.isNull(0, 16) ? 0 : sqliteConnector_.getData<int>(0, 16);
5292  td->storageType = storage_type;
5293  td->maxRollbackEpochs = sqliteConnector_.getData<int>(0, 18);
5294  td->is_system_table = sqliteConnector_.getData<bool>(0, 19);
5295  td->hasDeletedCol = false;
5296 
5297  if (td->isView) {
5298  updateViewUnlocked(*td);
5299  } else {
5300  td->fragmenter = nullptr;
5301  }
5302 
5303  if (auto ftd = dynamic_cast<foreign_storage::ForeignTable*>(td.get())) {
5304  reloadForeignTableUnlocked(*ftd);
5305  }
5306 
5307  return td.release();
5308 }
5309 
5310 void Catalog::setForeignServerProperty(const std::string& server_name,
5311  const std::string& property,
5312  const std::string& value) {
5313  cat_sqlite_lock sqlite_lock(getObjForLock());
5314  sqliteConnector_.query_with_text_params(
5315  "SELECT id from omnisci_foreign_servers where name = ?",
5316  std::vector<std::string>{server_name});
5317  auto num_rows = sqliteConnector_.getNumRows();
5318  if (num_rows > 0) {
5319  CHECK_EQ(size_t(1), num_rows);
5320  auto server_id = sqliteConnector_.getData<int32_t>(0, 0);
5321  sqliteConnector_.query_with_text_params(
5322  "UPDATE omnisci_foreign_servers SET " + property + " = ? WHERE id = ?",
5323  std::vector<std::string>{value, std::to_string(server_id)});
5324  } else {
5325  throw std::runtime_error{"Can not change property \"" + property +
5326  "\" for foreign server." + " Foreign server \"" +
5327  server_name + "\" is not found."};
5328  }
5329 }
5330 
5331 void Catalog::createDefaultServersIfNotExists() {
5336 
5337  auto local_csv_server = std::make_unique<foreign_storage::ForeignServer>(
5338  "default_local_delimited",
5340  options,
5342  local_csv_server->validate();
5343  createForeignServerNoLocks(std::move(local_csv_server), true);
5344 
5345 #ifdef ENABLE_IMPORT_PARQUET
5346  auto local_parquet_server = std::make_unique<foreign_storage::ForeignServer>(
5347  "default_local_parquet",
5349  options,
5351  local_parquet_server->validate();
5352  createForeignServerNoLocks(std::move(local_parquet_server), true);
5353 #endif
5354 
5355  auto local_regex_parser_server = std::make_unique<foreign_storage::ForeignServer>(
5356  "default_local_regex_parsed",
5358  options,
5360  local_regex_parser_server->validate();
5361  createForeignServerNoLocks(std::move(local_regex_parser_server), true);
5362 }
5363 
5364 // prepare a fresh file reload on next table access
5365 void Catalog::setForReload(const int32_t tableId) {
5366  const auto td = getMetadataForTable(tableId);
5367  for (const auto shard : getPhysicalTablesDescriptors(td)) {
5368  const auto tableEpoch = getTableEpoch(currentDB_.dbId, shard->tableId);
5369  setTableEpoch(currentDB_.dbId, shard->tableId, tableEpoch);
5370  }
5371 }
5372 
5373 // get a table's data dirs
5374 std::vector<std::string> Catalog::getTableDataDirectories(
5375  const TableDescriptor* td) const {
5376  const auto global_file_mgr = getDataMgr().getGlobalFileMgr();
5377  std::vector<std::string> file_paths;
5378  for (auto shard : getPhysicalTablesDescriptors(td)) {
5379  const auto file_mgr = dynamic_cast<File_Namespace::FileMgr*>(
5380  global_file_mgr->getFileMgr(currentDB_.dbId, shard->tableId));
5381  boost::filesystem::path file_path(file_mgr->getFileMgrBasePath());
5382  file_paths.push_back(file_path.filename().string());
5383  }
5384  return file_paths;
5385 }
5386 
5387 // get a column's dict dir basename
5388 std::string Catalog::getColumnDictDirectory(const ColumnDescriptor* cd,
5389  bool file_name_only) const {
5390  if ((cd->columnType.is_string() || cd->columnType.is_string_array()) &&
5392  cd->columnType.get_comp_param() > 0) {
5393  const auto dictId = cd->columnType.get_comp_param();
5394  const DictRef dictRef(currentDB_.dbId, dictId);
5395  const auto dit = dictDescriptorMapByRef_.find(dictRef);
5396  CHECK(dit != dictDescriptorMapByRef_.end());
5397  CHECK(dit->second);
5398  if (file_name_only) {
5399  boost::filesystem::path file_path(dit->second->dictFolderPath);
5400  return file_path.filename().string();
5401  } else {
5402  return dit->second->dictFolderPath;
5403  }
5404  }
5405  return std::string();
5406 }
5407 
5408 // get a table's dict dirs
5409 std::vector<std::string> Catalog::getTableDictDirectories(
5410  const TableDescriptor* td) const {
5411  std::vector<std::string> file_paths;
5412  for (auto cd : getAllColumnMetadataForTable(td->tableId, false, false, true)) {
5413  auto file_base = getColumnDictDirectory(cd);
5414  if (!file_base.empty() &&
5415  file_paths.end() == std::find(file_paths.begin(), file_paths.end(), file_base)) {
5416  file_paths.push_back(file_base);
5417  }
5418  }
5419  return file_paths;
5420 }
5421 
5422 std::set<std::string> Catalog::getTableDictDirectoryPaths(int32_t table_id) const {
5423  cat_read_lock read_lock(this);
5424  std::set<std::string> directory_paths;
5425  auto it = dict_columns_by_table_id_.find(table_id);
5426  if (it != dict_columns_by_table_id_.end()) {
5427  for (auto cd : it->second) {
5428  auto directory_path = getColumnDictDirectory(cd, false);
5429  if (!directory_path.empty()) {
5430  directory_paths.emplace(directory_path);
5431  }
5432  }
5433  }
5434  return directory_paths;
5435 }
5436 
5437 // returns table schema in a string
5438 // NOTE(sy): Might be able to replace dumpSchema() later with
5439 // dumpCreateTable() after a deeper review of the TableArchiver code.
5440 std::string Catalog::dumpSchema(const TableDescriptor* td) const {
5441  CHECK(!td->is_system_table);
5442  cat_read_lock read_lock(this);
5443 
5444  std::ostringstream os;
5445  os << "CREATE TABLE @T (";
5446  // gather column defines
5447  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
5448  std::string comma;
5449  std::vector<std::string> shared_dicts;
5450  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
5451  for (const auto cd : cds) {
5452  if (!(cd->isSystemCol || cd->isVirtualCol)) {
5453  const auto& ti = cd->columnType;
5454  os << comma << quoteIfRequired(cd->columnName);
5455  // CHAR is perculiar... better dump it as TEXT(32) like \d does
5456  if (ti.get_type() == SQLTypes::kCHAR) {
5457  os << " "
5458  << "TEXT";
5459  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
5460  os << " "
5461  << "TEXT[]";
5462  } else {
5463  os << " " << ti.get_type_name();
5464  }
5465  os << (ti.get_notnull() ? " NOT NULL" : "");
5466  if (cd->default_value.has_value()) {
5467  os << " DEFAULT " << cd->getDefaultValueLiteral();
5468  }
5469  if (ti.is_string() || (ti.is_array() && ti.get_subtype() == kTEXT)) {
5470  auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
5471  if (ti.get_compression() == kENCODING_DICT) {
5472  // if foreign reference, get referenced tab.col
5473  const auto dict_id = ti.get_comp_param();
5474  const DictRef dict_ref(currentDB_.dbId, dict_id);
5475  const auto dict_it = dictDescriptorMapByRef_.find(dict_ref);
5476  CHECK(dict_it != dictDescriptorMapByRef_.end());
5477  const auto dict_name = dict_it->second->dictName;
5478  // when migrating a table, any foreign dict ref will be dropped
5479  // and the first cd of a dict will become root of the dict
5480  if (dict_root_cds.end() == dict_root_cds.find(dict_name)) {
5481  dict_root_cds[dict_name] = cd;
5482  os << " ENCODING " << ti.get_compression_name() << "(" << (size * 8) << ")";
5483  } else {
5484  const auto dict_root_cd = dict_root_cds[dict_name];
5485  shared_dicts.push_back("SHARED DICTIONARY (" + cd->columnName +
5486  ") REFERENCES @T(" + dict_root_cd->columnName + ")");
5487  // "... shouldn't specify an encoding, it borrows from the referenced
5488  // column"
5489  }
5490  } else {
5491  os << " ENCODING NONE";
5492  }
5493  } else if (ti.is_date_in_days() ||
5494  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
5495  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
5496  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
5497  } else if (ti.is_geometry()) {
5498  if (ti.get_compression() == kENCODING_GEOINT) {
5499  os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
5500  << ")";
5501  } else {
5502  os << " ENCODING NONE";
5503  }
5504  }
5505  comma = ", ";
5506  }
5507  }
5508  // gather SHARED DICTIONARYs
5509  if (shared_dicts.size()) {
5510  os << ", " << boost::algorithm::join(shared_dicts, ", ");
5511  }
5512  // gather WITH options ...
5513  std::vector<std::string> with_options;
5514  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
5515  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
5516  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
5517  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
5518  with_options.emplace_back(td->hasDeletedCol ? "VACUUM='DELAYED'"
5519  : "VACUUM='IMMEDIATE'");
5520  if (!td->partitions.empty()) {
5521  with_options.push_back("PARTITIONS='" + td->partitions + "'");
5522  }
5523  if (td->nShards > 0) {
5524  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
5525  CHECK(shard_cd);
5526  os << ", SHARD KEY(" << shard_cd->columnName << ")";
5527  with_options.push_back(
5528  "SHARD_COUNT=" +
5529  std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
5530  }
5531  if (td->sortedColumnId > 0) {
5532  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
5533  CHECK(sort_cd);
5534  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
5535  }
5537  td->maxRollbackEpochs != -1) {
5538  with_options.push_back("MAX_ROLLBACK_EPOCHS=" +
5540  }
5541  os << ") WITH (" + boost::algorithm::join(with_options, ", ") + ");";
5542  return os.str();
5543 }
5544 
5545 #include "Parser/ReservedKeywords.h"
5546 
5548 inline bool contains_spaces(std::string_view str) {
5549  return std::find_if(str.begin(), str.end(), [](const unsigned char& ch) {
5550  return std::isspace(ch);
5551  }) != str.end();
5552 }
5553 
5556  std::string_view str,
5557  std::string_view chars = "`~!@#$%^&*()-=+[{]}\\|;:'\",<.>/?") {
5558  return str.find_first_of(chars) != std::string_view::npos;
5559 }
5560 
5562 inline bool is_reserved_sql_keyword(std::string_view str) {
5563  return reserved_keywords.find(to_upper(std::string(str))) != reserved_keywords.end();
5564 }
5565 
5566 // returns a "CREATE TABLE" statement in a string for "SHOW CREATE TABLE"
5567 std::string Catalog::dumpCreateTable(const TableDescriptor* td,
5568  bool multiline_formatting,
5569  bool dump_defaults) const {
5570  cat_read_lock read_lock(this);
5571  return dumpCreateTableUnlocked(td, multiline_formatting, dump_defaults);
5572 }
5573 
5574 std::optional<std::string> Catalog::dumpCreateTable(int32_t table_id,
5575  bool multiline_formatting,
5576  bool dump_defaults) const {
5577  cat_read_lock read_lock(this);
5578  const auto td = getMutableMetadataForTableUnlocked(table_id);
5579  if (!td) {
5580  return {};
5581  }
5582  return dumpCreateTableUnlocked(td, multiline_formatting, dump_defaults);
5583 }
5584 
5585 std::string Catalog::dumpCreateTableUnlocked(const TableDescriptor* td,
5586  bool multiline_formatting,
5587  bool dump_defaults) const {
5588  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
5589  std::ostringstream os;
5590 
5591  if (foreign_table && !td->is_system_table) {
5592  os << "CREATE FOREIGN TABLE " << td->tableName << " (";
5593  } else if (!td->isView) {
5594  os << "CREATE ";
5596  os << "TEMPORARY ";
5597  }
5598  os << "TABLE " + td->tableName + " (";
5599  } else {
5600  os << "CREATE VIEW " + td->tableName + " AS " << td->viewSQL;
5601  return os.str();
5602  }
5603  // scan column defines
5604  std::vector<std::string> additional_info;
5605  std::set<std::string> shared_dict_column_names;
5606 
5607  gatherAdditionalInfo(additional_info, shared_dict_column_names, td);
5608 
5609  // gather column defines
5610  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
5611  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
5612  bool first = true;
5613  for (const auto cd : cds) {
5614  if (!(cd->isSystemCol || cd->isVirtualCol)) {
5615  const auto& ti = cd->columnType;
5616  if (!first) {
5617  os << ",";
5618  if (!multiline_formatting) {
5619  os << " ";
5620  }
5621  } else {
5622  first = false;
5623  }
5624  if (multiline_formatting) {
5625  os << "\n ";
5626  }
5627  // column name
5628  os << quoteIfRequired(cd->columnName);
5629  // CHAR is perculiar... better dump it as TEXT(32) like \d does
5630  if (ti.get_type() == SQLTypes::kCHAR) {
5631  os << " "
5632  << "TEXT";
5633  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
5634  os << " "
5635  << "TEXT[]";
5636  } else {
5637  os << " " << ti.get_type_name();
5638  }
5639  os << (ti.get_notnull() ? " NOT NULL" : "");
5640  if (cd->default_value.has_value()) {
5641  os << " DEFAULT " << cd->getDefaultValueLiteral();
5642  }
5643  if (shared_dict_column_names.find(cd->columnName) ==
5644  shared_dict_column_names.end()) {
5645  // avoids "Column ... shouldn't specify an encoding, it borrows it
5646  // from the referenced column"
5647  if (ti.is_string() || (ti.is_array() && ti.get_subtype() == kTEXT)) {
5648  auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
5649  if (ti.get_compression() == kENCODING_DICT) {
5650  os << " ENCODING " << ti.get_compression_name() << "(" << (size * 8) << ")";
5651  } else {
5652  os << " ENCODING NONE";
5653  }
5654  } else if (ti.is_date_in_days() ||
5655  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
5656  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
5657  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
5658  } else if (ti.is_geometry()) {
5659  if (ti.get_compression() == kENCODING_GEOINT) {
5660  os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
5661  << ")";
5662  } else {
5663  os << " ENCODING NONE";
5664  }
5665  }
5666  }
5667  }
5668  }
5669  // gather SHARED DICTIONARYs
5670  if (additional_info.size()) {
5671  std::string comma;
5672  if (!multiline_formatting) {
5673  comma = ", ";
5674  } else {
5675  comma = ",\n ";
5676  }
5677  os << comma;
5678  os << boost::algorithm::join(additional_info, comma);
5679  }
5680  os << ")";
5681 
5682  std::vector<std::string> with_options;
5683  if (foreign_table && !td->is_system_table) {
5684  if (multiline_formatting) {
5685  os << "\n";
5686  } else {
5687  os << " ";
5688  }
5689  os << "SERVER " << foreign_table->foreign_server->name;
5690 
5691  // gather WITH options ...
5692  for (const auto& [option, value] : foreign_table->options) {
5693  with_options.emplace_back(option + "='" + value + "'");
5694  }
5695  }
5696 
5697  if (dump_defaults || td->maxFragRows != DEFAULT_FRAGMENT_ROWS) {
5698  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
5699  }
5700  if (dump_defaults || td->maxChunkSize != DEFAULT_MAX_CHUNK_SIZE) {
5701  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
5702  }
5703  if (!foreign_table && (dump_defaults || td->fragPageSize != DEFAULT_PAGE_SIZE)) {
5704  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
5705  }
5706  if (!foreign_table && (dump_defaults || td->maxRows != DEFAULT_MAX_ROWS)) {
5707  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
5708  }
5709  if ((dump_defaults || td->maxRollbackEpochs != DEFAULT_MAX_ROLLBACK_EPOCHS) &&
5710  td->maxRollbackEpochs != -1) {
5711  with_options.push_back("MAX_ROLLBACK_EPOCHS=" +
5713  }
5714  if (!foreign_table && (dump_defaults || !td->hasDeletedCol)) {
5715  with_options.emplace_back(td->hasDeletedCol ? "VACUUM='DELAYED'"
5716  : "VACUUM='IMMEDIATE'");
5717  }
5718  if (!foreign_table && !td->partitions.empty()) {
5719  with_options.push_back("PARTITIONS='" + td->partitions + "'");
5720  }
5721  if (!foreign_table && td->nShards > 0) {
5722  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
5723  CHECK(shard_cd);
5724  with_options.push_back(
5725  "SHARD_COUNT=" +
5726  std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
5727  }
5728  if (!foreign_table && td->sortedColumnId > 0) {
5729  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
5730  CHECK(sort_cd);
5731  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
5732  }
5733 
5734  if (!with_options.empty()) {
5735  if (!multiline_formatting) {
5736  os << " ";
5737  } else {
5738  os << "\n";
5739  }
5740  os << "WITH (" + boost::algorithm::join(with_options, ", ") + ")";
5741  }
5742  os << ";";
5743  return os.str();
5744 }
5745 
5746 std::string Catalog::dumpCreateServer(const std::string& name,
5747  bool multiline_formatting) const {
5748  cat_read_lock read_lock(this);
5749  auto server_it = foreignServerMap_.find(name);
5750  if (server_it == foreignServerMap_.end()) {
5751  throw std::runtime_error("Foreign server " + name + " does not exist.");
5752  }
5753  auto server = server_it->second.get();
5754  std::ostringstream os;
5755  os << "CREATE SERVER " << name << " FOREIGN DATA WRAPPER " << server->data_wrapper_type;
5756  std::vector<std::string> with_options;
5757  for (const auto& [option, value] : server->options) {
5758  with_options.emplace_back(option + "='" + value + "'");
5759  }
5760  if (!with_options.empty()) {
5761  if (!multiline_formatting) {
5762  os << " ";
5763  } else {
5764  os << "\n";
5765  }
5766  os << "WITH (" + boost::algorithm::join(with_options, ", ") + ")";
5767  }
5768  os << ";";
5769  return os.str();
5770 }
5771 
5772 bool Catalog::validateNonExistentTableOrView(const std::string& name,
5773  const bool if_not_exists) {
5774  if (getMetadataForTable(name, false)) {
5775  if (if_not_exists) {
5776  return false;
5777  }
5778  throw std::runtime_error("Table or View with name \"" + name + "\" already exists.");
5779  }
5780  return true;
5781 }
5782 
5783 std::vector<const TableDescriptor*> Catalog::getAllForeignTablesForRefresh() const {
5784  cat_read_lock read_lock(this);
5785  std::vector<const TableDescriptor*> tables;
5786  for (auto entry : tableDescriptorMapById_) {
5787  auto table_descriptor = entry.second;
5788  if (table_descriptor->storageType == StorageType::FOREIGN_TABLE) {
5789  auto foreign_table = dynamic_cast<foreign_storage::ForeignTable*>(table_descriptor);
5790  CHECK(foreign_table);
5791  auto timing_type_entry = foreign_table->options.find(
5793  CHECK(timing_type_entry != foreign_table->options.end());
5795  if (timing_type_entry->second ==
5797  foreign_table->next_refresh_time <= current_time) {
5798  tables.emplace_back(foreign_table);
5799  }
5800  }
5801  }
5802  return tables;
5803 }
5804 
5805 void Catalog::updateForeignTableRefreshTimes(const int32_t table_id) {
5806  cat_write_lock write_lock(this);
5807  cat_sqlite_lock sqlite_lock(getObjForLock());
5808  CHECK(tableDescriptorMapById_.find(table_id) != tableDescriptorMapById_.end());
5809  auto table_descriptor = tableDescriptorMapById_.find(table_id)->second;
5810  CHECK(table_descriptor);
5811  auto foreign_table = dynamic_cast<foreign_storage::ForeignTable*>(table_descriptor);
5812  CHECK(foreign_table);
5813  auto last_refresh_time = foreign_storage::RefreshTimeCalculator::getCurrentTime();
5814  auto next_refresh_time = get_next_refresh_time(*foreign_table);
5815  sqliteConnector_.query_with_text_params(
5816  "UPDATE omnisci_foreign_tables SET last_refresh_time = ?, next_refresh_time = ? "
5817  "WHERE table_id = ?",
5818  std::vector<std::string>{std::to_string(last_refresh_time),
5819  std::to_string(next_refresh_time),
5820  std::to_string(foreign_table->tableId)});
5821  foreign_table->last_refresh_time = last_refresh_time;
5822  foreign_table->next_refresh_time = next_refresh_time;
5823 }
5824 
5825 // TODO(Misiu): This function should be merged with setForeignServerOptions via
5826 // inheritance rather than replication similar functions.
5827 void Catalog::setForeignTableOptions(const std::string& table_name,
5828  foreign_storage::OptionsMap& options_map,
5829  bool clear_existing_options) {
5830  cat_write_lock write_lock(this);
5831  // update in-memory table
5832  auto foreign_table = getForeignTableUnlocked(table_name);
5833  auto saved_options = foreign_table->options;
5834  foreign_table->populateOptionsMap(std::move(options_map), clear_existing_options);
5835  try {
5836  foreign_table->validateOptionValues();
5837  } catch (const std::exception& e) {
5838  // validation did not succeed:
5839  // revert to saved options & throw exception
5840  foreign_table->options = saved_options;
5841  throw;
5842  }
5843  setForeignTableProperty(
5844  foreign_table, "options", foreign_table->getOptionsAsJsonString());
5845 }
5846 
5847 void Catalog::setForeignTableProperty(const foreign_storage::ForeignTable* table,
5848  const std::string& property,
5849  const std::string& value) {
5850  cat_sqlite_lock sqlite_lock(getObjForLock());
5851  sqliteConnector_.query_with_text_params(
5852  "SELECT table_id from omnisci_foreign_tables where table_id = ?",
5853  std::vector<std::string>{std::to_string(table->tableId)});
5854  auto num_rows = sqliteConnector_.getNumRows();
5855  if (num_rows > 0) {
5856  CHECK_EQ(size_t(1), num_rows);
5857  sqliteConnector_.query_with_text_params(
5858  "UPDATE omnisci_foreign_tables SET " + property + " = ? WHERE table_id = ?",
5859  std::vector<std::string>{value, std::to_string(table->tableId)});
5860  } else {
5861  throw std::runtime_error{"Can not change property \"" + property +
5862  "\" for foreign table." + " Foreign table \"" +
5863  table->tableName + "\" is not found."};
5864  }
5865 }
5866 
5867 std::string Catalog::quoteIfRequired(const std::string& column_name) const {
5868  if (is_reserved_sql_keyword(column_name) || contains_spaces(column_name) ||
5869  contains_sql_reserved_chars(column_name)) {
5870  return get_quoted_string(column_name, '"', '"');
5871  } else {
5872  return column_name;
5873  }
5874 }
5875 
5876 // this will gather information that represents the shared dictionary columns
5877 // as they are on the table NOW not at original creation
5878 void Catalog::gatherAdditionalInfo(std::vector<std::string>& additional_info,
5879  std::set<std::string>& shared_dict_column_names,
5880  const TableDescriptor* td) const {
5881  if (td->nShards > 0) {
5882  ColumnIdKey columnIdKey(td->tableId, td->shardedColumnId);
5883  auto scd = columnDescriptorMapById_.find(columnIdKey)->second;
5884  CHECK(scd);
5885  std::string txt = "SHARD KEY (" + quoteIfRequired(scd->columnName) + ")";
5886  additional_info.emplace_back(txt);
5887  }
5888  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
5889  for (const auto cd : cds) {
5890  if (!(cd->isSystemCol || cd->isVirtualCol)) {
5891  const SQLTypeInfo& ti = cd->columnType;
5892  if (ti.get_compression() != kENCODING_DICT) {
5893  continue;
5894  }
5895  auto dictId = ti.get_comp_param();
5896 
5897  // now we need to check how many other users have this dictionary
5898 
5899  DictRef dict_ref(currentDB_.dbId, dictId);
5900  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
5901  if (dictIt == dictDescriptorMapByRef_.end()) {
5902  LOG(ERROR) << "missing dictionary " << dictId << " for table " << td->tableName;
5903  continue;
5904  }
5905 
5906  const auto& dd = dictIt->second;
5907  if (dd->refcount > 1) {
5908  auto lowest_table = td->tableId;
5909  auto lowest_column = cd->columnId;
5910  std::string lowest_column_name;
5911  // we have multiple tables using this dictionary
5912  // find the other occurances and keep the "lowest"
5913  for (auto const& [key, val] : columnDescriptorMap_) {
5914  if (val->columnType.get_compression() == kENCODING_DICT &&
5915  val->columnType.get_comp_param() == dictId &&
5916  !(val->tableId == td->tableId && val->columnId == cd->columnId)) {
5917  if (val->tableId < lowest_table) {
5918  lowest_table = val->tableId;
5919  lowest_column = val->columnId;
5920  lowest_column_name = val->columnName;
5921  }
5922  if (val->columnId < lowest_column) {
5923  lowest_column = val->columnId;
5924  lowest_column_name = val->columnName;
5925  }
5926  }
5927  }
5928  if (lowest_table != td->tableId || lowest_column != cd->columnId) {
5929  // we are referencing a different tables dictionary
5930  auto lowest_td = tableDescriptorMapById_.find(lowest_table)->second;
5931  CHECK(lowest_td);
5932  std::string txt = "SHARED DICTIONARY (" + quoteIfRequired(cd->columnName) +
5933  ") REFERENCES " + lowest_td->tableName + "(" +
5934  quoteIfRequired(lowest_column_name) + ")";
5935 
5936  additional_info.emplace_back(txt);
5937  shared_dict_column_names.insert(cd->columnName);
5938  }
5939  }
5940  }
5941  }
5942 }
5943 
5944 int32_t Catalog::createCustomExpression(
5945  std::unique_ptr<CustomExpression> custom_expression) {
5946  cat_write_lock write_lock(this);
5947  cat_sqlite_lock sqlite_lock(getObjForLock());
5948  sqliteConnector_.query("BEGIN TRANSACTION");
5949  int32_t custom_expression_id{-1};
5950  try {
5951  auto data_source_type_str =
5952  CustomExpression::dataSourceTypeToString(custom_expression->data_source_type);
5953  auto data_source_id_str = std::to_string(custom_expression->data_source_id);
5954  std::string custom_expr_select_query{
5955  "SELECT id FROM omnisci_custom_expressions WHERE name = ? and data_source_type = "
5956  "? and data_source_id = ? and is_deleted = ?"};
5957  std::vector<std::string> custom_expr_select_params{custom_expression->name,
5958  data_source_type_str,
5959  data_source_id_str,
5960  std::to_string(false)};
5961  sqliteConnector_.query_with_text_params(custom_expr_select_query,
5962  custom_expr_select_params);
5963  if (sqliteConnector_.getNumRows() > 0) {
5964  throw std::runtime_error{
5965  "A custom expression with the given "
5966  "name and data source already exists."};
5967  }
5968  sqliteConnector_.query_with_text_params(
5969  "INSERT INTO omnisci_custom_expressions(name, expression_json, "
5970  "data_source_type, data_source_id, is_deleted) VALUES (?,?,?,?,?)",
5971  std::vector<std::string>{custom_expression->name,
5972  custom_expression->expression_json,
5973  data_source_type_str,
5974  data_source_id_str,
5975  std::to_string(false)});
5976  sqliteConnector_.query_with_text_params(custom_expr_select_query,
5977  custom_expr_select_params);
5978  CHECK_EQ(sqliteConnector_.getNumRows(), static_cast<size_t>(1));
5979  custom_expression->id = sqliteConnector_.getData<int32_t>(0, 0);
5980  custom_expression_id = custom_expression->id;
5981  CHECK(custom_expr_map_by_id_.find(custom_expression->id) ==
5982  custom_expr_map_by_id_.end());
5983  custom_expr_map_by_id_[custom_expression->id] = std::move(custom_expression);
5984  } catch (std::exception& e) {
5985  sqliteConnector_.query("ROLLBACK TRANSACTION");
5986  throw;
5987  }
5988  sqliteConnector_.query("END TRANSACTION");
5989  CHECK_GT(custom_expression_id, 0);
5990  return custom_expression_id;
5991 }
5992 
5993 const CustomExpression* Catalog::getCustomExpression(int32_t custom_expression_id) const {
5994  cat_read_lock read_lock(this);
5995  auto it = custom_expr_map_by_id_.find(custom_expression_id);
5996  if (it != custom_expr_map_by_id_.end()) {
5997  return it->second.get();
5998  }
5999  return nullptr;
6000 }
6001 
6002 const std::unique_ptr<const CustomExpression> Catalog::getCustomExpressionFromStorage(
6003  int32_t custom_expression_id) {
6004  cat_sqlite_lock sqlite_lock(getObjForLock());
6005  sqliteConnector_.query_with_text_params(
6006  "SELECT id, name, expression_json, data_source_type, data_source_id, "
6007  "is_deleted FROM omnisci_custom_expressions WHERE id = ?",
6008  std::vector<std::string>{to_string(custom_expression_id)});
6009  if (sqliteConnector_.getNumRows() > 0) {
6010  CHECK_EQ(sqliteConnector_.getNumRows(), static_cast<size_t>(1));
6011  return getCustomExpressionFromConnector(0);
6012  }
6013  return nullptr;
6014 }
6015 
6016 std::vector<const CustomExpression*> Catalog::getCustomExpressionsForUser(
6017  const UserMetadata& user) const {
6018  std::vector<const CustomExpression*> all_custom_expressions;
6019  {
6020  // Get custom expression pointers separately in order to avoid holding the catalog
6021  // read lock while checking privileges (which may cause a deadlock).
6022  cat_read_lock read_lock(this);
6023  for (const auto& [id, custom_expression] : custom_expr_map_by_id_) {
6024  all_custom_expressions.emplace_back(custom_expression.get());
6025  }
6026  }
6027 
6028  std::vector<const CustomExpression*> filtered_custom_expressions;
6029  for (const auto custom_expression : all_custom_expressions) {
6030  CHECK(custom_expression->data_source_type == DataSourceType::TABLE);
6031  DBObject db_object{custom_expression->data_source_id, TableDBObjectType};
6032  db_object.loadKey(*this);
6033  db_object.setPrivileges(AccessPrivileges::SELECT_FROM_TABLE);
6034  if (SysCatalog::instance().checkPrivileges(user, {db_object})) {
6035  filtered_custom_expressions.emplace_back(custom_expression);
6036  }
6037  }
6038  return filtered_custom_expressions;
6039 }
6040 
6041 void Catalog::updateCustomExpression(int32_t custom_expression_id,
6042  const std::string& expression_json) {
6043  cat_write_lock write_lock(this);
6044  cat_sqlite_lock sqlite_lock(getObjForLock());
6045  auto it = custom_expr_map_by_id_.find(custom_expression_id);
6046  if (it == custom_expr_map_by_id_.end() || it->second->is_deleted) {
6047  throw std::runtime_error{"Custom expression with id \"" +
6048  std::to_string(custom_expression_id) + "\" does not exist."};
6049  }
6050  auto old_expression_json = it->second->expression_json;
6051  sqliteConnector_.query("BEGIN TRANSACTION");
6052  try {
6053  sqliteConnector_.query_with_text_params(
6054  "SELECT id FROM omnisci_custom_expressions WHERE id = ?",
6055  std::vector<std::string>{std::to_string(custom_expression_id)});
6056  CHECK_EQ(sqliteConnector_.getNumRows(), static_cast<size_t>(1));
6057  sqliteConnector_.query_with_text_params(
6058  "UPDATE omnisci_custom_expressions SET expression_json = ? WHERE id = ?",
6059  std::vector<std::string>{expression_json, std::to_string(custom_expression_id)});
6060  it->second->expression_json = expression_json;
6061  } catch (std::exception& e) {
6062  sqliteConnector_.query("ROLLBACK TRANSACTION");
6063  it->second->expression_json = old_expression_json;
6064  throw;
6065  }
6066  sqliteConnector_.query("END TRANSACTION");
6067 }
6068 
6069 void Catalog::deleteCustomExpressions(const std::vector<int32_t>& custom_expression_ids,
6070  bool do_soft_delete) {
6071  cat_write_lock write_lock(this);
6072  cat_sqlite_lock sqlite_lock(getObjForLock());
6073 
6074  std::vector<int32_t> invalid_ids;
6075  for (const auto id : custom_expression_ids) {
6076  if (custom_expr_map_by_id_.find(id) == custom_expr_map_by_id_.end()) {
6077  invalid_ids.emplace_back(id);
6078  }
6079  }
6080  if (!invalid_ids.empty()) {
6081  throw std::runtime_error{"Custom expressions with ids: " + join(invalid_ids, ",") +
6082  " do not exist."};
6083  }
6084  sqliteConnector_.query("BEGIN TRANSACTION");
6085  try {
6086  for (const auto id : custom_expression_ids) {
6087  sqliteConnector_.query_with_text_params(
6088  "SELECT id FROM omnisci_custom_expressions WHERE id = ?",
6089  std::vector<std::string>{std::to_string(id)});
6090  CHECK_EQ(sqliteConnector_.getNumRows(), static_cast<size_t>(1));
6091  if (do_soft_delete) {
6092  sqliteConnector_.query_with_text_params(
6093  "UPDATE omnisci_custom_expressions SET is_deleted = ? WHERE id = ?",
6094  std::vector<std::string>{std::to_string(true), std::to_string(id)});
6095  } else {
6096  sqliteConnector_.query_with_text_params(
6097  "DELETE FROM omnisci_custom_expressions WHERE id = ?",
6098  std::vector<std::string>{std::to_string(id)});
6099  }
6100  }
6101 
6102  for (const auto id : custom_expression_ids) {
6103  if (do_soft_delete) {
6104  auto it = custom_expr_map_by_id_.find(id);
6105  CHECK(it != custom_expr_map_by_id_.end());
6106  it->second->is_deleted = true;
6107  } else {
6108  custom_expr_map_by_id_.erase(id);
6109  }
6110  }
6111  } catch (std::exception& e) {
6112  sqliteConnector_.query("ROLLBACK TRANSACTION");
6113  throw;
6114  }
6115  sqliteConnector_.query("END TRANSACTION");
6116 }
6117 
6118 namespace {
6119 int32_t validate_and_get_user_id(const std::string& user_name) {
6120  UserMetadata user;
6121  if (!SysCatalog::instance().getMetadataForUser(user_name, user)) {
6122  throw std::runtime_error{"User with username \"" + user_name + "\" does not exist."};
6123  }
6124  return user.userId;
6125 }
6126 
6128  int32_t db_id,
6129  int32_t new_owner_id,
6130  const std::map<int32_t, std::vector<DBObject>>& old_owner_db_objects) {
6131  std::stringstream result;
6132  for (const auto& [old_owner_id, db_objects] : old_owner_db_objects) {
6133  result << "db_id: " << db_id << ", new_owner_user_id: " << new_owner_id
6134  << ", old_owner_user_id: " << old_owner_id << ", db_objects: [";
6135  bool first_object{true};
6136  for (const auto& db_object : db_objects) {
6137  if (first_object) {
6138  first_object = false;
6139  } else {
6140  result << ", ";
6141  }
6142  result << "\"object_id: " << db_object.getObjectKey().objectId
6143  << ", object_type: " << DBObjectTypeToString(db_object.getType()) << "\"";
6144  }
6145  result << "]\n";
6146  }
6147  return result.str();
6148 }
6149 
6150 void add_db_object(const std::string& object_name,
6151  DBObjectType object_type,
6152  int32_t user_id,
6153  const AccessPrivileges& privileges,
6154  std::map<int32_t, std::vector<DBObject>>& db_objects) {
6155  DBObject db_object{object_name, object_type};
6156  db_object.setPrivileges(privileges);
6157  db_objects[user_id].emplace_back(db_object);
6158 }
6159 } // namespace
6160 
6161 void Catalog::reassignOwners(const std::set<std::string>& old_owners,
6162  const std::string& new_owner) {
6163  CHECK(!old_owners.empty());
6164  int32_t new_owner_id = validate_and_get_user_id(new_owner);
6165  std::map<int32_t, std::string> old_owners_user_name_by_id;
6166  std::set<int32_t> old_owner_ids;
6167  for (const auto& old_owner : old_owners) {
6168  auto old_owner_id = validate_and_get_user_id(old_owner);
6169  if (old_owner_id != new_owner_id) {
6170  old_owner_ids.emplace(old_owner_id);
6171  old_owners_user_name_by_id[old_owner_id] = old_owner;
6172  }
6173  }
6174 
6175  // An empty set after the above loop implies reassignment to the same user (i.e. all
6176  // users in the old_owners set is the same as new_owner). Do nothing in this case.
6177  if (old_owner_ids.empty()) {
6178  return;
6179  }
6180 
6181  std::map<int32_t, std::vector<DBObject>> old_owner_db_objects;
6182  {
6183  cat_write_lock write_lock(this);
6184  cat_sqlite_lock sqlite_lock(getObjForLock());
6185  sqliteConnector_.query("BEGIN TRANSACTION");
6186  try {
6187  for (const auto old_user_id : old_owner_ids) {
6188  sqliteConnector_.query_with_text_params(
6189  "UPDATE mapd_tables SET userid = ? WHERE userid = ?",
6190  std::vector<std::string>{std::to_string(new_owner_id),
6191  std::to_string(old_user_id)});
6192 
6193  sqliteConnector_.query_with_text_params(
6194  "UPDATE mapd_dashboards SET userid = ? WHERE userid = ?",
6195  std::vector<std::string>{std::to_string(new_owner_id),
6196  std::to_string(old_user_id)});
6197 
6198  if (g_enable_fsi) {
6199  sqliteConnector_.query_with_text_params(
6200  "UPDATE omnisci_foreign_servers SET owner_user_id = ? "
6201  "WHERE owner_user_id = ?",
6202  std::vector<std::string>{std::to_string(new_owner_id),
6203  std::to_string(old_user_id)});
6204  }
6205  }
6206 
6207  for (const auto& [table_name, td] : tableDescriptorMap_) {
6208  if (shared::contains(old_owner_ids, td->userId)) {
6209  if (td->isView) {
6210  add_db_object(td->tableName,
6212  td->userId,
6214  old_owner_db_objects);
6215  } else {
6216  add_db_object(td->tableName,
6218  td->userId,
6220  old_owner_db_objects);
6221  }
6222  td->userId = new_owner_id;
6223  }
6224  }
6225 
6226  DashboardDescriptorMap new_owner_dashboard_map;
6227  for (auto it = dashboardDescriptorMap_.begin();
6228  it != dashboardDescriptorMap_.end();) {
6229  if (auto dashboard = it->second;
6230  shared::contains(old_owner_ids, dashboard->userId)) {
6231  DBObject db_object{dashboard->dashboardId, DBObjectType::DashboardDBObjectType};
6232  db_object.setPrivileges(AccessPrivileges::ALL_DASHBOARD);
6233  old_owner_db_objects[dashboard->userId].emplace_back(db_object);
6234 
6235  // Dashboards in the dashboardDescriptorMap_ use keys with the format
6236  // "{user id}:{dashboard name}". Ensure that map entries are replaced
6237  // with the new owner's user id.
6238  std::string old_key{std::to_string(dashboard->userId) + ":" +
6239  dashboard->dashboardName};
6240  CHECK_EQ(it->first, old_key);
6241  std::string new_key{std::to_string(new_owner_id) + ":" +
6242  dashboard->dashboardName};
6243  CHECK(dashboardDescriptorMap_.find(new_key) == dashboardDescriptorMap_.end());
6244  new_owner_dashboard_map[new_key] = dashboard;
6245  dashboard->userId = new_owner_id;
6246  dashboard->user = new_owner;
6247  it = dashboardDescriptorMap_.erase(it);
6248  } else {
6249  it++;
6250  }
6251  }
6252  dashboardDescriptorMap_.merge(new_owner_dashboard_map);
6253 
6254  if (g_enable_fsi) {
6255  for (const auto& [server_name, server] : foreignServerMap_) {
6256  if (shared::contains(old_owner_ids, server->user_id)) {
6257  add_db_object(server->name,
6259  server->user_id,
6261  old_owner_db_objects);
6262  server->user_id = new_owner_id;
6263  }
6264  }
6265  }
6266 
6267  // Ensure new owner is set in the DB objects.
6268  for (auto& [old_owner_id, db_objects] : old_owner_db_objects) {
6269  for (auto& db_object : db_objects) {
6270  db_object.loadKey(*this);
6271  CHECK_EQ(db_object.getOwner(), new_owner_id);
6272  const auto& object_key = db_object.getObjectKey();
6273  CHECK_EQ(object_key.dbId, getDatabaseId());
6274  CHECK_NE(object_key.objectId, -1);
6275  }
6276  }
6277  } catch (std::exception& e) {
6278  sqliteConnector_.query("ROLLBACK TRANSACTION");
6279  restoreOldOwnersInMemory(
6280  old_owners_user_name_by_id, old_owner_db_objects, new_owner_id);
6281  throw;
6282  }
6283  sqliteConnector_.query("END TRANSACTION");
6284  }
6285 
6286  try {
6287  SysCatalog::instance().reassignObjectOwners(
6288  old_owner_db_objects, new_owner_id, *this);
6289  } catch (std::exception& e) {
6290  restoreOldOwners(old_owners_user_name_by_id, old_owner_db_objects, new_owner_id);
6291  throw;
6292  }
6293 }
6294 
6295 void Catalog::restoreOldOwners(
6296  const std::map<int32_t, std::string>& old_owners_user_name_by_id,
6297  const std::map<int32_t, std::vector<DBObject>>& old_owner_db_objects,
6298  int32_t new_owner_id) {
6299  cat_write_lock write_lock(this);
6300  cat_sqlite_lock sqlite_lock(getObjForLock());
6301  sqliteConnector_.query("BEGIN TRANSACTION");
6302  try {
6303  for (const auto& [old_owner_id, db_objects] : old_owner_db_objects) {
6304  for (const auto& db_object : db_objects) {
6305  auto object_id = db_object.getObjectKey().objectId;
6306  CHECK_GT(object_id, 0);
6307  std::vector<std::string> query_params{std::to_string(old_owner_id),
6308  std::to_string(new_owner_id),
6309  std::to_string(object_id)};
6310  auto object_type = db_object.getType();
6311  if (object_type == DBObjectType::TableDBObjectType ||
6312  object_type == DBObjectType::ViewDBObjectType) {
6313  sqliteConnector_.query_with_text_params(
6314  "UPDATE mapd_tables SET userid = ? WHERE userid = ? AND tableid = ?",
6315  query_params);
6316  } else if (object_type == DBObjectType::DashboardDBObjectType) {
6317  sqliteConnector_.query_with_text_params(
6318  "UPDATE mapd_dashboards SET userid = ? WHERE userid = ? AND id = ?",
6319  query_params);
6320  } else if (object_type == DBObjectType::ServerDBObjectType) {
6322  sqliteConnector_.query_with_text_params(
6323  "UPDATE omnisci_foreign_servers SET owner_user_id = ? "
6324  "WHERE owner_user_id = ? AND id = ?",
6325  query_params);
6326  } else {
6327  UNREACHABLE() << "Unexpected DB object type: " << static_cast<int>(object_type);
6328  }
6329  }
6330  }
6331  restoreOldOwnersInMemory(
6332  old_owners_user_name_by_id, old_owner_db_objects, new_owner_id);
6333  } catch (std::exception& e) {
6334  sqliteConnector_.query("ROLLBACK TRANSACTION");
6335  LOG(FATAL)
6336  << "Unable to restore database objects ownership after an error occurred. "
6337  "Database object ownership information may be in an inconsistent state. " +
6339  getDatabaseId(), new_owner_id, old_owner_db_objects);
6340  }
6341  sqliteConnector_.query("END TRANSACTION");
6342 }
6343 
6344 void Catalog::restoreOldOwnersInMemory(
6345  const std::map<int32_t, std::string>& old_owners_user_name_by_id,
6346  const std::map<int32_t, std::vector<DBObject>>& old_owner_db_objects,
6347  int32_t new_owner_id) {
6348  for (const auto& [old_owner_id, db_objects] : old_owner_db_objects) {
6349  for (const auto& db_object : db_objects) {
6350  auto object_id = db_object.getObjectKey().objectId;
6351  auto object_type = db_object.getType();
6352  if (object_type == DBObjectType::TableDBObjectType ||
6353  object_type == DBObjectType::ViewDBObjectType) {
6354  auto it = tableDescriptorMapById_.find(object_id);
6355  CHECK(it != tableDescriptorMapById_.end());
6356  CHECK(it->second);
6357  it->second->userId = old_owner_id;
6358  } else if (object_type == DBObjectType::DashboardDBObjectType) {
6359  auto it = dashboardDescriptorMap_.find(std::to_string(new_owner_id) + ":" +
6360  db_object.getName());
6361  CHECK(it != dashboardDescriptorMap_.end());
6362  CHECK(it->second);
6363  it->second->userId = old_owner_id;
6364  auto user_name_it = old_owners_user_name_by_id.find(old_owner_id);
6365  CHECK(user_name_it != old_owners_user_name_by_id.end());
6366  it->second->user = user_name_it->second;
6367  dashboardDescriptorMap_[std::to_string(old_owner_id) + ":" +
6368  db_object.getName()] = it->second;
6369  dashboardDescriptorMap_.erase(it);
6370  } else if (object_type == DBObjectType::ServerDBObjectType) {
6371  auto it = foreignServerMapById_.find(object_id);
6372  CHECK(it != foreignServerMapById_.end());
6373  CHECK(it->second);
6374  it->second->user_id = old_owner_id;
6375  } else {
6376  UNREACHABLE() << "Unexpected DB object type: " << static_cast<int>(object_type);
6377  }
6378  }
6379  }
6380 }
6381 
6382 void Catalog::conditionallyInitializeSystemObjects() {
6383  if (g_enable_system_tables && isInfoSchemaDb()) {
6384  initializeSystemServers();
6385  initializeSystemTables();
6386  }
6387 }
6388 
6389 bool Catalog::isInfoSchemaDb() const {
6390  return name() == shared::kInfoSchemaDbName;
6391 }
6392 
6393 void Catalog::initializeSystemServers() {
6394  createSystemTableServer(CATALOG_SERVER_NAME,
6396  createSystemTableServer(MEMORY_STATS_SERVER_NAME,
6398  createSystemTableServer(STORAGE_STATS_SERVER_NAME,
6400  createSystemTableServer(EXECUTOR_STATS_SERVER_NAME,
6402  createSystemTableServer(ML_METADATA_SERVER_NAME,
6404 
6406  foreign_storage::OptionsMap log_server_options;
6407  log_server_options
6411  logger::get_log_dir_path().string();
6412  createSystemTableServer(LOGS_SERVER_NAME,
6414  log_server_options);
6415  }
6416 }
6417 
6418 namespace {
6420  return {kTEXT, 0, 0, false, kENCODING_DICT, 32, kNULLT};
6421 }
6422 
6424  SQLTypeInfo sql_type_info{kARRAY, 0, 0, false, kENCODING_NONE, 0, type};
6425  sql_type_info.set_size(-1);
6426  return sql_type_info;
6427 }
6428 
6430  auto sql_type_info = get_var_array_type(kTEXT);
6431  sql_type_info.set_compression(kENCODING_DICT);
6432  sql_type_info.set_comp_param(32);
6433  return sql_type_info;
6434 }
6435 
6438  foreign_table.options[ForeignTable::REFRESH_TIMING_TYPE_KEY] =
6439  ForeignTable::MANUAL_REFRESH_TIMING_TYPE;
6440  foreign_table.options[ForeignTable::REFRESH_UPDATE_TYPE_KEY] =
6441  ForeignTable::APPEND_REFRESH_UPDATE_TYPE;
6443  foreign_table.options[AbstractFileStorageDataWrapper::ALLOW_FILE_ROLL_OFF_KEY] = "TRUE";
6444 }
6445 
6447  foreign_storage::ForeignTable& foreign_table) {
6448  // Each log entry should start with a timestamp.
6450  foreign_table.options[RegexFileBufferParser::LINE_START_REGEX_KEY] =
6451  "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{6}";
6453  foreign_table.options[AbstractFileStorageDataWrapper::REGEX_PATH_FILTER_KEY] =
6454  ".*heavydb\\.INFO\\..*";
6455  set_common_log_system_table_options(foreign_table);
6456 }
6457 
6459  int32_t db_id,
6460  int32_t table_id) {
6461  auto cache = data_mgr->getPersistentStorageMgr()->getDiskCache();
6462  if (cache) {
6463  cache->clearForTablePrefix({db_id, table_id});
6464  }
6465 }
6466 
6467 void drop_tables(Catalog& catalog, const std::vector<std::string>& table_names) {
6468  for (const auto& table_name : table_names) {
6469  if (auto td = catalog.getMetadataForTable(table_name)) {
6470  catalog.dropTable(td);
6471  }
6472  }
6473 }
6474 } // namespace
6475 
6476 void Catalog::initializeSystemTables() {
6477  initializeUsersSystemTable();
6478  initializeDatabasesSystemTable();
6479  initializePermissionsSystemTable();
6480  initializeRolesSystemTable();
6481  initializeTablesSystemTable();
6482  initializeDashboardsSystemTable();
6483  initializeRoleAssignmentsSystemTable();
6484  initializeMemorySummarySystemTable();
6485  initializeMemoryDetailsSystemTable();
6486  initializeStorageDetailsSystemTable();
6487  initializeExecutorResourcePoolSummarySystemTable();
6488  initializeMLModelMetadataSystemTable();
6489 
6491  initializeServerLogsSystemTables();
6492  initializeRequestLogsSystemTables();
6493  } else {
6494  drop_tables(*this,
6499  if (getForeignServer(LOGS_SERVER_NAME)) {
6500  dropForeignServer(LOGS_SERVER_NAME);
6501  }
6502  }
6503 }
6504 
6505 void Catalog::initializeUsersSystemTable() {
6506  auto [foreign_table, columns] =
6507  getSystemTableSchema(USERS_SYS_TABLE_NAME,
6508  CATALOG_SERVER_NAME,
6509  {{"user_id", {kINT}},
6510  {"user_name", get_encoded_text_type()},
6511  {"is_super_user", {kBOOLEAN}},
6512  {"default_db_id", {kINT}},
6513  {"default_db_name", get_encoded_text_type()},
6514  {"can_login", {kBOOLEAN}}},
6515  true);
6516  recreateSystemTableIfUpdated(foreign_table, columns);
6517 }
6518 
6519 void Catalog::initializeDatabasesSystemTable() {
6520  auto [foreign_table, columns] =
6521  getSystemTableSchema(DATABASES_SYS_TABLE_NAME,
6522  CATALOG_SERVER_NAME,
6523  {{"database_id", {kINT}},
6524  {"database_name", get_encoded_text_type()},
6525  {"owner_id", {kINT}},
6526  {"owner_user_name", get_encoded_text_type()}},
6527  true);
6528  recreateSystemTableIfUpdated(foreign_table, columns);
6529 }
6530 
6531 void Catalog::initializePermissionsSystemTable() {
6532  auto [foreign_table, columns] =
6533  getSystemTableSchema(PERMISSIONS_SYS_TABLE_NAME,
6534  CATALOG_SERVER_NAME,
6535  {{"role_name", get_encoded_text_type()},
6536  {"is_user_role", {kBOOLEAN}},
6537  {"database_id", {kINT}},
6538  {"database_name", get_encoded_text_type()},
6539  {"object_name", get_encoded_text_type()},
6540  {"object_id", {kINT}},
6541  {"object_owner_id", {kINT}},
6542  {"object_owner_user_name", get_encoded_text_type()},
6543  {"object_permission_type", get_encoded_text_type()},
6544  {"object_permissions", get_var_encoded_text_array_type()}},
6545  true);
6546  recreateSystemTableIfUpdated(foreign_table, columns);
6547 }
6548 
6549 void Catalog::initializeRolesSystemTable() {
6550  auto [foreign_table, columns] =
6551  getSystemTableSchema(ROLES_SYS_TABLE_NAME,
6552  CATALOG_SERVER_NAME,
6553  {{"role_name", get_encoded_text_type()}},
6554  true);
6555  recreateSystemTableIfUpdated(foreign_table, columns);
6556 }
6557 
6558 void Catalog::initializeTablesSystemTable() {
6559  auto [foreign_table, columns] =
6560  getSystemTableSchema(TABLES_SYS_TABLE_NAME,
6561  CATALOG_SERVER_NAME,
6562  {{"database_id", {kINT}},
6563  {"database_name", get_encoded_text_type()},
6564  {"table_id", {kINT}},
6565  {"table_name", get_encoded_text_type()},
6566  {"owner_id", {kINT}},
6567  {"owner_user_name", get_encoded_text_type()},
6568  {"column_count", {kINT}},
6569  {"table_type", get_encoded_text_type()},
6570  {"view_sql", get_encoded_text_type()},
6571  {"max_fragment_size", {kINT}},
6572  {"max_chunk_size", {kBIGINT}},
6573  {"fragment_page_size", {kINT}},
6574  {"max_rows", {kBIGINT}},
6575  {"max_rollback_epochs", {kINT}},
6576  {"shard_count", {kINT}},
6577  {"ddl_statement", get_encoded_text_type()}},
6578  true);
6579  recreateSystemTableIfUpdated(foreign_table, columns);
6580 }
6581 
6582 void Catalog::initializeDashboardsSystemTable() {
6583  auto [foreign_table, columns] =
6584  getSystemTableSchema(DASHBOARDS_SYS_TABLE_NAME,
6585  CATALOG_SERVER_NAME,
6586  {{"database_id", {kINT}},
6587  {"database_name", get_encoded_text_type()},
6588  {"dashboard_id", {kINT}},
6589  {"dashboard_name", get_encoded_text_type()},
6590  {"owner_id", {kINT}},
6591  {"owner_user_name", get_encoded_text_type()},
6592  {"last_updated_at", {kTIMESTAMP}},
6593  {"data_sources", get_var_encoded_text_array_type()}},
6594  true);
6595  recreateSystemTableIfUpdated(foreign_table, columns);
6596 }
6597 
6598 void Catalog::initializeRoleAssignmentsSystemTable() {
6599  auto [foreign_table, columns] = getSystemTableSchema(
6601  CATALOG_SERVER_NAME,
6602  {{"role_name", get_encoded_text_type()}, {"user_name", get_encoded_text_type()}},
6603  true);
6604  recreateSystemTableIfUpdated(foreign_table, columns);
6605 }
6606 
6607 void Catalog::initializeMemorySummarySystemTable() {
6608  auto [foreign_table, columns] =
6609  getSystemTableSchema(MEMORY_SUMMARY_SYS_TABLE_NAME,
6610  MEMORY_STATS_SERVER_NAME,
6611  {{"node", get_encoded_text_type()},
6612  {"device_id", {kINT}},
6613  {"device_type", get_encoded_text_type()},
6614  {"max_page_count", {kBIGINT}},
6615  {"page_size", {kBIGINT}},
6616  {"allocated_page_count", {kBIGINT}},
6617  {"used_page_count", {kBIGINT}},
6618  {"free_page_count", {kBIGINT}}},
6619  true);
6620  recreateSystemTableIfUpdated(foreign_table, columns);
6621 }
6622 
6623 void Catalog::initializeMemoryDetailsSystemTable() {
6624  auto [foreign_table, columns] =
6625  getSystemTableSchema(MEMORY_DETAILS_SYS_TABLE_NAME,
6626  MEMORY_STATS_SERVER_NAME,
6627  {{"node", get_encoded_text_type()},
6628  {"database_id", {kINT}},
6629  {"database_name", get_encoded_text_type()},
6630  {"table_id", {kINT}},
6631  {"table_name", get_encoded_text_type()},
6632  {"column_id", {kINT}},
6633  {"column_name", get_encoded_text_type()},
6634  {"chunk_key", get_var_array_type(kINT)},
6635  {"device_id", {kINT}},
6636  {"device_type", get_encoded_text_type()},
6637  {"memory_status", get_encoded_text_type()},
6638  {"page_count", {kBIGINT}},
6639  {"page_size", {kBIGINT}},
6640  {"slab_id", {kINT}},
6641  {"start_page", {kBIGINT}},
6642  {"last_touch_epoch", {kBIGINT}}},
6643  true);
6644  recreateSystemTableIfUpdated(foreign_table, columns);
6645 }
6646 
6647 void Catalog::initializeStorageDetailsSystemTable() {
6648  auto [foreign_table, columns] =
6649  getSystemTableSchema(STORAGE_DETAILS_SYS_TABLE_NAME,
6650  STORAGE_STATS_SERVER_NAME,
6651  {{"node", get_encoded_text_type()},
6652  {"database_id", {kINT}},
6653  {"database_name", get_encoded_text_type()},
6654  {"table_id", {kINT}},
6655  {"table_name", get_encoded_text_type()},
6656  {"epoch", {kINT}},
6657  {"epoch_floor", {kINT}},
6658  {"fragment_count", {kINT}},
6659  {"shard_id", {kINT}},
6660  {"data_file_count", {kINT}},
6661  {"metadata_file_count", {kINT}},
6662  {"total_data_file_size", {kBIGINT}},
6663  {"total_data_page_count", {kBIGINT}},
6664  {"total_free_data_page_count", {kBIGINT}},
6665  {"total_metadata_file_size", {kBIGINT}},
6666  {"total_metadata_page_count", {kBIGINT}},
6667  {"total_free_metadata_page_count", {kBIGINT}},
6668  {"total_dictionary_data_file_size", {kBIGINT}}},
6669  true);
6670  recreateSystemTableIfUpdated(foreign_table, columns);
6671 }
6672 
6673 void Catalog::initializeExecutorResourcePoolSummarySystemTable() {
6674  auto [foreign_table, columns] =
6675  getSystemTableSchema(EXECUTOR_RESOURCE_POOL_SUMMARY_SYS_TABLE_NAME,
6676  EXECUTOR_STATS_SERVER_NAME,
6677  {{"total_cpu_slots", {kINT}},
6678  {"total_gpu_slots", {kINT}},
6679  {"total_cpu_result_mem", {kBIGINT}},
6680  {"total_cpu_buffer_pool_mem", {kBIGINT}},
6681  {"total_gpu_buffer_pool_mem", {kBIGINT}},
6682  {"allocated_cpu_slots", {kINT}},
6683  {"allocated_gpu_slots", {kINT}},
6684  {"allocated_cpu_result_mem", {kBIGINT}},
6685  {"allocated_cpu_buffer_pool_mem", {kBIGINT}},
6686  {"allocated_gpu_buffer_pool_mem", {kBIGINT}},
6687  {"allocated_cpu_buffers", {kINT}},
6688  {"allocated_gpu_buffers", {kINT}},
6689  {"allocated_temp_cpu_buffer_pool_mem", {kBIGINT}},
6690  {"allocated_temp_gpu_buffer_pool_mem", {kBIGINT}},
6691  {"total_requests", {kBIGINT}},
6692  {"outstanding_requests", {kINT}},
6693  {"outstanding_cpu_slots_requests", {kINT}},
6694  {"outstanding_gpu_slots_requests", {kINT}},
6695  {"outstanding_cpu_result_mem_requests", {kINT}},
6696  {"outstanding_cpu_buffer_pool_mem_requests", {kINT}},
6697  {"outstanding_gpu_buffer_pool_mem_requests", {kINT}}},
6698  true);
6699  recreateSystemTableIfUpdated(foreign_table, columns);
6700 }
6701 
6702 void Catalog::initializeMLModelMetadataSystemTable() {
6703  auto [foreign_table, columns] =
6704  getSystemTableSchema(ML_MODEL_METADATA_SYS_TABLE_NAME,
6705  ML_METADATA_SERVER_NAME,
6706  {{"model_name", get_encoded_text_type()},
6707  {"model_type", get_encoded_text_type()},
6708  {"predicted", get_encoded_text_type()},
6709  {"features", get_var_encoded_text_array_type()},
6710  {"training_query", get_encoded_text_type()},
6711  {"num_logical_features", {kBIGINT}},
6712  {"num_physical_features", {kBIGINT}},
6713  {"num_categorical_features", {kBIGINT}},
6714  {"num_numeric_features", {kBIGINT}},
6715  {"train_fraction", {kDOUBLE}},
6716  {"eval_fraction", {kDOUBLE}}},
6717  true);
6718  recreateSystemTableIfUpdated(foreign_table, columns);
6719 }
6720 
6721 void Catalog::initializeServerLogsSystemTables() {
6722  auto [foreign_table, columns] =
6723  getSystemTableSchema(SERVER_LOGS_SYS_TABLE_NAME,
6724  LOGS_SERVER_NAME,
6725  {{"node", get_encoded_text_type()},
6726  {"log_timestamp", {kTIMESTAMP}},
6727  {"severity", get_encoded_text_type()},
6728  {"process_id", {kINT}},
6729  {"query_id", {kINT}},
6730  {"thread_id", {kINT}},
6731  {"file_location", get_encoded_text_type()},
6732  {"message", get_encoded_text_type()}},
6733  false);
6736  // Matches server logs like those seen in the "heavydb.INFO.20220518-210103.log" test
6737  // file.
6738  foreign_table.options[RegexFileBufferParser::LINE_REGEX_KEY] =
6739  "^([^\\s]+)\\s(\\w)\\s(\\d+)\\s(\\d+)\\s(\\d+)\\s([^\\s]+)\\s(.+)$";
6740  if (recreateSystemTableIfUpdated(foreign_table, columns)) {
6741  // Clear table cache if the table schema is updated
6742  clear_cached_table_data(dataMgr_.get(), currentDB_.dbId, foreign_table.tableId);
6743  }
6744 }
6745 
6746 void Catalog::initializeRequestLogsSystemTables() {
6747  auto [foreign_table, columns] =
6748  getSystemTableSchema(REQUEST_LOGS_SYS_TABLE_NAME,
6749  LOGS_SERVER_NAME,
6750  {{"log_timestamp", {kTIMESTAMP}},
6751  {"severity", get_encoded_text_type()},
6752  {"process_id", {kINT}},
6753  {"query_id", {kINT}},
6754  {"thread_id", {kINT}},
6755  {"file_location", get_encoded_text_type()},
6756  {"api_name", get_encoded_text_type()},
6757  {"request_duration_ms", {kBIGINT}},
6758  {"database_name", get_encoded_text_type()},
6759  {"user_name", get_encoded_text_type()},
6760  {"public_session_id", get_encoded_text_type()},
6761  {"query_string", get_encoded_text_type()},
6762  {"client", get_encoded_text_type()},
6763  {"dashboard_id", {kINT}},
6764  {"dashboard_name", get_encoded_text_type()},
6765  {"chart_id", {kINT}},
6766  {"execution_time_ms", {kBIGINT}},
6767  {"total_time_ms", {kBIGINT}}},
6768  false);
6771  // Matches request logs like those seen in the "heavydb.INFO.20220518-210103.log" test
6772  // file (specifically, lines containing " stdlog ").
6773  foreign_table.options[RegexFileBufferParser::LINE_REGEX_KEY] =
6774  "^([^\\s]+)\\s(\\w)\\s(\\d+)\\s(\\d+)\\s(\\d+)\\s([^\\s]+)\\s(?:stdlog)\\s(\\w+)"
6775  "\\s(?:\\d+)\\s(\\d+)\\s(\\w+)\\s([^\\s]+)\\s([^\\s]+)\\s(\\{[^\\}]+\\})\\s(\\{[^"
6776  "\\}]+\\})$";
6777  if (recreateSystemTableIfUpdated(foreign_table, columns)) {
6778  // Clear table cache if the table schema is updated
6779  clear_cached_table_data(dataMgr_.get(), currentDB_.dbId, foreign_table.tableId);
6780  }
6781 }
6782 
6783 void Catalog::initializeWebServerLogsSystemTables() {
6784  auto [foreign_table, columns] =
6785  getSystemTableSchema(WS_SERVER_LOGS_SYS_TABLE_NAME,
6786  LOGS_SERVER_NAME,
6787  {{"log_timestamp", {kTIMESTAMP}},
6788  {"severity", get_encoded_text_type()},
6789  {"message", get_encoded_text_type()}},
6790  false);
6791  set_common_log_system_table_options(foreign_table);
6793  foreign_table.options[AbstractFileStorageDataWrapper::REGEX_PATH_FILTER_KEY] =
6794  ".*heavy_web_server.*ALL\\..*";
6796  // Matches web server logs like those seen in the
6797  // "heavy_web_server.test.log.ALL.20220518-210103.307016" test file.
6798  foreign_table.options[RegexFileBufferParser::LINE_REGEX_KEY] =
6799  "^time=\"([^\"]+)\"\\slevel=([^\\s]+)\\smsg=\"([^\"]+)\"$";
6800  if (recreateSystemTableIfUpdated(foreign_table, columns)) {
6801  // Clear table cache if the table schema is updated
6802  clear_cached_table_data(dataMgr_.get(), currentDB_.dbId, foreign_table.tableId);
6803  }
6804 }
6805 
6806 void Catalog::initializeWebServerAccessLogsSystemTables() {
6807  auto [foreign_table, columns] =
6808  getSystemTableSchema(WS_SERVER_ACCESS_LOGS_SYS_TABLE_NAME,
6809  LOGS_SERVER_NAME,
6810  {{"ip_address", get_encoded_text_type()},
6811  {"log_timestamp", {kTIMESTAMP}},
6812  {"http_method", get_encoded_text_type()},
6813  {"endpoint", get_encoded_text_type()},
6814  {"http_status", {kSMALLINT}},
6815  {"response_size", {kBIGINT}}},
6816  false);
6817  set_common_log_system_table_options(foreign_table);
6819  foreign_table.options[AbstractFileStorageDataWrapper::REGEX_PATH_FILTER_KEY] =
6820  ".*heavy_web_server.*ACCESS\\..*";
6822  // Matches web server access logs like those seen in the
6823  // "heavy_web_server.test.log.ACCESS.20220518-210103.307016" test file.
6824  foreign_table.options[RegexFileBufferParser::LINE_REGEX_KEY] =
6825  "^(\\d+\\.\\d+\\.\\d+\\.\\d+)\\s+\\-\\s+\\-\\s+\\[([^\\]]+)\\]\\s+\"(\\w+)\\s+([^"
6826  "\\s]+)\\s+HTTP\\/1\\.1\"\\s+(\\d+)\\s+(\\d+)$";
6827  if (recreateSystemTableIfUpdated(foreign_table, columns)) {
6828  // Clear table cache if the table schema is updated
6829  clear_cached_table_data(dataMgr_.get(), currentDB_.dbId, foreign_table.tableId);
6830  }
6831 }
6832 
6833 void Catalog::createSystemTableServer(const std::string& server_name,
6834  const std::string& data_wrapper_type,
6835  const foreign_storage::OptionsMap& options) {
6836  auto server = std::make_unique<foreign_storage::ForeignServer>(
6837  server_name, data_wrapper_type, options, shared::kRootUserId);
6838  server->validate();
6839  auto stored_server = getForeignServer(server_name);
6840  if (stored_server && stored_server->options != server->options) {
6841  // Drop all tables for server before dropping server.
6842  auto tables = getAllForeignTablesForForeignServer(stored_server->id);
6843  for (const auto table : tables) {
6844  LOG(INFO) << "Dropping existing \"" << table->tableName << "\" system table for \""
6845  << server_name << "\" foreign server.";
6846  dropTable(table);
6847  }
6848  LOG(INFO) << "Dropping existing \"" << server_name
6849  << "\" system table foreign server.";
6850  dropForeignServer(server_name);
6851  stored_server = nullptr;
6852  }
6853  if (!stored_server) {
6854  LOG(INFO) << "Creating a new \"" << server_name << "\" system table foreign server.";
6855  createForeignServer(std::move(server), true);
6856  }
6857 }
6858 
6859 std::pair<foreign_storage::ForeignTable, std::list<ColumnDescriptor>>
6860 Catalog::getSystemTableSchema(
6861  const std::string& table_name,
6862  const std::string& server_name,
6863  const std::vector<std::pair<std::string, SQLTypeInfo>>& column_type_by_name,
6864  bool is_in_memory_system_table) {
6865  foreign_storage::ForeignTable foreign_table;
6866  foreign_table.tableName = table_name;
6867  foreign_table.nColumns = column_type_by_name.size();
6868  foreign_table.isView = false;
6869  foreign_table.is_system_table = true;
6870  foreign_table.is_in_memory_system_table = is_in_memory_system_table;
6871  foreign_table.fragmenter = nullptr;
6873  foreign_table.maxFragRows = DEFAULT_FRAGMENT_ROWS;
6874  foreign_table.maxChunkSize = DEFAULT_MAX_CHUNK_SIZE;
6875  foreign_table.fragPageSize = DEFAULT_PAGE_SIZE;
6876  foreign_table.maxRows = DEFAULT_MAX_ROWS;
6877  foreign_table.userId = shared::kRootUserId;
6878  foreign_table.storageType = StorageType::FOREIGN_TABLE;
6879  foreign_table.hasDeletedCol = false;
6880  foreign_table.keyMetainfo = "[]";
6881  foreign_table.fragments = "";
6882  foreign_table.partitions = "";
6883  foreign_table.foreign_server = getForeignServer(server_name);
6884  CHECK(foreign_table.foreign_server);
6885 
6886  list<ColumnDescriptor> columns;
6887  for (const auto& [column_name, column_type] : column_type_by_name) {
6888  columns.emplace_back();
6889  auto& cd = columns.back();
6890  cd.columnName = column_name;
6891  cd.columnType = column_type;
6892  cd.isSystemCol = false;
6893  cd.isVirtualCol = false;
6894  }
6895  return {foreign_table, columns};
6896 }
6897 
6898 bool Catalog::recreateSystemTableIfUpdated(foreign_storage::ForeignTable& foreign_table,
6899  const std::list<ColumnDescriptor>& columns) {
6900  auto stored_td = getMetadataForTable(foreign_table.tableName, false);
6901  bool should_recreate{false};
6902  if (stored_td) {
6903  auto stored_foreign_table =
6904  dynamic_cast<const foreign_storage::ForeignTable*>(stored_td);
6905  CHECK(stored_foreign_table);
6906  if (stored_foreign_table->foreign_server->name !=
6907  foreign_table.foreign_server->name ||
6908  stored_foreign_table->options != foreign_table.options) {
6909  should_recreate = true;
6910  } else {
6911  auto stored_columns =
6912  getAllColumnMetadataForTable(stored_td->tableId, false, false, false);
6913  if (stored_columns.size() != columns.size()) {
6914  should_recreate = true;
6915  } else {
6916  auto it_1 = stored_columns.begin();
6917  auto it_2 = columns.begin();
6918  for (; it_1 != stored_columns.end() && it_2 != columns.end(); it_1++, it_2++) {
6919  // Need a custom comparison here since we don't care if the dictionary comp
6920  // param has changed (comp can change because the column was assigned a
6921  // dictionary, whereas before it was just a compression number).
6922  if ((*it_1)->columnName != it_2->columnName ||
6923  (*it_1)->columnType.get_type() != it_2->columnType.get_type() ||
6924  (*it_1)->columnType.get_subtype() != it_2->columnType.get_subtype() ||
6925  (*it_1)->columnType.get_dimension() != it_2->columnType.get_dimension() ||
6926  (*it_1)->columnType.get_scale() != it_2->columnType.get_scale() ||
6927  (*it_1)->columnType.get_notnull() != it_2->columnType.get_notnull() ||
6928  (*it_1)->columnType.get_compression() !=
6929  it_2->columnType.get_compression() ||
6930  (*it_1)->columnType.get_size() != it_2->columnType.get_size()) {
6931  should_recreate = true;
6932  break;
6933  }
6934  }
6935  }
6936  }
6937  } else {
6938  should_recreate = true;
6939  }
6940  if (should_recreate) {
6941  if (stored_td) {
6942  LOG(INFO) << "Dropping existing \"" << foreign_table.tableName
6943  << "\" system table.";
6944  deleteTableCatalogMetadata(stored_td, {stored_td});
6945  }
6946  LOG(INFO) << "Creating a new \"" << foreign_table.tableName << "\" system table.";
6947  createTable(foreign_table, columns, {}, true);
6948  }
6949  return should_recreate;
6950 }
6951 
6952 namespace {
6953 std::string get_checked_table_name(const Catalog* catalog, const ColumnDescriptor* cd) {
6954  auto table_name_opt = catalog->getTableName(cd->tableId);
6955  CHECK(table_name_opt.has_value());
6956  return table_name_opt.value();
6957 }
6958 } // namespace
6959 
6960 void Catalog::updateInColumnMap(ColumnDescriptor* cd, ColumnDescriptor* old_cd) {
6961  shared::get_from_map(columnDescriptorMap_,
6962  ColumnKey{cd->tableId, to_upper(cd->columnName)}) = cd;
6963  shared::get_from_map(columnDescriptorMapById_, ColumnIdKey{cd->tableId, cd->columnId}) =
6964  cd;
6965  auto dict_it = dict_columns_by_table_id_.find(cd->tableId);
6966  if (dict_it != dict_columns_by_table_id_.end()) {
6967  auto& set = dict_it->second;
6968  for (auto it = set.begin(); it != set.end(); ++it) {
6969  if ((*it)->columnId == cd->columnId) {
6970  set.erase(it);
6971  break;
6972  }
6973  }
6974  }
6975  if (cd->columnType.is_dict_encoded_type()) {
6976  dict_columns_by_table_id_[cd->tableId].emplace(cd);
6977  }
6978 
6979  removeColumnDescriptor(old_cd);
6980  addColumnDescriptor(cd);
6981  calciteMgr_->updateMetadata(currentDB_.dbName, get_checked_table_name(this, cd));
6982 }
6983 
6984 void Catalog::addToColumnMap(ColumnDescriptor* cd) {
6985  columnDescriptorMap_[ColumnKey{cd->tableId, to_upper(cd->columnName)}] = cd;
6986  columnDescriptorMapById_[ColumnIdKey{cd->tableId, cd->columnId}] = cd;
6987  if (cd->columnType.is_dict_encoded_type()) {
6988  dict_columns_by_table_id_[cd->tableId].emplace(cd);
6989  }
6990 }
6991 
6992 void Catalog::removeFromColumnMap(ColumnDescriptor* cd) {
6993  if (cd->columnType.is_dict_encoded_type()) {
6994  dict_columns_by_table_id_[cd->tableId].erase(cd);
6995  }
6996  columnDescriptorMap_.erase(ColumnKey{cd->tableId, to_upper(cd->columnName)});
6997  columnDescriptorMapById_.erase(ColumnIdKey{cd->tableId, cd->columnId});
6998 }
6999 
7000 // TODO(Misiu): Replace most sqlite transactions with this idiom.
7001 template <typename F, typename... Args>
7002 void Catalog::execInTransaction(F&& f, Args&&... args) {
7003  cat_write_lock write_lock(this);
7005  sqliteConnector_.query("BEGIN TRANSACTION");
7006  try {
7007  (this->*f)(std::forward<Args>(args)...);
7008  } catch (std::exception&) {
7009  sqliteConnector_.query("ROLLBACK TRANSACTION");
7010  throw;
7011  }
7012  sqliteConnector_.query("END TRANSACTION");
7013 }
7014 } // namespace Catalog_Namespace
std::lock_guard< T > lock_guard
static constexpr const char * MEMORY_DETAILS_SYS_TABLE_NAME
Definition: Catalog.h:120
bool contains(const T &container, const U &element)
Definition: misc.h:195
int32_t maxRollbackEpochs
int64_t get_next_refresh_time(const foreign_storage::ForeignTable &foreign_table)
Definition: Catalog.cpp:2922
const Parser::SharedDictionaryDef compress_reference_path(Parser::SharedDictionaryDef cur_node, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
static std::set< std::string > reserved_keywords
void removeFromColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6992
static constexpr const char * WS_SERVER_ACCESS_LOGS_SYS_TABLE_NAME
Definition: Catalog.h:128
std::string virtualExpr
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:392
void set_compression(EncodingType c)
Definition: sqltypes.h:479
void set_size(int s)
Definition: sqltypes.h:476
#define CHECK_EQ(x, y)
Definition: Logger.h:301
const int MAPD_TEMP_TABLE_START_ID
Definition: Catalog.cpp:111
std::string partitions
std::vector< int > ChunkKey
Definition: types.h:36
std::vector< std::unique_ptr< lockmgr::AbstractLockContainer< const TableDescriptor * >>> LockedTableDescriptors
Definition: LockMgr.h:272
static constexpr const char * SERVER_LOGS_SYS_TABLE_NAME
Definition: Catalog.h:125
static constexpr const char * EXECUTOR_RESOURCE_POOL_SUMMARY_SYS_TABLE_NAME
Definition: Catalog.h:122
const int kRootUserId
static constexpr char const * REGEX_PARSER
void add_db_object(const std::string &object_name, DBObjectType object_type, int32_t user_id, const AccessPrivileges &privileges, std::map< int32_t, std::vector< DBObject >> &db_objects)
Definition: Catalog.cpp:6150
const std::string kDataDirectoryName
std::string dictFolderPath
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
HOST DEVICE int get_size() const
Definition: sqltypes.h:403
static TableSchemaLockMgr & instance()
Definition: LockMgr.h:40
~Catalog()
Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the ...
Definition: Catalog.cpp:231
std::string cat(Ts &&...args)
T getData(const int row, const int col)
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:143
SQLTypes
Definition: sqltypes.h:65
#define SPIMAP_MAGIC1
Definition: Catalog.h:82
std::string tableName
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:709
static const AccessPrivileges ALL_DATABASE
Definition: DBObject.h:151
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
bool g_enable_logs_system_tables
Definition: Catalog.cpp:100
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:470
static bool dropRenderGroupColumns(const Catalog_Namespace::TableDescriptorMapById &table_descriptors_by_id, Catalog_Namespace::Catalog *cat)
virtual void query_with_text_params(std::string const &query_only)
This file includes the class specification for the FILE manager (FileMgr), and related data structure...
DBObjectType
Definition: DBObject.h:40
static constexpr char const * INTERNAL_STORAGE_STATS
void set_common_log_system_table_options(foreign_storage::ForeignTable &foreign_table)
Definition: Catalog.cpp:6436
static constexpr const char * DASHBOARDS_SYS_TABLE_NAME
Definition: Catalog.h:114
#define LOG(tag)
Definition: Logger.h:285
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
#define CHUNK_KEY_DB_IDX
Definition: types.h:38
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:711
HOST DEVICE int get_scale() const
Definition: sqltypes.h:396
std::string storageType
PersistentStorageMgr * getPersistentStorageMgr() const
Definition: DataMgr.cpp:677
std::string join(T const &container, std::string const &delim)
#define DEFAULT_MAX_CHUNK_SIZE
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:722
#define UNREACHABLE()
Definition: Logger.h:338
This file includes the class specification for the FILE manager (FileMgr), and related data structure...
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:469
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
std::shared_ptr< Catalog > getDummyCatalog()
Definition: SysCatalog.h:369
const std::string & get_foreign_table() const
Definition: ParserNode.h:933
virtual void query(const std::string &queryString)
#define CHECK_GE(x, y)
Definition: Logger.h:306
fs::path get_log_dir_path()
Definition: Logger.cpp:904
void setObjectKey(const DBObjectKey &objectKey)
Definition: DBObject.h:225
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:162
std::string fragments
int32_t objectId
Definition: DBObject.h:55
const int MAPD_TEMP_DICT_START_ID
Definition: Catalog.cpp:113
Definition: Grantee.h:81
The InsertOrderFragmenter is a child class of AbstractFragmenter, and fragments data in insert order...
void checkDateInDaysColumnMigration()
Definition: Catalog.cpp:925
void clear_cached_table_data(const Data_Namespace::DataMgr *data_mgr, int32_t db_id, int32_t table_id)
Definition: Catalog.cpp:6458
void setPrivileges(const AccessPrivileges &privs)
Definition: DBObject.h:227
bool contains_spaces(std::string_view str)
returns true if the string contains one or more spaces
Definition: Catalog.cpp:5548
void reloadCatalogMetadataUnlocked(const std::map< int32_t, std::string > &user_name_by_user_id)
Definition: Catalog.cpp:1198
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:391
const std::string kInfoSchemaDbName
dsqliteMutex_(std::make_unique< heavyai::DistributedSharedMutex >(std::filesystem::path(basePath_)/shared::kLockfilesDirectoryName/shared::kCatalogDirectoryName/(currentDB_.dbName+".sqlite.lockfile")))
static const AccessPrivileges SELECT_FROM_TABLE
Definition: DBObject.h:160
std::vector< int > columnIdBySpi_
#define CHECK_GT(x, y)
Definition: Logger.h:305
static constexpr char const * INTERNAL_CATALOG
DeletedColumnPerTableMap deletedColumnPerTable_
Definition: Catalog.h:774
const ColumnDescriptor * get_foreign_col(const Catalog &cat, const Parser::SharedDictionaryDef &shared_dict_def)
Definition: Catalog.cpp:3953
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:710
DBObject * findDbObject(const DBObjectKey &objectKey, bool only_direct) const
Definition: Grantee.cpp:85
std::string to_string(char const *&&v)
std::shared_ptr< std::mutex > mutex_
static constexpr std::array< char const *, 5 > IN_MEMORY_DATA_WRAPPERS
bool contains_sql_reserved_chars(std::string_view str, std::string_view chars="`~!@#$%^&*()-=+[{]}\\|;:'\",<.>/?")
returns true if the string contains one or more OmniSci SQL reserved characters
Definition: Catalog.cpp:5555
bool is_in_memory_system_table
static const AccessPrivileges ALL_VIEW
Definition: DBObject.h:177
std::string chunks
void grantRoleBatch(const std::vector< std::string > &roles, const std::vector< std::string > &grantees)
constexpr double a
Definition: Utm.h:32
static constexpr const char * REQUEST_LOGS_SYS_TABLE_NAME
Definition: Catalog.h:126
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:797
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:129
#define SPIMAP_MAGIC2
Definition: Catalog.h:83
std::string name() const
Definition: Catalog.h:348
std::string dashboardSystemRoleName
This file contains the class specification and related data structures for Catalog.
void set_common_db_log_system_table_options(foreign_storage::ForeignTable &foreign_table)
Definition: Catalog.cpp:6446
EncodingType
Definition: sqltypes.h:240
foreign_storage::ForeignStorageCache * getDiskCache() const
#define DEFAULT_MAX_ROWS
int get_physical_cols() const
Definition: sqltypes.h:430
static constexpr const char * ROLES_SYS_TABLE_NAME
Definition: Catalog.h:117
static SysCatalog & instance()
Definition: SysCatalog.h:343
This file contains the class specification and related data structures for SysCatalog.
bool g_enable_s3_fsi
Definition: Catalog.cpp:97
std::string get_user_name_from_id(int32_t id, const std::map< int32_t, std::string > &user_name_by_user_id)
Definition: Catalog.cpp:1051
Classes representing a parse tree.
static int64_t getNextRefreshTime(const std::map< std::string, std::string, std::less<>> &foreign_table_options)
std::string get_quoted_string(const std::string &filename, char quote, char escape)
Quote a string while escaping any existing quotes in the string.
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:265
bool g_enable_system_tables
Definition: SysCatalog.cpp:64
static const std::string getForeignTableSchema(bool if_not_exists=false)
Definition: Catalog.cpp:782
std::string g_base_path
Definition: SysCatalog.cpp:62
#define INJECT_TIMER(DESC)
Definition: measure.h:96
#define CHECK_NE(x, y)
Definition: Logger.h:302
GetTablesType
Definition: Catalog.h:63
void populateOptionsMap(OptionsMap &&options_map, bool clear=false)
static constexpr char const * INTERNAL_ML_MODEL_METADATA
void set_scale(int s)
Definition: sqltypes.h:473
const std::string & get_foreign_column() const
Definition: ParserNode.h:935
void dropTable(const TableDescriptor *td)
Definition: Catalog.cpp:4282
std::string table_epochs_to_string(const std::vector< TableEpochInfo > &table_epochs)
Definition: Catalog.cpp:3868
std::vector< std::string > parse_underlying_dashboard_objects(const std::string &meta)
Encapsulates an enumeration of foreign data wrapper type strings.
std::unique_lock< T > unique_lock
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:39
static const int32_t MAPD_VERSION
Definition: release.h:32
Role * getRoleGrantee(const std::string &name) const
static const std::string getForeignServerSchema(bool if_not_exists=false)
Definition: Catalog.cpp:775
static const AccessPrivileges ALL_SERVER
Definition: DBObject.h:187
static constexpr const char * MEMORY_SUMMARY_SYS_TABLE_NAME
Definition: Catalog.h:119
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
void setOwner(int32_t userId)
Definition: DBObject.h:235
bool is_dict_encoded_type() const
Definition: sqltypes.h:653
std::string get_checked_table_name(const Catalog *catalog, const ColumnDescriptor *cd)
Definition: Catalog.cpp:6953
specifies the content in-memory of a row in the column metadata table
#define DEFAULT_MAX_ROLLBACK_EPOCHS
specifies the content in-memory of a row in the table metadata table
static constexpr const char * ROLE_ASSIGNMENTS_SYS_TABLE_NAME
Definition: Catalog.h:118
int32_t dictId
Definition: DictRef.h:14
std::list< UserMetadata > getAllUserMetadata()
static constexpr const char * USERS_SYS_TABLE_NAME
Definition: Catalog.h:112
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1724
std::string keyMetainfo
this
Definition: Execute.cpp:281
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
void replace_cached_table_name(std::map< std::string, int > &cachedTableMap, const std::string &curTableName, const std::string &newTableName, int tableId)
Definition: Catalog.cpp:4491
bool g_serialize_temp_tables
Definition: Catalog.cpp:106
static constexpr char const * INTERNAL_EXECUTOR_STATS
std::string DBObjectTypeToString(DBObjectType type)
Definition: DBObject.cpp:92
struct dict_ref_t DictRef
Definition: DictRef.h:54
static constexpr const char * DATABASES_SYS_TABLE_NAME
Definition: Catalog.h:115
std::string to_upper(const std::string &str)
#define DEFAULT_PAGE_SIZE
const DBObjectMap * getDbObjects(bool only_direct) const
Definition: Grantee.h:56
void set_comp_param(int p)
Definition: sqltypes.h:480
void loadKey()
Definition: DBObject.cpp:190
static constexpr const char * TABLES_SYS_TABLE_NAME
Definition: Catalog.h:113
const int DEFAULT_INITIAL_VERSION
Definition: Catalog.cpp:110
std::optional< std::string > default_value
Definition: sqltypes.h:79
V & get_from_map(std::map< K, V, comp > &map, const K &key)
Definition: misc.h:61
static const std::string physicalTableNameTag_
Definition: Catalog.h:729
int32_t g_distributed_leaf_idx
Definition: Catalog.cpp:98
void reloadTableMetadata(int table_id)
Definition: Catalog.cpp:1093
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:399
static const AccessPrivileges SELECT_FROM_VIEW
Definition: DBObject.h:180
bool table_is_temporary(const TableDescriptor *const td)
static constexpr const char * ML_MODEL_METADATA_SYS_TABLE_NAME
Definition: Catalog.h:124
void set_dimension(int d)
Definition: sqltypes.h:470
#define DEFAULT_FRAGMENT_ROWS
void setStringDictKey(const shared::StringDictKey &dict_key)
Definition: sqltypes.h:1061
std::map< std::string, std::shared_ptr< DashboardDescriptor >> DashboardDescriptorMap
Definition: Types.h:44
static const std::string getCustomExpressionsSchema(bool if_not_exists=false)
Definition: Catalog.cpp:790
static constexpr const char * PERMISSIONS_SYS_TABLE_NAME
Definition: Catalog.h:116
Fragmenter_Namespace::FragmenterType fragType
static constexpr char const * INTERNAL_MEMORY_STATS
int32_t dbId
Definition: DBObject.h:54
Data_Namespace::MemoryLevel persistenceLevel
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:393
static constexpr const char * STORAGE_DETAILS_SYS_TABLE_NAME
Definition: Catalog.h:121
std::string convert_object_owners_map_to_string(int32_t db_id, int32_t new_owner_id, const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects)
Definition: Catalog.cpp:6127
static const AccessPrivileges ALL_DASHBOARD
Definition: DBObject.h:169
torch::Tensor f(torch::Tensor x, torch::Tensor W_target, torch::Tensor b_target)
bool checkDropRenderGroupColumnsMigration()
Definition: Catalog.cpp:931
Definition: sqltypes.h:68
static const AccessPrivileges ALL_TABLE
Definition: DBObject.h:157
bool g_cache_string_hash
const std::string kCatalogDirectoryName
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:402
int32_t g_distributed_num_leaves
Definition: Catalog.cpp:99
void updateCustomExpressionsSchema()
Definition: Catalog.cpp:763
const ForeignServer * foreign_server
Definition: ForeignTable.h:57
static constexpr const char * REFRESH_TIMING_TYPE_KEY
Definition: ForeignTable.h:43
std::unique_ptr< heavyai::DistributedSharedMutex > dsqliteMutex_
Definition: Catalog.h:907
void set_notnull(bool n)
Definition: sqltypes.h:475
#define CHECK(condition)
Definition: Logger.h:291
bool is_geometry() const
Definition: sqltypes.h:595
static constexpr char const * CSV
std::map< std::string, std::string, std::less<>> OptionsMap
std::optional< std::string > getTableName(int32_t table_id) const
Definition: Catalog.cpp:1869
void reloadTableMetadataUnlocked(int table_id)
Definition: Catalog.cpp:1100
static void migrateDateInDaysMetadata(const Catalog_Namespace::TableDescriptorMapById &table_descriptors_by_id, const int database_id, Catalog_Namespace::Catalog *cat, SqliteConnector &sqlite)
static void clearExternalCaches(bool for_update, const TableDescriptor *td, const int current_db_id)
Definition: Execute.h:438
Descriptor for a dictionary for a string columne.
void updateLogicalToPhysicalTableLinkSchema()
Definition: Catalog.cpp:631
static constexpr int NULL_REFRESH_TIME
Definition: ForeignTable.h:55
FileBuffer Chunk
A Chunk is the fundamental unit of execution in Map-D.
Definition: FileMgr.h:80
std::unordered_map< std::string, std::vector< std::string > > getGranteesOfSharedDashboards(const std::vector< std::string > &dashboard_ids)
const std::string kLockfilesDirectoryName
int32_t permissionType
Definition: DBObject.h:53
void populateRoleDbObjects(const std::vector< DBObject > &objects)
std::string viewSQL
static const AccessPrivileges DELETE_DASHBOARD
Definition: DBObject.h:173
Definition: sqltypes.h:72
SQLTypeInfo columnType
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
void drop_tables(Catalog &catalog, const std::vector< std::string > &table_names)
Definition: Catalog.cpp:6467
bool is_string() const
Definition: sqltypes.h:559
string name
Definition: setup.in.py:72
size_t g_leaf_count
Definition: ParserNode.cpp:78
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:398
int32_t validate_and_get_user_id(const std::string &user_name)
Definition: Catalog.cpp:6119
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:210
void reloadDictionariesFromDiskUnlocked()
Definition: Catalog.cpp:5170
static constexpr char const * FOREIGN_TABLE
bool is_string_array() const
Definition: sqltypes.h:562
bool g_enable_fsi
Definition: Catalog.cpp:96
std::string columnName
static constexpr char const * INTERNAL_LOGS
static constexpr const char * WS_SERVER_LOGS_SYS_TABLE_NAME
Definition: Catalog.h:127
bool is_reserved_sql_keyword(std::string_view str)
returns true if the string equals an OmniSci SQL reserved keyword
Definition: Catalog.cpp:5562
std::map< int32_t, std::string > get_user_id_to_user_name_map()
Definition: Catalog.cpp:1042
#define IS_GEO(T)
Definition: sqltypes.h:310
static constexpr char const * PARQUET
virtual void renameDbObject(const DBObject &object)
Definition: Grantee.cpp:121
virtual size_t getNumRows() const
A selection of helper methods for File I/O.
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:727
Catalog()
Constructor builds a hollow catalog used during constructor of other catalogs.
Definition: Catalog.cpp:171
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:707
static thread_local bool thread_holds_read_lock
Definition: Catalog.h:913
bool is_array() const
Definition: sqltypes.h:583
#define VLOG(n)
Definition: Logger.h:388
void CheckAndExecuteMigrationsPostBuildMaps()
Definition: Catalog.cpp:1036
static constexpr const char * SCHEDULE_REFRESH_TIMING_TYPE
Definition: ForeignTable.h:53
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39
void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id)
Definition: Catalog.cpp:645
HOST DEVICE void set_type(SQLTypes t)
Definition: sqltypes.h:468