OmniSciDB  8a228a1076
Catalog.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
25 #include "Catalog.h"
26 #include "SysCatalog.h"
27 
28 #include <sys/wait.h>
29 
30 #include <algorithm>
31 #include <boost/algorithm/string/predicate.hpp>
32 #include <boost/filesystem.hpp>
33 #include <boost/range/adaptor/map.hpp>
34 #include <boost/version.hpp>
35 #include <cassert>
36 #include <cerrno>
37 #include <cstdio>
38 #include <cstring>
39 #include <exception>
40 #include <fstream>
41 #include <list>
42 #include <memory>
43 #include <random>
44 #include <regex>
45 #include <sstream>
46 #if BOOST_VERSION >= 106600
47 #include <boost/uuid/detail/sha1.hpp>
48 #else
49 #include <boost/uuid/sha1.hpp>
50 #endif
51 #include <rapidjson/document.h>
52 #include <rapidjson/istreamwrapper.h>
53 #include <rapidjson/ostreamwrapper.h>
54 #include <rapidjson/writer.h>
55 
56 #include "QueryEngine/Execute.h"
58 
62 #include "Fragmenter/Fragmenter.h"
64 #include "LockMgr/LockMgr.h"
66 #include "Parser/ParserNode.h"
67 #include "QueryEngine/Execute.h"
69 #include "Shared/File.h"
70 #include "Shared/StringTransform.h"
71 #include "Shared/TimeGM.h"
72 #include "Shared/measure.h"
74 
75 #include "MapDRelease.h"
76 #include "RWLocks.h"
78 
79 using Chunk_NS::Chunk;
82 using std::list;
83 using std::map;
84 using std::pair;
85 using std::runtime_error;
86 using std::string;
87 using std::vector;
88 
90 bool g_enable_fsi{false};
91 extern bool g_cache_string_hash;
92 
93 // Serialize temp tables to a json file in the Catalogs directory for Calcite parsing
94 // under unit testing.
96 
97 namespace Catalog_Namespace {
98 
99 const int DEFAULT_INITIAL_VERSION = 1; // start at version 1
101  1073741824; // 2^30, give room for over a billion non-temp tables
103  1073741824; // 2^30, give room for over a billion non-temp dictionaries
104 
105 const std::string Catalog::physicalTableNameTag_("_shard_#");
106 std::map<std::string, std::shared_ptr<Catalog>> Catalog::mapd_cat_map_;
107 
108 thread_local bool Catalog::thread_holds_read_lock = false;
109 
114 
115 // migration will be done as two step process this release
116 // will create and use new table
117 // next release will remove old table, doing this to have fall back path
118 // incase of migration failure
121  sqliteConnector_.query("BEGIN TRANSACTION");
122  try {
124  "SELECT name FROM sqlite_master WHERE type='table' AND name='mapd_dashboards'");
125  if (sqliteConnector_.getNumRows() != 0) {
126  // already done
127  sqliteConnector_.query("END TRANSACTION");
128  return;
129  }
131  "CREATE TABLE mapd_dashboards (id integer primary key autoincrement, name text , "
132  "userid integer references mapd_users, state text, image_hash text, update_time "
133  "timestamp, "
134  "metadata text, UNIQUE(userid, name) )");
135  // now copy content from old table to new table
137  "insert into mapd_dashboards (id, name , "
138  "userid, state, image_hash, update_time , "
139  "metadata) "
140  "SELECT viewid , name , userid, view_state, image_hash, update_time, "
141  "view_metadata "
142  "from mapd_frontend_views");
143  } catch (const std::exception& e) {
144  sqliteConnector_.query("ROLLBACK TRANSACTION");
145  throw;
146  }
147  sqliteConnector_.query("END TRANSACTION");
148 }
149 
150 namespace {
151 
152 inline auto table_json_filepath(const std::string& base_path,
153  const std::string& db_name) {
154  return boost::filesystem::path(base_path + "/mapd_catalogs/" + db_name +
155  "_temp_tables.json");
156 }
157 
158 } // namespace
159 
160 Catalog::Catalog(const string& basePath,
161  const DBMetadata& curDB,
162  std::shared_ptr<Data_Namespace::DataMgr> dataMgr,
163  const std::vector<LeafHostInfo>& string_dict_hosts,
164  std::shared_ptr<Calcite> calcite,
165  bool is_new_db)
166  : basePath_(basePath)
167  , sqliteConnector_(curDB.dbName, basePath + "/mapd_catalogs/")
168  , currentDB_(curDB)
169  , dataMgr_(dataMgr)
170  , string_dict_hosts_(string_dict_hosts)
171  , calciteMgr_(calcite)
172  , nextTempTableId_(MAPD_TEMP_TABLE_START_ID)
173  , nextTempDictId_(MAPD_TEMP_DICT_START_ID)
174  , sqliteMutex_()
175  , sharedMutex_()
178  if (!is_new_db) {
180  }
181  buildMaps();
182  if (!is_new_db) {
184  }
186  boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
187  }
188 }
189 
192  // must clean up heap-allocated TableDescriptor and ColumnDescriptor structs
193  for (TableDescriptorMap::iterator tableDescIt = tableDescriptorMap_.begin();
194  tableDescIt != tableDescriptorMap_.end();
195  ++tableDescIt) {
196  tableDescIt->second->fragmenter = nullptr;
197  delete tableDescIt->second;
198  }
199 
200  // TableDescriptorMapById points to the same descriptors. No need to delete
201 
202  for (ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.begin();
203  columnDescIt != columnDescriptorMap_.end();
204  ++columnDescIt) {
205  delete columnDescIt->second;
206  }
207 
208  // ColumnDescriptorMapById points to the same descriptors. No need to delete
209 
211  boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
212  }
213 }
214 
217  sqliteConnector_.query("BEGIN TRANSACTION");
218  try {
219  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_tables)");
220  std::vector<std::string> cols;
221  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
222  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
223  }
224  if (std::find(cols.begin(), cols.end(), std::string("max_chunk_size")) ==
225  cols.end()) {
226  string queryString("ALTER TABLE mapd_tables ADD max_chunk_size BIGINT DEFAULT " +
228  sqliteConnector_.query(queryString);
229  }
230  if (std::find(cols.begin(), cols.end(), std::string("shard_column_id")) ==
231  cols.end()) {
232  string queryString("ALTER TABLE mapd_tables ADD shard_column_id BIGINT DEFAULT " +
233  std::to_string(0));
234  sqliteConnector_.query(queryString);
235  }
236  if (std::find(cols.begin(), cols.end(), std::string("shard")) == cols.end()) {
237  string queryString("ALTER TABLE mapd_tables ADD shard BIGINT DEFAULT " +
238  std::to_string(-1));
239  sqliteConnector_.query(queryString);
240  }
241  if (std::find(cols.begin(), cols.end(), std::string("num_shards")) == cols.end()) {
242  string queryString("ALTER TABLE mapd_tables ADD num_shards BIGINT DEFAULT " +
243  std::to_string(0));
244  sqliteConnector_.query(queryString);
245  }
246  if (std::find(cols.begin(), cols.end(), std::string("key_metainfo")) == cols.end()) {
247  string queryString("ALTER TABLE mapd_tables ADD key_metainfo TEXT DEFAULT '[]'");
248  sqliteConnector_.query(queryString);
249  }
250  if (std::find(cols.begin(), cols.end(), std::string("userid")) == cols.end()) {
251  string queryString("ALTER TABLE mapd_tables ADD userid integer DEFAULT " +
253  sqliteConnector_.query(queryString);
254  }
255  if (std::find(cols.begin(), cols.end(), std::string("sort_column_id")) ==
256  cols.end()) {
258  "ALTER TABLE mapd_tables ADD sort_column_id INTEGER DEFAULT 0");
259  }
260  if (std::find(cols.begin(), cols.end(), std::string("storage_type")) == cols.end()) {
261  string queryString("ALTER TABLE mapd_tables ADD storage_type TEXT DEFAULT ''");
262  sqliteConnector_.query(queryString);
263  }
264  } catch (std::exception& e) {
265  sqliteConnector_.query("ROLLBACK TRANSACTION");
266  throw;
267  }
268  sqliteConnector_.query("END TRANSACTION");
269 }
270 
273  sqliteConnector_.query("BEGIN TRANSACTION");
274  try {
276  "select name from sqlite_master WHERE type='table' AND "
277  "name='mapd_version_history'");
278  if (sqliteConnector_.getNumRows() == 0) {
280  "CREATE TABLE mapd_version_history(version integer, migration_history text "
281  "unique)");
282  } else {
284  "select * from mapd_version_history where migration_history = "
285  "'notnull_fixlen_arrays'");
286  if (sqliteConnector_.getNumRows() != 0) {
287  // legacy fixlen arrays had migrated
288  // no need for further execution
289  sqliteConnector_.query("END TRANSACTION");
290  return;
291  }
292  }
293  // Insert check for migration
295  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
296  std::vector<std::string>{std::to_string(MAPD_VERSION), "notnull_fixlen_arrays"});
297  LOG(INFO) << "Updating mapd_columns, legacy fixlen arrays";
298  // Upating all fixlen array columns
299  string queryString("UPDATE mapd_columns SET is_notnull=1 WHERE coltype=" +
300  std::to_string(kARRAY) + " AND size>0;");
301  sqliteConnector_.query(queryString);
302  } catch (std::exception& e) {
303  sqliteConnector_.query("ROLLBACK TRANSACTION");
304  throw;
305  }
306  sqliteConnector_.query("END TRANSACTION");
307 }
308 
311  sqliteConnector_.query("BEGIN TRANSACTION");
312  try {
314  "select name from sqlite_master WHERE type='table' AND "
315  "name='mapd_version_history'");
316  if (sqliteConnector_.getNumRows() == 0) {
318  "CREATE TABLE mapd_version_history(version integer, migration_history text "
319  "unique)");
320  } else {
322  "select * from mapd_version_history where migration_history = "
323  "'notnull_geo_columns'");
324  if (sqliteConnector_.getNumRows() != 0) {
325  // legacy geo columns had migrated
326  // no need for further execution
327  sqliteConnector_.query("END TRANSACTION");
328  return;
329  }
330  }
331  // Insert check for migration
333  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
334  std::vector<std::string>{std::to_string(MAPD_VERSION), "notnull_geo_columns"});
335  LOG(INFO) << "Updating mapd_columns, legacy geo columns";
336  // Upating all geo columns
337  string queryString(
338  "UPDATE mapd_columns SET is_notnull=1 WHERE coltype=" + std::to_string(kPOINT) +
339  " OR coltype=" + std::to_string(kLINESTRING) + " OR coltype=" +
340  std::to_string(kPOLYGON) + " OR coltype=" + std::to_string(kMULTIPOLYGON) + ";");
341  sqliteConnector_.query(queryString);
342  } catch (std::exception& e) {
343  sqliteConnector_.query("ROLLBACK TRANSACTION");
344  throw;
345  }
346  sqliteConnector_.query("END TRANSACTION");
347 }
348 
351  sqliteConnector_.query("BEGIN TRANSACTION");
352  try {
353  // check table still exists
355  "SELECT name FROM sqlite_master WHERE type='table' AND "
356  "name='mapd_frontend_views'");
357  if (sqliteConnector_.getNumRows() == 0) {
358  // table does not exists
359  // no need to migrate
360  sqliteConnector_.query("END TRANSACTION");
361  return;
362  }
363  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_frontend_views)");
364  std::vector<std::string> cols;
365  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
366  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
367  }
368  if (std::find(cols.begin(), cols.end(), std::string("image_hash")) == cols.end()) {
369  sqliteConnector_.query("ALTER TABLE mapd_frontend_views ADD image_hash text");
370  }
371  if (std::find(cols.begin(), cols.end(), std::string("update_time")) == cols.end()) {
372  sqliteConnector_.query("ALTER TABLE mapd_frontend_views ADD update_time timestamp");
373  }
374  if (std::find(cols.begin(), cols.end(), std::string("view_metadata")) == cols.end()) {
375  sqliteConnector_.query("ALTER TABLE mapd_frontend_views ADD view_metadata text");
376  }
377  } catch (std::exception& e) {
378  sqliteConnector_.query("ROLLBACK TRANSACTION");
379  throw;
380  }
381  sqliteConnector_.query("END TRANSACTION");
382 }
383 
386  sqliteConnector_.query("BEGIN TRANSACTION");
387  try {
389  "CREATE TABLE IF NOT EXISTS mapd_links (linkid integer primary key, userid "
390  "integer references mapd_users, "
391  "link text unique, view_state text, update_time timestamp, view_metadata text)");
392  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_links)");
393  std::vector<std::string> cols;
394  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
395  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
396  }
397  if (std::find(cols.begin(), cols.end(), std::string("view_metadata")) == cols.end()) {
398  sqliteConnector_.query("ALTER TABLE mapd_links ADD view_metadata text");
399  }
400  } catch (const std::exception& e) {
401  sqliteConnector_.query("ROLLBACK TRANSACTION");
402  throw;
403  }
404  sqliteConnector_.query("END TRANSACTION");
405 }
406 
409  sqliteConnector_.query("BEGIN TRANSACTION");
410  try {
411  sqliteConnector_.query("UPDATE mapd_links SET userid = 0 WHERE userid IS NULL");
412  // check table still exists
414  "SELECT name FROM sqlite_master WHERE type='table' AND "
415  "name='mapd_frontend_views'");
416  if (sqliteConnector_.getNumRows() == 0) {
417  // table does not exists
418  // no need to migrate
419  sqliteConnector_.query("END TRANSACTION");
420  return;
421  }
423  "UPDATE mapd_frontend_views SET userid = 0 WHERE userid IS NULL");
424  } catch (const std::exception& e) {
425  sqliteConnector_.query("ROLLBACK TRANSACTION");
426  throw;
427  }
428  sqliteConnector_.query("END TRANSACTION");
429 }
430 
431 // introduce DB version into the tables table
432 // if the DB does not have a version reset all pagesizes to 2097152 to be compatible with
433 // old value
434 
437  if (currentDB_.dbName.length() == 0) {
438  // updateDictionaryNames dbName length is zero nothing to do here
439  return;
440  }
441  sqliteConnector_.query("BEGIN TRANSACTION");
442  try {
443  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_tables)");
444  std::vector<std::string> cols;
445  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
446  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
447  }
448  if (std::find(cols.begin(), cols.end(), std::string("version_num")) == cols.end()) {
449  LOG(INFO) << "Updating mapd_tables updatePageSize";
450  // No version number
451  // need to update the defaul tpagesize to old correct value
452  sqliteConnector_.query("UPDATE mapd_tables SET frag_page_size = 2097152 ");
453  // need to add new version info
454  string queryString("ALTER TABLE mapd_tables ADD version_num BIGINT DEFAULT " +
455  std::to_string(DEFAULT_INITIAL_VERSION));
456  sqliteConnector_.query(queryString);
457  }
458  } catch (std::exception& e) {
459  sqliteConnector_.query("ROLLBACK TRANSACTION");
460  throw;
461  }
462  sqliteConnector_.query("END TRANSACTION");
463 }
464 
467  sqliteConnector_.query("BEGIN TRANSACTION");
468  try {
469  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_columns)");
470  std::vector<std::string> cols;
471  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
472  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
473  }
474  if (std::find(cols.begin(), cols.end(), std::string("version_num")) == cols.end()) {
475  LOG(INFO) << "Updating mapd_columns updateDeletedColumnIndicator";
476  // need to add new version info
477  string queryString("ALTER TABLE mapd_columns ADD version_num BIGINT DEFAULT " +
478  std::to_string(DEFAULT_INITIAL_VERSION));
479  sqliteConnector_.query(queryString);
480  // need to add new column to table defintion to indicate deleted column, column used
481  // as bitmap for deleted rows.
483  "ALTER TABLE mapd_columns ADD is_deletedcol boolean default 0 ");
484  }
485  } catch (std::exception& e) {
486  sqliteConnector_.query("ROLLBACK TRANSACTION");
487  throw;
488  }
489  sqliteConnector_.query("END TRANSACTION");
490 }
491 
492 // introduce DB version into the dictionary tables
493 // if the DB does not have a version rename all dictionary tables
494 
497  if (currentDB_.dbName.length() == 0) {
498  // updateDictionaryNames dbName length is zero nothing to do here
499  return;
500  }
501  sqliteConnector_.query("BEGIN TRANSACTION");
502  try {
503  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_dictionaries)");
504  std::vector<std::string> cols;
505  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
506  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
507  }
508  if (std::find(cols.begin(), cols.end(), std::string("version_num")) == cols.end()) {
509  // No version number
510  // need to rename dictionaries
511  string dictQuery("SELECT dictid, name from mapd_dictionaries");
512  sqliteConnector_.query(dictQuery);
513  size_t numRows = sqliteConnector_.getNumRows();
514  for (size_t r = 0; r < numRows; ++r) {
515  int dictId = sqliteConnector_.getData<int>(r, 0);
516  std::string dictName = sqliteConnector_.getData<string>(r, 1);
517 
518  std::string oldName =
519  basePath_ + "/mapd_data/" + currentDB_.dbName + "_" + dictName;
520  std::string newName = basePath_ + "/mapd_data/DB_" +
521  std::to_string(currentDB_.dbId) + "_DICT_" +
522  std::to_string(dictId);
523 
524  int result = rename(oldName.c_str(), newName.c_str());
525 
526  if (result == 0) {
527  LOG(INFO) << "Dictionary upgrade: successfully renamed " << oldName << " to "
528  << newName;
529  } else {
530  LOG(ERROR) << "Failed to rename old dictionary directory " << oldName << " to "
531  << newName + " dbname '" << currentDB_.dbName << "' error code "
532  << std::to_string(result);
533  }
534  }
535  // need to add new version info
536  string queryString("ALTER TABLE mapd_dictionaries ADD version_num BIGINT DEFAULT " +
537  std::to_string(DEFAULT_INITIAL_VERSION));
538  sqliteConnector_.query(queryString);
539  }
540  } catch (std::exception& e) {
541  sqliteConnector_.query("ROLLBACK TRANSACTION");
542  throw;
543  }
544  sqliteConnector_.query("END TRANSACTION");
545 }
546 
549  sqliteConnector_.query("BEGIN TRANSACTION");
550  try {
552  "CREATE TABLE IF NOT EXISTS mapd_logical_to_physical("
553  "logical_table_id integer, physical_table_id integer)");
554  } catch (const std::exception& e) {
555  sqliteConnector_.query("ROLLBACK TRANSACTION");
556  throw;
557  }
558  sqliteConnector_.query("END TRANSACTION");
559 }
560 
561 void Catalog::updateLogicalToPhysicalTableMap(const int32_t logical_tb_id) {
562  /* this proc inserts/updates all pairs of (logical_tb_id, physical_tb_id) in
563  * sqlite mapd_logical_to_physical table for given logical_tb_id as needed
564  */
565 
567  sqliteConnector_.query("BEGIN TRANSACTION");
568  try {
569  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
570  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
571  const auto physicalTables = physicalTableIt->second;
572  CHECK(!physicalTables.empty());
573  for (size_t i = 0; i < physicalTables.size(); i++) {
574  int32_t physical_tb_id = physicalTables[i];
576  "INSERT OR REPLACE INTO mapd_logical_to_physical (logical_table_id, "
577  "physical_table_id) VALUES (?1, ?2)",
578  std::vector<std::string>{std::to_string(logical_tb_id),
579  std::to_string(physical_tb_id)});
580  }
581  }
582  } catch (std::exception& e) {
583  sqliteConnector_.query("ROLLBACK TRANSACTION");
584  throw;
585  }
586  sqliteConnector_.query("END TRANSACTION");
587 }
588 
591  sqliteConnector_.query("BEGIN TRANSACTION");
592  try {
593  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_dictionaries)");
594  std::vector<std::string> cols;
595  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
596  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
597  }
598  if (std::find(cols.begin(), cols.end(), std::string("refcount")) == cols.end()) {
599  sqliteConnector_.query("ALTER TABLE mapd_dictionaries ADD refcount DEFAULT 1");
600  }
601  } catch (std::exception& e) {
602  sqliteConnector_.query("ROLLBACK TRANSACTION");
603  throw;
604  }
605  sqliteConnector_.query("END TRANSACTION");
606 }
607 
610  sqliteConnector_.query("BEGIN TRANSACTION");
611  try {
615  } catch (std::exception& e) {
616  sqliteConnector_.query("ROLLBACK TRANSACTION");
617  throw;
618  }
619  sqliteConnector_.query("END TRANSACTION");
620 }
621 
623  std::vector<foreign_storage::ForeignTable> foreign_tables{};
624  {
626  sqliteConnector_.query("BEGIN TRANSACTION");
627  try {
629  "SELECT name FROM sqlite_master WHERE type='table' AND "
630  "name IN ('omnisci_foreign_servers', 'omnisci_foreign_tables', "
631  "'omnisci_user_mappings')");
632  if (sqliteConnector_.getNumRows() > 0) {
634  "SELECT tableid, name, isview, storage_type FROM mapd_tables "
635  "WHERE storage_type = 'FOREIGN_TABLE'");
637  for (size_t r = 0; r < num_rows; r++) {
638  foreign_storage::ForeignTable foreign_table{};
639  foreign_table.tableId = sqliteConnector_.getData<int>(r, 0);
640  foreign_table.tableName = sqliteConnector_.getData<std::string>(r, 1);
641  foreign_table.isView = sqliteConnector_.getData<bool>(r, 2);
642  foreign_table.storageType = sqliteConnector_.getData<std::string>(r, 3);
643  foreign_tables.emplace_back(foreign_table);
644  }
645  for (auto& foreign_table : foreign_tables) {
646  tableDescriptorMap_[to_upper(foreign_table.tableName)] = &foreign_table;
647  tableDescriptorMapById_[foreign_table.tableId] = &foreign_table;
648  executeDropTableSqliteQueries(&foreign_table);
649  }
650  sqliteConnector_.query("SELECT COUNT(*) FROM omnisci_foreign_tables");
651  CHECK_EQ(size_t(1), sqliteConnector_.getNumRows());
652  CHECK_EQ(0, sqliteConnector_.getData<int>(0, 0));
653 
654  sqliteConnector_.query("DROP TABLE omnisci_foreign_tables");
655  sqliteConnector_.query("DROP TABLE omnisci_foreign_servers");
656  }
657  } catch (std::exception& e) {
658  sqliteConnector_.query("ROLLBACK TRANSACTION");
659  throw;
660  }
661  sqliteConnector_.query("END TRANSACTION");
662  }
663 
664  for (auto& foreign_table : foreign_tables) {
666  DBObject(foreign_table.tableName, TableDBObjectType), this);
667  tableDescriptorMap_.erase(to_upper(foreign_table.tableName));
668  tableDescriptorMapById_.erase(foreign_table.tableId);
669  }
670 }
671 
672 const std::string Catalog::getForeignServerSchema(bool if_not_exists) {
673  return "CREATE TABLE " + (if_not_exists ? std::string{"IF NOT EXISTS "} : "") +
674  "omnisci_foreign_servers(id integer primary key, name text unique, " +
675  "data_wrapper_type text, owner_user_id integer, creation_time integer, " +
676  "options text)";
677 }
678 
679 const std::string Catalog::getForeignTableSchema(bool if_not_exists) {
680  return "CREATE TABLE " + (if_not_exists ? std::string{"IF NOT EXISTS "} : "") +
681  "omnisci_foreign_tables(table_id integer unique, server_id integer, " +
682  "options text, FOREIGN KEY(table_id) REFERENCES mapd_tables(tableid), " +
683  "FOREIGN KEY(server_id) REFERENCES omnisci_foreign_servers(id))";
684 }
685 
688  sqliteConnector_.query("BEGIN TRANSACTION");
689  std::vector<DBObject> objects;
690  try {
692  "SELECT name FROM sqlite_master WHERE type='table' AND "
693  "name='mapd_record_ownership_marker'");
694  // check if mapd catalog - marker exists
695  if (sqliteConnector_.getNumRows() != 0 && currentDB_.dbId == 1) {
696  // already done
697  sqliteConnector_.query("END TRANSACTION");
698  return;
699  }
700  // check if different catalog - marker exists
701  else if (sqliteConnector_.getNumRows() != 0 && currentDB_.dbId != 1) {
702  sqliteConnector_.query("SELECT dummy FROM mapd_record_ownership_marker");
703  // Check if migration is being performed on existing non mapd catalogs
704  // Older non mapd dbs will have table but no record in them
705  if (sqliteConnector_.getNumRows() != 0) {
706  // already done
707  sqliteConnector_.query("END TRANSACTION");
708  return;
709  }
710  }
711  // marker not exists - create one
712  else {
713  sqliteConnector_.query("CREATE TABLE mapd_record_ownership_marker (dummy integer)");
714  }
715 
716  DBMetadata db;
717  CHECK(SysCatalog::instance().getMetadataForDB(currentDB_.dbName, db));
718  // place dbId as a refernce for migration being performed
720  "INSERT INTO mapd_record_ownership_marker (dummy) VALUES (?1)",
721  std::vector<std::string>{std::to_string(db.dbOwner)});
722 
723  static const std::map<const DBObjectType, const AccessPrivileges>
724  object_level_all_privs_lookup{
730 
731  // grant owner all permissions on DB
732  DBObjectKey key;
733  key.dbId = currentDB_.dbId;
734  auto _key_place = [&key](auto type) {
735  key.permissionType = type;
736  return key;
737  };
738  for (auto& it : object_level_all_privs_lookup) {
739  objects.emplace_back(_key_place(it.first), it.second, db.dbOwner);
740  objects.back().setName(currentDB_.dbName);
741  }
742 
743  {
744  // other users tables and views
745  string tableQuery(
746  "SELECT tableid, name, userid, isview FROM mapd_tables WHERE userid > 0");
747  sqliteConnector_.query(tableQuery);
748  size_t numRows = sqliteConnector_.getNumRows();
749  for (size_t r = 0; r < numRows; ++r) {
750  int32_t tableid = sqliteConnector_.getData<int>(r, 0);
751  std::string tableName = sqliteConnector_.getData<string>(r, 1);
752  int32_t ownerid = sqliteConnector_.getData<int>(r, 2);
753  bool isview = sqliteConnector_.getData<bool>(r, 3);
754 
757  DBObjectKey key;
758  key.dbId = currentDB_.dbId;
759  key.objectId = tableid;
760  key.permissionType = type;
761 
762  DBObject obj(tableName, type);
763  obj.setObjectKey(key);
764  obj.setOwner(ownerid);
767 
768  objects.push_back(obj);
769  }
770  }
771 
772  {
773  // other users dashboards
774  string tableQuery("SELECT id, name, userid FROM mapd_dashboards WHERE userid > 0");
775  sqliteConnector_.query(tableQuery);
776  size_t numRows = sqliteConnector_.getNumRows();
777  for (size_t r = 0; r < numRows; ++r) {
778  int32_t dashId = sqliteConnector_.getData<int>(r, 0);
779  std::string dashName = sqliteConnector_.getData<string>(r, 1);
780  int32_t ownerid = sqliteConnector_.getData<int>(r, 2);
781 
783  DBObjectKey key;
784  key.dbId = currentDB_.dbId;
785  key.objectId = dashId;
786  key.permissionType = type;
787 
788  DBObject obj(dashName, type);
789  obj.setObjectKey(key);
790  obj.setOwner(ownerid);
792 
793  objects.push_back(obj);
794  }
795  }
796  } catch (const std::exception& e) {
797  sqliteConnector_.query("ROLLBACK TRANSACTION");
798  throw;
799  }
800  sqliteConnector_.query("END TRANSACTION");
801 
802  // now apply the objects to the syscat to track the permisisons
803  // moved outside transaction to avoid lock in sqlite
804  try {
806  } catch (const std::exception& e) {
807  LOG(ERROR) << " Issue during migration of DB " << name() << " issue was " << e.what();
808  throw std::runtime_error(" Issue during migration of DB " + name() + " issue was " +
809  e.what());
810  // will need to remove the mapd_record_ownership_marker table and retry
811  }
812 }
813 
818 }
819 
821  std::unordered_map<std::string, std::pair<int, std::string>> dashboards;
822  std::vector<std::string> dashboard_ids;
823  static const std::string migration_name{"dashboard_roles_migration"};
824  {
826  sqliteConnector_.query("BEGIN TRANSACTION");
827  try {
828  // migration_history should be present in all catalogs by now
829  // if not then would be created before this migration
831  "select * from mapd_version_history where migration_history = '" +
832  migration_name + "'");
833  if (sqliteConnector_.getNumRows() != 0) {
834  // no need for further execution
835  sqliteConnector_.query("END TRANSACTION");
836  return;
837  }
838  LOG(INFO) << "Performing dashboard internal roles Migration.";
839  sqliteConnector_.query("select id, userid, metadata from mapd_dashboards");
840  for (size_t i = 0; i < sqliteConnector_.getNumRows(); ++i) {
843  sqliteConnector_.getData<string>(i, 0)))) {
844  // Successfully created roles during previous migration/crash
845  // No need to include them
846  continue;
847  }
848  dashboards[sqliteConnector_.getData<string>(i, 0)] = std::make_pair(
849  sqliteConnector_.getData<int>(i, 1), sqliteConnector_.getData<string>(i, 2));
850  dashboard_ids.push_back(sqliteConnector_.getData<string>(i, 0));
851  }
852  } catch (const std::exception& e) {
853  sqliteConnector_.query("ROLLBACK TRANSACTION");
854  throw;
855  }
856  sqliteConnector_.query("END TRANSACTION");
857  }
858  // All current grantees with shared dashboards.
859  const auto active_grantees =
861 
862  try {
863  // NOTE(wamsi): Transactionally unsafe
864  for (auto dash : dashboards) {
865  createOrUpdateDashboardSystemRole(dash.second.second,
866  dash.second.first,
868  std::to_string(currentDB_.dbId), dash.first));
869  auto result = active_grantees.find(dash.first);
870  if (result != active_grantees.end()) {
873  dash.first)},
874  result->second);
875  }
876  }
878  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
879  std::vector<std::string>{std::to_string(MAPD_VERSION), migration_name});
880  } catch (const std::exception& e) {
881  LOG(ERROR) << "Failed to create dashboard system roles during migration: "
882  << e.what();
883  throw;
884  }
885  LOG(INFO) << "Successfully created dashboard system roles during migration.";
886 }
887 
898  updatePageSize();
902 
903  if (g_enable_fsi) {
905  } else {
907  }
908 }
909 
913 }
914 
915 namespace {
916 std::string getUserFromId(const int32_t id) {
917  UserMetadata user;
918  if (SysCatalog::instance().getMetadataForUserById(id, user)) {
919  return user.userName;
920  }
921  // a user could be deleted and a dashboard still exist?
922  return "Unknown";
923 }
924 } // namespace
925 
929 
930  string dictQuery(
931  "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
932  sqliteConnector_.query(dictQuery);
933  size_t numRows = sqliteConnector_.getNumRows();
934  for (size_t r = 0; r < numRows; ++r) {
935  int dictId = sqliteConnector_.getData<int>(r, 0);
936  std::string dictName = sqliteConnector_.getData<string>(r, 1);
937  int dictNBits = sqliteConnector_.getData<int>(r, 2);
938  bool is_shared = sqliteConnector_.getData<bool>(r, 3);
939  int refcount = sqliteConnector_.getData<int>(r, 4);
940  std::string fname = basePath_ + "/mapd_data/DB_" + std::to_string(currentDB_.dbId) +
941  "_DICT_" + std::to_string(dictId);
942  DictRef dict_ref(currentDB_.dbId, dictId);
943  DictDescriptor* dd = new DictDescriptor(
944  dict_ref, dictName, dictNBits, is_shared, refcount, fname, false);
945  dictDescriptorMapByRef_[dict_ref].reset(dd);
946  }
947 
948  string tableQuery(
949  "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
950  "max_chunk_size, frag_page_size, "
951  "max_rows, partitions, shard_column_id, shard, num_shards, key_metainfo, userid, "
952  "sort_column_id, storage_type "
953  "from mapd_tables");
954  sqliteConnector_.query(tableQuery);
955  numRows = sqliteConnector_.getNumRows();
956  for (size_t r = 0; r < numRows; ++r) {
957  TableDescriptor* td;
958  const auto& storage_type = sqliteConnector_.getData<string>(r, 17);
959  if (!storage_type.empty() &&
960  (!g_enable_fsi || storage_type != StorageType::FOREIGN_TABLE)) {
961  const auto table_id = sqliteConnector_.getData<int>(r, 0);
962  const auto& table_name = sqliteConnector_.getData<string>(r, 1);
963  LOG(FATAL) << "Unable to read Catalog metadata: storage type is currently not a "
964  "supported table option (table "
965  << table_name << " [" << table_id << "] in database "
966  << currentDB_.dbName << ").";
967  }
968 
969  if (storage_type == StorageType::FOREIGN_TABLE) {
971  } else {
972  td = new TableDescriptor();
973  }
974 
975  td->storageType = storage_type;
976  td->tableId = sqliteConnector_.getData<int>(r, 0);
977  td->tableName = sqliteConnector_.getData<string>(r, 1);
978  td->nColumns = sqliteConnector_.getData<int>(r, 2);
979  td->isView = sqliteConnector_.getData<bool>(r, 3);
980  td->fragments = sqliteConnector_.getData<string>(r, 4);
981  td->fragType =
983  td->maxFragRows = sqliteConnector_.getData<int>(r, 6);
984  td->maxChunkSize = sqliteConnector_.getData<int>(r, 7);
985  td->fragPageSize = sqliteConnector_.getData<int>(r, 8);
986  td->maxRows = sqliteConnector_.getData<int64_t>(r, 9);
987  td->partitions = sqliteConnector_.getData<string>(r, 10);
988  td->shardedColumnId = sqliteConnector_.getData<int>(r, 11);
989  td->shard = sqliteConnector_.getData<int>(r, 12);
990  td->nShards = sqliteConnector_.getData<int>(r, 13);
991  td->keyMetainfo = sqliteConnector_.getData<string>(r, 14);
992  td->userId = sqliteConnector_.getData<int>(r, 15);
993  td->sortedColumnId =
994  sqliteConnector_.isNull(r, 16) ? 0 : sqliteConnector_.getData<int>(r, 16);
995  if (!td->isView) {
996  td->fragmenter = nullptr;
997  }
998  td->hasDeletedCol = false;
999 
1001  tableDescriptorMapById_[td->tableId] = td;
1002  }
1003 
1004  if (g_enable_fsi) {
1007  }
1008 
1009  string columnQuery(
1010  "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
1011  "is_notnull, compression, comp_param, "
1012  "size, chunks, is_systemcol, is_virtualcol, virtual_expr, is_deletedcol from "
1013  "mapd_columns ORDER BY tableid, "
1014  "columnid");
1015  sqliteConnector_.query(columnQuery);
1016  numRows = sqliteConnector_.getNumRows();
1017  int32_t skip_physical_cols = 0;
1018  for (size_t r = 0; r < numRows; ++r) {
1019  ColumnDescriptor* cd = new ColumnDescriptor();
1020  cd->tableId = sqliteConnector_.getData<int>(r, 0);
1021  cd->columnId = sqliteConnector_.getData<int>(r, 1);
1022  cd->columnName = sqliteConnector_.getData<string>(r, 2);
1026  cd->columnType.set_scale(sqliteConnector_.getData<int>(r, 6));
1030  cd->columnType.set_size(sqliteConnector_.getData<int>(r, 10));
1031  cd->chunks = sqliteConnector_.getData<string>(r, 11);
1032  cd->isSystemCol = sqliteConnector_.getData<bool>(r, 12);
1033  cd->isVirtualCol = sqliteConnector_.getData<bool>(r, 13);
1034  cd->virtualExpr = sqliteConnector_.getData<string>(r, 14);
1035  cd->isDeletedCol = sqliteConnector_.getData<bool>(r, 15);
1036  cd->isGeoPhyCol = skip_physical_cols > 0;
1037  ColumnKey columnKey(cd->tableId, to_upper(cd->columnName));
1038  columnDescriptorMap_[columnKey] = cd;
1039  ColumnIdKey columnIdKey(cd->tableId, cd->columnId);
1040  columnDescriptorMapById_[columnIdKey] = cd;
1041 
1042  if (skip_physical_cols <= 0) {
1043  skip_physical_cols = cd->columnType.get_physical_cols();
1044  }
1045 
1046  auto td_itr = tableDescriptorMapById_.find(cd->tableId);
1047  CHECK(td_itr != tableDescriptorMapById_.end());
1048 
1049  if (cd->isDeletedCol) {
1050  td_itr->second->hasDeletedCol = true;
1051  setDeletedColumnUnlocked(td_itr->second, cd);
1052  } else if (cd->columnType.is_geometry() || skip_physical_cols-- <= 0) {
1053  tableDescriptorMapById_[cd->tableId]->columnIdBySpi_.push_back(cd->columnId);
1054  }
1055  }
1056  // sort columnIdBySpi_ based on columnId
1057  for (auto& tit : tableDescriptorMapById_) {
1058  std::sort(tit.second->columnIdBySpi_.begin(),
1059  tit.second->columnIdBySpi_.end(),
1060  [](const size_t a, const size_t b) -> bool { return a < b; });
1061  }
1062 
1063  string viewQuery("SELECT tableid, sql FROM mapd_views");
1064  sqliteConnector_.query(viewQuery);
1065  numRows = sqliteConnector_.getNumRows();
1066  for (size_t r = 0; r < numRows; ++r) {
1067  int32_t tableId = sqliteConnector_.getData<int>(r, 0);
1068  TableDescriptor* td = tableDescriptorMapById_[tableId];
1069  td->viewSQL = sqliteConnector_.getData<string>(r, 1);
1070  td->fragmenter = nullptr;
1071  }
1072 
1073  string frontendViewQuery(
1074  "SELECT id, state, name, image_hash, strftime('%Y-%m-%dT%H:%M:%SZ', update_time), "
1075  "userid, "
1076  "metadata "
1077  "FROM mapd_dashboards");
1078  sqliteConnector_.query(frontendViewQuery);
1079  numRows = sqliteConnector_.getNumRows();
1080  for (size_t r = 0; r < numRows; ++r) {
1081  std::shared_ptr<DashboardDescriptor> vd = std::make_shared<DashboardDescriptor>();
1082  vd->dashboardId = sqliteConnector_.getData<int>(r, 0);
1083  vd->dashboardState = sqliteConnector_.getData<string>(r, 1);
1084  vd->dashboardName = sqliteConnector_.getData<string>(r, 2);
1085  vd->imageHash = sqliteConnector_.getData<string>(r, 3);
1086  vd->updateTime = sqliteConnector_.getData<string>(r, 4);
1087  vd->userId = sqliteConnector_.getData<int>(r, 5);
1088  vd->dashboardMetadata = sqliteConnector_.getData<string>(r, 6);
1089  vd->user = getUserFromId(vd->userId);
1090  vd->dashboardSystemRoleName = generate_dashboard_system_rolename(
1092  dashboardDescriptorMap_[std::to_string(vd->userId) + ":" + vd->dashboardName] = vd;
1093  }
1094 
1095  string linkQuery(
1096  "SELECT linkid, userid, link, view_state, strftime('%Y-%m-%dT%H:%M:%SZ', "
1097  "update_time), view_metadata "
1098  "FROM mapd_links");
1099  sqliteConnector_.query(linkQuery);
1100  numRows = sqliteConnector_.getNumRows();
1101  for (size_t r = 0; r < numRows; ++r) {
1102  LinkDescriptor* ld = new LinkDescriptor();
1103  ld->linkId = sqliteConnector_.getData<int>(r, 0);
1104  ld->userId = sqliteConnector_.getData<int>(r, 1);
1105  ld->link = sqliteConnector_.getData<string>(r, 2);
1106  ld->viewState = sqliteConnector_.getData<string>(r, 3);
1107  ld->updateTime = sqliteConnector_.getData<string>(r, 4);
1108  ld->viewMetadata = sqliteConnector_.getData<string>(r, 5);
1110  linkDescriptorMapById_[ld->linkId] = ld;
1111  }
1112 
1113  /* rebuild map linking logical tables to corresponding physical ones */
1114  string logicalToPhysicalTableMapQuery(
1115  "SELECT logical_table_id, physical_table_id "
1116  "FROM mapd_logical_to_physical");
1117  sqliteConnector_.query(logicalToPhysicalTableMapQuery);
1118  numRows = sqliteConnector_.getNumRows();
1119  for (size_t r = 0; r < numRows; ++r) {
1120  int32_t logical_tb_id = sqliteConnector_.getData<int>(r, 0);
1121  int32_t physical_tb_id = sqliteConnector_.getData<int>(r, 1);
1122  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
1123  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
1124  /* add new entity to the map logicalToPhysicalTableMapById_ */
1125  std::vector<int32_t> physicalTables;
1126  physicalTables.push_back(physical_tb_id);
1127  const auto it_ok =
1128  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
1129  CHECK(it_ok.second);
1130  } else {
1131  /* update map logicalToPhysicalTableMapById_ */
1132  physicalTableIt->second.push_back(physical_tb_id);
1133  }
1134  }
1135 }
1136 
1138  const list<ColumnDescriptor>& columns,
1139  const list<DictDescriptor>& dicts) {
1140  cat_write_lock write_lock(this);
1141  TableDescriptor* new_td;
1142 
1143  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
1144  if (foreign_table) {
1145  auto new_foreign_table = new foreign_storage::ForeignTable();
1146  *new_foreign_table = *foreign_table;
1147  new_td = new_foreign_table;
1148  } else {
1149  new_td = new TableDescriptor();
1150  *new_td = *td;
1151  }
1152 
1153  new_td->mutex_ = std::make_shared<std::mutex>();
1154  tableDescriptorMap_[to_upper(td->tableName)] = new_td;
1155  tableDescriptorMapById_[td->tableId] = new_td;
1156  for (auto cd : columns) {
1157  ColumnDescriptor* new_cd = new ColumnDescriptor();
1158  *new_cd = cd;
1159  ColumnKey columnKey(new_cd->tableId, to_upper(new_cd->columnName));
1160  columnDescriptorMap_[columnKey] = new_cd;
1161  ColumnIdKey columnIdKey(new_cd->tableId, new_cd->columnId);
1162  columnDescriptorMapById_[columnIdKey] = new_cd;
1163 
1164  // Add deleted column to the map
1165  if (cd.isDeletedCol) {
1166  CHECK(new_td->hasDeletedCol);
1167  setDeletedColumnUnlocked(new_td, new_cd);
1168  }
1169  }
1170 
1171  std::sort(new_td->columnIdBySpi_.begin(),
1172  new_td->columnIdBySpi_.end(),
1173  [](const size_t a, const size_t b) -> bool { return a < b; });
1174 
1175  std::unique_ptr<StringDictionaryClient> client;
1176  DictRef dict_ref(currentDB_.dbId, -1);
1177  if (!string_dict_hosts_.empty()) {
1178  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1179  }
1180  for (auto dd : dicts) {
1181  if (!dd.dictRef.dictId) {
1182  // Dummy entry created for a shard of a logical table, nothing to do.
1183  continue;
1184  }
1185  dict_ref.dictId = dd.dictRef.dictId;
1186  if (client) {
1187  client->create(dict_ref, dd.dictIsTemp);
1188  }
1189  DictDescriptor* new_dd = new DictDescriptor(dd);
1190  dictDescriptorMapByRef_[dict_ref].reset(new_dd);
1191  if (!dd.dictIsTemp) {
1192  boost::filesystem::create_directory(new_dd->dictFolderPath);
1193  }
1194  }
1195 }
1196 
1197 void Catalog::removeTableFromMap(const string& tableName,
1198  const int tableId,
1199  const bool is_on_error) {
1200  cat_write_lock write_lock(this);
1201  TableDescriptorMapById::iterator tableDescIt = tableDescriptorMapById_.find(tableId);
1202  if (tableDescIt == tableDescriptorMapById_.end()) {
1203  throw runtime_error("Table " + tableName + " does not exist.");
1204  }
1205 
1206  TableDescriptor* td = tableDescIt->second;
1207 
1208  if (td->hasDeletedCol) {
1209  const auto ret = deletedColumnPerTable_.erase(td);
1210  CHECK_EQ(ret, size_t(1));
1211  }
1212 
1213  tableDescriptorMapById_.erase(tableDescIt);
1214  tableDescriptorMap_.erase(to_upper(tableName));
1215  td->fragmenter = nullptr;
1216 
1218  delete td;
1219 
1220  std::unique_ptr<StringDictionaryClient> client;
1221  if (SysCatalog::instance().isAggregator()) {
1222  CHECK(!string_dict_hosts_.empty());
1223  DictRef dict_ref(currentDB_.dbId, -1);
1224  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1225  }
1226 
1227  // delete all column descriptors for the table
1228  // no more link columnIds to sequential indexes!
1229  for (auto cit = columnDescriptorMapById_.begin();
1230  cit != columnDescriptorMapById_.end();) {
1231  if (tableId != std::get<0>(cit->first)) {
1232  ++cit;
1233  } else {
1234  int i = std::get<1>(cit++->first);
1235  ColumnIdKey cidKey(tableId, i);
1236  ColumnDescriptorMapById::iterator colDescIt = columnDescriptorMapById_.find(cidKey);
1237  ColumnDescriptor* cd = colDescIt->second;
1238  columnDescriptorMapById_.erase(colDescIt);
1239  ColumnKey cnameKey(tableId, to_upper(cd->columnName));
1240  columnDescriptorMap_.erase(cnameKey);
1241  const int dictId = cd->columnType.get_comp_param();
1242  // Dummy dictionaries created for a shard of a logical table have the id set to
1243  // zero.
1244  if (cd->columnType.get_compression() == kENCODING_DICT && dictId) {
1245  INJECT_TIMER(removingDicts);
1246  DictRef dict_ref(currentDB_.dbId, dictId);
1247  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
1248  // If we're removing this table due to an error, it is possible that the string
1249  // dictionary reference was never populated. Don't crash, just continue cleaning
1250  // up the TableDescriptor and ColumnDescriptors
1251  if (!is_on_error) {
1252  CHECK(dictIt != dictDescriptorMapByRef_.end());
1253  } else {
1254  if (dictIt == dictDescriptorMapByRef_.end()) {
1255  continue;
1256  }
1257  }
1258  const auto& dd = dictIt->second;
1259  CHECK_GE(dd->refcount, 1);
1260  --dd->refcount;
1261  if (!dd->refcount) {
1262  dd->stringDict.reset();
1263  if (!isTemp) {
1264  File_Namespace::renameForDelete(dd->dictFolderPath);
1265  }
1266  if (client) {
1267  client->drop(dict_ref);
1268  }
1269  dictDescriptorMapByRef_.erase(dictIt);
1270  }
1271  }
1272 
1273  delete cd;
1274  }
1275  }
1276 }
1277 
1279  cat_write_lock write_lock(this);
1281 }
1282 
1284  cat_write_lock write_lock(this);
1286  std::make_shared<DashboardDescriptor>(vd);
1287 }
1288 
1289 std::vector<DBObject> Catalog::parseDashboardObjects(const std::string& view_meta,
1290  const int& user_id) {
1291  std::vector<DBObject> objects;
1292  DBObjectKey key;
1293  key.dbId = currentDB_.dbId;
1294  auto _key_place = [&key](auto type, auto id) {
1295  key.permissionType = type;
1296  key.objectId = id;
1297  return key;
1298  };
1299  for (auto object_name : parse_underlying_dashboard_objects(view_meta)) {
1300  auto td = getMetadataForTable(object_name);
1301  if (!td) {
1302  // Parsed object source is not present in current database
1303  // LOG the info and ignore
1304  LOG(INFO) << "Ignoring dashboard source Table/View: " << object_name
1305  << " no longer exists in current DB.";
1306  continue;
1307  }
1308  // Dashboard source can be Table or View
1309  const auto object_type = td->isView ? ViewDBObjectType : TableDBObjectType;
1310  const auto priv = td->isView ? AccessPrivileges::SELECT_FROM_VIEW
1312  objects.emplace_back(_key_place(object_type, td->tableId), priv, user_id);
1313  objects.back().setObjectType(td->isView ? ViewDBObjectType : TableDBObjectType);
1314  objects.back().setName(td->tableName);
1315  }
1316  return objects;
1317 }
1318 
1319 void Catalog::createOrUpdateDashboardSystemRole(const std::string& view_meta,
1320  const int32_t& user_id,
1321  const std::string& dash_role_name) {
1322  auto objects = parseDashboardObjects(view_meta, user_id);
1323  Role* rl = SysCatalog::instance().getRoleGrantee(dash_role_name);
1324  if (!rl) {
1325  // Dashboard role does not exist
1326  // create role and grant privileges
1327  // NOTE(wamsi): Transactionally unsafe
1328  SysCatalog::instance().createRole(dash_role_name, false);
1329  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1330  } else {
1331  // Dashboard system role already exists
1332  // Add/remove privileges on objects
1333  auto ex_objects = rl->getDbObjects(true);
1334  for (auto key : *ex_objects | boost::adaptors::map_keys) {
1335  if (key.permissionType != TableDBObjectType &&
1336  key.permissionType != ViewDBObjectType) {
1337  continue;
1338  }
1339  bool found = false;
1340  for (auto obj : objects) {
1341  found = key == obj.getObjectKey() ? true : false;
1342  if (found) {
1343  break;
1344  }
1345  }
1346  if (!found) {
1347  // revoke privs on object since the object is no
1348  // longer used by the dashboard as source
1349  // NOTE(wamsi): Transactionally unsafe
1351  dash_role_name, *rl->findDbObject(key, true), *this);
1352  }
1353  }
1354  // Update privileges on remaining objects
1355  // NOTE(wamsi): Transactionally unsafe
1356  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1357  }
1358 }
1359 
1361  cat_write_lock write_lock(this);
1362  LinkDescriptor* new_ld = new LinkDescriptor();
1363  *new_ld = ld;
1365  linkDescriptorMapById_[ld.linkId] = new_ld;
1366 }
1367 
1369  auto time_ms = measure<>::execution([&]() {
1370  // instanciate table fragmenter upon first use
1371  // assume only insert order fragmenter is supported
1373  vector<Chunk> chunkVec;
1374  list<const ColumnDescriptor*> columnDescs;
1375  getAllColumnMetadataForTableImpl(td, columnDescs, true, false, true);
1376  Chunk::translateColumnDescriptorsToChunkVec(columnDescs, chunkVec);
1377  ChunkKey chunkKeyPrefix = {currentDB_.dbId, td->tableId};
1378  if (td->sortedColumnId > 0) {
1379  td->fragmenter = std::make_shared<SortedOrderFragmenter>(chunkKeyPrefix,
1380  chunkVec,
1381  dataMgr_.get(),
1382  const_cast<Catalog*>(this),
1383  td->tableId,
1384  td->shard,
1385  td->maxFragRows,
1386  td->maxChunkSize,
1387  td->fragPageSize,
1388  td->maxRows,
1389  td->persistenceLevel);
1390  } else {
1391  td->fragmenter = std::make_shared<InsertOrderFragmenter>(chunkKeyPrefix,
1392  chunkVec,
1393  dataMgr_.get(),
1394  const_cast<Catalog*>(this),
1395  td->tableId,
1396  td->shard,
1397  td->maxFragRows,
1398  td->maxChunkSize,
1399  td->fragPageSize,
1400  td->maxRows,
1401  td->persistenceLevel,
1402  !td->storageType.empty());
1403  }
1404  });
1405  LOG(INFO) << "Instantiating Fragmenter for table " << td->tableName << " took "
1406  << time_ms << "ms";
1407 }
1408 
1409 const TableDescriptor* Catalog::getMetadataForTable(const string& tableName,
1410  const bool populateFragmenter) const {
1411  // we give option not to populate fragmenter (default true/yes) as it can be heavy for
1412  // pure metadata calls
1413  cat_read_lock read_lock(this);
1414  auto tableDescIt = tableDescriptorMap_.find(to_upper(tableName));
1415  if (tableDescIt == tableDescriptorMap_.end()) { // check to make sure table exists
1416  return nullptr;
1417  }
1418  TableDescriptor* td = tableDescIt->second;
1419  std::unique_lock<std::mutex> td_lock(*td->mutex_.get());
1420  if (populateFragmenter && td->fragmenter == nullptr && !td->isView) {
1422  }
1423  return td; // returns pointer to table descriptor
1424 }
1425 
1427  int tableId,
1428  const bool populateFragmenter) const {
1429  auto tableDescIt = tableDescriptorMapById_.find(tableId);
1430  if (tableDescIt == tableDescriptorMapById_.end()) { // check to make sure table exists
1431  return nullptr;
1432  }
1433  TableDescriptor* td = tableDescIt->second;
1434  if (populateFragmenter) {
1435  std::unique_lock<std::mutex> td_lock(*td->mutex_.get());
1436  if (td->fragmenter == nullptr && !td->isView) {
1438  }
1439  }
1440  return td; // returns pointer to table descriptor
1441 }
1442 
1444  bool populateFragmenter) const {
1445  cat_read_lock read_lock(this);
1446  return getMetadataForTableImpl(tableId, populateFragmenter);
1447 }
1448 
1450  const bool load_dict) const {
1451  cat_read_lock read_lock(this);
1452  return getMetadataForDictUnlocked(dict_id, load_dict);
1453 }
1454 
1456  const bool loadDict) const {
1457  const DictRef dictRef(currentDB_.dbId, dictId);
1458  auto dictDescIt = dictDescriptorMapByRef_.find(dictRef);
1459  if (dictDescIt ==
1460  dictDescriptorMapByRef_.end()) { // check to make sure dictionary exists
1461  return nullptr;
1462  }
1463  auto& dd = dictDescIt->second;
1464 
1465  if (loadDict) {
1466  std::lock_guard string_dict_lock(*dd->string_dict_mutex);
1467  if (!dd->stringDict) {
1468  auto time_ms = measure<>::execution([&]() {
1469  if (string_dict_hosts_.empty()) {
1470  if (dd->dictIsTemp) {
1471  dd->stringDict = std::make_shared<StringDictionary>(
1472  dd->dictFolderPath, true, true, g_cache_string_hash);
1473  } else {
1474  dd->stringDict = std::make_shared<StringDictionary>(
1475  dd->dictFolderPath, false, true, g_cache_string_hash);
1476  }
1477  } else {
1478  dd->stringDict =
1479  std::make_shared<StringDictionary>(string_dict_hosts_.front(), dd->dictRef);
1480  }
1481  });
1482  LOG(INFO) << "Time to load Dictionary " << dd->dictRef.dbId << "_"
1483  << dd->dictRef.dictId << " was " << time_ms << "ms";
1484  }
1485  }
1486 
1487  return dd.get();
1488 }
1489 
1490 const std::vector<LeafHostInfo>& Catalog::getStringDictionaryHosts() const {
1491  return string_dict_hosts_;
1492 }
1493 
1495  const string& columnName) const {
1496  cat_read_lock read_lock(this);
1497 
1498  ColumnKey columnKey(tableId, to_upper(columnName));
1499  auto colDescIt = columnDescriptorMap_.find(columnKey);
1500  if (colDescIt ==
1501  columnDescriptorMap_.end()) { // need to check to make sure column exists for table
1502  return nullptr;
1503  }
1504  return colDescIt->second;
1505 }
1506 
1507 const ColumnDescriptor* Catalog::getMetadataForColumn(int table_id, int column_id) const {
1508  cat_read_lock read_lock(this);
1509  return getMetadataForColumnUnlocked(table_id, column_id);
1510 }
1511 
1513  int columnId) const {
1514  ColumnIdKey columnIdKey(tableId, columnId);
1515  auto colDescIt = columnDescriptorMapById_.find(columnIdKey);
1516  if (colDescIt == columnDescriptorMapById_
1517  .end()) { // need to check to make sure column exists for table
1518  return nullptr;
1519  }
1520  return colDescIt->second;
1521 }
1522 
1523 const int Catalog::getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const {
1524  const auto tabDescIt = tableDescriptorMapById_.find(table_id);
1525  CHECK(tableDescriptorMapById_.end() != tabDescIt);
1526  const auto& columnIdBySpi = tabDescIt->second->columnIdBySpi_;
1527 
1528  auto spx = spi;
1529  int phi = 0;
1530  if (spx >= SPIMAP_MAGIC1) // see Catalog.h
1531  {
1532  phi = (spx - SPIMAP_MAGIC1) % SPIMAP_MAGIC2;
1533  spx = (spx - SPIMAP_MAGIC1) / SPIMAP_MAGIC2;
1534  }
1535 
1536  CHECK(0 < spx && spx <= columnIdBySpi.size());
1537  return columnIdBySpi[spx - 1] + phi;
1538 }
1539 
1540 const int Catalog::getColumnIdBySpi(const int table_id, const size_t spi) const {
1541  cat_read_lock read_lock(this);
1542  return getColumnIdBySpiUnlocked(table_id, spi);
1543 }
1544 
1546  const size_t spi) const {
1547  cat_read_lock read_lock(this);
1548 
1549  const auto columnId = getColumnIdBySpiUnlocked(tableId, spi);
1550  ColumnIdKey columnIdKey(tableId, columnId);
1551  const auto colDescIt = columnDescriptorMapById_.find(columnIdKey);
1552  return columnDescriptorMapById_.end() == colDescIt ? nullptr : colDescIt->second;
1553 }
1554 
1555 void Catalog::deleteMetadataForDashboards(const std::vector<int32_t> dashboard_ids,
1556  const UserMetadata& user) {
1557  std::stringstream invalid_ids, restricted_ids;
1558 
1559  for (int32_t dashboard_id : dashboard_ids) {
1560  if (!getMetadataForDashboard(dashboard_id)) {
1561  invalid_ids << (!invalid_ids.str().empty() ? ", " : "") << dashboard_id;
1562  continue;
1563  }
1564  DBObject object(dashboard_id, DashboardDBObjectType);
1565  object.loadKey(*this);
1566  object.setPrivileges(AccessPrivileges::DELETE_DASHBOARD);
1567  std::vector<DBObject> privs = {object};
1568  if (!SysCatalog::instance().checkPrivileges(user, privs))
1569  restricted_ids << (!restricted_ids.str().empty() ? ", " : "") << dashboard_id;
1570  }
1571 
1572  if (invalid_ids.str().size() > 0 || restricted_ids.str().size() > 0) {
1573  std::stringstream error_message;
1574  error_message << "Delete dashboard(s) failed with error(s):";
1575  if (invalid_ids.str().size() > 0)
1576  error_message << "\nDashboard id: " << invalid_ids.str()
1577  << " - Dashboard id does not exist";
1578  if (restricted_ids.str().size() > 0)
1579  error_message
1580  << "\nDashboard id: " << restricted_ids.str()
1581  << " - User should be either owner of dashboard or super user to delete it";
1582  throw std::runtime_error(error_message.str());
1583  }
1584  std::vector<DBObject> dash_objs;
1585 
1586  for (int32_t dashboard_id : dashboard_ids) {
1587  dash_objs.push_back(DBObject(dashboard_id, DashboardDBObjectType));
1588  }
1589  // BE-5245: Transactionally unsafe (like other combined Catalog/Syscatalog operations)
1591  {
1592  cat_write_lock write_lock(this);
1594 
1595  sqliteConnector_.query("BEGIN TRANSACTION");
1596  try {
1597  for (int32_t dashboard_id : dashboard_ids) {
1598  auto dash = getMetadataForDashboard(dashboard_id);
1599  // Dash should still exist if revokeDBObjectPrivileges passed but throw and
1600  // rollback if already deleted
1601  if (!dash) {
1602  throw std::runtime_error(
1603  std::string("Delete dashboard(s) failed with error(s):\nDashboard id: ") +
1604  std::to_string(dashboard_id) + " - Dashboard id does not exist ");
1605  }
1606  std::string user_id = std::to_string(dash->userId);
1607  std::string dash_name = dash->dashboardName;
1608  auto viewDescIt = dashboardDescriptorMap_.find(user_id + ":" + dash_name);
1609  dashboardDescriptorMap_.erase(viewDescIt);
1611  "DELETE FROM mapd_dashboards WHERE name = ? and userid = ?",
1612  std::vector<std::string>{dash_name, user_id});
1613  }
1614  } catch (std::exception& e) {
1615  sqliteConnector_.query("ROLLBACK TRANSACTION");
1616  throw;
1617  }
1618  sqliteConnector_.query("END TRANSACTION");
1619  }
1620 }
1621 
1623  const string& userId,
1624  const string& dashName) const {
1625  cat_read_lock read_lock(this);
1626 
1627  auto viewDescIt = dashboardDescriptorMap_.find(userId + ":" + dashName);
1628  if (viewDescIt == dashboardDescriptorMap_.end()) { // check to make sure view exists
1629  return nullptr;
1630  }
1631  return viewDescIt->second.get(); // returns pointer to view descriptor
1632 }
1633 
1635  cat_read_lock read_lock(this);
1636  std::string userId;
1637  std::string name;
1638  bool found{false};
1639  {
1640  for (auto descp : dashboardDescriptorMap_) {
1641  auto dash = descp.second.get();
1642  if (dash->dashboardId == id) {
1643  userId = std::to_string(dash->userId);
1644  name = dash->dashboardName;
1645  found = true;
1646  break;
1647  }
1648  }
1649  }
1650  if (found) {
1651  return getMetadataForDashboard(userId, name);
1652  }
1653  return nullptr;
1654 }
1655 
1656 const LinkDescriptor* Catalog::getMetadataForLink(const string& link) const {
1657  cat_read_lock read_lock(this);
1658  auto linkDescIt = linkDescriptorMap_.find(link);
1659  if (linkDescIt == linkDescriptorMap_.end()) { // check to make sure view exists
1660  return nullptr;
1661  }
1662  return linkDescIt->second; // returns pointer to view descriptor
1663 }
1664 
1666  cat_read_lock read_lock(this);
1667  auto linkDescIt = linkDescriptorMapById_.find(linkId);
1668  if (linkDescIt == linkDescriptorMapById_.end()) { // check to make sure view exists
1669  return nullptr;
1670  }
1671  return linkDescIt->second;
1672 }
1673 
1675  const TableDescriptor* td,
1676  list<const ColumnDescriptor*>& columnDescriptors,
1677  const bool fetchSystemColumns,
1678  const bool fetchVirtualColumns,
1679  const bool fetchPhysicalColumns) const {
1680  int32_t skip_physical_cols = 0;
1681  for (const auto& columnDescriptor : columnDescriptorMapById_) {
1682  if (!fetchPhysicalColumns && skip_physical_cols > 0) {
1683  --skip_physical_cols;
1684  continue;
1685  }
1686  auto cd = columnDescriptor.second;
1687  if (cd->tableId != td->tableId) {
1688  continue;
1689  }
1690  if (!fetchSystemColumns && cd->isSystemCol) {
1691  continue;
1692  }
1693  if (!fetchVirtualColumns && cd->isVirtualCol) {
1694  continue;
1695  }
1696  if (!fetchPhysicalColumns) {
1697  const auto& col_ti = cd->columnType;
1698  skip_physical_cols = col_ti.get_physical_cols();
1699  }
1700  columnDescriptors.push_back(cd);
1701  }
1702 }
1703 
1704 std::list<const ColumnDescriptor*> Catalog::getAllColumnMetadataForTable(
1705  const int tableId,
1706  const bool fetchSystemColumns,
1707  const bool fetchVirtualColumns,
1708  const bool fetchPhysicalColumns) const {
1709  cat_read_lock read_lock(this);
1711  tableId, fetchSystemColumns, fetchVirtualColumns, fetchPhysicalColumns);
1712 }
1713 
1714 std::list<const ColumnDescriptor*> Catalog::getAllColumnMetadataForTableUnlocked(
1715  const int tableId,
1716  const bool fetchSystemColumns,
1717  const bool fetchVirtualColumns,
1718  const bool fetchPhysicalColumns) const {
1719  std::list<const ColumnDescriptor*> columnDescriptors;
1720  const TableDescriptor* td =
1721  getMetadataForTableImpl(tableId, false); // dont instantiate fragmenter
1723  columnDescriptors,
1724  fetchSystemColumns,
1725  fetchVirtualColumns,
1726  fetchPhysicalColumns);
1727  return columnDescriptors;
1728 }
1729 
1730 list<const TableDescriptor*> Catalog::getAllTableMetadata() const {
1731  cat_read_lock read_lock(this);
1732  list<const TableDescriptor*> table_list;
1733  for (auto p : tableDescriptorMapById_) {
1734  table_list.push_back(p.second);
1735  }
1736  return table_list;
1737 }
1738 
1739 list<const DashboardDescriptor*> Catalog::getAllDashboardsMetadata() const {
1740  list<const DashboardDescriptor*> view_list;
1741  for (auto p : dashboardDescriptorMap_) {
1742  view_list.push_back(p.second.get());
1743  }
1744  return view_list;
1745 }
1746 
1748  const auto& td = *tableDescriptorMapById_[cd.tableId];
1749  list<DictDescriptor> dds;
1750  setColumnDictionary(cd, dds, td, true);
1751  auto& dd = dds.back();
1752  CHECK(dd.dictRef.dictId);
1753 
1754  std::unique_ptr<StringDictionaryClient> client;
1755  if (!string_dict_hosts_.empty()) {
1756  client.reset(new StringDictionaryClient(
1757  string_dict_hosts_.front(), DictRef(currentDB_.dbId, -1), true));
1758  }
1759  if (client) {
1760  client->create(dd.dictRef, dd.dictIsTemp);
1761  }
1762 
1763  DictDescriptor* new_dd = new DictDescriptor(dd);
1764  dictDescriptorMapByRef_[dd.dictRef].reset(new_dd);
1765  if (!dd.dictIsTemp) {
1766  boost::filesystem::create_directory(new_dd->dictFolderPath);
1767  }
1768  return dd.dictRef;
1769 }
1770 
1772  cat_write_lock write_lock(this);
1774  if (!(cd.columnType.is_string() || cd.columnType.is_string_array())) {
1775  return;
1776  }
1777  if (!(cd.columnType.get_compression() == kENCODING_DICT)) {
1778  return;
1779  }
1780  const auto dictId = cd.columnType.get_comp_param();
1781  CHECK_GT(dictId, 0);
1782  // decrement and zero check dict ref count
1783  const auto td = getMetadataForTable(cd.tableId);
1784  CHECK(td);
1786  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
1787  std::to_string(dictId));
1789  "SELECT refcount FROM mapd_dictionaries WHERE dictid = ?", std::to_string(dictId));
1790  const auto refcount = sqliteConnector_.getData<int>(0, 0);
1791  VLOG(3) << "Dictionary " << dictId << "from dropped table has reference count "
1792  << refcount;
1793  if (refcount > 0) {
1794  return;
1795  }
1796  const DictRef dictRef(currentDB_.dbId, dictId);
1797  sqliteConnector_.query_with_text_param("DELETE FROM mapd_dictionaries WHERE dictid = ?",
1798  std::to_string(dictId));
1799  File_Namespace::renameForDelete(basePath_ + "/mapd_data/DB_" +
1800  std::to_string(currentDB_.dbId) + "_DICT_" +
1801  std::to_string(dictId));
1802 
1803  std::unique_ptr<StringDictionaryClient> client;
1804  if (!string_dict_hosts_.empty()) {
1805  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dictRef, true));
1806  }
1807  if (client) {
1808  client->drop(dictRef);
1809  }
1810 
1811  dictDescriptorMapByRef_.erase(dictRef);
1812 }
1813 
1815  std::map<int, StringDictionary*>& stringDicts) {
1816  // learn 'committed' ColumnDescriptor of this column
1817  auto cit = columnDescriptorMap_.find(ColumnKey(cd.tableId, to_upper(cd.columnName)));
1818  CHECK(cit != columnDescriptorMap_.end());
1819  auto& ccd = *cit->second;
1820 
1821  if (!(ccd.columnType.is_string() || ccd.columnType.is_string_array())) {
1822  return;
1823  }
1824  if (!(ccd.columnType.get_compression() == kENCODING_DICT)) {
1825  return;
1826  }
1827  if (!(ccd.columnType.get_comp_param() > 0)) {
1828  return;
1829  }
1830 
1831  auto dictId = ccd.columnType.get_comp_param();
1832  getMetadataForDict(dictId);
1833 
1834  const DictRef dictRef(currentDB_.dbId, dictId);
1835  auto dit = dictDescriptorMapByRef_.find(dictRef);
1836  CHECK(dit != dictDescriptorMapByRef_.end());
1837  CHECK(dit->second);
1838  CHECK(dit->second.get()->stringDict);
1839  stringDicts[ccd.columnId] = dit->second.get()->stringDict.get();
1840 }
1841 
1843  cat_write_lock write_lock(this);
1844  // caller must handle sqlite/chunk transaction TOGETHER
1845  cd.tableId = td.tableId;
1846  if (td.nShards > 0 && td.shard < 0) {
1847  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
1848  auto shard_cd = cd;
1849  addColumn(*shard, shard_cd);
1850  }
1851  }
1853  addDictionary(cd);
1854  }
1855 
1857  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
1858  "colscale, is_notnull, "
1859  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
1860  "is_deletedcol) "
1861  "VALUES (?, "
1862  "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
1863  "?, ?, ?, "
1864  "?, "
1865  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1866  std::vector<std::string>{std::to_string(td.tableId),
1867  std::to_string(td.tableId),
1868  cd.columnName,
1877  "",
1880  cd.virtualExpr,
1882 
1884  "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
1885  std::vector<std::string>{std::to_string(td.tableId)});
1886 
1888  "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
1889  std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
1890  cd.columnId = sqliteConnector_.getData<int>(0, 0);
1891 
1892  ++tableDescriptorMapById_[td.tableId]->nColumns;
1893  auto ncd = new ColumnDescriptor(cd);
1896  columnDescriptorsForRoll.emplace_back(nullptr, ncd);
1897 }
1898 
1900  cat_write_lock write_lock(this);
1902  // caller must handle sqlite/chunk transaction TOGETHER
1904  "DELETE FROM mapd_columns where tableid = ? and columnid = ?",
1905  std::vector<std::string>{std::to_string(td.tableId), std::to_string(cd.columnId)});
1906 
1908  "UPDATE mapd_tables SET ncolumns = ncolumns - 1 WHERE tableid = ?",
1909  std::vector<std::string>{std::to_string(td.tableId)});
1910 
1911  ColumnDescriptorMap::iterator columnDescIt =
1913  CHECK(columnDescIt != columnDescriptorMap_.end());
1914 
1915  columnDescriptorsForRoll.emplace_back(columnDescIt->second, nullptr);
1916 
1917  columnDescriptorMap_.erase(columnDescIt);
1919  --tableDescriptorMapById_[td.tableId]->nColumns;
1920  // for each shard
1921  if (td.nShards > 0 && td.shard < 0) {
1922  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
1923  const auto shard_cd = getMetadataForColumn(shard->tableId, cd.columnId);
1924  CHECK(shard_cd);
1925  dropColumn(*shard, *shard_cd);
1926  }
1927  }
1928 }
1929 
1930 void Catalog::roll(const bool forward) {
1931  cat_write_lock write_lock(this);
1932  std::set<const TableDescriptor*> tds;
1933 
1934  for (const auto& cdr : columnDescriptorsForRoll) {
1935  auto ocd = cdr.first;
1936  auto ncd = cdr.second;
1937  CHECK(ocd || ncd);
1938  auto tabDescIt = tableDescriptorMapById_.find((ncd ? ncd : ocd)->tableId);
1939  CHECK(tableDescriptorMapById_.end() != tabDescIt);
1940  auto td = tabDescIt->second;
1941  auto& vc = td->columnIdBySpi_;
1942  if (forward) {
1943  if (ocd) {
1944  if (nullptr == ncd ||
1945  ncd->columnType.get_comp_param() != ocd->columnType.get_comp_param()) {
1946  delDictionary(*ocd);
1947  }
1948 
1949  vc.erase(std::remove(vc.begin(), vc.end(), ocd->columnId), vc.end());
1950 
1951  delete ocd;
1952  }
1953  if (ncd) {
1954  // append columnId if its new and not phy geo
1955  if (vc.end() == std::find(vc.begin(), vc.end(), ncd->columnId)) {
1956  if (!ncd->isGeoPhyCol) {
1957  vc.push_back(ncd->columnId);
1958  }
1959  }
1960  }
1961  tds.insert(td);
1962  } else {
1963  if (ocd) {
1964  columnDescriptorMap_[ColumnKey(ocd->tableId, to_upper(ocd->columnName))] = ocd;
1965  columnDescriptorMapById_[ColumnIdKey(ocd->tableId, ocd->columnId)] = ocd;
1966  }
1967  // roll back the dict of new column
1968  if (ncd) {
1969  columnDescriptorMap_.erase(ColumnKey(ncd->tableId, to_upper(ncd->columnName)));
1970  columnDescriptorMapById_.erase(ColumnIdKey(ncd->tableId, ncd->columnId));
1971  if (nullptr == ocd ||
1972  ocd->columnType.get_comp_param() != ncd->columnType.get_comp_param()) {
1973  delDictionary(*ncd);
1974  }
1975  delete ncd;
1976  }
1977  }
1978  }
1979  columnDescriptorsForRoll.clear();
1980 
1981  if (forward) {
1982  for (const auto td : tds) {
1983  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
1984  }
1985  }
1986 }
1987 
1989  list<ColumnDescriptor>& columns) {
1990  const auto& col_ti = cd.columnType;
1991  if (IS_GEO(col_ti.get_type())) {
1992  switch (col_ti.get_type()) {
1993  case kPOINT: {
1994  ColumnDescriptor physical_cd_coords(true);
1995  physical_cd_coords.columnName = cd.columnName + "_coords";
1996  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
1997  // Raw data: compressed/uncompressed coords
1998  coords_ti.set_subtype(kTINYINT);
1999  size_t unit_size;
2000  if (col_ti.get_compression() == kENCODING_GEOINT &&
2001  col_ti.get_comp_param() == 32) {
2002  unit_size = 4 * sizeof(int8_t);
2003  } else {
2004  CHECK(col_ti.get_compression() == kENCODING_NONE);
2005  unit_size = 8 * sizeof(int8_t);
2006  }
2007  coords_ti.set_size(2 * unit_size);
2008  physical_cd_coords.columnType = coords_ti;
2009  columns.push_back(physical_cd_coords);
2010 
2011  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2012 
2013  break;
2014  }
2015  case kLINESTRING: {
2016  ColumnDescriptor physical_cd_coords(true);
2017  physical_cd_coords.columnName = cd.columnName + "_coords";
2018  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2019  // Raw data: compressed/uncompressed coords
2020  coords_ti.set_subtype(kTINYINT);
2021  physical_cd_coords.columnType = coords_ti;
2022  columns.push_back(physical_cd_coords);
2023 
2024  ColumnDescriptor physical_cd_bounds(true);
2025  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2026  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2027  bounds_ti.set_subtype(kDOUBLE);
2028  bounds_ti.set_size(4 * sizeof(double));
2029  physical_cd_bounds.columnType = bounds_ti;
2030  columns.push_back(physical_cd_bounds);
2031 
2032  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2033 
2034  break;
2035  }
2036  case kPOLYGON: {
2037  ColumnDescriptor physical_cd_coords(true);
2038  physical_cd_coords.columnName = cd.columnName + "_coords";
2039  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2040  // Raw data: compressed/uncompressed coords
2041  coords_ti.set_subtype(kTINYINT);
2042  physical_cd_coords.columnType = coords_ti;
2043  columns.push_back(physical_cd_coords);
2044 
2045  ColumnDescriptor physical_cd_ring_sizes(true);
2046  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
2047  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2048  ring_sizes_ti.set_subtype(kINT);
2049  physical_cd_ring_sizes.columnType = ring_sizes_ti;
2050  columns.push_back(physical_cd_ring_sizes);
2051 
2052  ColumnDescriptor physical_cd_bounds(true);
2053  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2054  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2055  bounds_ti.set_subtype(kDOUBLE);
2056  bounds_ti.set_size(4 * sizeof(double));
2057  physical_cd_bounds.columnType = bounds_ti;
2058  columns.push_back(physical_cd_bounds);
2059 
2060  ColumnDescriptor physical_cd_render_group(true);
2061  physical_cd_render_group.columnName = cd.columnName + "_render_group";
2062  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, col_ti.get_notnull());
2063  physical_cd_render_group.columnType = render_group_ti;
2064  columns.push_back(physical_cd_render_group);
2065 
2066  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2067 
2068  break;
2069  }
2070  case kMULTIPOLYGON: {
2071  ColumnDescriptor physical_cd_coords(true);
2072  physical_cd_coords.columnName = cd.columnName + "_coords";
2073  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2074  // Raw data: compressed/uncompressed coords
2075  coords_ti.set_subtype(kTINYINT);
2076  physical_cd_coords.columnType = coords_ti;
2077  columns.push_back(physical_cd_coords);
2078 
2079  ColumnDescriptor physical_cd_ring_sizes(true);
2080  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
2081  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2082  ring_sizes_ti.set_subtype(kINT);
2083  physical_cd_ring_sizes.columnType = ring_sizes_ti;
2084  columns.push_back(physical_cd_ring_sizes);
2085 
2086  ColumnDescriptor physical_cd_poly_rings(true);
2087  physical_cd_poly_rings.columnName = cd.columnName + "_poly_rings";
2088  SQLTypeInfo poly_rings_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2089  poly_rings_ti.set_subtype(kINT);
2090  physical_cd_poly_rings.columnType = poly_rings_ti;
2091  columns.push_back(physical_cd_poly_rings);
2092 
2093  ColumnDescriptor physical_cd_bounds(true);
2094  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2095  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2096  bounds_ti.set_subtype(kDOUBLE);
2097  bounds_ti.set_size(4 * sizeof(double));
2098  physical_cd_bounds.columnType = bounds_ti;
2099  columns.push_back(physical_cd_bounds);
2100 
2101  ColumnDescriptor physical_cd_render_group(true);
2102  physical_cd_render_group.columnName = cd.columnName + "_render_group";
2103  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, col_ti.get_notnull());
2104  physical_cd_render_group.columnType = render_group_ti;
2105  columns.push_back(physical_cd_render_group);
2106 
2107  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2108 
2109  break;
2110  }
2111  default:
2112  throw runtime_error("Unrecognized geometry type.");
2113  break;
2114  }
2115  }
2116 }
2117 
2119  TableDescriptor& td,
2120  const list<ColumnDescriptor>& cols,
2121  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs,
2122  bool isLogicalTable) {
2123  cat_write_lock write_lock(this);
2124  list<ColumnDescriptor> cds = cols;
2125  list<DictDescriptor> dds;
2126  std::set<std::string> toplevel_column_names;
2127  list<ColumnDescriptor> columns;
2128 
2129  if (!td.storageType.empty() &&
2132  throw std::runtime_error("Only temporary tables can be backed by foreign storage.");
2133  }
2135  }
2136 
2137  for (auto cd : cds) {
2138  if (cd.columnName == "rowid") {
2139  throw std::runtime_error(
2140  "Cannot create column with name rowid. rowid is a system defined column.");
2141  }
2142  columns.push_back(cd);
2143  toplevel_column_names.insert(cd.columnName);
2144  if (cd.columnType.is_geometry()) {
2145  expandGeoColumn(cd, columns);
2146  }
2147  }
2148  cds.clear();
2149 
2150  ColumnDescriptor cd;
2151  // add row_id column -- Must be last column in the table
2152  cd.columnName = "rowid";
2153  cd.isSystemCol = true;
2154  cd.columnType = SQLTypeInfo(kBIGINT, true);
2155 #ifdef MATERIALIZED_ROWID
2156  cd.isVirtualCol = false;
2157 #else
2158  cd.isVirtualCol = true;
2159  cd.virtualExpr = "MAPD_FRAG_ID * MAPD_ROWS_PER_FRAG + MAPD_FRAG_ROW_ID";
2160 #endif
2161  columns.push_back(cd);
2162  toplevel_column_names.insert(cd.columnName);
2163 
2164  if (td.hasDeletedCol) {
2165  ColumnDescriptor cd_del;
2166  cd_del.columnName = "$deleted$";
2167  cd_del.isSystemCol = true;
2168  cd_del.isVirtualCol = false;
2169  cd_del.columnType = SQLTypeInfo(kBOOLEAN, true);
2170  cd_del.isDeletedCol = true;
2171 
2172  columns.push_back(cd_del);
2173  }
2174 
2175  td.nColumns = columns.size();
2177  sqliteConnector_.query("BEGIN TRANSACTION");
2179  try {
2181  R"(INSERT INTO mapd_tables (name, userid, ncolumns, isview, fragments, frag_type, max_frag_rows, max_chunk_size, frag_page_size, max_rows, partitions, shard_column_id, shard, num_shards, sort_column_id, storage_type, key_metainfo) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?))",
2182  std::vector<std::string>{td.tableName,
2183  std::to_string(td.userId),
2185  std::to_string(td.isView),
2186  "",
2191  std::to_string(td.maxRows),
2192  td.partitions,
2194  std::to_string(td.shard),
2195  std::to_string(td.nShards),
2197  td.storageType,
2198  td.keyMetainfo});
2199 
2200  // now get the auto generated tableid
2202  "SELECT tableid FROM mapd_tables WHERE name = ?", td.tableName);
2203  td.tableId = sqliteConnector_.getData<int>(0, 0);
2204  int colId = 1;
2205  for (auto cd : columns) {
2207  const bool is_foreign_col =
2208  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2209  if (!is_foreign_col) {
2210  setColumnDictionary(cd, dds, td, isLogicalTable);
2211  }
2212  }
2213 
2214  if (toplevel_column_names.count(cd.columnName)) {
2215  // make up colId gap for sanity test (begin with 1 bc much code depends on it!)
2216  if (colId > 1) {
2217  colId += g_test_against_columnId_gap;
2218  }
2219  if (!cd.isGeoPhyCol) {
2220  td.columnIdBySpi_.push_back(colId);
2221  }
2222  }
2223 
2225  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, "
2226  "coldim, colscale, is_notnull, "
2227  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
2228  "virtual_expr, is_deletedcol) "
2229  "VALUES (?, ?, ?, ?, ?, "
2230  "?, "
2231  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2232  std::vector<std::string>{std::to_string(td.tableId),
2233  std::to_string(colId),
2234  cd.columnName,
2243  "",
2246  cd.virtualExpr,
2248  cd.tableId = td.tableId;
2249  cd.columnId = colId++;
2250  cds.push_back(cd);
2251  }
2252  if (td.isView) {
2254  "INSERT INTO mapd_views (tableid, sql) VALUES (?,?)",
2255  std::vector<std::string>{std::to_string(td.tableId), td.viewSQL});
2256  }
2258  auto& foreignTable = dynamic_cast<foreign_storage::ForeignTable&>(td);
2260  "INSERT INTO omnisci_foreign_tables (table_id, server_id, options) VALUES "
2261  "(?, ?, ?)",
2262  std::vector<std::string>{std::to_string(foreignTable.tableId),
2263  std::to_string(foreignTable.foreign_server->id),
2264  foreignTable.getOptionsAsJsonString()});
2265  }
2266  } catch (std::exception& e) {
2267  sqliteConnector_.query("ROLLBACK TRANSACTION");
2268  throw;
2269  }
2270  } else { // Temporary table
2271  td.tableId = nextTempTableId_++;
2272  int colId = 1;
2273  for (auto cd : columns) {
2275  const bool is_foreign_col =
2276  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2277 
2278  if (!is_foreign_col) {
2279  // Create a new temporary dictionary
2280  std::string fileName("");
2281  std::string folderPath("");
2283  nextTempDictId_++;
2284  DictDescriptor dd(dict_ref,
2285  fileName,
2287  false,
2288  1,
2289  folderPath,
2290  true); // Is dictName (2nd argument) used?
2291  dds.push_back(dd);
2292  if (!cd.columnType.is_array()) {
2294  }
2295  cd.columnType.set_comp_param(dict_ref.dictId);
2296  }
2297  }
2298  if (toplevel_column_names.count(cd.columnName)) {
2299  // make up colId gap for sanity test (begin with 1 bc much code depends on it!)
2300  if (colId > 1) {
2301  colId += g_test_against_columnId_gap;
2302  }
2303  if (!cd.isGeoPhyCol) {
2304  td.columnIdBySpi_.push_back(colId);
2305  }
2306  }
2307  cd.tableId = td.tableId;
2308  cd.columnId = colId++;
2309  cds.push_back(cd);
2310  }
2311 
2313  serializeTableJsonUnlocked(&td, cds);
2314  }
2315  }
2316 
2317  try {
2318  addTableToMap(&td, cds, dds);
2319  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
2320  if (!td.storageType.empty() && td.storageType != StorageType::FOREIGN_TABLE) {
2322  }
2323  } catch (std::exception& e) {
2324  sqliteConnector_.query("ROLLBACK TRANSACTION");
2325  removeTableFromMap(td.tableName, td.tableId, true);
2326  throw;
2327  }
2328  sqliteConnector_.query("END TRANSACTION");
2329 }
2330 
2332  const std::list<ColumnDescriptor>& cds) const {
2333  // relies on the catalog write lock
2334  using namespace rapidjson;
2335 
2336  VLOG(1) << "Serializing temporary table " << td->tableName << " to JSON for Calcite.";
2337 
2338  const auto db_name = currentDB_.dbName;
2339  const auto file_path = table_json_filepath(basePath_, db_name);
2340 
2341  Document d;
2342  if (boost::filesystem::exists(file_path)) {
2343  // look for an existing file for this database
2344  std::ifstream reader(file_path.string());
2345  CHECK(reader.is_open());
2346  IStreamWrapper json_read_wrapper(reader);
2347  d.ParseStream(json_read_wrapper);
2348  } else {
2349  d.SetObject();
2350  }
2351  CHECK(d.IsObject());
2352  CHECK(!d.HasMember(StringRef(td->tableName.c_str())));
2353 
2354  Value table(kObjectType);
2355  table.AddMember(
2356  "name", Value().SetString(StringRef(td->tableName.c_str())), d.GetAllocator());
2357  table.AddMember("id", Value().SetInt(td->tableId), d.GetAllocator());
2358  table.AddMember("columns", Value(kArrayType), d.GetAllocator());
2359 
2360  for (const auto& cd : cds) {
2361  Value column(kObjectType);
2362  column.AddMember(
2363  "name", Value().SetString(StringRef(cd.columnName)), d.GetAllocator());
2364  column.AddMember("coltype",
2365  Value().SetInt(static_cast<int>(cd.columnType.get_type())),
2366  d.GetAllocator());
2367  column.AddMember("colsubtype",
2368  Value().SetInt(static_cast<int>(cd.columnType.get_subtype())),
2369  d.GetAllocator());
2370  column.AddMember(
2371  "coldim", Value().SetInt(cd.columnType.get_dimension()), d.GetAllocator());
2372  column.AddMember(
2373  "colscale", Value().SetInt(cd.columnType.get_scale()), d.GetAllocator());
2374  column.AddMember(
2375  "is_notnull", Value().SetBool(cd.columnType.get_notnull()), d.GetAllocator());
2376  column.AddMember("is_systemcol", Value().SetBool(cd.isSystemCol), d.GetAllocator());
2377  column.AddMember("is_virtualcol", Value().SetBool(cd.isVirtualCol), d.GetAllocator());
2378  column.AddMember("is_deletedcol", Value().SetBool(cd.isDeletedCol), d.GetAllocator());
2379  table["columns"].PushBack(column, d.GetAllocator());
2380  }
2381  d.AddMember(StringRef(td->tableName.c_str()), table, d.GetAllocator());
2382 
2383  // Overwrite the existing file
2384  std::ofstream writer(file_path.string(), writer.trunc | writer.out);
2385  CHECK(writer.is_open());
2386  OStreamWrapper json_wrapper(writer);
2387 
2388  Writer<OStreamWrapper> json_writer(json_wrapper);
2389  d.Accept(json_writer);
2390  writer.close();
2391 }
2392 
2393 void Catalog::dropTableFromJsonUnlocked(const std::string& table_name) const {
2394  // relies on the catalog write lock
2395  using namespace rapidjson;
2396 
2397  VLOG(1) << "Dropping temporary table " << table_name << " to JSON for Calcite.";
2398 
2399  const auto db_name = currentDB_.dbName;
2400  const auto file_path = table_json_filepath(basePath_, db_name);
2401 
2402  CHECK(boost::filesystem::exists(file_path));
2403  Document d;
2404 
2405  std::ifstream reader(file_path.string());
2406  CHECK(reader.is_open());
2407  IStreamWrapper json_read_wrapper(reader);
2408  d.ParseStream(json_read_wrapper);
2409 
2410  CHECK(d.IsObject());
2411  auto table_name_ref = StringRef(table_name.c_str());
2412  CHECK(d.HasMember(table_name_ref));
2413  CHECK(d.RemoveMember(table_name_ref));
2414 
2415  // Overwrite the existing file
2416  std::ofstream writer(file_path.string(), writer.trunc | writer.out);
2417  CHECK(writer.is_open());
2418  OStreamWrapper json_wrapper(writer);
2419 
2420  Writer<OStreamWrapper> json_writer(json_wrapper);
2421  d.Accept(json_writer);
2422  writer.close();
2423 }
2424 
2426  std::unique_ptr<foreign_storage::ForeignServer> foreign_server,
2427  bool if_not_exists) {
2428  cat_write_lock write_lock(this);
2430  createForeignServerNoLocks(std::move(foreign_server), if_not_exists);
2431 }
2432 
2434  std::unique_ptr<foreign_storage::ForeignServer> foreign_server,
2435  bool if_not_exists) {
2437  "SELECT name from omnisci_foreign_servers where name = ?",
2438  std::vector<std::string>{foreign_server->name});
2439 
2440  if (sqliteConnector_.getNumRows() == 0) {
2441  foreign_server->creation_time = std::time(nullptr);
2443  "INSERT INTO omnisci_foreign_servers (name, data_wrapper_type, owner_user_id, "
2444  "creation_time, "
2445  "options) "
2446  "VALUES (?, ?, ?, ?, ?)",
2447  std::vector<std::string>{foreign_server->name,
2448  foreign_server->data_wrapper_type,
2449  std::to_string(foreign_server->user_id),
2450  std::to_string(foreign_server->creation_time),
2451  foreign_server->getOptionsAsJsonString()});
2453  "SELECT id from omnisci_foreign_servers where name = ?",
2454  std::vector<std::string>{foreign_server->name});
2455  CHECK_EQ(sqliteConnector_.getNumRows(), size_t(1));
2456  foreign_server->id = sqliteConnector_.getData<int32_t>(0, 0);
2457  } else if (!if_not_exists) {
2458  throw std::runtime_error{"A foreign server with name \"" + foreign_server->name +
2459  "\" already exists."};
2460  }
2461 
2462  std::shared_ptr<foreign_storage::ForeignServer> foreign_server_shared =
2463  std::move(foreign_server);
2464  foreignServerMap_[foreign_server_shared->name] = foreign_server_shared;
2465  foreignServerMapById_[foreign_server_shared->id] = foreign_server_shared;
2466 }
2467 
2469  const std::string& server_name) const {
2470  foreign_storage::ForeignServer* foreign_server = nullptr;
2471  cat_read_lock read_lock(this);
2472  if (foreignServerMap_.find(server_name) != foreignServerMap_.end()) {
2473  foreign_server = foreignServerMap_.find(server_name)->second.get();
2474  }
2475  return foreign_server;
2476 }
2477 
2478 const std::unique_ptr<const foreign_storage::ForeignServer>
2479 Catalog::getForeignServerFromStorage(const std::string& server_name) {
2480  std::unique_ptr<foreign_storage::ForeignServer> foreign_server = nullptr;
2483  "SELECT id, name, data_wrapper_type, options, owner_user_id, creation_time "
2484  "FROM omnisci_foreign_servers WHERE name = ?",
2485  std::vector<std::string>{server_name});
2486  if (sqliteConnector_.getNumRows() > 0) {
2487  foreign_server = std::make_unique<foreign_storage::ForeignServer>(
2488  sqliteConnector_.getData<int>(0, 0),
2489  sqliteConnector_.getData<std::string>(0, 1),
2490  sqliteConnector_.getData<std::string>(0, 2),
2491  sqliteConnector_.getData<std::string>(0, 3),
2492  sqliteConnector_.getData<std::int32_t>(0, 4),
2493  sqliteConnector_.getData<std::int32_t>(0, 5));
2494  }
2495  return foreign_server;
2496 }
2497 
2498 void Catalog::changeForeignServerOwner(const std::string& server_name,
2499  const int new_owner_id) {
2500  cat_write_lock write_lock(this);
2501  foreign_storage::ForeignServer* foreign_server =
2502  foreignServerMap_.find(server_name)->second.get();
2503  CHECK(foreign_server);
2504  setForeignServerProperty(server_name, "owner_user_id", std::to_string(new_owner_id));
2505  // update in-memory server
2506  foreign_server->user_id = new_owner_id;
2507 }
2508 
2509 void Catalog::setForeignServerDataWrapper(const std::string& server_name,
2510  const std::string& data_wrapper) {
2511  cat_write_lock write_lock(this);
2512  auto data_wrapper_type = to_upper(data_wrapper);
2513  // update in-memory server
2514  foreign_storage::ForeignServer* foreign_server =
2515  foreignServerMap_.find(server_name)->second.get();
2516  CHECK(foreign_server);
2517  std::string saved_data_wrapper_type = foreign_server->data_wrapper_type;
2518  foreign_server->data_wrapper_type = data_wrapper_type;
2519  try {
2520  foreign_server->validate();
2521  } catch (const std::exception& e) {
2522  // validation did not succeed:
2523  // revert to saved data_wrapper_type & throw exception
2524  foreign_server->data_wrapper_type = saved_data_wrapper_type;
2525  throw;
2526  }
2527  setForeignServerProperty(server_name, "data_wrapper_type", data_wrapper_type);
2528 }
2529 
2530 void Catalog::setForeignServerOptions(const std::string& server_name,
2531  const std::string& options) {
2532  cat_write_lock write_lock(this);
2533  // update in-memory server
2534  foreign_storage::ForeignServer* foreign_server =
2535  foreignServerMap_.find(server_name)->second.get();
2536  CHECK(foreign_server);
2537  std::string saved_options_string = foreign_server->getOptionsAsJsonString();
2538  foreign_server->populateOptionsMap(options, true);
2539  try {
2540  foreign_server->validate();
2541  } catch (const std::exception& e) {
2542  // validation did not succeed:
2543  // revert to saved options & throw exception
2544  foreign_server->populateOptionsMap(saved_options_string, true);
2545  throw;
2546  }
2547  setForeignServerProperty(server_name, "options", options);
2548 }
2549 
2550 void Catalog::renameForeignServer(const std::string& server_name,
2551  const std::string& name) {
2552  cat_write_lock write_lock(this);
2553  auto foreign_server_it = foreignServerMap_.find(server_name);
2554  CHECK(foreign_server_it != foreignServerMap_.end());
2555  setForeignServerProperty(server_name, "name", name);
2556  auto foreign_server_shared = foreign_server_it->second;
2557  foreign_server_shared->name = name;
2558  foreignServerMap_[name] = foreign_server_shared;
2559  foreignServerMap_.erase(foreign_server_it);
2560 }
2561 
2562 void Catalog::dropForeignServer(const std::string& server_name) {
2563  cat_write_lock write_lock(this);
2565 
2567  "SELECT id from omnisci_foreign_servers where name = ?",
2568  std::vector<std::string>{server_name});
2570  if (num_rows > 0) {
2571  CHECK_EQ(size_t(1), num_rows);
2572  auto server_id = sqliteConnector_.getData<int32_t>(0, 0);
2574  "SELECT table_id from omnisci_foreign_tables where server_id = ?",
2575  std::to_string(server_id));
2576  if (sqliteConnector_.getNumRows() > 0) {
2577  throw std::runtime_error{"Foreign server \"" + server_name +
2578  "\" is referenced "
2579  "by existing foreign tables and cannot be dropped."};
2580  }
2581  sqliteConnector_.query("BEGIN TRANSACTION");
2582  try {
2584  "DELETE FROM omnisci_foreign_servers WHERE name = ?",
2585  std::vector<std::string>{server_name});
2586  } catch (const std::exception& e) {
2587  sqliteConnector_.query("ROLLBACK TRANSACTION");
2588  throw;
2589  }
2590  sqliteConnector_.query("END TRANSACTION");
2591  foreignServerMap_.erase(server_name);
2592  foreignServerMapById_.erase(server_id);
2593  }
2594 }
2595 
2597  const rapidjson::Value* filters,
2598  const UserMetadata& user,
2599  std::vector<const foreign_storage::ForeignServer*>& results) {
2600  sys_read_lock syscat_read_lock(&SysCatalog::instance());
2601  cat_read_lock read_lock(this);
2603  // Customer facing and internal SQlite names
2604  std::map<std::string, std::string> col_names{{"server_name", "name"},
2605  {"data_wrapper", "data_wrapper_type"},
2606  {"created_at", "creation_time"},
2607  {"options", "options"}};
2608 
2609  // TODO add "owner" when FSI privilege is implemented
2610  std::stringstream filter_string;
2611  std::vector<std::string> arguments;
2612 
2613  if (filters != nullptr) {
2614  // Create SQL WHERE clause for SQLite query
2615  int num_filters = 0;
2616  filter_string << " WHERE";
2617  for (auto& filter_def : filters->GetArray()) {
2618  if (num_filters > 0) {
2619  filter_string << " " << std::string(filter_def["chain"].GetString());
2620  ;
2621  }
2622 
2623  if (col_names.find(std::string(filter_def["attribute"].GetString())) ==
2624  col_names.end()) {
2625  throw std::runtime_error{"Attribute with name \"" +
2626  std::string(filter_def["attribute"].GetString()) +
2627  "\" does not exist."};
2628  }
2629 
2630  filter_string << " " << col_names[std::string(filter_def["attribute"].GetString())];
2631 
2632  bool equals_operator = false;
2633  if (std::strcmp(filter_def["operation"].GetString(), "EQUALS") == 0) {
2634  filter_string << " = ? ";
2635  equals_operator = true;
2636  } else {
2637  filter_string << " LIKE ? ";
2638  }
2639 
2640  bool timestamp_column =
2641  (std::strcmp(filter_def["attribute"].GetString(), "created_at") == 0);
2642 
2643  if (timestamp_column && !equals_operator) {
2644  throw std::runtime_error{"LIKE operator is incompatible with TIMESTAMP data"};
2645  }
2646 
2647  if (timestamp_column && equals_operator) {
2648  arguments.push_back(std::to_string(
2649  DateTimeStringValidate<kTIMESTAMP>()(filter_def["value"].GetString(), 0)));
2650  } else {
2651  arguments.push_back(filter_def["value"].GetString());
2652  }
2653 
2654  num_filters++;
2655  }
2656  }
2657  // Create select query for the omnisci_foreign_servers table
2658  std::string query = std::string("SELECT name from omnisci_foreign_servers ");
2659  query += filter_string.str();
2660 
2661  sqliteConnector_.query_with_text_params(query, arguments);
2663 
2664  if (sqliteConnector_.getNumRows() == 0)
2665  return;
2666 
2668  // Return pointers to objects
2669  results.reserve(num_rows);
2670  for (size_t row = 0; row < num_rows; ++row) {
2671  const foreign_storage::ForeignServer* foreign_server =
2672  getForeignServer(sqliteConnector_.getData<std::string>(row, 0));
2673 
2674  CHECK(foreign_server != nullptr);
2675 
2676  DBObject dbObject(foreign_server->name, ServerDBObjectType);
2677  dbObject.loadKey(*this);
2678  std::vector<DBObject> privObjects = {dbObject};
2679  if (!SysCatalog::instance().hasAnyPrivileges(user, privObjects)) {
2680  // skip server, as there are no privileges to access it
2681  continue;
2682  }
2683  results.push_back(foreign_server);
2684  }
2685 }
2686 
2687 // returns the table epoch or -1 if there is something wrong with the shared epoch
2688 int32_t Catalog::getTableEpoch(const int32_t db_id, const int32_t table_id) const {
2689  cat_read_lock read_lock(this);
2690  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(table_id);
2691  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2692  // check all shards have same checkpoint
2693  const auto physicalTables = physicalTableIt->second;
2694  CHECK(!physicalTables.empty());
2695  size_t curr_epoch = 0;
2696  for (size_t i = 0; i < physicalTables.size(); i++) {
2697  int32_t physical_tb_id = physicalTables[i];
2698  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2699  CHECK(phys_td);
2700  if (i == 0) {
2701  curr_epoch = dataMgr_->getTableEpoch(db_id, physical_tb_id);
2702  } else {
2703  if (curr_epoch != dataMgr_->getTableEpoch(db_id, physical_tb_id)) {
2704  // oh dear the leaves do not agree on the epoch for this table
2705  LOG(ERROR) << "Epochs on shards do not all agree on table id " << table_id
2706  << " db id " << db_id << " epoch " << curr_epoch << " leaf_epoch "
2707  << dataMgr_->getTableEpoch(db_id, physical_tb_id);
2708  return -1;
2709  }
2710  }
2711  }
2712  return curr_epoch;
2713  } else {
2714  return dataMgr_->getTableEpoch(db_id, table_id);
2715  }
2716 }
2717 
2718 void Catalog::setTableEpoch(const int db_id, const int table_id, int new_epoch) {
2719  cat_read_lock read_lock(this);
2720  LOG(INFO) << "Set table epoch db:" << db_id << " Table ID " << table_id
2721  << " back to new epoch " << new_epoch;
2722  removeChunks(table_id);
2723  dataMgr_->setTableEpoch(db_id, table_id, new_epoch);
2724 
2725  // check if sharded
2726  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(table_id);
2727  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2728  const auto physicalTables = physicalTableIt->second;
2729  CHECK(!physicalTables.empty());
2730  for (size_t i = 0; i < physicalTables.size(); i++) {
2731  int32_t physical_tb_id = physicalTables[i];
2732  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2733  CHECK(phys_td);
2734  LOG(INFO) << "Set sharded table epoch db:" << db_id << " Table ID "
2735  << physical_tb_id << " back to new epoch " << new_epoch;
2736  removeChunks(physical_tb_id);
2737  dataMgr_->setTableEpoch(db_id, physical_tb_id, new_epoch);
2738  }
2739  }
2740 }
2741 
2743  cat_read_lock read_lock(this);
2744  const auto it = deletedColumnPerTable_.find(td);
2745  return it != deletedColumnPerTable_.end() ? it->second : nullptr;
2746 }
2747 
2749  int delete_column_id) const {
2750  // check if there are rows deleted by examining the deletedColumn metadata
2751  CHECK(td);
2752 
2753  if (table_is_temporary(td)) {
2754  auto fragmenter = td->fragmenter;
2755  CHECK(fragmenter);
2756  return fragmenter->hasDeletedRows(delete_column_id);
2757  } else {
2758  ChunkKey chunk_key_prefix = {currentDB_.dbId, td->tableId, delete_column_id};
2759  ChunkMetadataVector chunk_metadata_vec;
2760  dataMgr_->getChunkMetadataVecForKeyPrefix(chunk_metadata_vec, chunk_key_prefix);
2761  int64_t chunk_max{0};
2762 
2763  for (auto chunk_metadata : chunk_metadata_vec) {
2764  chunk_max = chunk_metadata.second->chunkStats.max.tinyintval;
2765  // delete has occured
2766  if (chunk_max == 1) {
2767  return true;
2768  }
2769  }
2770  return false;
2771  }
2772 }
2773 
2775  const TableDescriptor* td) const {
2776  cat_read_lock read_lock(this);
2777 
2778  const auto it = deletedColumnPerTable_.find(td);
2779  // if not a table that supports delete return nullptr, nothing more to do
2780  if (it == deletedColumnPerTable_.end()) {
2781  return nullptr;
2782  }
2783  const ColumnDescriptor* cd = it->second;
2784 
2785  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
2786 
2787  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2788  // check all shards
2789  const auto physicalTables = physicalTableIt->second;
2790  CHECK(!physicalTables.empty());
2791  for (size_t i = 0; i < physicalTables.size(); i++) {
2792  int32_t physical_tb_id = physicalTables[i];
2793  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2794  CHECK(phys_td);
2795  if (checkMetadataForDeletedRecs(phys_td, cd->columnId)) {
2796  return cd;
2797  }
2798  }
2799  } else {
2800  if (checkMetadataForDeletedRecs(td, cd->columnId)) {
2801  return cd;
2802  }
2803  }
2804  // no deletes so far recorded in metadata
2805  return nullptr;
2806 }
2807 
2809  cat_write_lock write_lock(this);
2810  setDeletedColumnUnlocked(td, cd);
2811 }
2812 
2814  const ColumnDescriptor* cd) {
2815  cat_write_lock write_lock(this);
2816  const auto it_ok = deletedColumnPerTable_.emplace(td, cd);
2817  CHECK(it_ok.second);
2818 }
2819 
2820 namespace {
2821 
2823  const Catalog& cat,
2824  const Parser::SharedDictionaryDef& shared_dict_def) {
2825  const auto& table_name = shared_dict_def.get_foreign_table();
2826  const auto td = cat.getMetadataForTable(table_name);
2827  CHECK(td);
2828  const auto& foreign_col_name = shared_dict_def.get_foreign_column();
2829  return cat.getMetadataForColumn(td->tableId, foreign_col_name);
2830 }
2831 
2832 } // namespace
2833 
2835  Parser::SharedDictionaryDef shared_dict_def,
2836  const bool persist_reference) {
2837  cat_write_lock write_lock(this);
2838  const auto foreign_ref_col = get_foreign_col(*this, shared_dict_def);
2839  CHECK(foreign_ref_col);
2840  referencing_column.columnType = foreign_ref_col->columnType;
2841  const int dict_id = referencing_column.columnType.get_comp_param();
2842  const DictRef dict_ref(currentDB_.dbId, dict_id);
2843  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
2844  CHECK(dictIt != dictDescriptorMapByRef_.end());
2845  const auto& dd = dictIt->second;
2846  CHECK_GE(dd->refcount, 1);
2847  ++dd->refcount;
2848  if (persist_reference) {
2851  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
2852  {std::to_string(dict_id)});
2853  }
2854 }
2855 
2857  ColumnDescriptor& cd,
2858  std::list<ColumnDescriptor>& cdd,
2859  std::list<DictDescriptor>& dds,
2860  const TableDescriptor td,
2861  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs) {
2862  cat_write_lock write_lock(this);
2864 
2865  if (shared_dict_defs.empty()) {
2866  return false;
2867  }
2868  for (const auto& shared_dict_def : shared_dict_defs) {
2869  // check if the current column is a referencing column
2870  const auto& column = shared_dict_def.get_column();
2871  if (cd.columnName == column) {
2872  if (!shared_dict_def.get_foreign_table().compare(td.tableName)) {
2873  // Dictionaries are being shared in table to be created
2874  const auto& ref_column = shared_dict_def.get_foreign_column();
2875  auto colIt =
2876  std::find_if(cdd.begin(), cdd.end(), [ref_column](const ColumnDescriptor it) {
2877  return !ref_column.compare(it.columnName);
2878  });
2879  CHECK(colIt != cdd.end());
2880  cd.columnType = colIt->columnType;
2881 
2882  const int dict_id = colIt->columnType.get_comp_param();
2883  CHECK_GE(dict_id, 1);
2884  auto dictIt = std::find_if(
2885  dds.begin(), dds.end(), [this, dict_id](const DictDescriptor it) {
2886  return it.dictRef.dbId == this->currentDB_.dbId &&
2887  it.dictRef.dictId == dict_id;
2888  });
2889  if (dictIt != dds.end()) {
2890  // There exists dictionary definition of a dictionary column
2891  CHECK_GE(dictIt->refcount, 1);
2892  ++dictIt->refcount;
2893  if (!table_is_temporary(&td)) {
2894  // Persist reference count
2896  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
2897  {std::to_string(dict_id)});
2898  }
2899  } else {
2900  // The dictionary is referencing a column which is referencing a column in
2901  // diffrent table
2902  auto root_dict_def = compress_reference_path(shared_dict_def, shared_dict_defs);
2903  addReferenceToForeignDict(cd, root_dict_def, !table_is_temporary(&td));
2904  }
2905  } else {
2906  const auto& foreign_table_name = shared_dict_def.get_foreign_table();
2907  const auto foreign_td = getMetadataForTable(foreign_table_name, false);
2908  if (table_is_temporary(foreign_td)) {
2909  if (!table_is_temporary(&td)) {
2910  throw std::runtime_error(
2911  "Only temporary tables can share dictionaries with other temporary "
2912  "tables.");
2913  }
2914  addReferenceToForeignDict(cd, shared_dict_def, false);
2915  } else {
2916  addReferenceToForeignDict(cd, shared_dict_def, !table_is_temporary(&td));
2917  }
2918  }
2919  return true;
2920  }
2921  }
2922  return false;
2923 }
2924 
2926  std::list<DictDescriptor>& dds,
2927  const TableDescriptor& td,
2928  const bool isLogicalTable) {
2929  cat_write_lock write_lock(this);
2930 
2931  std::string dictName{"Initial_key"};
2932  int dictId{0};
2933  std::string folderPath;
2934  if (isLogicalTable) {
2936 
2938  "INSERT INTO mapd_dictionaries (name, nbits, is_shared, refcount) VALUES (?, ?, "
2939  "?, 1)",
2940  std::vector<std::string>{
2941  dictName, std::to_string(cd.columnType.get_comp_param()), "0"});
2943  "SELECT dictid FROM mapd_dictionaries WHERE name = ?", dictName);
2944  dictId = sqliteConnector_.getData<int>(0, 0);
2945  dictName = td.tableName + "_" + cd.columnName + "_dict" + std::to_string(dictId);
2947  "UPDATE mapd_dictionaries SET name = ? WHERE name = 'Initial_key'", dictName);
2948  folderPath = basePath_ + "/mapd_data/DB_" + std::to_string(currentDB_.dbId) +
2949  "_DICT_" + std::to_string(dictId);
2950  }
2952  dictId,
2953  dictName,
2955  false,
2956  1,
2957  folderPath,
2958  false);
2959  dds.push_back(dd);
2960  if (!cd.columnType.is_array()) {
2962  }
2963  cd.columnType.set_comp_param(dictId);
2964 }
2965 
2967  TableDescriptor& td,
2968  const list<ColumnDescriptor>& cols,
2969  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs) {
2970  cat_write_lock write_lock(this);
2971 
2972  /* create logical table */
2973  TableDescriptor* tdl = &td;
2974  createTable(*tdl, cols, shared_dict_defs, true); // create logical table
2975  int32_t logical_tb_id = tdl->tableId;
2976 
2977  /* create physical tables and link them to the logical table */
2978  std::vector<int32_t> physicalTables;
2979  for (int32_t i = 1; i <= td.nShards; i++) {
2980  TableDescriptor* tdp = &td;
2982  tdp->shard = i - 1;
2983  createTable(*tdp, cols, shared_dict_defs, false); // create physical table
2984  int32_t physical_tb_id = tdp->tableId;
2985 
2986  /* add physical table to the vector of physical tables */
2987  physicalTables.push_back(physical_tb_id);
2988  }
2989 
2990  if (!physicalTables.empty()) {
2991  /* add logical to physical tables correspondence to the map */
2992  const auto it_ok =
2993  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
2994  CHECK(it_ok.second);
2995  /* update sqlite mapd_logical_to_physical in sqlite database */
2996  if (!table_is_temporary(&td)) {
2997  updateLogicalToPhysicalTableMap(logical_tb_id);
2998  }
2999  }
3000 }
3001 
3003  cat_write_lock write_lock(this);
3004 
3005  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
3006  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
3007  // truncate all corresponding physical tables if this is a logical table
3008  const auto physicalTables = physicalTableIt->second;
3009  CHECK(!physicalTables.empty());
3010  for (size_t i = 0; i < physicalTables.size(); i++) {
3011  int32_t physical_tb_id = physicalTables[i];
3012  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
3013  CHECK(phys_td);
3014  doTruncateTable(phys_td);
3015  }
3016  }
3017  doTruncateTable(td);
3018 }
3019 
3021  cat_write_lock write_lock(this);
3022 
3023  const int tableId = td->tableId;
3024  // must destroy fragmenter before deleteChunks is called.
3025  if (td->fragmenter != nullptr) {
3026  auto tableDescIt = tableDescriptorMapById_.find(tableId);
3027  CHECK(tableDescIt != tableDescriptorMapById_.end());
3028  tableDescIt->second->fragmenter = nullptr;
3029  CHECK(td->fragmenter == nullptr);
3030  }
3031  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
3032  // assuming deleteChunksWithPrefix is atomic
3033  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
3034  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
3035 
3036  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
3037 
3038  std::unique_ptr<StringDictionaryClient> client;
3039  if (SysCatalog::instance().isAggregator()) {
3040  CHECK(!string_dict_hosts_.empty());
3041  DictRef dict_ref(currentDB_.dbId, -1);
3042  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
3043  }
3044  // clean up any dictionaries
3045  // delete all column descriptors for the table
3046  for (const auto& columnDescriptor : columnDescriptorMapById_) {
3047  auto cd = columnDescriptor.second;
3048  if (cd->tableId != td->tableId) {
3049  continue;
3050  }
3051  const int dict_id = cd->columnType.get_comp_param();
3052  // Dummy dictionaries created for a shard of a logical table have the id set to zero.
3053  if (cd->columnType.get_compression() == kENCODING_DICT && dict_id) {
3054  const DictRef dict_ref(currentDB_.dbId, dict_id);
3055  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
3056  CHECK(dictIt != dictDescriptorMapByRef_.end());
3057  const auto& dd = dictIt->second;
3058  CHECK_GE(dd->refcount, 1);
3059  // if this is the only table using this dict reset the dict
3060  if (dd->refcount == 1) {
3061  // close the dictionary
3062  dd->stringDict.reset();
3063  File_Namespace::renameForDelete(dd->dictFolderPath);
3064  if (client) {
3065  client->drop(dd->dictRef);
3066  }
3067  if (!dd->dictIsTemp) {
3068  boost::filesystem::create_directory(dd->dictFolderPath);
3069  }
3070  }
3071 
3072  DictDescriptor* new_dd = new DictDescriptor(dd->dictRef,
3073  dd->dictName,
3074  dd->dictNBits,
3075  dd->dictIsShared,
3076  dd->refcount,
3077  dd->dictFolderPath,
3078  dd->dictIsTemp);
3079  dictDescriptorMapByRef_.erase(dictIt);
3080  // now create new Dict -- need to figure out what to do here for temp tables
3081  if (client) {
3082  client->create(new_dd->dictRef, new_dd->dictIsTemp);
3083  }
3084  dictDescriptorMapByRef_[new_dd->dictRef].reset(new_dd);
3086  }
3087  }
3088 }
3089 
3090 void Catalog::removeFragmenterForTable(const int table_id) {
3091  cat_write_lock write_lock(this);
3092  auto td = getMetadataForTable(table_id);
3093  if (td->fragmenter != nullptr) {
3094  auto tableDescIt = tableDescriptorMapById_.find(table_id);
3095  CHECK(tableDescIt != tableDescriptorMapById_.end());
3096  tableDescIt->second->fragmenter = nullptr;
3097  CHECK(td->fragmenter == nullptr);
3098  }
3099 }
3100 
3101 // used by rollback_table_epoch to clean up in memory artifacts after a rollback
3102 void Catalog::removeChunks(const int table_id) {
3103  auto td = getMetadataForTable(table_id);
3104 
3105  if (td->fragmenter != nullptr) {
3107  if (td->fragmenter != nullptr) {
3108  auto tableDescIt = tableDescriptorMapById_.find(table_id);
3109  CHECK(tableDescIt != tableDescriptorMapById_.end());
3110  tableDescIt->second->fragmenter = nullptr;
3111  CHECK(td->fragmenter == nullptr);
3112  }
3113  }
3114 
3115  // remove the chunks from in memory structures
3116  ChunkKey chunkKey = {currentDB_.dbId, table_id};
3117 
3118  dataMgr_->deleteChunksWithPrefix(chunkKey, MemoryLevel::CPU_LEVEL);
3119  dataMgr_->deleteChunksWithPrefix(chunkKey, MemoryLevel::GPU_LEVEL);
3120 }
3121 
3125  cat_write_lock write_lock(this);
3127  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
3128  sqliteConnector_.query("BEGIN TRANSACTION");
3129  try {
3130  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
3131  // remove all corresponding physical tables if this is a logical table
3132  const auto physicalTables = physicalTableIt->second;
3133  CHECK(!physicalTables.empty());
3134  for (size_t i = 0; i < physicalTables.size(); i++) {
3135  int32_t physical_tb_id = physicalTables[i];
3136  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
3137  CHECK(phys_td);
3138  doDropTable(phys_td);
3139  }
3140 
3141  // remove corresponding record from the logicalToPhysicalTableMap in sqlite database
3143  "DELETE FROM mapd_logical_to_physical WHERE logical_table_id = ?",
3144  std::to_string(td->tableId));
3146  }
3147  doDropTable(td);
3148  } catch (std::exception& e) {
3149  sqliteConnector_.query("ROLLBACK TRANSACTION");
3150  throw;
3151  }
3152  sqliteConnector_.query("END TRANSACTION");
3153 }
3154 
3159  }
3161 }
3162 
3164  const int tableId = td->tableId;
3165  sqliteConnector_.query_with_text_param("DELETE FROM mapd_tables WHERE tableid = ?",
3166  std::to_string(tableId));
3168  "select comp_param from mapd_columns where compression = ? and tableid = ?",
3169  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
3170  int numRows = sqliteConnector_.getNumRows();
3171  std::vector<int> dict_id_list;
3172  for (int r = 0; r < numRows; ++r) {
3173  dict_id_list.push_back(sqliteConnector_.getData<int>(r, 0));
3174  }
3175  for (auto dict_id : dict_id_list) {
3177  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
3178  std::vector<std::string>{std::to_string(dict_id)});
3179  }
3181  "DELETE FROM mapd_dictionaries WHERE dictid in (select comp_param from "
3182  "mapd_columns where compression = ? "
3183  "and tableid = ?) and refcount = 0",
3184  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
3185  sqliteConnector_.query_with_text_param("DELETE FROM mapd_columns WHERE tableid = ?",
3186  std::to_string(tableId));
3187  if (td->isView) {
3188  sqliteConnector_.query_with_text_param("DELETE FROM mapd_views WHERE tableid = ?",
3189  std::to_string(tableId));
3190  }
3193  "DELETE FROM omnisci_foreign_tables WHERE table_id = ?", std::to_string(tableId));
3194  }
3195 }
3196 
3197 void Catalog::renamePhysicalTable(const TableDescriptor* td, const string& newTableName) {
3198  cat_write_lock write_lock(this);
3200 
3201  sqliteConnector_.query("BEGIN TRANSACTION");
3202  try {
3204  "UPDATE mapd_tables SET name = ? WHERE tableid = ?",
3205  std::vector<std::string>{newTableName, std::to_string(td->tableId)});
3206  } catch (std::exception& e) {
3207  sqliteConnector_.query("ROLLBACK TRANSACTION");
3208  throw;
3209  }
3210  sqliteConnector_.query("END TRANSACTION");
3211  TableDescriptorMap::iterator tableDescIt =
3213  CHECK(tableDescIt != tableDescriptorMap_.end());
3214  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
3215  // Get table descriptor to change it
3216  TableDescriptor* changeTd = tableDescIt->second;
3217  changeTd->tableName = newTableName;
3218  tableDescriptorMap_.erase(tableDescIt); // erase entry under old name
3219  tableDescriptorMap_[to_upper(newTableName)] = changeTd;
3220  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
3221 }
3222 
3223 void Catalog::renameTable(const TableDescriptor* td, const string& newTableName) {
3224  {
3225  cat_write_lock write_lock(this);
3227  // rename all corresponding physical tables if this is a logical table
3228  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
3229  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
3230  const auto physicalTables = physicalTableIt->second;
3231  CHECK(!physicalTables.empty());
3232  for (size_t i = 0; i < physicalTables.size(); i++) {
3233  int32_t physical_tb_id = physicalTables[i];
3234  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
3235  CHECK(phys_td);
3236  std::string newPhysTableName =
3237  generatePhysicalTableName(newTableName, static_cast<int32_t>(i + 1));
3238  renamePhysicalTable(phys_td, newPhysTableName);
3239  }
3240  }
3241  renamePhysicalTable(td, newTableName);
3242  }
3243  {
3244  DBObject object(newTableName, TableDBObjectType);
3245  // update table name in direct and effective priv map
3246  DBObjectKey key;
3247  key.dbId = currentDB_.dbId;
3248  key.objectId = td->tableId;
3249  key.permissionType = static_cast<int>(DBObjectType::TableDBObjectType);
3250  object.setObjectKey(key);
3251  auto objdescs = SysCatalog::instance().getMetadataForObject(
3252  currentDB_.dbId, static_cast<int>(DBObjectType::TableDBObjectType), td->tableId);
3253  for (auto obj : objdescs) {
3254  Grantee* grnt = SysCatalog::instance().getGrantee(obj->roleName);
3255  if (grnt) {
3256  grnt->renameDbObject(object);
3257  }
3258  }
3260  }
3261 }
3262 
3264  const ColumnDescriptor* cd,
3265  const string& newColumnName) {
3266  cat_write_lock write_lock(this);
3268  sqliteConnector_.query("BEGIN TRANSACTION");
3269  try {
3270  for (int i = 0; i <= cd->columnType.get_physical_cols(); ++i) {
3271  auto cdx = getMetadataForColumn(td->tableId, cd->columnId + i);
3272  CHECK(cdx);
3273  std::string new_column_name = cdx->columnName;
3274  new_column_name.replace(0, cd->columnName.size(), newColumnName);
3276  "UPDATE mapd_columns SET name = ? WHERE tableid = ? AND columnid = ?",
3277  std::vector<std::string>{new_column_name,
3278  std::to_string(td->tableId),
3279  std::to_string(cdx->columnId)});
3280  }
3281  } catch (std::exception& e) {
3282  sqliteConnector_.query("ROLLBACK TRANSACTION");
3283  throw;
3284  }
3285  sqliteConnector_.query("END TRANSACTION");
3286  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
3287  for (int i = 0; i <= cd->columnType.get_physical_cols(); ++i) {
3288  auto cdx = getMetadataForColumn(td->tableId, cd->columnId + i);
3289  CHECK(cdx);
3290  ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.find(
3291  std::make_tuple(td->tableId, to_upper(cdx->columnName)));
3292  CHECK(columnDescIt != columnDescriptorMap_.end());
3293  ColumnDescriptor* changeCd = columnDescIt->second;
3294  changeCd->columnName.replace(0, cd->columnName.size(), newColumnName);
3295  columnDescriptorMap_.erase(columnDescIt); // erase entry under old name
3296  columnDescriptorMap_[std::make_tuple(td->tableId, to_upper(changeCd->columnName))] =
3297  changeCd;
3298  }
3299  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
3300 }
3301 
3303  cat_write_lock write_lock(this);
3305  sqliteConnector_.query("BEGIN TRANSACTION");
3306  try {
3307  // TODO(andrew): this should be an upsert
3309  "SELECT id FROM mapd_dashboards WHERE name = ? and userid = ?",
3310  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
3311  if (sqliteConnector_.getNumRows() > 0) {
3313  "UPDATE mapd_dashboards SET state = ?, image_hash = ?, metadata = ?, "
3314  "update_time = "
3315  "datetime('now') where name = ? "
3316  "and userid = ?",
3317  std::vector<std::string>{vd.dashboardState,
3318  vd.imageHash,
3319  vd.dashboardMetadata,
3320  vd.dashboardName,
3321  std::to_string(vd.userId)});
3322  } else {
3324  "INSERT INTO mapd_dashboards (name, state, image_hash, metadata, "
3325  "update_time, "
3326  "userid) "
3327  "VALUES "
3328  "(?,?,?,?, "
3329  "datetime('now'), ?)",
3330  std::vector<std::string>{vd.dashboardName,
3331  vd.dashboardState,
3332  vd.imageHash,
3333  vd.dashboardMetadata,
3334  std::to_string(vd.userId)});
3335  }
3336  } catch (std::exception& e) {
3337  sqliteConnector_.query("ROLLBACK TRANSACTION");
3338  throw;
3339  }
3340  sqliteConnector_.query("END TRANSACTION");
3341 
3342  // now get the auto generated dashboardId
3343  try {
3345  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_dashboards "
3346  "WHERE name = ? and userid = ?",
3347  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
3348  vd.dashboardId = sqliteConnector_.getData<int>(0, 0);
3349  vd.updateTime = sqliteConnector_.getData<std::string>(0, 1);
3350  } catch (std::exception& e) {
3351  throw;
3352  }
3356  sqlite_lock.unlock();
3357  write_lock.unlock();
3358  // NOTE(wamsi): Transactionally unsafe
3361  return vd.dashboardId;
3362 }
3363 
3365  cat_write_lock write_lock(this);
3367 
3368  sqliteConnector_.query("BEGIN TRANSACTION");
3369  try {
3371  "SELECT id FROM mapd_dashboards WHERE id = ?",
3372  std::vector<std::string>{std::to_string(vd.dashboardId)});
3373  if (sqliteConnector_.getNumRows() > 0) {
3375  "UPDATE mapd_dashboards SET name = ?, state = ?, image_hash = ?, metadata = ?, "
3376  "userid = ?, update_time = datetime('now') where id = ? ",
3377  std::vector<std::string>{vd.dashboardName,
3378  vd.dashboardState,
3379  vd.imageHash,
3380  vd.dashboardMetadata,
3381  std::to_string(vd.userId),
3383  } else {
3384  LOG(ERROR) << "Error replacing dashboard id " << vd.dashboardId
3385  << " does not exist in db";
3386  throw runtime_error("Error replacing dashboard id " +
3387  std::to_string(vd.dashboardId) + " does not exist in db");
3388  }
3389  } catch (std::exception& e) {
3390  sqliteConnector_.query("ROLLBACK TRANSACTION");
3391  throw;
3392  }
3393  sqliteConnector_.query("END TRANSACTION");
3394 
3395  bool found{false};
3396  for (auto descp : dashboardDescriptorMap_) {
3397  auto dash = descp.second.get();
3398  if (dash->dashboardId == vd.dashboardId) {
3399  found = true;
3400  auto viewDescIt = dashboardDescriptorMap_.find(std::to_string(dash->userId) + ":" +
3401  dash->dashboardName);
3402  if (viewDescIt ==
3403  dashboardDescriptorMap_.end()) { // check to make sure view exists
3404  LOG(ERROR) << "No metadata for dashboard for user " << dash->userId
3405  << " dashboard " << dash->dashboardName << " does not exist in map";
3406  throw runtime_error("No metadata for dashboard for user " +
3407  std::to_string(dash->userId) + " dashboard " +
3408  dash->dashboardName + " does not exist in map");
3409  }
3410  dashboardDescriptorMap_.erase(viewDescIt);
3411  break;
3412  }
3413  }
3414  if (!found) {
3415  LOG(ERROR) << "Error replacing dashboard id " << vd.dashboardId
3416  << " does not exist in map";
3417  throw runtime_error("Error replacing dashboard id " + std::to_string(vd.dashboardId) +
3418  " does not exist in map");
3419  }
3420 
3421  // now reload the object
3423  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM "
3424  "mapd_dashboards "
3425  "WHERE id = ?",
3426  std::vector<std::string>{std::to_string(vd.dashboardId)});
3427  vd.updateTime = sqliteConnector_.getData<string>(0, 1);
3431  sqlite_lock.unlock();
3432  write_lock.unlock();
3433  // NOTE(wamsi): Transactionally unsafe
3436 }
3437 
3438 std::string Catalog::calculateSHA1(const std::string& data) {
3439  boost::uuids::detail::sha1 sha1;
3440  unsigned int digest[5];
3441  sha1.process_bytes(data.c_str(), data.length());
3442  sha1.get_digest(digest);
3443  std::stringstream ss;
3444  for (size_t i = 0; i < 5; i++) {
3445  ss << std::hex << digest[i];
3446  }
3447  return ss.str();
3448 }
3449 
3450 std::string Catalog::createLink(LinkDescriptor& ld, size_t min_length) {
3451  cat_write_lock write_lock(this);
3453  sqliteConnector_.query("BEGIN TRANSACTION");
3454  try {
3456  .substr(0, 8);
3458  "SELECT linkid FROM mapd_links WHERE link = ? and userid = ?",
3459  std::vector<std::string>{ld.link, std::to_string(ld.userId)});
3460  if (sqliteConnector_.getNumRows() > 0) {
3462  "UPDATE mapd_links SET update_time = datetime('now') WHERE userid = ? AND link "
3463  "= ?",
3464  std::vector<std::string>{std::to_string(ld.userId), ld.link});
3465  } else {
3467  "INSERT INTO mapd_links (userid, link, view_state, view_metadata, update_time) "
3468  "VALUES (?,?,?,?, "
3469  "datetime('now'))",
3470  std::vector<std::string>{
3471  std::to_string(ld.userId), ld.link, ld.viewState, ld.viewMetadata});
3472  }
3473  // now get the auto generated dashid
3475  "SELECT linkid, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_links "
3476  "WHERE link = ?",
3477  ld.link);
3478  ld.linkId = sqliteConnector_.getData<int>(0, 0);
3479  ld.updateTime = sqliteConnector_.getData<std::string>(0, 1);
3480  } catch (std::exception& e) {
3481  sqliteConnector_.query("ROLLBACK TRANSACTION");
3482  throw;
3483  }
3484  sqliteConnector_.query("END TRANSACTION");
3485  addLinkToMap(ld);
3486  return ld.link;
3487 }
3488 
3490  const TableDescriptor* td) const {
3491  cat_read_lock read_lock(this);
3492 
3493  const auto column_descriptors =
3494  getAllColumnMetadataForTable(td->tableId, false, true, true);
3495 
3496  const ColumnDescriptor* shard_cd{nullptr};
3497  int i = 1;
3498  for (auto cd_itr = column_descriptors.begin(); cd_itr != column_descriptors.end();
3499  ++cd_itr, ++i) {
3500  if (i == td->shardedColumnId) {
3501  shard_cd = *cd_itr;
3502  }
3503  }
3504  return shard_cd;
3505 }
3506 
3507 std::vector<const TableDescriptor*> Catalog::getPhysicalTablesDescriptors(
3508  const TableDescriptor* logicalTableDesc) const {
3509  cat_read_lock read_lock(this);
3510  const auto physicalTableIt =
3511  logicalToPhysicalTableMapById_.find(logicalTableDesc->tableId);
3512  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
3513  return {logicalTableDesc};
3514  }
3515 
3516  const auto physicalTablesIds = physicalTableIt->second;
3517  CHECK(!physicalTablesIds.empty());
3518  std::vector<const TableDescriptor*> physicalTables;
3519  for (size_t i = 0; i < physicalTablesIds.size(); i++) {
3520  physicalTables.push_back(getMetadataForTable(physicalTablesIds[i]));
3521  }
3522 
3523  return physicalTables;
3524 }
3525 
3526 const std::map<int, const ColumnDescriptor*> Catalog::getDictionaryToColumnMapping() {
3527  cat_read_lock read_lock(this);
3528 
3529  std::map<int, const ColumnDescriptor*> mapping;
3530 
3531  const auto tables = getAllTableMetadata();
3532  for (const auto td : tables) {
3533  if (td->shard >= 0) {
3534  // skip shards, they're not standalone tables
3535  continue;
3536  }
3537 
3538  for (auto& cd : getAllColumnMetadataForTable(td->tableId, false, false, true)) {
3539  const auto& ti = cd->columnType;
3540  if (ti.is_string()) {
3541  if (ti.get_compression() == kENCODING_DICT) {
3542  // if foreign reference, get referenced tab.col
3543  const auto dict_id = ti.get_comp_param();
3544 
3545  // ignore temp (negative) dictionaries
3546  if (dict_id > 0 && mapping.end() == mapping.find(dict_id)) {
3547  mapping[dict_id] = cd;
3548  }
3549  }
3550  }
3551  }
3552  }
3553 
3554  return mapping;
3555 }
3556 
3557 std::vector<std::string> Catalog::getTableNamesForUser(
3558  const UserMetadata& user_metadata,
3559  const GetTablesType get_tables_type) const {
3560  sys_read_lock syscat_read_lock(&SysCatalog::instance());
3561  cat_read_lock read_lock(this);
3562 
3563  std::vector<std::string> table_names;
3564  const auto tables = getAllTableMetadata();
3565  for (const auto td : tables) {
3566  if (td->shard >= 0) {
3567  // skip shards, they're not standalone tables
3568  continue;
3569  }
3570  switch (get_tables_type) {
3571  case GET_PHYSICAL_TABLES: {
3572  if (td->isView) {
3573  continue;
3574  }
3575  break;
3576  }
3577  case GET_VIEWS: {
3578  if (!td->isView) {
3579  continue;
3580  }
3581  }
3582  default:
3583  break;
3584  }
3585  DBObject dbObject(td->tableName, td->isView ? ViewDBObjectType : TableDBObjectType);
3586  dbObject.loadKey(*this);
3587  std::vector<DBObject> privObjects = {dbObject};
3588  if (!SysCatalog::instance().hasAnyPrivileges(user_metadata, privObjects)) {
3589  // skip table, as there are no privileges to access it
3590  continue;
3591  }
3592  table_names.push_back(td->tableName);
3593  }
3594  return table_names;
3595 }
3596 
3597 int Catalog::getLogicalTableId(const int physicalTableId) const {
3598  cat_read_lock read_lock(this);
3599  for (const auto& l : logicalToPhysicalTableMapById_) {
3600  if (l.second.end() != std::find_if(l.second.begin(),
3601  l.second.end(),
3602  [&](decltype(*l.second.begin()) tid) -> bool {
3603  return physicalTableId == tid;
3604  })) {
3605  return l.first;
3606  }
3607  }
3608  return physicalTableId;
3609 }
3610 
3611 void Catalog::checkpoint(const int logicalTableId) const {
3612  const auto td = getMetadataForTable(logicalTableId);
3613  const auto shards = getPhysicalTablesDescriptors(td);
3614  for (const auto shard : shards) {
3615  getDataMgr().checkpoint(getCurrentDB().dbId, shard->tableId);
3616  }
3617 }
3618 
3620  cat_write_lock write_lock(this);
3621  // Physically erase all tables and dictionaries from disc and memory
3622  const auto tables = getAllTableMetadata();
3623  for (const auto table : tables) {
3624  eraseTablePhysicalData(table);
3625  }
3626  // Physically erase database metadata
3627  boost::filesystem::remove(basePath_ + "/mapd_catalogs/" + currentDB_.dbName);
3628  calciteMgr_->updateMetadata(currentDB_.dbName, "");
3629 }
3630 
3632  const int tableId = td->tableId;
3633  // must destroy fragmenter before deleteChunks is called.
3634  if (td->fragmenter != nullptr) {
3635  auto tableDescIt = tableDescriptorMapById_.find(tableId);
3636  CHECK(tableDescIt != tableDescriptorMapById_.end());
3637  {
3638  INJECT_TIMER(deleting_fragmenter);
3639  tableDescIt->second->fragmenter = nullptr;
3640  }
3641  }
3642  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
3643  {
3644  INJECT_TIMER(deleteChunksWithPrefix);
3645  // assuming deleteChunksWithPrefix is atomic
3646  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
3647  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
3648  }
3649  if (!td->isView) {
3650  INJECT_TIMER(Remove_Table);
3651  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
3652  }
3653  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
3654  {
3655  INJECT_TIMER(removeTableFromMap_);
3656  removeTableFromMap(td->tableName, tableId);
3657  }
3658 }
3659 
3660 std::string Catalog::generatePhysicalTableName(const std::string& logicalTableName,
3661  const int32_t& shardNumber) {
3662  std::string physicalTableName =
3663  logicalTableName + physicalTableNameTag_ + std::to_string(shardNumber);
3664  return (physicalTableName);
3665 }
3666 
3667 void Catalog::set(const std::string& dbName, std::shared_ptr<Catalog> cat) {
3668  mapd_cat_map_[dbName] = cat;
3669 }
3670 
3671 std::shared_ptr<Catalog> Catalog::get(const std::string& dbName) {
3672  auto cat_it = mapd_cat_map_.find(dbName);
3673  if (cat_it != mapd_cat_map_.end()) {
3674  return cat_it->second;
3675  }
3676  return nullptr;
3677 }
3678 
3679 std::shared_ptr<Catalog> Catalog::get(const int32_t db_id) {
3680  for (const auto& entry : mapd_cat_map_) {
3681  if (entry.second->currentDB_.dbId == db_id) {
3682  return entry.second;
3683  }
3684  }
3685  return nullptr;
3686 }
3687 
3688 std::shared_ptr<Catalog> Catalog::get(const string& basePath,
3689  const DBMetadata& curDB,
3690  std::shared_ptr<Data_Namespace::DataMgr> dataMgr,
3691  const std::vector<LeafHostInfo>& string_dict_hosts,
3692  std::shared_ptr<Calcite> calcite,
3693  bool is_new_db) {
3694  auto cat = Catalog::get(curDB.dbName);
3695 
3696  if (cat) {
3697  return cat;
3698  } else {
3699  cat = std::make_shared<Catalog>(
3700  basePath, curDB, dataMgr, string_dict_hosts, calcite, is_new_db);
3701  Catalog::set(curDB.dbName, cat);
3702  return cat;
3703  }
3704 }
3705 
3706 void Catalog::remove(const std::string& dbName) {
3707  mapd_cat_map_.erase(dbName);
3708 }
3709 
3710 void Catalog::vacuumDeletedRows(const int logicalTableId) const {
3711  // shard here to serve request from TableOptimizer and elsewhere
3712  const auto td = getMetadataForTable(logicalTableId);
3713  const auto shards = getPhysicalTablesDescriptors(td);
3714  for (const auto shard : shards) {
3715  vacuumDeletedRows(shard);
3716  }
3717 }
3718 
3720  // "if not a table that supports delete return nullptr, nothing more to do"
3721  const ColumnDescriptor* cd = getDeletedColumn(td);
3722  if (nullptr == cd) {
3723  return;
3724  }
3725  // vacuum chunks which show sign of deleted rows in metadata
3726  ChunkKey chunkKeyPrefix = {currentDB_.dbId, td->tableId, cd->columnId};
3727  ChunkMetadataVector chunkMetadataVec;
3728  dataMgr_->getChunkMetadataVecForKeyPrefix(chunkMetadataVec, chunkKeyPrefix);
3729  for (auto cm : chunkMetadataVec) {
3730  // "delete has occured"
3731  if (cm.second->chunkStats.max.tinyintval == 1) {
3732  UpdelRoll updel_roll;
3733  updel_roll.catalog = this;
3734  updel_roll.logicalTableId = getLogicalTableId(td->tableId);
3736  const auto cd = getMetadataForColumn(td->tableId, cm.first[2]);
3737  const auto chunk = Chunk_NS::Chunk::getChunk(cd,
3738  &getDataMgr(),
3739  cm.first,
3740  updel_roll.memoryLevel,
3741  0,
3742  cm.second->numBytes,
3743  cm.second->numElements);
3744  td->fragmenter->compactRows(this,
3745  td,
3746  cm.first[3],
3747  td->fragmenter->getVacuumOffsets(chunk),
3748  updel_roll.memoryLevel,
3749  updel_roll);
3750  updel_roll.commitUpdate();
3751  }
3752  }
3753 }
3754 
3757  "SELECT id, name, data_wrapper_type, options, owner_user_id, creation_time FROM "
3758  "omnisci_foreign_servers");
3760  for (size_t row = 0; row < num_rows; row++) {
3761  auto foreign_server = std::make_shared<foreign_storage::ForeignServer>(
3762  sqliteConnector_.getData<int>(row, 0),
3763  sqliteConnector_.getData<std::string>(row, 1),
3764  sqliteConnector_.getData<std::string>(row, 2),
3765  sqliteConnector_.getData<std::string>(row, 3),
3766  sqliteConnector_.getData<std::int32_t>(row, 4),
3767  sqliteConnector_.getData<std::int32_t>(row, 5));
3768  foreignServerMap_[foreign_server->name] = foreign_server;
3769  foreignServerMapById_[foreign_server->id] = foreign_server;
3770  }
3771 }
3772 
3775  "SELECT table_id, server_id, options from omnisci_foreign_tables");
3777  for (size_t r = 0; r < num_rows; r++) {
3778  const auto table_id = sqliteConnector_.getData<int32_t>(r, 0);
3779  const auto server_id = sqliteConnector_.getData<int32_t>(r, 1);
3780  const auto& options = sqliteConnector_.getData<std::string>(r, 2);
3781 
3782  CHECK(tableDescriptorMapById_.find(table_id) != tableDescriptorMapById_.end());
3783  auto foreign_table =
3784  dynamic_cast<foreign_storage::ForeignTable*>(tableDescriptorMapById_[table_id]);
3785  CHECK(foreign_table);
3786  foreign_table->foreign_server = foreignServerMapById_[server_id].get();
3787  CHECK(foreign_table->foreign_server);
3788  foreign_table->populateOptionsMap(options);
3789  }
3790 }
3791 
3792 void Catalog::setForeignServerProperty(const std::string& server_name,
3793  const std::string& property,
3794  const std::string& value) {
3797  "SELECT id from omnisci_foreign_servers where name = ?",
3798  std::vector<std::string>{server_name});
3800  if (num_rows > 0) {
3801  CHECK_EQ(size_t(1), num_rows);
3802  auto server_id = sqliteConnector_.getData<int32_t>(0, 0);
3804  "UPDATE omnisci_foreign_servers SET " + property + " = ? WHERE id = ?",
3805  std::vector<std::string>{value, std::to_string(server_id)});
3806  } else {
3807  throw std::runtime_error{"Can not change property \"" + property +
3808  "\" for foreign server." + " Foreign server \"" +
3809  server_name + "\" is not found."};
3810  }
3811 }
3812 
3815  std::map<std::string, std::string, std::less<>> options;
3816  options[std::string{ForeignServer::STORAGE_TYPE_KEY}] =
3817  ForeignServer::LOCAL_FILE_STORAGE_TYPE;
3818  options[std::string{ForeignServer::BASE_PATH_KEY}] =
3819  boost::filesystem::path::preferred_separator;
3820 
3821  auto local_csv_server =
3822  std::make_unique<ForeignServer>("omnisci_local_csv",
3824  options,
3826  local_csv_server->validate();
3827  createForeignServerNoLocks(std::move(local_csv_server), true);
3828 
3829  auto local_parquet_server =
3830  std::make_unique<ForeignServer>("omnisci_local_parquet",
3832  options,
3834  local_parquet_server->validate();
3835  createForeignServerNoLocks(std::move(local_parquet_server), true);
3836 }
3837 
3838 // prepare a fresh file reload on next table access
3839 void Catalog::setForReload(const int32_t tableId) {
3840  cat_read_lock read_lock(this);
3841  const auto td = getMetadataForTable(tableId);
3842  for (const auto shard : getPhysicalTablesDescriptors(td)) {
3843  const auto tableEpoch = getTableEpoch(currentDB_.dbId, shard->tableId);
3844  setTableEpoch(currentDB_.dbId, shard->tableId, tableEpoch);
3845  }
3846 }
3847 
3848 // get a table's data dirs
3849 std::vector<std::string> Catalog::getTableDataDirectories(
3850  const TableDescriptor* td) const {
3851  const auto global_file_mgr = getDataMgr().getGlobalFileMgr();
3852  std::vector<std::string> file_paths;
3853  for (auto shard : getPhysicalTablesDescriptors(td)) {
3854  const auto file_mgr = dynamic_cast<File_Namespace::FileMgr*>(
3855  global_file_mgr->getFileMgr(currentDB_.dbId, shard->tableId));
3856  boost::filesystem::path file_path(file_mgr->getFileMgrBasePath());
3857  file_paths.push_back(file_path.filename().string());
3858  }
3859  return file_paths;
3860 }
3861 
3862 // get a column's dict dir basename
3864  if ((cd->columnType.is_string() || cd->columnType.is_string_array()) &&
3866  cd->columnType.get_comp_param() > 0) {
3867  const auto dictId = cd->columnType.get_comp_param();
3868  const DictRef dictRef(currentDB_.dbId, dictId);
3869  const auto dit = dictDescriptorMapByRef_.find(dictRef);
3870  CHECK(dit != dictDescriptorMapByRef_.end());
3871  CHECK(dit->second);
3872  boost::filesystem::path file_path(dit->second->dictFolderPath);
3873  return file_path.filename().string();
3874  }
3875  return std::string();
3876 }
3877 
3878 // get a table's dict dirs
3879 std::vector<std::string> Catalog::getTableDictDirectories(
3880  const TableDescriptor* td) const {
3881  std::vector<std::string> file_paths;
3882  for (auto cd : getAllColumnMetadataForTable(td->tableId, false, false, true)) {
3883  auto file_base = getColumnDictDirectory(cd);
3884  if (!file_base.empty() &&
3885  file_paths.end() == std::find(file_paths.begin(), file_paths.end(), file_base)) {
3886  file_paths.push_back(file_base);
3887  }
3888  }
3889  return file_paths;
3890 }
3891 
3892 // returns table schema in a string
3893 // NOTE(sy): Might be able to replace dumpSchema() later with
3894 // dumpCreateTable() after a deeper review of the TableArchiver code.
3895 std::string Catalog::dumpSchema(const TableDescriptor* td) const {
3896  cat_read_lock read_lock(this);
3897 
3898  std::ostringstream os;
3899  os << "CREATE TABLE @T (";
3900  // gather column defines
3901  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
3902  std::string comma;
3903  std::vector<std::string> shared_dicts;
3904  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
3905  for (const auto cd : cds) {
3906  if (!(cd->isSystemCol || cd->isVirtualCol)) {
3907  const auto& ti = cd->columnType;
3908  os << comma << cd->columnName;
3909  // CHAR is perculiar... better dump it as TEXT(32) like \d does
3910  if (ti.get_type() == SQLTypes::kCHAR) {
3911  os << " "
3912  << "TEXT";
3913  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
3914  os << " "
3915  << "TEXT[]";
3916  } else {
3917  os << " " << ti.get_type_name();
3918  }
3919  os << (ti.get_notnull() ? " NOT NULL" : "");
3920  if (ti.is_string()) {
3921  if (ti.get_compression() == kENCODING_DICT) {
3922  // if foreign reference, get referenced tab.col
3923  const auto dict_id = ti.get_comp_param();
3924  const DictRef dict_ref(currentDB_.dbId, dict_id);
3925  const auto dict_it = dictDescriptorMapByRef_.find(dict_ref);
3926  CHECK(dict_it != dictDescriptorMapByRef_.end());
3927  const auto dict_name = dict_it->second->dictName;
3928  // when migrating a table, any foreign dict ref will be dropped
3929  // and the first cd of a dict will become root of the dict
3930  if (dict_root_cds.end() == dict_root_cds.find(dict_name)) {
3931  dict_root_cds[dict_name] = cd;
3932  os << " ENCODING " << ti.get_compression_name() << "(" << (ti.get_size() * 8)
3933  << ")";
3934  } else {
3935  const auto dict_root_cd = dict_root_cds[dict_name];
3936  shared_dicts.push_back("SHARED DICTIONARY (" + cd->columnName +
3937  ") REFERENCES @T(" + dict_root_cd->columnName + ")");
3938  // "... shouldn't specify an encoding, it borrows from the referenced column"
3939  }
3940  } else {
3941  os << " ENCODING NONE";
3942  }
3943  } else if (ti.is_date_in_days() ||
3944  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
3945  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
3946  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
3947  } else if (ti.is_geometry()) {
3948  if (ti.get_compression() == kENCODING_GEOINT) {
3949  os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
3950  << ")";
3951  } else {
3952  os << " ENCODING NONE";
3953  }
3954  }
3955  comma = ", ";
3956  }
3957  }
3958  // gather SHARED DICTIONARYs
3959  if (shared_dicts.size()) {
3960  os << ", " << boost::algorithm::join(shared_dicts, ", ");
3961  }
3962  // gather WITH options ...
3963  std::vector<std::string> with_options;
3964  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
3965  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
3966  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
3967  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
3968  with_options.emplace_back(td->hasDeletedCol ? "VACUUM='DELAYED'"
3969  : "VACUUM='IMMEDIATE'");
3970  if (!td->partitions.empty()) {
3971  with_options.push_back("PARTITIONS='" + td->partitions + "'");
3972  }
3973  if (td->nShards > 0) {
3974  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
3975  CHECK(shard_cd);
3976  os << ", SHARD KEY(" << shard_cd->columnName << ")";
3977  with_options.push_back(
3978  "SHARD_COUNT=" +
3979  std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
3980  }
3981  if (td->sortedColumnId > 0) {
3982  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
3983  CHECK(sort_cd);
3984  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
3985  }
3986  os << ") WITH (" + boost::algorithm::join(with_options, ", ") + ");";
3987  return os.str();
3988 }
3989 
3990 namespace {
3991 
3992 void unserialize_key_metainfo(std::vector<std::string>& shared_dicts,
3993  std::set<std::string>& shared_dict_column_names,
3994  const std::string keyMetainfo) {
3995  rapidjson::Document document;
3996  document.Parse(keyMetainfo.c_str());
3997  CHECK(!document.HasParseError());
3998  CHECK(document.IsArray());
3999  for (auto it = document.Begin(); it != document.End(); ++it) {
4000  const auto& key_with_spec_json = *it;
4001  CHECK(key_with_spec_json.IsObject());
4002  const std::string type = key_with_spec_json["type"].GetString();
4003  const std::string name = key_with_spec_json["name"].GetString();
4004  auto key_with_spec = type + " (" + name + ")";
4005  if (type == "SHARED DICTIONARY") {
4006  shared_dict_column_names.insert(name);
4007  key_with_spec += " REFERENCES ";
4008  const std::string foreign_table = key_with_spec_json["foreign_table"].GetString();
4009  const std::string foreign_column = key_with_spec_json["foreign_column"].GetString();
4010  key_with_spec += foreign_table + "(" + foreign_column + ")";
4011  } else {
4012  CHECK(type == "SHARD KEY");
4013  }
4014  shared_dicts.push_back(key_with_spec);
4015  }
4016 }
4017 
4018 } // namespace
4019 
4020 #include "Parser/ReservedKeywords.h"
4021 
4023 inline bool contains_spaces(std::string_view str) {
4024  return std::find_if(str.begin(), str.end(), [](const unsigned char& ch) {
4025  return std::isspace(ch);
4026  }) != str.end();
4027 }
4028 
4031  std::string_view str,
4032  std::string_view chars = "`~!@#$%^&*()-=+[{]}\\|;:'\",<.>/?") {
4033  return str.find_first_of(chars) != std::string_view::npos;
4034 }
4035 
4037 inline bool is_reserved_sql_keyword(std::string_view str) {
4038  return reserved_keywords.find(to_upper(std::string(str))) != reserved_keywords.end();
4039 }
4040 
4041 // returns a "CREATE TABLE" statement in a string for "SHOW CREATE TABLE"
4043  bool multiline_formatting,
4044  bool dump_defaults) const {
4045  cat_read_lock read_lock(this);
4046 
4047  std::ostringstream os;
4048 
4049  if (!td->isView) {
4050  os << "CREATE ";
4052  os << "TEMPORARY ";
4053  }
4054  os << "TABLE " + td->tableName + " (";
4055  } else {
4056  os << "CREATE VIEW " + td->tableName + " AS " << td->viewSQL;
4057  return os.str();
4058  }
4059  // scan column defines
4060  std::vector<std::string> shared_dicts;
4061  std::set<std::string> shared_dict_column_names;
4062  unserialize_key_metainfo(shared_dicts, shared_dict_column_names, td->keyMetainfo);
4063  // gather column defines
4064  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
4065  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
4066  bool first = true;
4067  for (const auto cd : cds) {
4068  if (!(cd->isSystemCol || cd->isVirtualCol)) {
4069  const auto& ti = cd->columnType;
4070  if (!first) {
4071  os << ",";
4072  if (!multiline_formatting) {
4073  os << " ";
4074  }
4075  } else {
4076  first = false;
4077  }
4078  if (multiline_formatting) {
4079  os << "\n ";
4080  }
4081  // column name
4082  bool column_name_needs_quotes = is_reserved_sql_keyword(cd->columnName) ||
4083  contains_spaces(cd->columnName) ||
4084  contains_sql_reserved_chars(cd->columnName);
4085  if (!column_name_needs_quotes) {
4086  os << cd->columnName;
4087  } else {
4088  os << "\"" << cd->columnName << "\"";
4089  }
4090  // CHAR is perculiar... better dump it as TEXT(32) like \d does
4091  if (ti.get_type() == SQLTypes::kCHAR) {
4092  os << " "
4093  << "TEXT";
4094  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
4095  os << " "
4096  << "TEXT[]";
4097  } else {
4098  os << " " << ti.get_type_name();
4099  }
4100  os << (ti.get_notnull() ? " NOT NULL" : "");
4101  if (shared_dict_column_names.find(cd->columnName) ==
4102  shared_dict_column_names.end()) {
4103  // avoids "Exception: Column ... shouldn't specify an encoding, it borrows it from
4104  // the referenced column"
4105  if (ti.is_string()) {
4106  if (ti.get_compression() == kENCODING_DICT) {
4107  os << " ENCODING " << ti.get_compression_name() << "(" << (ti.get_size() * 8)
4108  << ")";
4109  } else {
4110  os << " ENCODING NONE";
4111  }
4112  } else if (ti.is_date_in_days() ||
4113  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
4114  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
4115  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
4116  } else if (ti.is_geometry()) {
4117  if (ti.get_compression() == kENCODING_GEOINT) {
4118  os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
4119  << ")";
4120  } else {
4121  os << " ENCODING NONE";
4122  }
4123  }
4124  }
4125  }
4126  }
4127  // gather SHARED DICTIONARYs
4128  if (shared_dicts.size()) {
4129  std::string comma;
4130  if (!multiline_formatting) {
4131  comma = ", ";
4132  } else {
4133  comma = ",\n ";
4134  }
4135  os << comma;
4136  os << boost::algorithm::join(shared_dicts, comma);
4137  }
4138  // gather WITH options ...
4139  std::vector<std::string> with_options;
4140  if (dump_defaults || td->maxFragRows != DEFAULT_FRAGMENT_ROWS) {
4141  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
4142  }
4143  if (dump_defaults || td->maxChunkSize != DEFAULT_MAX_CHUNK_SIZE) {
4144  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
4145  }
4146  if (dump_defaults || td->fragPageSize != DEFAULT_PAGE_SIZE) {
4147  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
4148  }
4149  if (dump_defaults || td->maxRows != DEFAULT_MAX_ROWS) {
4150  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
4151  }
4152  if (dump_defaults || !td->hasDeletedCol) {
4153  with_options.push_back(td->hasDeletedCol ? "VACUUM='DELAYED'" : "VACUUM='IMMEDIATE'");
4154  }
4155  if (!td->partitions.empty()) {
4156  with_options.push_back("PARTITIONS='" + td->partitions + "'");
4157  }
4158  if (td->nShards > 0) {
4159  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
4160  CHECK(shard_cd);
4161  with_options.push_back(
4162  "SHARD_COUNT=" +
4163  std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
4164  }
4165  if (td->sortedColumnId > 0) {
4166  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
4167  CHECK(sort_cd);
4168  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
4169  }
4170  os << ")";
4171  if (!with_options.empty()) {
4172  if (!multiline_formatting) {
4173  os << " ";
4174  } else {
4175  os << "\n";
4176  }
4177  os << "WITH (" + boost::algorithm::join(with_options, ", ") + ")";
4178  }
4179  os << ";";
4180  return os.str();
4181 }
4182 
4184  const bool if_not_exists) {
4185  if (getMetadataForTable(name, false)) {
4186  if (if_not_exists) {
4187  return false;
4188  }
4189  throw std::runtime_error("Table or View with name \"" + name + "\" already exists.");
4190  }
4191  return true;
4192 }
4193 
4194 } // namespace Catalog_Namespace
void serializeTableJsonUnlocked(const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
Definition: Catalog.cpp:2331
const Parser::SharedDictionaryDef compress_reference_path(Parser::SharedDictionaryDef cur_node, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
static std::set< std::string > reserved_keywords
Data_Namespace::MemoryLevel memoryLevel
Definition: UpdelRoll.h:65
std::string virtualExpr
void set_compression(EncodingType c)
Definition: sqltypes.h:359
void set_size(int s)
Definition: sqltypes.h:357
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:1730
void deleteMetadataForDashboards(const std::vector< int32_t > ids, const UserMetadata &user)
Definition: Catalog.cpp:1555
#define CHECK_EQ(x, y)
Definition: Logger.h:205
void executeDropTableSqliteQueries(const TableDescriptor *td)
Definition: Catalog.cpp:3163
const int MAPD_TEMP_TABLE_START_ID
Definition: Catalog.cpp:100
std::string partitions
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:3631
std::string dictFolderPath
std::tuple< int, std::string > ColumnKey
Definition: Types.h:36
void doDropTable(const TableDescriptor *td)
Definition: Catalog.cpp:3155
~Catalog()
Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the ...
Definition: Catalog.cpp:190
bool is_array() const
Definition: sqltypes.h:424
T getData(const int row, const int col)
bool is_string() const
Definition: sqltypes.h:416
mapd_shared_mutex sharedMutex_
Definition: Catalog.h:519
void commitUpdate()
const int8_t const int64_t * num_rows
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:86
SQLTypes
Definition: sqltypes.h:40
const std::vector< LeafHostInfo > & getStringDictionaryHosts() const
Definition: Catalog.cpp:1490
#define SPIMAP_MAGIC1
Definition: Catalog.h:73
std::string tableName
void addReferenceToForeignDict(ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def, const bool persist_reference)
Definition: Catalog.cpp:2834
void setDeletedColumn(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:2808
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:463
static const AccessPrivileges ALL_DATABASE
Definition: DBObject.h:152
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:2118
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:407
virtual void query_with_text_params(std::string const &query_only)
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
DBObjectType
Definition: DBObject.h:42
void createForeignServer(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2425
std::string getOptionsAsJsonString() const
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:196
void addColumn(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:1842
void checkpoint(const int db_id, const int tb_id)
Definition: DataMgr.cpp:498
void roll(const bool forward)
Definition: Catalog.cpp:1930
bool validateNonExistentTableOrView(const std::string &name, const bool if_not_exists)
Definition: Catalog.cpp:4183
#define LOG(tag)
Definition: Logger.h:188
SqliteConnector sqliteConnector_
Definition: Catalog.h:472
static std::shared_ptr< Catalog > get(const std::string &dbName)
Definition: Catalog.cpp:3671
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:476
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:465
std::string storageType
void revokeDBObjectPrivilegesFromAll(DBObject object, Catalog *catalog)
const ColumnDescriptor * getDeletedColumnIfRowsDeleted(const TableDescriptor *td) const
Definition: Catalog.cpp:2774
void revokeDBObjectPrivileges(const std::string &grantee, const DBObject &object, const Catalog_Namespace::Catalog &catalog)
std::string join(T const &container, std::string const &delim)
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:268
Catalog(const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
Constructor - takes basePath to already extant data directory for writing.
Definition: Catalog.cpp:160
#define DEFAULT_MAX_CHUNK_SIZE
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:474
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:350
DBObject * findDbObject(const DBObjectKey &objectKey, bool only_direct) const
Definition: Grantee.cpp:72
void populateOptionsMap(const rapidjson::Value &ddl_options)
virtual void query(const std::string &queryString)
#define CHECK_GE(x, y)
Definition: Logger.h:210
void setObjectKey(const DBObjectKey &objectKey)
Definition: DBObject.h:215
int32_t getTableEpoch(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:2688
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:152
HOST DEVICE int get_size() const
Definition: sqltypes.h:269
std::string fragments
void setForReload(const int32_t tableId)
Definition: Catalog.cpp:3839
int32_t objectId
Definition: DBObject.h:57
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
void replaceDashboard(DashboardDescriptor &vd)
Definition: Catalog.cpp:3364
void unserialize_key_metainfo(std::vector< std::string > &shared_dicts, std::set< std::string > &shared_dict_column_names, const std::string keyMetainfo)
Definition: Catalog.cpp:3992
const int MAPD_TEMP_DICT_START_ID
Definition: Catalog.cpp:102
void getForeignServersForUser(const rapidjson::Value *filters, const UserMetadata &user, std::vector< const foreign_storage::ForeignServer *> &results)
Definition: Catalog.cpp:2596
Definition: Grantee.h:76
std::string generatePhysicalTableName(const std::string &logicalTableName, const int32_t &shardNumber)
Definition: Catalog.cpp:3660
The InsertOrderFragmenter is a child class of AbstractFragmenter, and fragments data in insert order...
void dropColumn(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:1899
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logicalTableDesc) const
Definition: Catalog.cpp:3507
void checkDateInDaysColumnMigration()
Definition: Catalog.cpp:814
void setPrivileges(const AccessPrivileges &privs)
Definition: DBObject.h:217
void addFrontendViewToMap(DashboardDescriptor &vd)
Definition: Catalog.cpp:1278
bool contains_spaces(std::string_view str)
returns true if the string contains one or more spaces
Definition: Catalog.cpp:4023
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:267
void vacuumDeletedRows(const TableDescriptor *td) const
Definition: Catalog.cpp:3719
static const AccessPrivileges SELECT_FROM_TABLE
Definition: DBObject.h:161
std::vector< int > columnIdBySpi_
std::string dumpCreateTable(const TableDescriptor *td, bool multiline_formatting=true, bool dump_defaults=false) const
Definition: Catalog.cpp:4042
#define CHECK_GT(x, y)
Definition: Logger.h:209
Grantee * getGrantee(const std::string &name) const
void changeForeignServerOwner(const std::string &server_name, const int new_owner_id)
Definition: Catalog.cpp:2498
std::atomic< std::thread::id > thread_holding_sqlite_lock
Definition: Catalog.h:520
HOST DEVICE int get_scale() const
Definition: sqltypes.h:264
void renameColumn(const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
Definition: Catalog.cpp:3263
DeletedColumnPerTableMap deletedColumnPerTable_
Definition: Catalog.h:495
const ColumnDescriptor * get_foreign_col(const Catalog &cat, const Parser::SharedDictionaryDef &shared_dict_def)
Definition: Catalog.cpp:2822
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:464
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:466
void setForeignServerOptions(const std::string &server_name, const std::string &options)
Definition: Catalog.cpp:2530
std::string to_string(char const *&&v)
std::shared_ptr< std::mutex > mutex_
void addTableToMap(const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
Definition: Catalog.cpp:1137
std::string dumpSchema(const TableDescriptor *td) const
Definition: Catalog.cpp:3895
static const AccessPrivileges ALL_VIEW
Definition: DBObject.h:178
std::string chunks
void grantRoleBatch(const std::vector< std::string > &roles, const std::vector< std::string > &grantees)
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:686
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:119
std::list< const ColumnDescriptor * > getAllColumnMetadataForTableUnlocked(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Definition: Catalog.cpp:1714
#define SPIMAP_MAGIC2
Definition: Catalog.h:74
const ColumnDescriptor * getShardColumnMetadataForTable(const TableDescriptor *td) const
Definition: Catalog.cpp:3489
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:266
void createRole(const std::string &roleName, const bool &userPrivateRole=false)
std::string dashboardSystemRoleName
This file contains the class specification and related data structures for Catalog.
int getLogicalTableId(const int physicalTableId) const
Definition: Catalog.cpp:3597
std::vector< std::string > getTableDictDirectories(const TableDescriptor *td) const
Definition: Catalog.cpp:3879
EncodingType
Definition: sqltypes.h:156
#define DEFAULT_MAX_ROWS
void checkpoint(const int logicalTableId) const
Definition: Catalog.cpp:3611
static SysCatalog & instance()
Definition: SysCatalog.h:288
This file contains the class specification and related data structures for SysCatalog.
std::string cat(Ts &&... args)
Classes representing a parse tree.
const Catalog_Namespace::Catalog * catalog
Definition: UpdelRoll.h:63
std::string calculateSHA1(const std::string &data)
Definition: Catalog.cpp:3438
std::string getUserFromId(const int32_t id)
Definition: Catalog.cpp:916
void addLinkToMap(LinkDescriptor &ld)
Definition: Catalog.cpp:1360
static const std::string getForeignTableSchema(bool if_not_exists=false)
Definition: Catalog.cpp:679
#define INJECT_TIMER(DESC)
Definition: measure.h:91
std::string getColumnDictDirectory(const ColumnDescriptor *cd) const
Definition: Catalog.cpp:3863
void delDictionary(const ColumnDescriptor &cd)
Definition: Catalog.cpp:1771
const std::string & get_foreign_column() const
Definition: ParserNode.h:901
GetTablesType
Definition: Catalog.h:61
const TableDescriptor * getMetadataForTableImpl(int tableId, const bool populateFragmenter) const
Definition: Catalog.cpp:1426
void renameObjectsInDescriptorMap(DBObject &object, const Catalog_Namespace::Catalog &cat)
void removeChunks(const int table_id)
Definition: Catalog.cpp:3102
void set_scale(int s)
Definition: sqltypes.h:354
void dropForeignServer(const std::string &server_name)
Definition: Catalog.cpp:2562
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:260
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:2813
void dropTable(const TableDescriptor *td)
Definition: Catalog.cpp:3122
std::vector< std::string > parse_underlying_dashboard_objects(const std::string &meta)
static void migrateDateInDaysMetadata(const Catalog_Namespace::TableDescriptorMapById &table_descriptors_by_id, const int database_id, const Catalog_Namespace::Catalog *cat, SqliteConnector &sqlite)
static const int32_t MAPD_VERSION
Definition: release.h:33
static const std::string getForeignServerSchema(bool if_not_exists=false)
Definition: Catalog.cpp:672
static const AccessPrivileges ALL_SERVER
Definition: DBObject.h:188
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:462
void setOwner(int32_t userId)
Definition: DBObject.h:225
static std::shared_ptr< Chunk > getChunk(const ColumnDescriptor *cd, DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId, const size_t num_bytes, const size_t num_elems)
Definition: Chunk.cpp:28
int get_physical_cols() const
Definition: sqltypes.h:280
void grantDBObjectPrivilegesBatch(const std::vector< std::string > &grantees, const std::vector< DBObject > &objects, const Catalog_Namespace::Catalog &catalog)
specifies the content in-memory of a row in the column metadata table
int32_t dictId
Definition: DictRef.h:10
static void registerTable(Catalog_Namespace::Catalog *catalog, const TableDescriptor &td, const std::list< ColumnDescriptor > &cols)
ids are created
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:195
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:1988
bool contains_sql_reserved_chars(std::string_view str, std::string_view chars="`~!@#$%^&*()-=+[{]}\;:'<.>/?")
returns true if the string contains one or more OmniSci SQL reserved characters
Definition: Catalog.cpp:4030
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1319
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:470
std::string keyMetainfo
int g_test_against_columnId_gap
Definition: Catalog.cpp:89
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1449
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
bool g_serialize_temp_tables
Definition: Catalog.cpp:95
struct dict_ref_t DictRef
Definition: DictRef.h:37
void createShardedTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:2966
ForeignServerMap foreignServerMap_
Definition: Catalog.h:469
std::string to_upper(const std::string &str)
#define DEFAULT_PAGE_SIZE
void set_comp_param(int p)
Definition: sqltypes.h:360
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2433
void loadKey()
Definition: DBObject.cpp:179
const int DEFAULT_INITIAL_VERSION
Definition: Catalog.cpp:99
const bool checkMetadataForDeletedRecs(const TableDescriptor *td, int column_id) const
Definition: Catalog.cpp:2748
bool is_geometry() const
Definition: sqltypes.h:428
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:477
static const std::string physicalTableNameTag_
Definition: Catalog.h:481
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1283
static void remove(const std::string &dbName)
Definition: Catalog.cpp:3706
const ColumnDescriptor * getMetadataForColumnUnlocked(int tableId, int columnId) const
Definition: Catalog.cpp:1512
static const AccessPrivileges SELECT_FROM_VIEW
Definition: DBObject.h:181
bool table_is_temporary(const TableDescriptor *const td)
bool is_string_array() const
Definition: sqltypes.h:417
std::vector< std::string > getTableNamesForUser(const UserMetadata &user, const GetTablesType get_tables_type) const
Definition: Catalog.cpp:3557
void setTableEpoch(const int db_id, const int table_id, const int new_epoch)
Definition: Catalog.cpp:2718
static void set(const std::string &dbName, std::shared_ptr< Catalog > cat)
Definition: Catalog.cpp:3667
std::vector< ObjectRoleDescriptor * > getMetadataForObject(int32_t dbId, int32_t dbType, int32_t objectId) const
const DBObjectMap * getDbObjects(bool only_direct) const
Definition: Grantee.h:56
void set_dimension(int d)
Definition: sqltypes.h:351
#define DEFAULT_FRAGMENT_ROWS
const int OMNISCI_ROOT_USER_ID
Definition: SysCatalog.h:60
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:261
Fragmenter_Namespace::FragmenterType fragType
int32_t dbId
Definition: DBObject.h:56
Data_Namespace::MemoryLevel persistenceLevel
void dropTableFromJsonUnlocked(const std::string &table_name) const
Definition: Catalog.cpp:2393
std::string name() const
Definition: Catalog.h:251
const std::string & get_foreign_table() const
Definition: ParserNode.h:899
int32_t createDashboard(DashboardDescriptor &vd)
Definition: Catalog.cpp:3302
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1197
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
Definition: DataMgr.cpp:543
static const AccessPrivileges ALL_DASHBOARD
Definition: DBObject.h:170
Definition: sqltypes.h:43
static const AccessPrivileges ALL_TABLE
Definition: DBObject.h:158
int logicalTableId
Definition: UpdelRoll.h:64
bool g_cache_string_hash
Definition: Execute.cpp:93
bool isNull(const int row, const int col) const
const ColumnDescriptor * getDeletedColumn(const TableDescriptor *td) const
Definition: