OmniSciDB  7bf56492aa
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Catalog.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
25 #include "Catalog.h"
26 #include "SysCatalog.h"
27 
28 #include <sys/wait.h>
29 
30 #include <algorithm>
31 #include <boost/algorithm/string/predicate.hpp>
32 #include <boost/filesystem.hpp>
33 #include <boost/range/adaptor/map.hpp>
34 #include <boost/version.hpp>
35 #include <cassert>
36 #include <cerrno>
37 #include <cstdio>
38 #include <cstring>
39 #include <exception>
40 #include <fstream>
41 #include <list>
42 #include <memory>
43 #include <random>
44 #include <regex>
45 #include <sstream>
46 #if BOOST_VERSION >= 106600
47 #include <boost/uuid/detail/sha1.hpp>
48 #else
49 #include <boost/uuid/sha1.hpp>
50 #endif
51 #include <rapidjson/document.h>
52 #include <rapidjson/istreamwrapper.h>
53 #include <rapidjson/ostreamwrapper.h>
54 #include <rapidjson/writer.h>
55 
56 #include "QueryEngine/Execute.h"
58 
62 #include "Fragmenter/Fragmenter.h"
64 #include "LockMgr/LockMgr.h"
66 #include "Parser/ParserNode.h"
67 #include "QueryEngine/Execute.h"
69 #include "Shared/File.h"
70 #include "Shared/StringTransform.h"
71 #include "Shared/measure.h"
73 
74 #include "MapDRelease.h"
75 #include "RWLocks.h"
77 
78 using Chunk_NS::Chunk;
81 using std::list;
82 using std::map;
83 using std::pair;
84 using std::runtime_error;
85 using std::string;
86 using std::vector;
87 
89 bool g_enable_fsi{false};
90 extern bool g_cache_string_hash;
91 
92 // Serialize temp tables to a json file in the Catalogs directory for Calcite parsing
93 // under unit testing.
95 
96 namespace Catalog_Namespace {
97 
98 const int DEFAULT_INITIAL_VERSION = 1; // start at version 1
100  1073741824; // 2^30, give room for over a billion non-temp tables
102  1073741824; // 2^30, give room for over a billion non-temp dictionaries
103 
104 const std::string Catalog::physicalTableNameTag_("_shard_#");
105 std::map<std::string, std::shared_ptr<Catalog>> Catalog::mapd_cat_map_;
106 
107 thread_local bool Catalog::thread_holds_read_lock = false;
108 
112 
113 // migration will be done as two step process this release
114 // will create and use new table
115 // next release will remove old table, doing this to have fall back path
116 // incase of migration failure
119  sqliteConnector_.query("BEGIN TRANSACTION");
120  try {
122  "SELECT name FROM sqlite_master WHERE type='table' AND name='mapd_dashboards'");
123  if (sqliteConnector_.getNumRows() != 0) {
124  // already done
125  sqliteConnector_.query("END TRANSACTION");
126  return;
127  }
129  "CREATE TABLE mapd_dashboards (id integer primary key autoincrement, name text , "
130  "userid integer references mapd_users, state text, image_hash text, update_time "
131  "timestamp, "
132  "metadata text, UNIQUE(userid, name) )");
133  // now copy content from old table to new table
135  "insert into mapd_dashboards (id, name , "
136  "userid, state, image_hash, update_time , "
137  "metadata) "
138  "SELECT viewid , name , userid, view_state, image_hash, update_time, "
139  "view_metadata "
140  "from mapd_frontend_views");
141  } catch (const std::exception& e) {
142  sqliteConnector_.query("ROLLBACK TRANSACTION");
143  throw;
144  }
145  sqliteConnector_.query("END TRANSACTION");
146 }
147 
148 namespace {
149 
150 inline auto table_json_filepath(const std::string& base_path,
151  const std::string& db_name) {
152  return boost::filesystem::path(base_path + "/mapd_catalogs/" + db_name +
153  "_temp_tables.json");
154 }
155 
156 } // namespace
157 
158 Catalog::Catalog(const string& basePath,
159  const DBMetadata& curDB,
160  std::shared_ptr<Data_Namespace::DataMgr> dataMgr,
161  const std::vector<LeafHostInfo>& string_dict_hosts,
162  std::shared_ptr<Calcite> calcite,
163  bool is_new_db)
164  : basePath_(basePath)
165  , sqliteConnector_(curDB.dbName, basePath + "/mapd_catalogs/")
166  , currentDB_(curDB)
167  , dataMgr_(dataMgr)
168  , string_dict_hosts_(string_dict_hosts)
169  , calciteMgr_(calcite)
170  , nextTempTableId_(MAPD_TEMP_TABLE_START_ID)
171  , nextTempDictId_(MAPD_TEMP_DICT_START_ID)
172  , sqliteMutex_()
173  , sharedMutex_()
174  , thread_holding_sqlite_lock()
175  , thread_holding_write_lock() {
176  if (!is_new_db) {
178  }
179  buildMaps();
180  if (!is_new_db) {
182  }
184  boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
185  }
186 }
187 
190  // must clean up heap-allocated TableDescriptor and ColumnDescriptor structs
191  for (TableDescriptorMap::iterator tableDescIt = tableDescriptorMap_.begin();
192  tableDescIt != tableDescriptorMap_.end();
193  ++tableDescIt) {
194  if (tableDescIt->second->fragmenter != nullptr) {
195  delete tableDescIt->second->fragmenter;
196  }
197  delete tableDescIt->second;
198  }
199 
200  // TableDescriptorMapById points to the same descriptors. No need to delete
201 
202  for (ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.begin();
203  columnDescIt != columnDescriptorMap_.end();
204  ++columnDescIt) {
205  delete columnDescIt->second;
206  }
207 
208  // ColumnDescriptorMapById points to the same descriptors. No need to delete
209 
211  boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
212  }
213 }
214 
217  sqliteConnector_.query("BEGIN TRANSACTION");
218  try {
219  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_tables)");
220  std::vector<std::string> cols;
221  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
222  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
223  }
224  if (std::find(cols.begin(), cols.end(), std::string("max_chunk_size")) ==
225  cols.end()) {
226  string queryString("ALTER TABLE mapd_tables ADD max_chunk_size BIGINT DEFAULT " +
228  sqliteConnector_.query(queryString);
229  }
230  if (std::find(cols.begin(), cols.end(), std::string("shard_column_id")) ==
231  cols.end()) {
232  string queryString("ALTER TABLE mapd_tables ADD shard_column_id BIGINT DEFAULT " +
233  std::to_string(0));
234  sqliteConnector_.query(queryString);
235  }
236  if (std::find(cols.begin(), cols.end(), std::string("shard")) == cols.end()) {
237  string queryString("ALTER TABLE mapd_tables ADD shard BIGINT DEFAULT " +
238  std::to_string(-1));
239  sqliteConnector_.query(queryString);
240  }
241  if (std::find(cols.begin(), cols.end(), std::string("num_shards")) == cols.end()) {
242  string queryString("ALTER TABLE mapd_tables ADD num_shards BIGINT DEFAULT " +
243  std::to_string(0));
244  sqliteConnector_.query(queryString);
245  }
246  if (std::find(cols.begin(), cols.end(), std::string("key_metainfo")) == cols.end()) {
247  string queryString("ALTER TABLE mapd_tables ADD key_metainfo TEXT DEFAULT '[]'");
248  sqliteConnector_.query(queryString);
249  }
250  if (std::find(cols.begin(), cols.end(), std::string("userid")) == cols.end()) {
251  string queryString("ALTER TABLE mapd_tables ADD userid integer DEFAULT " +
253  sqliteConnector_.query(queryString);
254  }
255  if (std::find(cols.begin(), cols.end(), std::string("sort_column_id")) ==
256  cols.end()) {
258  "ALTER TABLE mapd_tables ADD sort_column_id INTEGER DEFAULT 0");
259  }
260  if (std::find(cols.begin(), cols.end(), std::string("storage_type")) == cols.end()) {
261  string queryString("ALTER TABLE mapd_tables ADD storage_type TEXT DEFAULT ''");
262  sqliteConnector_.query(queryString);
263  }
264  } catch (std::exception& e) {
265  sqliteConnector_.query("ROLLBACK TRANSACTION");
266  throw;
267  }
268  sqliteConnector_.query("END TRANSACTION");
269 }
270 
273  sqliteConnector_.query("BEGIN TRANSACTION");
274  try {
276  "select name from sqlite_master WHERE type='table' AND "
277  "name='mapd_version_history'");
278  if (sqliteConnector_.getNumRows() == 0) {
280  "CREATE TABLE mapd_version_history(version integer, migration_history text "
281  "unique)");
282  } else {
284  "select * from mapd_version_history where migration_history = "
285  "'notnull_fixlen_arrays'");
286  if (sqliteConnector_.getNumRows() != 0) {
287  // legacy fixlen arrays had migrated
288  // no need for further execution
289  sqliteConnector_.query("END TRANSACTION");
290  return;
291  }
292  }
293  // Insert check for migration
295  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
296  std::vector<std::string>{std::to_string(MAPD_VERSION), "notnull_fixlen_arrays"});
297  LOG(INFO) << "Updating mapd_columns, legacy fixlen arrays";
298  // Upating all fixlen array columns
299  string queryString("UPDATE mapd_columns SET is_notnull=1 WHERE coltype=" +
300  std::to_string(kARRAY) + " AND size>0;");
301  sqliteConnector_.query(queryString);
302  } catch (std::exception& e) {
303  sqliteConnector_.query("ROLLBACK TRANSACTION");
304  throw;
305  }
306  sqliteConnector_.query("END TRANSACTION");
307 }
308 
311  sqliteConnector_.query("BEGIN TRANSACTION");
312  try {
314  "select name from sqlite_master WHERE type='table' AND "
315  "name='mapd_version_history'");
316  if (sqliteConnector_.getNumRows() == 0) {
318  "CREATE TABLE mapd_version_history(version integer, migration_history text "
319  "unique)");
320  } else {
322  "select * from mapd_version_history where migration_history = "
323  "'notnull_geo_columns'");
324  if (sqliteConnector_.getNumRows() != 0) {
325  // legacy geo columns had migrated
326  // no need for further execution
327  sqliteConnector_.query("END TRANSACTION");
328  return;
329  }
330  }
331  // Insert check for migration
333  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
334  std::vector<std::string>{std::to_string(MAPD_VERSION), "notnull_geo_columns"});
335  LOG(INFO) << "Updating mapd_columns, legacy geo columns";
336  // Upating all geo columns
337  string queryString(
338  "UPDATE mapd_columns SET is_notnull=1 WHERE coltype=" + std::to_string(kPOINT) +
339  " OR coltype=" + std::to_string(kLINESTRING) + " OR coltype=" +
340  std::to_string(kPOLYGON) + " OR coltype=" + std::to_string(kMULTIPOLYGON) + ";");
341  sqliteConnector_.query(queryString);
342  } catch (std::exception& e) {
343  sqliteConnector_.query("ROLLBACK TRANSACTION");
344  throw;
345  }
346  sqliteConnector_.query("END TRANSACTION");
347 }
348 
351  sqliteConnector_.query("BEGIN TRANSACTION");
352  try {
353  // check table still exists
355  "SELECT name FROM sqlite_master WHERE type='table' AND "
356  "name='mapd_frontend_views'");
357  if (sqliteConnector_.getNumRows() == 0) {
358  // table does not exists
359  // no need to migrate
360  sqliteConnector_.query("END TRANSACTION");
361  return;
362  }
363  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_frontend_views)");
364  std::vector<std::string> cols;
365  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
366  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
367  }
368  if (std::find(cols.begin(), cols.end(), std::string("image_hash")) == cols.end()) {
369  sqliteConnector_.query("ALTER TABLE mapd_frontend_views ADD image_hash text");
370  }
371  if (std::find(cols.begin(), cols.end(), std::string("update_time")) == cols.end()) {
372  sqliteConnector_.query("ALTER TABLE mapd_frontend_views ADD update_time timestamp");
373  }
374  if (std::find(cols.begin(), cols.end(), std::string("view_metadata")) == cols.end()) {
375  sqliteConnector_.query("ALTER TABLE mapd_frontend_views ADD view_metadata text");
376  }
377  } catch (std::exception& e) {
378  sqliteConnector_.query("ROLLBACK TRANSACTION");
379  throw;
380  }
381  sqliteConnector_.query("END TRANSACTION");
382 }
383 
386  sqliteConnector_.query("BEGIN TRANSACTION");
387  try {
389  "CREATE TABLE IF NOT EXISTS mapd_links (linkid integer primary key, userid "
390  "integer references mapd_users, "
391  "link text unique, view_state text, update_time timestamp, view_metadata text)");
392  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_links)");
393  std::vector<std::string> cols;
394  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
395  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
396  }
397  if (std::find(cols.begin(), cols.end(), std::string("view_metadata")) == cols.end()) {
398  sqliteConnector_.query("ALTER TABLE mapd_links ADD view_metadata text");
399  }
400  } catch (const std::exception& e) {
401  sqliteConnector_.query("ROLLBACK TRANSACTION");
402  throw;
403  }
404  sqliteConnector_.query("END TRANSACTION");
405 }
406 
409  sqliteConnector_.query("BEGIN TRANSACTION");
410  try {
411  sqliteConnector_.query("UPDATE mapd_links SET userid = 0 WHERE userid IS NULL");
412  // check table still exists
414  "SELECT name FROM sqlite_master WHERE type='table' AND "
415  "name='mapd_frontend_views'");
416  if (sqliteConnector_.getNumRows() == 0) {
417  // table does not exists
418  // no need to migrate
419  sqliteConnector_.query("END TRANSACTION");
420  return;
421  }
423  "UPDATE mapd_frontend_views SET userid = 0 WHERE userid IS NULL");
424  } catch (const std::exception& e) {
425  sqliteConnector_.query("ROLLBACK TRANSACTION");
426  throw;
427  }
428  sqliteConnector_.query("END TRANSACTION");
429 }
430 
431 // introduce DB version into the tables table
432 // if the DB does not have a version reset all pagesizes to 2097152 to be compatible with
433 // old value
434 
437  if (currentDB_.dbName.length() == 0) {
438  // updateDictionaryNames dbName length is zero nothing to do here
439  return;
440  }
441  sqliteConnector_.query("BEGIN TRANSACTION");
442  try {
443  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_tables)");
444  std::vector<std::string> cols;
445  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
446  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
447  }
448  if (std::find(cols.begin(), cols.end(), std::string("version_num")) == cols.end()) {
449  LOG(INFO) << "Updating mapd_tables updatePageSize";
450  // No version number
451  // need to update the defaul tpagesize to old correct value
452  sqliteConnector_.query("UPDATE mapd_tables SET frag_page_size = 2097152 ");
453  // need to add new version info
454  string queryString("ALTER TABLE mapd_tables ADD version_num BIGINT DEFAULT " +
456  sqliteConnector_.query(queryString);
457  }
458  } catch (std::exception& e) {
459  sqliteConnector_.query("ROLLBACK TRANSACTION");
460  throw;
461  }
462  sqliteConnector_.query("END TRANSACTION");
463 }
464 
467  sqliteConnector_.query("BEGIN TRANSACTION");
468  try {
469  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_columns)");
470  std::vector<std::string> cols;
471  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
472  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
473  }
474  if (std::find(cols.begin(), cols.end(), std::string("version_num")) == cols.end()) {
475  LOG(INFO) << "Updating mapd_columns updateDeletedColumnIndicator";
476  // need to add new version info
477  string queryString("ALTER TABLE mapd_columns ADD version_num BIGINT DEFAULT " +
479  sqliteConnector_.query(queryString);
480  // need to add new column to table defintion to indicate deleted column, column used
481  // as bitmap for deleted rows.
483  "ALTER TABLE mapd_columns ADD is_deletedcol boolean default 0 ");
484  }
485  } catch (std::exception& e) {
486  sqliteConnector_.query("ROLLBACK TRANSACTION");
487  throw;
488  }
489  sqliteConnector_.query("END TRANSACTION");
490 }
491 
492 // introduce DB version into the dictionary tables
493 // if the DB does not have a version rename all dictionary tables
494 
497  if (currentDB_.dbName.length() == 0) {
498  // updateDictionaryNames dbName length is zero nothing to do here
499  return;
500  }
501  sqliteConnector_.query("BEGIN TRANSACTION");
502  try {
503  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_dictionaries)");
504  std::vector<std::string> cols;
505  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
506  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
507  }
508  if (std::find(cols.begin(), cols.end(), std::string("version_num")) == cols.end()) {
509  // No version number
510  // need to rename dictionaries
511  string dictQuery("SELECT dictid, name from mapd_dictionaries");
512  sqliteConnector_.query(dictQuery);
513  size_t numRows = sqliteConnector_.getNumRows();
514  for (size_t r = 0; r < numRows; ++r) {
515  int dictId = sqliteConnector_.getData<int>(r, 0);
516  std::string dictName = sqliteConnector_.getData<string>(r, 1);
517 
518  std::string oldName =
519  basePath_ + "/mapd_data/" + currentDB_.dbName + "_" + dictName;
520  std::string newName = basePath_ + "/mapd_data/DB_" +
521  std::to_string(currentDB_.dbId) + "_DICT_" +
522  std::to_string(dictId);
523 
524  int result = rename(oldName.c_str(), newName.c_str());
525 
526  if (result == 0) {
527  LOG(INFO) << "Dictionary upgrade: successfully renamed " << oldName << " to "
528  << newName;
529  } else {
530  LOG(ERROR) << "Failed to rename old dictionary directory " << oldName << " to "
531  << newName + " dbname '" << currentDB_.dbName << "' error code "
532  << std::to_string(result);
533  }
534  }
535  // need to add new version info
536  string queryString("ALTER TABLE mapd_dictionaries ADD version_num BIGINT DEFAULT " +
538  sqliteConnector_.query(queryString);
539  }
540  } catch (std::exception& e) {
541  sqliteConnector_.query("ROLLBACK TRANSACTION");
542  throw;
543  }
544  sqliteConnector_.query("END TRANSACTION");
545 }
546 
549  sqliteConnector_.query("BEGIN TRANSACTION");
550  try {
552  "CREATE TABLE IF NOT EXISTS mapd_logical_to_physical("
553  "logical_table_id integer, physical_table_id integer)");
554  } catch (const std::exception& e) {
555  sqliteConnector_.query("ROLLBACK TRANSACTION");
556  throw;
557  }
558  sqliteConnector_.query("END TRANSACTION");
559 }
560 
561 void Catalog::updateLogicalToPhysicalTableMap(const int32_t logical_tb_id) {
562  /* this proc inserts/updates all pairs of (logical_tb_id, physical_tb_id) in
563  * sqlite mapd_logical_to_physical table for given logical_tb_id as needed
564  */
565 
567  sqliteConnector_.query("BEGIN TRANSACTION");
568  try {
569  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
570  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
571  const auto physicalTables = physicalTableIt->second;
572  CHECK(!physicalTables.empty());
573  for (size_t i = 0; i < physicalTables.size(); i++) {
574  int32_t physical_tb_id = physicalTables[i];
576  "INSERT OR REPLACE INTO mapd_logical_to_physical (logical_table_id, "
577  "physical_table_id) VALUES (?1, ?2)",
578  std::vector<std::string>{std::to_string(logical_tb_id),
579  std::to_string(physical_tb_id)});
580  }
581  }
582  } catch (std::exception& e) {
583  sqliteConnector_.query("ROLLBACK TRANSACTION");
584  throw;
585  }
586  sqliteConnector_.query("END TRANSACTION");
587 }
588 
591  sqliteConnector_.query("BEGIN TRANSACTION");
592  try {
593  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_dictionaries)");
594  std::vector<std::string> cols;
595  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
596  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
597  }
598  if (std::find(cols.begin(), cols.end(), std::string("refcount")) == cols.end()) {
599  sqliteConnector_.query("ALTER TABLE mapd_dictionaries ADD refcount DEFAULT 1");
600  }
601  } catch (std::exception& e) {
602  sqliteConnector_.query("ROLLBACK TRANSACTION");
603  throw;
604  }
605  sqliteConnector_.query("END TRANSACTION");
606 }
607 
610  sqliteConnector_.query("BEGIN TRANSACTION");
611  try {
613  "CREATE TABLE IF NOT EXISTS omnisci_foreign_servers("
614  "id integer primary key, "
615  "name text unique, "
616  "data_wrapper_type text, "
617  "owner_user_id integer, "
618  "options text)");
621  "CREATE TABLE IF NOT EXISTS omnisci_foreign_tables("
622  "table_id integer unique, "
623  "server_id integer, "
624  "options text, "
625  "FOREIGN KEY(table_id) REFERENCES mapd_tables(tableid), "
626  "FOREIGN KEY(server_id) REFERENCES omnisci_foreign_servers(id))");
627  } catch (std::exception& e) {
628  sqliteConnector_.query("ROLLBACK TRANSACTION");
629  throw;
630  }
631  sqliteConnector_.query("END TRANSACTION");
632 }
633 
635  std::vector<foreign_storage::ForeignTable> foreign_tables{};
636  {
638  sqliteConnector_.query("BEGIN TRANSACTION");
639  try {
641  "SELECT name FROM sqlite_master WHERE type='table' AND "
642  "name IN ('omnisci_foreign_servers', 'omnisci_foreign_tables')");
643  if (sqliteConnector_.getNumRows() > 0) {
645  "SELECT tableid, name, isview, storage_type FROM mapd_tables "
646  "WHERE storage_type = 'FOREIGN_TABLE'");
648  for (size_t r = 0; r < num_rows; r++) {
649  foreign_storage::ForeignTable foreign_table{};
650  foreign_table.tableId = sqliteConnector_.getData<int>(r, 0);
651  foreign_table.tableName = sqliteConnector_.getData<std::string>(r, 1);
652  foreign_table.isView = sqliteConnector_.getData<bool>(r, 2);
653  foreign_table.storageType = sqliteConnector_.getData<std::string>(r, 3);
654  foreign_tables.emplace_back(foreign_table);
655  }
656  for (auto& foreign_table : foreign_tables) {
657  tableDescriptorMap_[to_upper(foreign_table.tableName)] = &foreign_table;
658  tableDescriptorMapById_[foreign_table.tableId] = &foreign_table;
659  executeDropTableSqliteQueries(&foreign_table);
660  }
661  sqliteConnector_.query("SELECT COUNT(*) FROM omnisci_foreign_tables");
662  CHECK_EQ(size_t(1), sqliteConnector_.getNumRows());
663  CHECK_EQ(0, sqliteConnector_.getData<int>(0, 0));
664 
665  sqliteConnector_.query("DROP TABLE omnisci_foreign_tables");
666  sqliteConnector_.query("DROP TABLE omnisci_foreign_servers");
667  }
668  } catch (std::exception& e) {
669  sqliteConnector_.query("ROLLBACK TRANSACTION");
670  throw;
671  }
672  sqliteConnector_.query("END TRANSACTION");
673  }
674 
675  for (auto& foreign_table : foreign_tables) {
677  DBObject(foreign_table.tableName, TableDBObjectType), this);
678  tableDescriptorMap_.erase(to_upper(foreign_table.tableName));
679  tableDescriptorMapById_.erase(foreign_table.tableId);
680  }
681 }
682 
685  sqliteConnector_.query("BEGIN TRANSACTION");
686  std::vector<DBObject> objects;
687  try {
689  "SELECT name FROM sqlite_master WHERE type='table' AND "
690  "name='mapd_record_ownership_marker'");
691  // check if mapd catalog - marker exists
692  if (sqliteConnector_.getNumRows() != 0 && currentDB_.dbId == 1) {
693  // already done
694  sqliteConnector_.query("END TRANSACTION");
695  return;
696  }
697  // check if different catalog - marker exists
698  else if (sqliteConnector_.getNumRows() != 0 && currentDB_.dbId != 1) {
699  sqliteConnector_.query("SELECT dummy FROM mapd_record_ownership_marker");
700  // Check if migration is being performed on existing non mapd catalogs
701  // Older non mapd dbs will have table but no record in them
702  if (sqliteConnector_.getNumRows() != 0) {
703  // already done
704  sqliteConnector_.query("END TRANSACTION");
705  return;
706  }
707  }
708  // marker not exists - create one
709  else {
710  sqliteConnector_.query("CREATE TABLE mapd_record_ownership_marker (dummy integer)");
711  }
712 
713  DBMetadata db;
714  CHECK(SysCatalog::instance().getMetadataForDB(currentDB_.dbName, db));
715  // place dbId as a refernce for migration being performed
717  "INSERT INTO mapd_record_ownership_marker (dummy) VALUES (?1)",
718  std::vector<std::string>{std::to_string(db.dbOwner)});
719 
720  static const std::map<const DBObjectType, const AccessPrivileges>
721  object_level_all_privs_lookup{
727 
728  // grant owner all permissions on DB
729  DBObjectKey key;
730  key.dbId = currentDB_.dbId;
731  auto _key_place = [&key](auto type) {
732  key.permissionType = type;
733  return key;
734  };
735  for (auto& it : object_level_all_privs_lookup) {
736  objects.emplace_back(_key_place(it.first), it.second, db.dbOwner);
737  objects.back().setName(currentDB_.dbName);
738  }
739 
740  {
741  // other users tables and views
742  string tableQuery(
743  "SELECT tableid, name, userid, isview FROM mapd_tables WHERE userid > 0");
744  sqliteConnector_.query(tableQuery);
745  size_t numRows = sqliteConnector_.getNumRows();
746  for (size_t r = 0; r < numRows; ++r) {
747  int32_t tableid = sqliteConnector_.getData<int>(r, 0);
748  std::string tableName = sqliteConnector_.getData<string>(r, 1);
749  int32_t ownerid = sqliteConnector_.getData<int>(r, 2);
750  bool isview = sqliteConnector_.getData<bool>(r, 3);
751 
754  DBObjectKey key;
755  key.dbId = currentDB_.dbId;
756  key.objectId = tableid;
757  key.permissionType = type;
758 
759  DBObject obj(tableName, type);
760  obj.setObjectKey(key);
761  obj.setOwner(ownerid);
764 
765  objects.push_back(obj);
766  }
767  }
768 
769  {
770  // other users dashboards
771  string tableQuery("SELECT id, name, userid FROM mapd_dashboards WHERE userid > 0");
772  sqliteConnector_.query(tableQuery);
773  size_t numRows = sqliteConnector_.getNumRows();
774  for (size_t r = 0; r < numRows; ++r) {
775  int32_t dashId = sqliteConnector_.getData<int>(r, 0);
776  std::string dashName = sqliteConnector_.getData<string>(r, 1);
777  int32_t ownerid = sqliteConnector_.getData<int>(r, 2);
778 
780  DBObjectKey key;
781  key.dbId = currentDB_.dbId;
782  key.objectId = dashId;
783  key.permissionType = type;
784 
785  DBObject obj(dashName, type);
786  obj.setObjectKey(key);
787  obj.setOwner(ownerid);
789 
790  objects.push_back(obj);
791  }
792  }
793  } catch (const std::exception& e) {
794  sqliteConnector_.query("ROLLBACK TRANSACTION");
795  throw;
796  }
797  sqliteConnector_.query("END TRANSACTION");
798 
799  // now apply the objects to the syscat to track the permisisons
800  // moved outside transaction to avoid lock in sqlite
801  try {
803  } catch (const std::exception& e) {
804  LOG(ERROR) << " Issue during migration of DB " << name() << " issue was " << e.what();
805  throw std::runtime_error(" Issue during migration of DB " + name() + " issue was " +
806  e.what());
807  // will need to remove the mapd_record_ownership_marker table and retry
808  }
809 }
810 
815 }
816 
818  std::unordered_map<std::string, std::pair<int, std::string>> dashboards;
819  std::vector<std::string> dashboard_ids;
820  static const std::string migration_name{"dashboard_roles_migration"};
821  {
823  sqliteConnector_.query("BEGIN TRANSACTION");
824  try {
825  // migration_history should be present in all catalogs by now
826  // if not then would be created before this migration
828  "select * from mapd_version_history where migration_history = '" +
829  migration_name + "'");
830  if (sqliteConnector_.getNumRows() != 0) {
831  // no need for further execution
832  sqliteConnector_.query("END TRANSACTION");
833  return;
834  }
835  LOG(INFO) << "Performing dashboard internal roles Migration.";
836  sqliteConnector_.query("select id, userid, metadata from mapd_dashboards");
837  for (size_t i = 0; i < sqliteConnector_.getNumRows(); ++i) {
840  sqliteConnector_.getData<string>(i, 0)))) {
841  // Successfully created roles during previous migration/crash
842  // No need to include them
843  continue;
844  }
845  dashboards[sqliteConnector_.getData<string>(i, 0)] = std::make_pair(
846  sqliteConnector_.getData<int>(i, 1), sqliteConnector_.getData<string>(i, 2));
847  dashboard_ids.push_back(sqliteConnector_.getData<string>(i, 0));
848  }
849  } catch (const std::exception& e) {
850  sqliteConnector_.query("ROLLBACK TRANSACTION");
851  throw;
852  }
853  sqliteConnector_.query("END TRANSACTION");
854  }
855  // All current grantees with shared dashboards.
856  const auto active_grantees =
858 
859  try {
860  // NOTE(wamsi): Transactionally unsafe
861  for (auto dash : dashboards) {
862  createOrUpdateDashboardSystemRole(dash.second.second,
863  dash.second.first,
865  std::to_string(currentDB_.dbId), dash.first));
866  auto result = active_grantees.find(dash.first);
867  if (result != active_grantees.end()) {
870  dash.first)},
871  result->second);
872  }
873  }
875  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
876  std::vector<std::string>{std::to_string(MAPD_VERSION), migration_name});
877  } catch (const std::exception& e) {
878  LOG(ERROR) << "Failed to create dashboard system roles during migration: "
879  << e.what();
880  throw;
881  }
882  LOG(INFO) << "Successfully created dashboard system roles during migration.";
883 }
884 
895  updatePageSize();
899 
900  if (g_enable_fsi) {
902  } else {
904  }
905 }
906 
910 }
911 
912 namespace {
913 std::string getUserFromId(const int32_t id) {
914  UserMetadata user;
915  if (SysCatalog::instance().getMetadataForUserById(id, user)) {
916  return user.userName;
917  }
918  // a user could be deleted and a dashboard still exist?
919  return "Unknown";
920 }
921 } // namespace
922 
926 
927  string dictQuery(
928  "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
929  sqliteConnector_.query(dictQuery);
930  size_t numRows = sqliteConnector_.getNumRows();
931  for (size_t r = 0; r < numRows; ++r) {
932  int dictId = sqliteConnector_.getData<int>(r, 0);
933  std::string dictName = sqliteConnector_.getData<string>(r, 1);
934  int dictNBits = sqliteConnector_.getData<int>(r, 2);
935  bool is_shared = sqliteConnector_.getData<bool>(r, 3);
936  int refcount = sqliteConnector_.getData<int>(r, 4);
937  std::string fname = basePath_ + "/mapd_data/DB_" + std::to_string(currentDB_.dbId) +
938  "_DICT_" + std::to_string(dictId);
939  DictRef dict_ref(currentDB_.dbId, dictId);
940  DictDescriptor* dd = new DictDescriptor(
941  dict_ref, dictName, dictNBits, is_shared, refcount, fname, false);
942  dictDescriptorMapByRef_[dict_ref].reset(dd);
943  }
944 
945  string tableQuery(
946  "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
947  "max_chunk_size, frag_page_size, "
948  "max_rows, partitions, shard_column_id, shard, num_shards, key_metainfo, userid, "
949  "sort_column_id, storage_type "
950  "from mapd_tables");
951  sqliteConnector_.query(tableQuery);
952  numRows = sqliteConnector_.getNumRows();
953  for (size_t r = 0; r < numRows; ++r) {
954  TableDescriptor* td;
955  const auto& storage_type = sqliteConnector_.getData<string>(r, 17);
956  if (!storage_type.empty() &&
957  (!g_enable_fsi || storage_type != StorageType::FOREIGN_TABLE)) {
958  const auto table_id = sqliteConnector_.getData<int>(r, 0);
959  const auto& table_name = sqliteConnector_.getData<string>(r, 1);
960  LOG(FATAL) << "Unable to read Catalog metadata: storage type is currently not a "
961  "supported table option (table "
962  << table_name << " [" << table_id << "] in database "
963  << currentDB_.dbName << ").";
964  }
965 
966  if (storage_type == StorageType::FOREIGN_TABLE) {
968  } else {
969  td = new TableDescriptor();
970  }
971 
972  td->storageType = storage_type;
973  td->tableId = sqliteConnector_.getData<int>(r, 0);
974  td->tableName = sqliteConnector_.getData<string>(r, 1);
975  td->nColumns = sqliteConnector_.getData<int>(r, 2);
976  td->isView = sqliteConnector_.getData<bool>(r, 3);
977  td->fragments = sqliteConnector_.getData<string>(r, 4);
978  td->fragType =
980  td->maxFragRows = sqliteConnector_.getData<int>(r, 6);
981  td->maxChunkSize = sqliteConnector_.getData<int>(r, 7);
982  td->fragPageSize = sqliteConnector_.getData<int>(r, 8);
983  td->maxRows = sqliteConnector_.getData<int64_t>(r, 9);
984  td->partitions = sqliteConnector_.getData<string>(r, 10);
985  td->shardedColumnId = sqliteConnector_.getData<int>(r, 11);
986  td->shard = sqliteConnector_.getData<int>(r, 12);
987  td->nShards = sqliteConnector_.getData<int>(r, 13);
988  td->keyMetainfo = sqliteConnector_.getData<string>(r, 14);
989  td->userId = sqliteConnector_.getData<int>(r, 15);
990  td->sortedColumnId =
991  sqliteConnector_.isNull(r, 16) ? 0 : sqliteConnector_.getData<int>(r, 16);
992  if (!td->isView) {
993  td->fragmenter = nullptr;
994  }
995  td->hasDeletedCol = false;
996 
999  }
1000 
1001  if (g_enable_fsi) {
1004  }
1005 
1006  string columnQuery(
1007  "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
1008  "is_notnull, compression, comp_param, "
1009  "size, chunks, is_systemcol, is_virtualcol, virtual_expr, is_deletedcol from "
1010  "mapd_columns ORDER BY tableid, "
1011  "columnid");
1012  sqliteConnector_.query(columnQuery);
1013  numRows = sqliteConnector_.getNumRows();
1014  int32_t skip_physical_cols = 0;
1015  for (size_t r = 0; r < numRows; ++r) {
1016  ColumnDescriptor* cd = new ColumnDescriptor();
1017  cd->tableId = sqliteConnector_.getData<int>(r, 0);
1018  cd->columnId = sqliteConnector_.getData<int>(r, 1);
1019  cd->columnName = sqliteConnector_.getData<string>(r, 2);
1023  cd->columnType.set_scale(sqliteConnector_.getData<int>(r, 6));
1027  cd->columnType.set_size(sqliteConnector_.getData<int>(r, 10));
1028  cd->chunks = sqliteConnector_.getData<string>(r, 11);
1029  cd->isSystemCol = sqliteConnector_.getData<bool>(r, 12);
1030  cd->isVirtualCol = sqliteConnector_.getData<bool>(r, 13);
1031  cd->virtualExpr = sqliteConnector_.getData<string>(r, 14);
1032  cd->isDeletedCol = sqliteConnector_.getData<bool>(r, 15);
1033  cd->isGeoPhyCol = skip_physical_cols > 0;
1034  ColumnKey columnKey(cd->tableId, to_upper(cd->columnName));
1035  columnDescriptorMap_[columnKey] = cd;
1036  ColumnIdKey columnIdKey(cd->tableId, cd->columnId);
1037  columnDescriptorMapById_[columnIdKey] = cd;
1038 
1039  if (skip_physical_cols <= 0) {
1040  skip_physical_cols = cd->columnType.get_physical_cols();
1041  }
1042 
1043  auto td_itr = tableDescriptorMapById_.find(cd->tableId);
1044  CHECK(td_itr != tableDescriptorMapById_.end());
1045 
1046  if (cd->isDeletedCol) {
1047  td_itr->second->hasDeletedCol = true;
1048  setDeletedColumnUnlocked(td_itr->second, cd);
1049  } else if (cd->columnType.is_geometry() || skip_physical_cols-- <= 0) {
1050  tableDescriptorMapById_[cd->tableId]->columnIdBySpi_.push_back(cd->columnId);
1051  }
1052  }
1053  // sort columnIdBySpi_ based on columnId
1054  for (auto& tit : tableDescriptorMapById_) {
1055  std::sort(tit.second->columnIdBySpi_.begin(),
1056  tit.second->columnIdBySpi_.end(),
1057  [](const size_t a, const size_t b) -> bool { return a < b; });
1058  }
1059 
1060  string viewQuery("SELECT tableid, sql FROM mapd_views");
1061  sqliteConnector_.query(viewQuery);
1062  numRows = sqliteConnector_.getNumRows();
1063  for (size_t r = 0; r < numRows; ++r) {
1064  int32_t tableId = sqliteConnector_.getData<int>(r, 0);
1065  TableDescriptor* td = tableDescriptorMapById_[tableId];
1066  td->viewSQL = sqliteConnector_.getData<string>(r, 1);
1067  td->fragmenter = nullptr;
1068  }
1069 
1070  string frontendViewQuery(
1071  "SELECT id, state, name, image_hash, strftime('%Y-%m-%dT%H:%M:%SZ', update_time), "
1072  "userid, "
1073  "metadata "
1074  "FROM mapd_dashboards");
1075  sqliteConnector_.query(frontendViewQuery);
1076  numRows = sqliteConnector_.getNumRows();
1077  for (size_t r = 0; r < numRows; ++r) {
1078  std::shared_ptr<DashboardDescriptor> vd = std::make_shared<DashboardDescriptor>();
1079  vd->dashboardId = sqliteConnector_.getData<int>(r, 0);
1080  vd->dashboardState = sqliteConnector_.getData<string>(r, 1);
1081  vd->dashboardName = sqliteConnector_.getData<string>(r, 2);
1082  vd->imageHash = sqliteConnector_.getData<string>(r, 3);
1083  vd->updateTime = sqliteConnector_.getData<string>(r, 4);
1084  vd->userId = sqliteConnector_.getData<int>(r, 5);
1085  vd->dashboardMetadata = sqliteConnector_.getData<string>(r, 6);
1086  vd->user = getUserFromId(vd->userId);
1087  vd->dashboardSystemRoleName = generate_dashboard_system_rolename(
1089  dashboardDescriptorMap_[std::to_string(vd->userId) + ":" + vd->dashboardName] = vd;
1090  }
1091 
1092  string linkQuery(
1093  "SELECT linkid, userid, link, view_state, strftime('%Y-%m-%dT%H:%M:%SZ', "
1094  "update_time), view_metadata "
1095  "FROM mapd_links");
1096  sqliteConnector_.query(linkQuery);
1097  numRows = sqliteConnector_.getNumRows();
1098  for (size_t r = 0; r < numRows; ++r) {
1099  LinkDescriptor* ld = new LinkDescriptor();
1100  ld->linkId = sqliteConnector_.getData<int>(r, 0);
1101  ld->userId = sqliteConnector_.getData<int>(r, 1);
1102  ld->link = sqliteConnector_.getData<string>(r, 2);
1103  ld->viewState = sqliteConnector_.getData<string>(r, 3);
1104  ld->updateTime = sqliteConnector_.getData<string>(r, 4);
1105  ld->viewMetadata = sqliteConnector_.getData<string>(r, 5);
1107  linkDescriptorMapById_[ld->linkId] = ld;
1108  }
1109 
1110  /* rebuild map linking logical tables to corresponding physical ones */
1111  string logicalToPhysicalTableMapQuery(
1112  "SELECT logical_table_id, physical_table_id "
1113  "FROM mapd_logical_to_physical");
1114  sqliteConnector_.query(logicalToPhysicalTableMapQuery);
1115  numRows = sqliteConnector_.getNumRows();
1116  for (size_t r = 0; r < numRows; ++r) {
1117  int32_t logical_tb_id = sqliteConnector_.getData<int>(r, 0);
1118  int32_t physical_tb_id = sqliteConnector_.getData<int>(r, 1);
1119  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
1120  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
1121  /* add new entity to the map logicalToPhysicalTableMapById_ */
1122  std::vector<int32_t> physicalTables;
1123  physicalTables.push_back(physical_tb_id);
1124  const auto it_ok =
1125  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
1126  CHECK(it_ok.second);
1127  } else {
1128  /* update map logicalToPhysicalTableMapById_ */
1129  physicalTableIt->second.push_back(physical_tb_id);
1130  }
1131  }
1132 }
1133 
1135  const list<ColumnDescriptor>& columns,
1136  const list<DictDescriptor>& dicts) {
1137  cat_write_lock write_lock(this);
1138  TableDescriptor* new_td;
1139 
1140  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
1141  if (foreign_table) {
1142  auto new_foreign_table = new foreign_storage::ForeignTable();
1143  *new_foreign_table = *foreign_table;
1144  new_td = new_foreign_table;
1145  } else {
1146  new_td = new TableDescriptor();
1147  *new_td = *td;
1148  }
1149 
1150  new_td->mutex_ = std::make_shared<std::mutex>();
1151  tableDescriptorMap_[to_upper(td->tableName)] = new_td;
1152  tableDescriptorMapById_[td->tableId] = new_td;
1153  for (auto cd : columns) {
1154  ColumnDescriptor* new_cd = new ColumnDescriptor();
1155  *new_cd = cd;
1156  ColumnKey columnKey(new_cd->tableId, to_upper(new_cd->columnName));
1157  columnDescriptorMap_[columnKey] = new_cd;
1158  ColumnIdKey columnIdKey(new_cd->tableId, new_cd->columnId);
1159  columnDescriptorMapById_[columnIdKey] = new_cd;
1160 
1161  // Add deleted column to the map
1162  if (cd.isDeletedCol) {
1163  CHECK(new_td->hasDeletedCol);
1164  setDeletedColumnUnlocked(new_td, new_cd);
1165  }
1166  }
1167 
1168  std::sort(new_td->columnIdBySpi_.begin(),
1169  new_td->columnIdBySpi_.end(),
1170  [](const size_t a, const size_t b) -> bool { return a < b; });
1171 
1172  std::unique_ptr<StringDictionaryClient> client;
1173  DictRef dict_ref(currentDB_.dbId, -1);
1174  if (!string_dict_hosts_.empty()) {
1175  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1176  }
1177  for (auto dd : dicts) {
1178  if (!dd.dictRef.dictId) {
1179  // Dummy entry created for a shard of a logical table, nothing to do.
1180  continue;
1181  }
1182  dict_ref.dictId = dd.dictRef.dictId;
1183  if (client) {
1184  client->create(dict_ref, dd.dictIsTemp);
1185  }
1186  DictDescriptor* new_dd = new DictDescriptor(dd);
1187  dictDescriptorMapByRef_[dict_ref].reset(new_dd);
1188  if (!dd.dictIsTemp) {
1189  boost::filesystem::create_directory(new_dd->dictFolderPath);
1190  }
1191  }
1192 }
1193 
1194 void Catalog::removeTableFromMap(const string& tableName,
1195  const int tableId,
1196  const bool is_on_error) {
1197  cat_write_lock write_lock(this);
1198  TableDescriptorMapById::iterator tableDescIt = tableDescriptorMapById_.find(tableId);
1199  if (tableDescIt == tableDescriptorMapById_.end()) {
1200  throw runtime_error("Table " + tableName + " does not exist.");
1201  }
1202 
1203  TableDescriptor* td = tableDescIt->second;
1204 
1205  if (td->hasDeletedCol) {
1206  const auto ret = deletedColumnPerTable_.erase(td);
1207  CHECK_EQ(ret, size_t(1));
1208  }
1209 
1210  tableDescriptorMapById_.erase(tableDescIt);
1211  tableDescriptorMap_.erase(to_upper(tableName));
1212  if (td->fragmenter != nullptr) {
1213  delete td->fragmenter;
1214  }
1216  delete td;
1217 
1218  std::unique_ptr<StringDictionaryClient> client;
1219  if (SysCatalog::instance().isAggregator()) {
1220  CHECK(!string_dict_hosts_.empty());
1221  DictRef dict_ref(currentDB_.dbId, -1);
1222  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1223  }
1224 
1225  // delete all column descriptors for the table
1226  // no more link columnIds to sequential indexes!
1227  for (auto cit = columnDescriptorMapById_.begin();
1228  cit != columnDescriptorMapById_.end();) {
1229  if (tableId != std::get<0>(cit->first)) {
1230  ++cit;
1231  } else {
1232  int i = std::get<1>(cit++->first);
1233  ColumnIdKey cidKey(tableId, i);
1234  ColumnDescriptorMapById::iterator colDescIt = columnDescriptorMapById_.find(cidKey);
1235  ColumnDescriptor* cd = colDescIt->second;
1236  columnDescriptorMapById_.erase(colDescIt);
1237  ColumnKey cnameKey(tableId, to_upper(cd->columnName));
1238  columnDescriptorMap_.erase(cnameKey);
1239  const int dictId = cd->columnType.get_comp_param();
1240  // Dummy dictionaries created for a shard of a logical table have the id set to
1241  // zero.
1242  if (cd->columnType.get_compression() == kENCODING_DICT && dictId) {
1243  INJECT_TIMER(removingDicts);
1244  DictRef dict_ref(currentDB_.dbId, dictId);
1245  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
1246  // If we're removing this table due to an error, it is possible that the string
1247  // dictionary reference was never populated. Don't crash, just continue cleaning
1248  // up the TableDescriptor and ColumnDescriptors
1249  if (!is_on_error) {
1250  CHECK(dictIt != dictDescriptorMapByRef_.end());
1251  } else {
1252  if (dictIt == dictDescriptorMapByRef_.end()) {
1253  continue;
1254  }
1255  }
1256  const auto& dd = dictIt->second;
1257  CHECK_GE(dd->refcount, 1);
1258  --dd->refcount;
1259  if (!dd->refcount) {
1260  dd->stringDict.reset();
1261  if (!isTemp) {
1262  File_Namespace::renameForDelete(dd->dictFolderPath);
1263  }
1264  if (client) {
1265  client->drop(dict_ref);
1266  }
1267  dictDescriptorMapByRef_.erase(dictIt);
1268  }
1269  }
1270 
1271  delete cd;
1272  }
1273  }
1274 }
1275 
1277  cat_write_lock write_lock(this);
1279 }
1280 
1282  cat_write_lock write_lock(this);
1284  std::make_shared<DashboardDescriptor>(vd);
1285 }
1286 
1287 std::vector<DBObject> Catalog::parseDashboardObjects(const std::string& view_meta,
1288  const int& user_id) {
1289  std::vector<DBObject> objects;
1290  DBObjectKey key;
1291  key.dbId = currentDB_.dbId;
1292  auto _key_place = [&key](auto type, auto id) {
1293  key.permissionType = type;
1294  key.objectId = id;
1295  return key;
1296  };
1297  for (auto object_name : parse_underlying_dashboard_objects(view_meta)) {
1298  auto td = getMetadataForTable(object_name);
1299  if (!td) {
1300  // Parsed object source is not present in current database
1301  // LOG the info and ignore
1302  LOG(INFO) << "Ignoring dashboard source Table/View: " << object_name
1303  << " no longer exists in current DB.";
1304  continue;
1305  }
1306  // Dashboard source can be Table or View
1307  const auto object_type = td->isView ? ViewDBObjectType : TableDBObjectType;
1308  const auto priv = td->isView ? AccessPrivileges::SELECT_FROM_VIEW
1310  objects.emplace_back(_key_place(object_type, td->tableId), priv, user_id);
1311  objects.back().setObjectType(td->isView ? ViewDBObjectType : TableDBObjectType);
1312  objects.back().setName(td->tableName);
1313  }
1314  return objects;
1315 }
1316 
1317 void Catalog::createOrUpdateDashboardSystemRole(const std::string& view_meta,
1318  const int32_t& user_id,
1319  const std::string& dash_role_name) {
1320  auto objects = parseDashboardObjects(view_meta, user_id);
1321  Role* rl = SysCatalog::instance().getRoleGrantee(dash_role_name);
1322  if (!rl) {
1323  // Dashboard role does not exist
1324  // create role and grant privileges
1325  // NOTE(wamsi): Transactionally unsafe
1326  SysCatalog::instance().createRole(dash_role_name, false);
1327  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1328  } else {
1329  // Dashboard system role already exists
1330  // Add/remove privileges on objects
1331  auto ex_objects = rl->getDbObjects(true);
1332  for (auto key : *ex_objects | boost::adaptors::map_keys) {
1333  if (key.permissionType != TableDBObjectType &&
1334  key.permissionType != ViewDBObjectType) {
1335  continue;
1336  }
1337  bool found = false;
1338  for (auto obj : objects) {
1339  found = key == obj.getObjectKey() ? true : false;
1340  if (found) {
1341  break;
1342  }
1343  }
1344  if (!found) {
1345  // revoke privs on object since the object is no
1346  // longer used by the dashboard as source
1347  // NOTE(wamsi): Transactionally unsafe
1349  dash_role_name, *rl->findDbObject(key, true), *this);
1350  }
1351  }
1352  // Update privileges on remaining objects
1353  // NOTE(wamsi): Transactionally unsafe
1354  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1355  }
1356 }
1357 
1359  cat_write_lock write_lock(this);
1360  LinkDescriptor* new_ld = new LinkDescriptor();
1361  *new_ld = ld;
1363  linkDescriptorMapById_[ld.linkId] = new_ld;
1364 }
1365 
1367  auto time_ms = measure<>::execution([&]() {
1368  // instanciate table fragmenter upon first use
1369  // assume only insert order fragmenter is supported
1371  vector<Chunk> chunkVec;
1372  list<const ColumnDescriptor*> columnDescs;
1373  getAllColumnMetadataForTable(td, columnDescs, true, false, true);
1374  Chunk::translateColumnDescriptorsToChunkVec(columnDescs, chunkVec);
1375  ChunkKey chunkKeyPrefix = {currentDB_.dbId, td->tableId};
1376  if (td->sortedColumnId > 0) {
1377  td->fragmenter = new SortedOrderFragmenter(chunkKeyPrefix,
1378  chunkVec,
1379  dataMgr_.get(),
1380  const_cast<Catalog*>(this),
1381  td->tableId,
1382  td->shard,
1383  td->maxFragRows,
1384  td->maxChunkSize,
1385  td->fragPageSize,
1386  td->maxRows,
1387  td->persistenceLevel);
1388  } else {
1389  td->fragmenter = new InsertOrderFragmenter(chunkKeyPrefix,
1390  chunkVec,
1391  dataMgr_.get(),
1392  const_cast<Catalog*>(this),
1393  td->tableId,
1394  td->shard,
1395  td->maxFragRows,
1396  td->maxChunkSize,
1397  td->fragPageSize,
1398  td->maxRows,
1399  td->persistenceLevel,
1400  !td->storageType.empty());
1401  }
1402  });
1403  LOG(INFO) << "Instantiating Fragmenter for table " << td->tableName << " took "
1404  << time_ms << "ms";
1405 }
1406 
1407 const TableDescriptor* Catalog::getMetadataForTable(const string& tableName,
1408  const bool populateFragmenter) const {
1409  // we give option not to populate fragmenter (default true/yes) as it can be heavy for
1410  // pure metadata calls
1411  cat_read_lock read_lock(this);
1412  auto tableDescIt = tableDescriptorMap_.find(to_upper(tableName));
1413  if (tableDescIt == tableDescriptorMap_.end()) { // check to make sure table exists
1414  return nullptr;
1415  }
1416  TableDescriptor* td = tableDescIt->second;
1417  std::unique_lock<std::mutex> td_lock(*td->mutex_.get());
1418  if (populateFragmenter && td->fragmenter == nullptr && !td->isView) {
1420  }
1421  return td; // returns pointer to table descriptor
1422 }
1423 
1425  int tableId,
1426  const bool populateFragmenter) const {
1427  cat_read_lock read_lock(this);
1428  auto tableDescIt = tableDescriptorMapById_.find(tableId);
1429  if (tableDescIt == tableDescriptorMapById_.end()) { // check to make sure table exists
1430  return nullptr;
1431  }
1432  TableDescriptor* td = tableDescIt->second;
1433  std::unique_lock<std::mutex> td_lock(*td->mutex_.get());
1434  if (populateFragmenter && td->fragmenter == nullptr && !td->isView) {
1436  }
1437  return td; // returns pointer to table descriptor
1438 }
1439 
1441  return getMetadataForTableImpl(tableId, true);
1442 }
1443 
1445  const bool loadDict) const {
1446  const DictRef dictRef(currentDB_.dbId, dictId);
1447  cat_read_lock read_lock(this);
1448  auto dictDescIt = dictDescriptorMapByRef_.find(dictRef);
1449  if (dictDescIt ==
1450  dictDescriptorMapByRef_.end()) { // check to make sure dictionary exists
1451  return nullptr;
1452  }
1453  auto& dd = dictDescIt->second;
1454 
1455  if (loadDict) {
1457  if (!dd->stringDict) {
1458  auto time_ms = measure<>::execution([&]() {
1459  if (string_dict_hosts_.empty()) {
1460  if (dd->dictIsTemp) {
1461  dd->stringDict = std::make_shared<StringDictionary>(
1462  dd->dictFolderPath, true, true, g_cache_string_hash);
1463  } else {
1464  dd->stringDict = std::make_shared<StringDictionary>(
1465  dd->dictFolderPath, false, true, g_cache_string_hash);
1466  }
1467  } else {
1468  dd->stringDict =
1469  std::make_shared<StringDictionary>(string_dict_hosts_.front(), dd->dictRef);
1470  }
1471  });
1472  LOG(INFO) << "Time to load Dictionary " << dd->dictRef.dbId << "_"
1473  << dd->dictRef.dictId << " was " << time_ms << "ms";
1474  }
1475  }
1476 
1477  return dd.get();
1478 }
1479 
1480 const std::vector<LeafHostInfo>& Catalog::getStringDictionaryHosts() const {
1481  return string_dict_hosts_;
1482 }
1483 
1485  const string& columnName) const {
1486  cat_read_lock read_lock(this);
1487 
1488  ColumnKey columnKey(tableId, to_upper(columnName));
1489  auto colDescIt = columnDescriptorMap_.find(columnKey);
1490  if (colDescIt ==
1491  columnDescriptorMap_.end()) { // need to check to make sure column exists for table
1492  return nullptr;
1493  }
1494  return colDescIt->second;
1495 }
1496 
1497 const ColumnDescriptor* Catalog::getMetadataForColumn(int tableId, int columnId) const {
1498  cat_read_lock read_lock(this);
1499 
1500  ColumnIdKey columnIdKey(tableId, columnId);
1501  auto colDescIt = columnDescriptorMapById_.find(columnIdKey);
1502  if (colDescIt == columnDescriptorMapById_
1503  .end()) { // need to check to make sure column exists for table
1504  return nullptr;
1505  }
1506  return colDescIt->second;
1507 }
1508 
1509 const int Catalog::getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const {
1510  const auto tabDescIt = tableDescriptorMapById_.find(table_id);
1511  CHECK(tableDescriptorMapById_.end() != tabDescIt);
1512  const auto& columnIdBySpi = tabDescIt->second->columnIdBySpi_;
1513 
1514  auto spx = spi;
1515  int phi = 0;
1516  if (spx >= SPIMAP_MAGIC1) // see Catalog.h
1517  {
1518  phi = (spx - SPIMAP_MAGIC1) % SPIMAP_MAGIC2;
1519  spx = (spx - SPIMAP_MAGIC1) / SPIMAP_MAGIC2;
1520  }
1521 
1522  CHECK(0 < spx && spx <= columnIdBySpi.size());
1523  return columnIdBySpi[spx - 1] + phi;
1524 }
1525 
1526 const int Catalog::getColumnIdBySpi(const int table_id, const size_t spi) const {
1527  cat_read_lock read_lock(this);
1528  return getColumnIdBySpiUnlocked(table_id, spi);
1529 }
1530 
1532  const size_t spi) const {
1533  cat_read_lock read_lock(this);
1534 
1535  const auto columnId = getColumnIdBySpiUnlocked(tableId, spi);
1536  ColumnIdKey columnIdKey(tableId, columnId);
1537  const auto colDescIt = columnDescriptorMapById_.find(columnIdKey);
1538  return columnDescriptorMapById_.end() == colDescIt ? nullptr : colDescIt->second;
1539 }
1540 
1541 void Catalog::deleteMetadataForDashboard(const std::string& userId,
1542  const std::string& dashName) {
1543  cat_write_lock write_lock(this);
1544 
1545  auto viewDescIt = dashboardDescriptorMap_.find(userId + ":" + dashName);
1546  if (viewDescIt == dashboardDescriptorMap_.end()) { // check to make sure view exists
1547  LOG(ERROR) << "No metadata for dashboard for user " << userId << " dashboard "
1548  << dashName << " does not exist in map";
1549  throw runtime_error("No metadata for dashboard for user " + userId + " dashboard " +
1550  dashName + " does not exist in map");
1551  }
1552  // found view in Map now remove it
1553  dashboardDescriptorMap_.erase(viewDescIt);
1554  // remove from DB
1556  sqliteConnector_.query("BEGIN TRANSACTION");
1557  try {
1559  "DELETE FROM mapd_dashboards WHERE name = ? and userid = ?",
1560  std::vector<std::string>{dashName, userId});
1561  } catch (std::exception& e) {
1562  sqliteConnector_.query("ROLLBACK TRANSACTION");
1563  throw;
1564  }
1565  sqliteConnector_.query("END TRANSACTION");
1566 }
1567 
1569  const string& userId,
1570  const string& dashName) const {
1571  cat_read_lock read_lock(this);
1572 
1573  auto viewDescIt = dashboardDescriptorMap_.find(userId + ":" + dashName);
1574  if (viewDescIt == dashboardDescriptorMap_.end()) { // check to make sure view exists
1575  return nullptr;
1576  }
1577  return viewDescIt->second.get(); // returns pointer to view descriptor
1578 }
1579 
1581  cat_read_lock read_lock(this);
1582  std::string userId;
1583  std::string name;
1584  bool found{false};
1585  {
1586  for (auto descp : dashboardDescriptorMap_) {
1587  auto dash = descp.second.get();
1588  if (dash->dashboardId == id) {
1589  userId = std::to_string(dash->userId);
1590  name = dash->dashboardName;
1591  found = true;
1592  break;
1593  }
1594  }
1595  }
1596  if (found) {
1597  return getMetadataForDashboard(userId, name);
1598  }
1599  return nullptr;
1600 }
1601 
1602 void Catalog::deleteMetadataForDashboard(const int32_t id) {
1603  std::string userId;
1604  std::string name;
1605  bool found{false};
1606  {
1607  cat_read_lock read_lock(this);
1608  for (auto descp : dashboardDescriptorMap_) {
1609  auto dash = descp.second.get();
1610  if (dash->dashboardId == id) {
1611  userId = std::to_string(dash->userId);
1612  name = dash->dashboardName;
1613  found = true;
1614  break;
1615  }
1616  }
1617  }
1618  if (found) {
1619  // TODO: transactionally unsafe
1621  DBObject(id, DashboardDBObjectType), this);
1622  deleteMetadataForDashboard(userId, name);
1623  }
1624 }
1625 
1626 const LinkDescriptor* Catalog::getMetadataForLink(const string& link) const {
1627  cat_read_lock read_lock(this);
1628  auto linkDescIt = linkDescriptorMap_.find(link);
1629  if (linkDescIt == linkDescriptorMap_.end()) { // check to make sure view exists
1630  return nullptr;
1631  }
1632  return linkDescIt->second; // returns pointer to view descriptor
1633 }
1634 
1636  cat_read_lock read_lock(this);
1637  auto linkDescIt = linkDescriptorMapById_.find(linkId);
1638  if (linkDescIt == linkDescriptorMapById_.end()) { // check to make sure view exists
1639  return nullptr;
1640  }
1641  return linkDescIt->second;
1642 }
1643 
1645  const TableDescriptor* td,
1646  list<const ColumnDescriptor*>& columnDescriptors,
1647  const bool fetchSystemColumns,
1648  const bool fetchVirtualColumns,
1649  const bool fetchPhysicalColumns) const {
1650  cat_read_lock read_lock(this);
1651  int32_t skip_physical_cols = 0;
1652  for (const auto& columnDescriptor : columnDescriptorMapById_) {
1653  if (!fetchPhysicalColumns && skip_physical_cols > 0) {
1654  --skip_physical_cols;
1655  continue;
1656  }
1657  auto cd = columnDescriptor.second;
1658  if (cd->tableId != td->tableId) {
1659  continue;
1660  }
1661  if (!fetchSystemColumns && cd->isSystemCol) {
1662  continue;
1663  }
1664  if (!fetchVirtualColumns && cd->isVirtualCol) {
1665  continue;
1666  }
1667  if (!fetchPhysicalColumns) {
1668  const auto& col_ti = cd->columnType;
1669  skip_physical_cols = col_ti.get_physical_cols();
1670  }
1671  columnDescriptors.push_back(cd);
1672  }
1673 }
1674 
1675 list<const ColumnDescriptor*> Catalog::getAllColumnMetadataForTable(
1676  const int tableId,
1677  const bool fetchSystemColumns,
1678  const bool fetchVirtualColumns,
1679  const bool fetchPhysicalColumns) const {
1680  cat_read_lock read_lock(this);
1681  list<const ColumnDescriptor*> columnDescriptors;
1682  const TableDescriptor* td =
1683  getMetadataForTableImpl(tableId, false); // dont instantiate fragmenter
1685  columnDescriptors,
1686  fetchSystemColumns,
1687  fetchVirtualColumns,
1688  fetchPhysicalColumns);
1689  return columnDescriptors;
1690 }
1691 
1692 list<const TableDescriptor*> Catalog::getAllTableMetadata() const {
1693  cat_read_lock read_lock(this);
1694  list<const TableDescriptor*> table_list;
1695  for (auto p : tableDescriptorMapById_) {
1696  table_list.push_back(p.second);
1697  }
1698  return table_list;
1699 }
1700 
1701 list<const DashboardDescriptor*> Catalog::getAllDashboardsMetadata() const {
1702  list<const DashboardDescriptor*> view_list;
1703  for (auto p : dashboardDescriptorMap_) {
1704  view_list.push_back(p.second.get());
1705  }
1706  return view_list;
1707 }
1708 
1710  const auto& td = *tableDescriptorMapById_[cd.tableId];
1711  list<DictDescriptor> dds;
1712  setColumnDictionary(cd, dds, td, true);
1713  auto& dd = dds.back();
1714  CHECK(dd.dictRef.dictId);
1715 
1716  std::unique_ptr<StringDictionaryClient> client;
1717  if (!string_dict_hosts_.empty()) {
1718  client.reset(new StringDictionaryClient(
1719  string_dict_hosts_.front(), DictRef(currentDB_.dbId, -1), true));
1720  }
1721  if (client) {
1722  client->create(dd.dictRef, dd.dictIsTemp);
1723  }
1724 
1725  DictDescriptor* new_dd = new DictDescriptor(dd);
1726  dictDescriptorMapByRef_[dd.dictRef].reset(new_dd);
1727  if (!dd.dictIsTemp) {
1728  boost::filesystem::create_directory(new_dd->dictFolderPath);
1729  }
1730  return dd.dictRef;
1731 }
1732 
1734  cat_write_lock write_lock(this);
1736  if (!(cd.columnType.is_string() || cd.columnType.is_string_array())) {
1737  return;
1738  }
1739  if (!(cd.columnType.get_compression() == kENCODING_DICT)) {
1740  return;
1741  }
1742  const auto dictId = cd.columnType.get_comp_param();
1743  CHECK_GT(dictId, 0);
1744  // decrement and zero check dict ref count
1745  const auto td = getMetadataForTable(cd.tableId);
1746  CHECK(td);
1748  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
1749  std::to_string(dictId));
1751  "SELECT refcount FROM mapd_dictionaries WHERE dictid = ?", std::to_string(dictId));
1752  const auto refcount = sqliteConnector_.getData<int>(0, 0);
1753  VLOG(3) << "Dictionary " << dictId << "from dropped table has reference count "
1754  << refcount;
1755  if (refcount > 0) {
1756  return;
1757  }
1758  const DictRef dictRef(currentDB_.dbId, dictId);
1759  sqliteConnector_.query_with_text_param("DELETE FROM mapd_dictionaries WHERE dictid = ?",
1760  std::to_string(dictId));
1761  File_Namespace::renameForDelete(basePath_ + "/mapd_data/DB_" +
1762  std::to_string(currentDB_.dbId) + "_DICT_" +
1763  std::to_string(dictId));
1764 
1765  std::unique_ptr<StringDictionaryClient> client;
1766  if (!string_dict_hosts_.empty()) {
1767  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dictRef, true));
1768  }
1769  if (client) {
1770  client->drop(dictRef);
1771  }
1772 
1773  dictDescriptorMapByRef_.erase(dictRef);
1774 }
1775 
1777  std::map<int, StringDictionary*>& stringDicts) {
1778  // learn 'committed' ColumnDescriptor of this column
1779  auto cit = columnDescriptorMap_.find(ColumnKey(cd.tableId, to_upper(cd.columnName)));
1780  CHECK(cit != columnDescriptorMap_.end());
1781  auto& ccd = *cit->second;
1782 
1783  if (!(ccd.columnType.is_string() || ccd.columnType.is_string_array())) {
1784  return;
1785  }
1786  if (!(ccd.columnType.get_compression() == kENCODING_DICT)) {
1787  return;
1788  }
1789  if (!(ccd.columnType.get_comp_param() > 0)) {
1790  return;
1791  }
1792 
1793  auto dictId = ccd.columnType.get_comp_param();
1794  getMetadataForDict(dictId);
1795 
1796  const DictRef dictRef(currentDB_.dbId, dictId);
1797  auto dit = dictDescriptorMapByRef_.find(dictRef);
1798  CHECK(dit != dictDescriptorMapByRef_.end());
1799  CHECK(dit->second);
1800  CHECK(dit->second.get()->stringDict);
1801  stringDicts[ccd.columnId] = dit->second.get()->stringDict.get();
1802 }
1803 
1805  cat_write_lock write_lock(this);
1806  // caller must handle sqlite/chunk transaction TOGETHER
1807  cd.tableId = td.tableId;
1808  if (td.nShards > 0 && td.shard < 0) {
1809  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
1810  auto shard_cd = cd;
1811  addColumn(*shard, shard_cd);
1812  }
1813  }
1815  addDictionary(cd);
1816  }
1817 
1819  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
1820  "colscale, is_notnull, "
1821  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
1822  "is_deletedcol) "
1823  "VALUES (?, "
1824  "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
1825  "?, ?, ?, "
1826  "?, "
1827  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1828  std::vector<std::string>{std::to_string(td.tableId),
1829  std::to_string(td.tableId),
1830  cd.columnName,
1839  "",
1842  cd.virtualExpr,
1844 
1846  "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
1847  std::vector<std::string>{std::to_string(td.tableId)});
1848 
1850  "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
1851  std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
1852  cd.columnId = sqliteConnector_.getData<int>(0, 0);
1853 
1854  ++tableDescriptorMapById_[td.tableId]->nColumns;
1855  auto ncd = new ColumnDescriptor(cd);
1857  columnDescriptorMapById_[ColumnIdKey(cd.tableId, cd.columnId)] = ncd;
1858  columnDescriptorsForRoll.emplace_back(nullptr, ncd);
1859 }
1860 
1862  cat_write_lock write_lock(this);
1864  // caller must handle sqlite/chunk transaction TOGETHER
1866  "DELETE FROM mapd_columns where tableid = ? and columnid = ?",
1867  std::vector<std::string>{std::to_string(td.tableId), std::to_string(cd.columnId)});
1868 
1870  "UPDATE mapd_tables SET ncolumns = ncolumns - 1 WHERE tableid = ?",
1871  std::vector<std::string>{std::to_string(td.tableId)});
1872 
1873  ColumnDescriptorMap::iterator columnDescIt =
1875  CHECK(columnDescIt != columnDescriptorMap_.end());
1876 
1877  columnDescriptorsForRoll.emplace_back(columnDescIt->second, nullptr);
1878 
1879  columnDescriptorMap_.erase(columnDescIt);
1880  columnDescriptorMapById_.erase(ColumnIdKey(cd.tableId, cd.columnId));
1881  --tableDescriptorMapById_[td.tableId]->nColumns;
1882  // for each shard
1883  if (td.nShards > 0 && td.shard < 0) {
1884  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
1885  const auto shard_cd = getMetadataForColumn(shard->tableId, cd.columnId);
1886  CHECK(shard_cd);
1887  dropColumn(*shard, *shard_cd);
1888  }
1889  }
1890 }
1891 
1892 void Catalog::roll(const bool forward) {
1893  cat_write_lock write_lock(this);
1894  std::set<const TableDescriptor*> tds;
1895 
1896  for (const auto& cdr : columnDescriptorsForRoll) {
1897  auto ocd = cdr.first;
1898  auto ncd = cdr.second;
1899  CHECK(ocd || ncd);
1900  auto tabDescIt = tableDescriptorMapById_.find((ncd ? ncd : ocd)->tableId);
1901  CHECK(tableDescriptorMapById_.end() != tabDescIt);
1902  auto td = tabDescIt->second;
1903  auto& vc = td->columnIdBySpi_;
1904  if (forward) {
1905  if (ocd) {
1906  if (nullptr == ncd ||
1907  ncd->columnType.get_comp_param() != ocd->columnType.get_comp_param()) {
1908  delDictionary(*ocd);
1909  }
1910 
1911  vc.erase(std::remove(vc.begin(), vc.end(), ocd->columnId), vc.end());
1912 
1913  delete ocd;
1914  }
1915  if (ncd) {
1916  // append columnId if its new and not phy geo
1917  if (vc.end() == std::find(vc.begin(), vc.end(), ncd->columnId)) {
1918  if (!ncd->isGeoPhyCol) {
1919  vc.push_back(ncd->columnId);
1920  }
1921  }
1922  }
1923  tds.insert(td);
1924  } else {
1925  if (ocd) {
1926  columnDescriptorMap_[ColumnKey(ocd->tableId, to_upper(ocd->columnName))] = ocd;
1927  columnDescriptorMapById_[ColumnIdKey(ocd->tableId, ocd->columnId)] = ocd;
1928  }
1929  // roll back the dict of new column
1930  if (ncd) {
1931  columnDescriptorMap_.erase(ColumnKey(ncd->tableId, to_upper(ncd->columnName)));
1932  columnDescriptorMapById_.erase(ColumnIdKey(ncd->tableId, ncd->columnId));
1933  if (nullptr == ocd ||
1934  ocd->columnType.get_comp_param() != ncd->columnType.get_comp_param()) {
1935  delDictionary(*ncd);
1936  }
1937  delete ncd;
1938  }
1939  }
1940  }
1941  columnDescriptorsForRoll.clear();
1942 
1943  if (forward) {
1944  for (const auto td : tds) {
1945  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
1946  }
1947  }
1948 }
1949 
1951  list<ColumnDescriptor>& columns) {
1952  const auto& col_ti = cd.columnType;
1953  if (IS_GEO(col_ti.get_type())) {
1954  switch (col_ti.get_type()) {
1955  case kPOINT: {
1956  ColumnDescriptor physical_cd_coords(true);
1957  physical_cd_coords.columnName = cd.columnName + "_coords";
1958  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
1959  // Raw data: compressed/uncompressed coords
1960  coords_ti.set_subtype(kTINYINT);
1961  size_t unit_size;
1962  if (col_ti.get_compression() == kENCODING_GEOINT &&
1963  col_ti.get_comp_param() == 32) {
1964  unit_size = 4 * sizeof(int8_t);
1965  } else {
1966  CHECK(col_ti.get_compression() == kENCODING_NONE);
1967  unit_size = 8 * sizeof(int8_t);
1968  }
1969  coords_ti.set_size(2 * unit_size);
1970  physical_cd_coords.columnType = coords_ti;
1971  columns.push_back(physical_cd_coords);
1972 
1973  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1974 
1975  break;
1976  }
1977  case kLINESTRING: {
1978  ColumnDescriptor physical_cd_coords(true);
1979  physical_cd_coords.columnName = cd.columnName + "_coords";
1980  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
1981  // Raw data: compressed/uncompressed coords
1982  coords_ti.set_subtype(kTINYINT);
1983  physical_cd_coords.columnType = coords_ti;
1984  columns.push_back(physical_cd_coords);
1985 
1986  ColumnDescriptor physical_cd_bounds(true);
1987  physical_cd_bounds.columnName = cd.columnName + "_bounds";
1988  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
1989  bounds_ti.set_subtype(kDOUBLE);
1990  bounds_ti.set_size(4 * sizeof(double));
1991  physical_cd_bounds.columnType = bounds_ti;
1992  columns.push_back(physical_cd_bounds);
1993 
1994  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1995 
1996  break;
1997  }
1998  case kPOLYGON: {
1999  ColumnDescriptor physical_cd_coords(true);
2000  physical_cd_coords.columnName = cd.columnName + "_coords";
2001  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2002  // Raw data: compressed/uncompressed coords
2003  coords_ti.set_subtype(kTINYINT);
2004  physical_cd_coords.columnType = coords_ti;
2005  columns.push_back(physical_cd_coords);
2006 
2007  ColumnDescriptor physical_cd_ring_sizes(true);
2008  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
2009  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2010  ring_sizes_ti.set_subtype(kINT);
2011  physical_cd_ring_sizes.columnType = ring_sizes_ti;
2012  columns.push_back(physical_cd_ring_sizes);
2013 
2014  ColumnDescriptor physical_cd_bounds(true);
2015  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2016  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2017  bounds_ti.set_subtype(kDOUBLE);
2018  bounds_ti.set_size(4 * sizeof(double));
2019  physical_cd_bounds.columnType = bounds_ti;
2020  columns.push_back(physical_cd_bounds);
2021 
2022  ColumnDescriptor physical_cd_render_group(true);
2023  physical_cd_render_group.columnName = cd.columnName + "_render_group";
2024  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, col_ti.get_notnull());
2025  physical_cd_render_group.columnType = render_group_ti;
2026  columns.push_back(physical_cd_render_group);
2027 
2028  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2029 
2030  break;
2031  }
2032  case kMULTIPOLYGON: {
2033  ColumnDescriptor physical_cd_coords(true);
2034  physical_cd_coords.columnName = cd.columnName + "_coords";
2035  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2036  // Raw data: compressed/uncompressed coords
2037  coords_ti.set_subtype(kTINYINT);
2038  physical_cd_coords.columnType = coords_ti;
2039  columns.push_back(physical_cd_coords);
2040 
2041  ColumnDescriptor physical_cd_ring_sizes(true);
2042  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
2043  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2044  ring_sizes_ti.set_subtype(kINT);
2045  physical_cd_ring_sizes.columnType = ring_sizes_ti;
2046  columns.push_back(physical_cd_ring_sizes);
2047 
2048  ColumnDescriptor physical_cd_poly_rings(true);
2049  physical_cd_poly_rings.columnName = cd.columnName + "_poly_rings";
2050  SQLTypeInfo poly_rings_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2051  poly_rings_ti.set_subtype(kINT);
2052  physical_cd_poly_rings.columnType = poly_rings_ti;
2053  columns.push_back(physical_cd_poly_rings);
2054 
2055  ColumnDescriptor physical_cd_bounds(true);
2056  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2057  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2058  bounds_ti.set_subtype(kDOUBLE);
2059  bounds_ti.set_size(4 * sizeof(double));
2060  physical_cd_bounds.columnType = bounds_ti;
2061  columns.push_back(physical_cd_bounds);
2062 
2063  ColumnDescriptor physical_cd_render_group(true);
2064  physical_cd_render_group.columnName = cd.columnName + "_render_group";
2065  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, col_ti.get_notnull());
2066  physical_cd_render_group.columnType = render_group_ti;
2067  columns.push_back(physical_cd_render_group);
2068 
2069  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2070 
2071  break;
2072  }
2073  default:
2074  throw runtime_error("Unrecognized geometry type.");
2075  break;
2076  }
2077  }
2078 }
2079 
2081  TableDescriptor& td,
2082  const list<ColumnDescriptor>& cols,
2083  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs,
2084  bool isLogicalTable) {
2085  cat_write_lock write_lock(this);
2086  list<ColumnDescriptor> cds = cols;
2087  list<DictDescriptor> dds;
2088  std::set<std::string> toplevel_column_names;
2089  list<ColumnDescriptor> columns;
2090 
2091  if (!td.storageType.empty() &&
2094  throw std::runtime_error("Only temporary tables can be backed by foreign storage.");
2095  }
2097  }
2098 
2099  for (auto cd : cds) {
2100  if (cd.columnName == "rowid") {
2101  throw std::runtime_error(
2102  "Cannot create column with name rowid. rowid is a system defined column.");
2103  }
2104  columns.push_back(cd);
2105  toplevel_column_names.insert(cd.columnName);
2106  if (cd.columnType.is_geometry()) {
2107  expandGeoColumn(cd, columns);
2108  }
2109  }
2110  cds.clear();
2111 
2112  ColumnDescriptor cd;
2113  // add row_id column -- Must be last column in the table
2114  cd.columnName = "rowid";
2115  cd.isSystemCol = true;
2116  cd.columnType = SQLTypeInfo(kBIGINT, true);
2117 #ifdef MATERIALIZED_ROWID
2118  cd.isVirtualCol = false;
2119 #else
2120  cd.isVirtualCol = true;
2121  cd.virtualExpr = "MAPD_FRAG_ID * MAPD_ROWS_PER_FRAG + MAPD_FRAG_ROW_ID";
2122 #endif
2123  columns.push_back(cd);
2124  toplevel_column_names.insert(cd.columnName);
2125 
2126  if (td.hasDeletedCol) {
2127  ColumnDescriptor cd_del;
2128  cd_del.columnName = "$deleted$";
2129  cd_del.isSystemCol = true;
2130  cd_del.isVirtualCol = false;
2131  cd_del.columnType = SQLTypeInfo(kBOOLEAN, true);
2132  cd_del.isDeletedCol = true;
2133 
2134  columns.push_back(cd_del);
2135  }
2136 
2137  td.nColumns = columns.size();
2139  sqliteConnector_.query("BEGIN TRANSACTION");
2141  try {
2143  R"(INSERT INTO mapd_tables (name, userid, ncolumns, isview, fragments, frag_type, max_frag_rows, max_chunk_size, frag_page_size, max_rows, partitions, shard_column_id, shard, num_shards, sort_column_id, storage_type, key_metainfo) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?))",
2144  std::vector<std::string>{td.tableName,
2145  std::to_string(td.userId),
2147  std::to_string(td.isView),
2148  "",
2153  std::to_string(td.maxRows),
2154  td.partitions,
2156  std::to_string(td.shard),
2157  std::to_string(td.nShards),
2159  td.storageType,
2160  td.keyMetainfo});
2161 
2162  // now get the auto generated tableid
2164  "SELECT tableid FROM mapd_tables WHERE name = ?", td.tableName);
2165  td.tableId = sqliteConnector_.getData<int>(0, 0);
2166  int colId = 1;
2167  for (auto cd : columns) {
2169  const bool is_foreign_col =
2170  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2171  if (!is_foreign_col) {
2172  setColumnDictionary(cd, dds, td, isLogicalTable);
2173  }
2174  }
2175 
2176  if (toplevel_column_names.count(cd.columnName)) {
2177  // make up colId gap for sanity test (begin with 1 bc much code depends on it!)
2178  if (colId > 1) {
2179  colId += g_test_against_columnId_gap;
2180  }
2181  if (!cd.isGeoPhyCol) {
2182  td.columnIdBySpi_.push_back(colId);
2183  }
2184  }
2185 
2187  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, "
2188  "coldim, colscale, is_notnull, "
2189  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
2190  "virtual_expr, is_deletedcol) "
2191  "VALUES (?, ?, ?, ?, ?, "
2192  "?, "
2193  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2194  std::vector<std::string>{std::to_string(td.tableId),
2195  std::to_string(colId),
2196  cd.columnName,
2205  "",
2208  cd.virtualExpr,
2210  cd.tableId = td.tableId;
2211  cd.columnId = colId++;
2212  cds.push_back(cd);
2213  }
2214  if (td.isView) {
2216  "INSERT INTO mapd_views (tableid, sql) VALUES (?,?)",
2217  std::vector<std::string>{std::to_string(td.tableId), td.viewSQL});
2218  }
2220  auto& foreignTable = dynamic_cast<foreign_storage::ForeignTable&>(td);
2222  "INSERT INTO omnisci_foreign_tables (table_id, server_id, options) VALUES "
2223  "(?, ?, ?)",
2224  std::vector<std::string>{std::to_string(foreignTable.tableId),
2225  std::to_string(foreignTable.foreign_server->id),
2226  foreignTable.getOptionsAsJsonString()});
2227  }
2228  } catch (std::exception& e) {
2229  sqliteConnector_.query("ROLLBACK TRANSACTION");
2230  throw;
2231  }
2232  } else { // Temporary table
2233  td.tableId = nextTempTableId_++;
2234  int colId = 1;
2235  for (auto cd : columns) {
2236  auto col_ti = cd.columnType;
2237  if (IS_GEO(col_ti.get_type())) {
2238  throw runtime_error("Geometry types in temporary tables are not supported.");
2239  }
2240 
2242  const bool is_foreign_col =
2243  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2244 
2245  if (!is_foreign_col) {
2246  // Create a new temporary dictionary
2247  std::string fileName("");
2248  std::string folderPath("");
2250  nextTempDictId_++;
2251  DictDescriptor dd(dict_ref,
2252  fileName,
2254  false,
2255  1,
2256  folderPath,
2257  true); // Is dictName (2nd argument) used?
2258  dds.push_back(dd);
2259  if (!cd.columnType.is_array()) {
2261  }
2262  cd.columnType.set_comp_param(dict_ref.dictId);
2263  }
2264  }
2265  cd.tableId = td.tableId;
2266  cd.columnId = colId++;
2267  td.columnIdBySpi_.push_back(cd.columnId);
2268  cds.push_back(cd);
2269  }
2270 
2272  serializeTableJsonUnlocked(&td, cds);
2273  }
2274  }
2275 
2276  try {
2277  addTableToMap(&td, cds, dds);
2278  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
2279  if (!td.storageType.empty() && td.storageType != StorageType::FOREIGN_TABLE) {
2281  }
2282  } catch (std::exception& e) {
2283  sqliteConnector_.query("ROLLBACK TRANSACTION");
2284  removeTableFromMap(td.tableName, td.tableId, true);
2285  throw;
2286  }
2287  sqliteConnector_.query("END TRANSACTION");
2288 }
2289 
2291  const std::list<ColumnDescriptor>& cds) const {
2292  // relies on the catalog write lock
2293  using namespace rapidjson;
2294 
2295  VLOG(1) << "Serializing temporary table " << td->tableName << " to JSON for Calcite.";
2296 
2297  const auto db_name = currentDB_.dbName;
2298  const auto file_path = table_json_filepath(basePath_, db_name);
2299 
2300  Document d;
2301  if (boost::filesystem::exists(file_path)) {
2302  // look for an existing file for this database
2303  std::ifstream reader(file_path.string());
2304  CHECK(reader.is_open());
2305  IStreamWrapper json_read_wrapper(reader);
2306  d.ParseStream(json_read_wrapper);
2307  } else {
2308  d.SetObject();
2309  }
2310  CHECK(d.IsObject());
2311  CHECK(!d.HasMember(StringRef(td->tableName.c_str())));
2312 
2313  Value table(kObjectType);
2314  table.AddMember(
2315  "name", Value().SetString(StringRef(td->tableName.c_str())), d.GetAllocator());
2316  table.AddMember("id", Value().SetInt(td->tableId), d.GetAllocator());
2317  table.AddMember("columns", Value(kArrayType), d.GetAllocator());
2318 
2319  for (const auto& cd : cds) {
2320  Value column(kObjectType);
2321  column.AddMember(
2322  "name", Value().SetString(StringRef(cd.columnName)), d.GetAllocator());
2323  column.AddMember("coltype",
2324  Value().SetInt(static_cast<int>(cd.columnType.get_type())),
2325  d.GetAllocator());
2326  column.AddMember("colsubtype",
2327  Value().SetInt(static_cast<int>(cd.columnType.get_subtype())),
2328  d.GetAllocator());
2329  column.AddMember(
2330  "coldim", Value().SetInt(cd.columnType.get_dimension()), d.GetAllocator());
2331  column.AddMember(
2332  "colscale", Value().SetInt(cd.columnType.get_scale()), d.GetAllocator());
2333  column.AddMember(
2334  "is_notnull", Value().SetBool(cd.columnType.get_notnull()), d.GetAllocator());
2335  column.AddMember("is_systemcol", Value().SetBool(cd.isSystemCol), d.GetAllocator());
2336  column.AddMember("is_virtualcol", Value().SetBool(cd.isVirtualCol), d.GetAllocator());
2337  column.AddMember("is_deletedcol", Value().SetBool(cd.isDeletedCol), d.GetAllocator());
2338  table["columns"].PushBack(column, d.GetAllocator());
2339  }
2340  d.AddMember(StringRef(td->tableName.c_str()), table, d.GetAllocator());
2341 
2342  // Overwrite the existing file
2343  std::ofstream writer(file_path.string(), writer.trunc | writer.out);
2344  CHECK(writer.is_open());
2345  OStreamWrapper json_wrapper(writer);
2346 
2347  Writer<OStreamWrapper> json_writer(json_wrapper);
2348  d.Accept(json_writer);
2349  writer.close();
2350 }
2351 
2352 void Catalog::dropTableFromJsonUnlocked(const std::string& table_name) const {
2353  // relies on the catalog write lock
2354  using namespace rapidjson;
2355 
2356  VLOG(1) << "Dropping temporary table " << table_name << " to JSON for Calcite.";
2357 
2358  const auto db_name = currentDB_.dbName;
2359  const auto file_path = table_json_filepath(basePath_, db_name);
2360 
2361  CHECK(boost::filesystem::exists(file_path));
2362  Document d;
2363 
2364  std::ifstream reader(file_path.string());
2365  CHECK(reader.is_open());
2366  IStreamWrapper json_read_wrapper(reader);
2367  d.ParseStream(json_read_wrapper);
2368 
2369  CHECK(d.IsObject());
2370  auto table_name_ref = StringRef(table_name.c_str());
2371  CHECK(d.HasMember(table_name_ref));
2372  CHECK(d.RemoveMember(table_name_ref));
2373 
2374  // Overwrite the existing file
2375  std::ofstream writer(file_path.string(), writer.trunc | writer.out);
2376  CHECK(writer.is_open());
2377  OStreamWrapper json_wrapper(writer);
2378 
2379  Writer<OStreamWrapper> json_writer(json_wrapper);
2380  d.Accept(json_writer);
2381  writer.close();
2382 }
2383 
2385  std::unique_ptr<foreign_storage::ForeignServer> foreign_server,
2386  bool if_not_exists) {
2387  cat_write_lock write_lock(this);
2389  createForeignServerNoLocks(std::move(foreign_server), if_not_exists);
2390 }
2391 
2393  std::unique_ptr<foreign_storage::ForeignServer> foreign_server,
2394  bool if_not_exists) {
2396  "SELECT name from omnisci_foreign_servers where name = ?",
2397  std::vector<std::string>{foreign_server->name});
2398 
2399  if (sqliteConnector_.getNumRows() == 0) {
2401  "INSERT INTO omnisci_foreign_servers (name, data_wrapper_type, owner_user_id, "
2402  "options) "
2403  "VALUES (?, ?, ?, ?)",
2404  std::vector<std::string>{foreign_server->name,
2405  foreign_server->data_wrapper.name,
2406  std::to_string(foreign_server->user_id),
2407  foreign_server->getOptionsAsJsonString()});
2409  "SELECT id from omnisci_foreign_servers where name = ?",
2410  std::vector<std::string>{foreign_server->name});
2411  CHECK_EQ(sqliteConnector_.getNumRows(), size_t(1));
2412  foreign_server->id = sqliteConnector_.getData<int>(0, 0);
2413  } else if (!if_not_exists) {
2414  throw std::runtime_error{"A foreign server with name \"" + foreign_server->name +
2415  "\" already exists."};
2416  }
2417 
2418  std::shared_ptr<foreign_storage::ForeignServer> foreign_server_shared =
2419  std::move(foreign_server);
2420  foreignServerMap_[foreign_server_shared->name] = foreign_server_shared;
2421  foreignServerMapById_[foreign_server_shared->id] = foreign_server_shared;
2422 }
2423 
2425  const std::string& server_name) const {
2426  foreign_storage::ForeignServer* foreign_server = nullptr;
2427  cat_read_lock read_lock(this);
2428  if (foreignServerMap_.find(server_name) != foreignServerMap_.end()) {
2429  foreign_server = foreignServerMap_.find(server_name)->second.get();
2430  }
2431  return foreign_server;
2432 }
2433 
2435  const std::string& server_name) {
2436  foreign_storage::ForeignServer* foreign_server = nullptr;
2437  cat_write_lock write_lock(this);
2440  "SELECT id, name, data_wrapper_type, options, owner_user_id "
2441  "FROM omnisci_foreign_servers WHERE name = ?",
2442  std::vector<std::string>{server_name});
2443  if (sqliteConnector_.getNumRows() > 0) {
2444  auto server = std::make_shared<foreign_storage::ForeignServer>(
2446  server->id = sqliteConnector_.getData<int>(0, 0);
2447  server->name = sqliteConnector_.getData<std::string>(0, 1);
2448  server->user_id = sqliteConnector_.getData<std::int32_t>(0, 4);
2449  server->populateOptionsMap(sqliteConnector_.getData<std::string>(0, 3));
2450  foreign_server = server.get();
2451  foreignServerMap_[server->name] = server;
2452  foreignServerMapById_[server->id] = server;
2453  }
2454  return foreign_server;
2455 }
2456 
2457 void Catalog::dropForeignServer(const std::string& server_name) {
2458  cat_write_lock write_lock(this);
2459  cat_sqlite_lock sqlite_lock(this);
2460 
2462  "SELECT id from omnisci_foreign_servers where name = ?",
2463  std::vector<std::string>{server_name});
2465  if (num_rows > 0) {
2466  CHECK_EQ(size_t(1), num_rows);
2467  auto server_id = sqliteConnector_.getData<int>(0, 0);
2469  "SELECT table_id from omnisci_foreign_tables where server_id = ?",
2470  std::to_string(server_id));
2471  if (sqliteConnector_.getNumRows() > 0) {
2472  throw std::runtime_error{"Foreign server \"" + server_name +
2473  "\" is referenced "
2474  "by existing foreign tables and cannot be dropped."};
2475  }
2477  "DELETE FROM omnisci_foreign_servers WHERE name = ?",
2478  std::vector<std::string>{server_name});
2479  foreignServerMap_.erase(server_name);
2480  foreignServerMapById_.erase(server_id);
2481  }
2482 }
2483 
2484 // returns the table epoch or -1 if there is something wrong with the shared epoch
2485 int32_t Catalog::getTableEpoch(const int32_t db_id, const int32_t table_id) const {
2486  cat_read_lock read_lock(this);
2487  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(table_id);
2488  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2489  // check all shards have same checkpoint
2490  const auto physicalTables = physicalTableIt->second;
2491  CHECK(!physicalTables.empty());
2492  size_t curr_epoch = 0;
2493  for (size_t i = 0; i < physicalTables.size(); i++) {
2494  int32_t physical_tb_id = physicalTables[i];
2495  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2496  CHECK(phys_td);
2497  if (i == 0) {
2498  curr_epoch = dataMgr_->getTableEpoch(db_id, physical_tb_id);
2499  } else {
2500  if (curr_epoch != dataMgr_->getTableEpoch(db_id, physical_tb_id)) {
2501  // oh dear the leaves do not agree on the epoch for this table
2502  LOG(ERROR) << "Epochs on shards do not all agree on table id " << table_id
2503  << " db id " << db_id << " epoch " << curr_epoch << " leaf_epoch "
2504  << dataMgr_->getTableEpoch(db_id, physical_tb_id);
2505  return -1;
2506  }
2507  }
2508  }
2509  return curr_epoch;
2510  } else {
2511  return dataMgr_->getTableEpoch(db_id, table_id);
2512  }
2513 }
2514 
2515 void Catalog::setTableEpoch(const int db_id, const int table_id, int new_epoch) {
2516  cat_read_lock read_lock(this);
2517  LOG(INFO) << "Set table epoch db:" << db_id << " Table ID " << table_id
2518  << " back to new epoch " << new_epoch;
2519  removeChunks(table_id);
2520  dataMgr_->setTableEpoch(db_id, table_id, new_epoch);
2521 
2522  // check if sharded
2523  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(table_id);
2524  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2525  const auto physicalTables = physicalTableIt->second;
2526  CHECK(!physicalTables.empty());
2527  for (size_t i = 0; i < physicalTables.size(); i++) {
2528  int32_t physical_tb_id = physicalTables[i];
2529  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2530  CHECK(phys_td);
2531  LOG(INFO) << "Set sharded table epoch db:" << db_id << " Table ID "
2532  << physical_tb_id << " back to new epoch " << new_epoch;
2533  removeChunks(physical_tb_id);
2534  dataMgr_->setTableEpoch(db_id, physical_tb_id, new_epoch);
2535  }
2536  }
2537 }
2538 
2540  cat_read_lock read_lock(this);
2541  const auto it = deletedColumnPerTable_.find(td);
2542  return it != deletedColumnPerTable_.end() ? it->second : nullptr;
2543 }
2544 
2546  int delete_column_id) const {
2547  // check if there are rows deleted by examining the deletedColumn metadata
2548  CHECK(td);
2549 
2550  if (table_is_temporary(td)) {
2551  auto fragmenter = td->fragmenter;
2552  CHECK(fragmenter);
2553  return fragmenter->hasDeletedRows(delete_column_id);
2554  } else {
2555  ChunkKey chunk_key_prefix = {currentDB_.dbId, td->tableId, delete_column_id};
2556  std::vector<std::pair<ChunkKey, ChunkMetadata>> chunk_metadata_vec;
2557  dataMgr_->getChunkMetadataVecForKeyPrefix(chunk_metadata_vec, chunk_key_prefix);
2558  int64_t chunk_max{0};
2559 
2560  for (auto chunk_metadata : chunk_metadata_vec) {
2561  chunk_max = chunk_metadata.second.chunkStats.max.tinyintval;
2562  // delete has occured
2563  if (chunk_max == 1) {
2564  return true;
2565  }
2566  }
2567  return false;
2568  }
2569 }
2570 
2572  const TableDescriptor* td) const {
2573  cat_read_lock read_lock(this);
2574 
2575  const auto it = deletedColumnPerTable_.find(td);
2576  // if not a table that supports delete return nullptr, nothing more to do
2577  if (it == deletedColumnPerTable_.end()) {
2578  return nullptr;
2579  }
2580  const ColumnDescriptor* cd = it->second;
2581 
2582  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
2583 
2584  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2585  // check all shards
2586  const auto physicalTables = physicalTableIt->second;
2587  CHECK(!physicalTables.empty());
2588  for (size_t i = 0; i < physicalTables.size(); i++) {
2589  int32_t physical_tb_id = physicalTables[i];
2590  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2591  CHECK(phys_td);
2592  if (checkMetadataForDeletedRecs(phys_td, cd->columnId)) {
2593  return cd;
2594  }
2595  }
2596  } else {
2597  if (checkMetadataForDeletedRecs(td, cd->columnId)) {
2598  return cd;
2599  }
2600  }
2601  // no deletes so far recorded in metadata
2602  return nullptr;
2603 }
2604 
2606  cat_write_lock write_lock(this);
2607  setDeletedColumnUnlocked(td, cd);
2608 }
2609 
2611  const ColumnDescriptor* cd) {
2612  cat_write_lock write_lock(this);
2613  const auto it_ok = deletedColumnPerTable_.emplace(td, cd);
2614  CHECK(it_ok.second);
2615 }
2616 
2617 namespace {
2618 
2620  const Catalog& cat,
2621  const Parser::SharedDictionaryDef& shared_dict_def) {
2622  const auto& table_name = shared_dict_def.get_foreign_table();
2623  const auto td = cat.getMetadataForTable(table_name);
2624  CHECK(td);
2625  const auto& foreign_col_name = shared_dict_def.get_foreign_column();
2626  return cat.getMetadataForColumn(td->tableId, foreign_col_name);
2627 }
2628 
2629 } // namespace
2630 
2632  Parser::SharedDictionaryDef shared_dict_def,
2633  const bool persist_reference) {
2634  cat_write_lock write_lock(this);
2635  const auto foreign_ref_col = get_foreign_col(*this, shared_dict_def);
2636  CHECK(foreign_ref_col);
2637  referencing_column.columnType = foreign_ref_col->columnType;
2638  const int dict_id = referencing_column.columnType.get_comp_param();
2639  const DictRef dict_ref(currentDB_.dbId, dict_id);
2640  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
2641  CHECK(dictIt != dictDescriptorMapByRef_.end());
2642  const auto& dd = dictIt->second;
2643  CHECK_GE(dd->refcount, 1);
2644  ++dd->refcount;
2645  if (persist_reference) {
2646  cat_sqlite_lock sqlite_lock(this);
2648  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
2649  {std::to_string(dict_id)});
2650  }
2651 }
2652 
2654  ColumnDescriptor& cd,
2655  std::list<ColumnDescriptor>& cdd,
2656  std::list<DictDescriptor>& dds,
2657  const TableDescriptor td,
2658  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs) {
2659  cat_write_lock write_lock(this);
2660  cat_sqlite_lock sqlite_lock(this);
2661 
2662  if (shared_dict_defs.empty()) {
2663  return false;
2664  }
2665  for (const auto& shared_dict_def : shared_dict_defs) {
2666  // check if the current column is a referencing column
2667  const auto& column = shared_dict_def.get_column();
2668  if (cd.columnName == column) {
2669  if (!shared_dict_def.get_foreign_table().compare(td.tableName)) {
2670  // Dictionaries are being shared in table to be created
2671  const auto& ref_column = shared_dict_def.get_foreign_column();
2672  auto colIt =
2673  std::find_if(cdd.begin(), cdd.end(), [ref_column](const ColumnDescriptor it) {
2674  return !ref_column.compare(it.columnName);
2675  });
2676  CHECK(colIt != cdd.end());
2677  cd.columnType = colIt->columnType;
2678 
2679  const int dict_id = colIt->columnType.get_comp_param();
2680  CHECK_GE(dict_id, 1);
2681  auto dictIt = std::find_if(
2682  dds.begin(), dds.end(), [this, dict_id](const DictDescriptor it) {
2683  return it.dictRef.dbId == this->currentDB_.dbId &&
2684  it.dictRef.dictId == dict_id;
2685  });
2686  if (dictIt != dds.end()) {
2687  // There exists dictionary definition of a dictionary column
2688  CHECK_GE(dictIt->refcount, 1);
2689  ++dictIt->refcount;
2690  if (!table_is_temporary(&td)) {
2691  // Persist reference count
2693  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
2694  {std::to_string(dict_id)});
2695  }
2696  } else {
2697  // The dictionary is referencing a column which is referencing a column in
2698  // diffrent table
2699  auto root_dict_def = compress_reference_path(shared_dict_def, shared_dict_defs);
2700  addReferenceToForeignDict(cd, root_dict_def, !table_is_temporary(&td));
2701  }
2702  } else {
2703  const auto& foreign_table_name = shared_dict_def.get_foreign_table();
2704  const auto foreign_td = getMetadataForTable(foreign_table_name, false);
2705  if (table_is_temporary(foreign_td)) {
2706  if (!table_is_temporary(&td)) {
2707  throw std::runtime_error(
2708  "Only temporary tables can share dictionaries with other temporary "
2709  "tables.");
2710  }
2711  addReferenceToForeignDict(cd, shared_dict_def, false);
2712  } else {
2713  addReferenceToForeignDict(cd, shared_dict_def, !table_is_temporary(&td));
2714  }
2715  }
2716  return true;
2717  }
2718  }
2719  return false;
2720 }
2721 
2723  std::list<DictDescriptor>& dds,
2724  const TableDescriptor& td,
2725  const bool isLogicalTable) {
2726  cat_write_lock write_lock(this);
2727 
2728  std::string dictName{"Initial_key"};
2729  int dictId{0};
2730  std::string folderPath;
2731  if (isLogicalTable) {
2732  cat_sqlite_lock sqlite_lock(this);
2733 
2735  "INSERT INTO mapd_dictionaries (name, nbits, is_shared, refcount) VALUES (?, ?, "
2736  "?, 1)",
2737  std::vector<std::string>{
2738  dictName, std::to_string(cd.columnType.get_comp_param()), "0"});
2740  "SELECT dictid FROM mapd_dictionaries WHERE name = ?", dictName);
2741  dictId = sqliteConnector_.getData<int>(0, 0);
2742  dictName = td.tableName + "_" + cd.columnName + "_dict" + std::to_string(dictId);
2744  "UPDATE mapd_dictionaries SET name = ? WHERE name = 'Initial_key'", dictName);
2745  folderPath = basePath_ + "/mapd_data/DB_" + std::to_string(currentDB_.dbId) +
2746  "_DICT_" + std::to_string(dictId);
2747  }
2749  dictId,
2750  dictName,
2752  false,
2753  1,
2754  folderPath,
2755  false);
2756  dds.push_back(dd);
2757  if (!cd.columnType.is_array()) {
2759  }
2760  cd.columnType.set_comp_param(dictId);
2761 }
2762 
2764  TableDescriptor& td,
2765  const list<ColumnDescriptor>& cols,
2766  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs) {
2767  cat_write_lock write_lock(this);
2768 
2769  /* create logical table */
2770  TableDescriptor tdl(td);
2771  createTable(tdl, cols, shared_dict_defs, true); // create logical table
2772  int32_t logical_tb_id = tdl.tableId;
2773 
2774  /* create physical tables and link them to the logical table */
2775  std::vector<int32_t> physicalTables;
2776  for (int32_t i = 1; i <= td.nShards; i++) {
2777  TableDescriptor tdp(td);
2779  tdp.shard = i - 1;
2780  createTable(tdp, cols, shared_dict_defs, false); // create physical table
2781  int32_t physical_tb_id = tdp.tableId;
2782 
2783  /* add physical table to the vector of physical tables */
2784  physicalTables.push_back(physical_tb_id);
2785  }
2786 
2787  if (!physicalTables.empty()) {
2788  /* add logical to physical tables correspondence to the map */
2789  const auto it_ok =
2790  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
2791  CHECK(it_ok.second);
2792  /* update sqlite mapd_logical_to_physical in sqlite database */
2793  if (!table_is_temporary(&td)) {
2794  updateLogicalToPhysicalTableMap(logical_tb_id);
2795  }
2796  }
2797 }
2798 
2800  cat_write_lock write_lock(this);
2801 
2802  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
2803  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2804  // truncate all corresponding physical tables if this is a logical table
2805  const auto physicalTables = physicalTableIt->second;
2806  CHECK(!physicalTables.empty());
2807  for (size_t i = 0; i < physicalTables.size(); i++) {
2808  int32_t physical_tb_id = physicalTables[i];
2809  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2810  CHECK(phys_td);
2811  doTruncateTable(phys_td);
2812  }
2813  }
2814  doTruncateTable(td);
2815 }
2816 
2818  cat_write_lock write_lock(this);
2819 
2820  const int tableId = td->tableId;
2821  // must destroy fragmenter before deleteChunks is called.
2822  if (td->fragmenter != nullptr) {
2823  auto tableDescIt = tableDescriptorMapById_.find(tableId);
2824  delete td->fragmenter;
2825  tableDescIt->second->fragmenter = nullptr; // get around const-ness
2826  }
2827  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
2828  // assuming deleteChunksWithPrefix is atomic
2829  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
2830  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
2831 
2832  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
2833 
2834  std::unique_ptr<StringDictionaryClient> client;
2835  if (SysCatalog::instance().isAggregator()) {
2836  CHECK(!string_dict_hosts_.empty());
2837  DictRef dict_ref(currentDB_.dbId, -1);
2838  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
2839  }
2840  // clean up any dictionaries
2841  // delete all column descriptors for the table
2842  for (const auto& columnDescriptor : columnDescriptorMapById_) {
2843  auto cd = columnDescriptor.second;
2844  if (cd->tableId != td->tableId) {
2845  continue;
2846  }
2847  const int dict_id = cd->columnType.get_comp_param();
2848  // Dummy dictionaries created for a shard of a logical table have the id set to zero.
2849  if (cd->columnType.get_compression() == kENCODING_DICT && dict_id) {
2850  const DictRef dict_ref(currentDB_.dbId, dict_id);
2851  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
2852  CHECK(dictIt != dictDescriptorMapByRef_.end());
2853  const auto& dd = dictIt->second;
2854  CHECK_GE(dd->refcount, 1);
2855  // if this is the only table using this dict reset the dict
2856  if (dd->refcount == 1) {
2857  // close the dictionary
2858  dd->stringDict.reset();
2859  File_Namespace::renameForDelete(dd->dictFolderPath);
2860  if (client) {
2861  client->drop(dd->dictRef);
2862  }
2863  if (!dd->dictIsTemp) {
2864  boost::filesystem::create_directory(dd->dictFolderPath);
2865  }
2866  }
2867 
2868  DictDescriptor* new_dd = new DictDescriptor(dd->dictRef,
2869  dd->dictName,
2870  dd->dictNBits,
2871  dd->dictIsShared,
2872  dd->refcount,
2873  dd->dictFolderPath,
2874  dd->dictIsTemp);
2875  dictDescriptorMapByRef_.erase(dictIt);
2876  // now create new Dict -- need to figure out what to do here for temp tables
2877  if (client) {
2878  client->create(new_dd->dictRef, new_dd->dictIsTemp);
2879  }
2880  dictDescriptorMapByRef_[new_dd->dictRef].reset(new_dd);
2882  }
2883  }
2884 }
2885 
2886 // used by rollback_table_epoch to clean up in memory artifacts after a rollback
2887 void Catalog::removeChunks(const int table_id) {
2888  auto td = getMetadataForTable(table_id);
2889 
2890  if (td->fragmenter != nullptr) {
2891  cat_sqlite_lock sqlite_lock(this);
2892  if (td->fragmenter != nullptr) {
2893  auto tableDescIt = tableDescriptorMapById_.find(table_id);
2894  delete td->fragmenter;
2895  tableDescIt->second->fragmenter = nullptr; // get around const-ness
2896  }
2897  }
2898 
2899  // remove the chunks from in memory structures
2900  ChunkKey chunkKey = {currentDB_.dbId, table_id};
2901 
2902  dataMgr_->deleteChunksWithPrefix(chunkKey, MemoryLevel::CPU_LEVEL);
2903  dataMgr_->deleteChunksWithPrefix(chunkKey, MemoryLevel::GPU_LEVEL);
2904 }
2905 
2909  cat_write_lock write_lock(this);
2910  cat_sqlite_lock sqlite_lock(this);
2911  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
2912  sqliteConnector_.query("BEGIN TRANSACTION");
2913  try {
2914  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2915  // remove all corresponding physical tables if this is a logical table
2916  const auto physicalTables = physicalTableIt->second;
2917  CHECK(!physicalTables.empty());
2918  for (size_t i = 0; i < physicalTables.size(); i++) {
2919  int32_t physical_tb_id = physicalTables[i];
2920  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2921  CHECK(phys_td);
2922  doDropTable(phys_td);
2923  }
2924 
2925  // remove corresponding record from the logicalToPhysicalTableMap in sqlite database
2927  "DELETE FROM mapd_logical_to_physical WHERE logical_table_id = ?",
2928  std::to_string(td->tableId));
2930  }
2931  doDropTable(td);
2932  } catch (std::exception& e) {
2933  sqliteConnector_.query("ROLLBACK TRANSACTION");
2934  throw;
2935  }
2936  sqliteConnector_.query("END TRANSACTION");
2937 }
2938 
2943  }
2945 }
2946 
2948  const int tableId = td->tableId;
2949  sqliteConnector_.query_with_text_param("DELETE FROM mapd_tables WHERE tableid = ?",
2950  std::to_string(tableId));
2952  "select comp_param from mapd_columns where compression = ? and tableid = ?",
2953  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
2954  int numRows = sqliteConnector_.getNumRows();
2955  std::vector<int> dict_id_list;
2956  for (int r = 0; r < numRows; ++r) {
2957  dict_id_list.push_back(sqliteConnector_.getData<int>(r, 0));
2958  }
2959  for (auto dict_id : dict_id_list) {
2961  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
2962  std::vector<std::string>{std::to_string(dict_id)});
2963  }
2965  "DELETE FROM mapd_dictionaries WHERE dictid in (select comp_param from "
2966  "mapd_columns where compression = ? "
2967  "and tableid = ?) and refcount = 0",
2968  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
2969  sqliteConnector_.query_with_text_param("DELETE FROM mapd_columns WHERE tableid = ?",
2970  std::to_string(tableId));
2971  if (td->isView) {
2972  sqliteConnector_.query_with_text_param("DELETE FROM mapd_views WHERE tableid = ?",
2973  std::to_string(tableId));
2974  }
2977  "DELETE FROM omnisci_foreign_tables WHERE table_id = ?", std::to_string(tableId));
2978  }
2979 }
2980 
2981 void Catalog::renamePhysicalTable(const TableDescriptor* td, const string& newTableName) {
2982  cat_write_lock write_lock(this);
2983  cat_sqlite_lock sqlite_lock(this);
2984 
2985  sqliteConnector_.query("BEGIN TRANSACTION");
2986  try {
2988  "UPDATE mapd_tables SET name = ? WHERE tableid = ?",
2989  std::vector<std::string>{newTableName, std::to_string(td->tableId)});
2990  } catch (std::exception& e) {
2991  sqliteConnector_.query("ROLLBACK TRANSACTION");
2992  throw;
2993  }
2994  sqliteConnector_.query("END TRANSACTION");
2995  TableDescriptorMap::iterator tableDescIt =
2997  CHECK(tableDescIt != tableDescriptorMap_.end());
2998  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
2999  // Get table descriptor to change it
3000  TableDescriptor* changeTd = tableDescIt->second;
3001  changeTd->tableName = newTableName;
3002  tableDescriptorMap_.erase(tableDescIt); // erase entry under old name
3003  tableDescriptorMap_[to_upper(newTableName)] = changeTd;
3004  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
3005 }
3006 
3007 void Catalog::renameTable(const TableDescriptor* td, const string& newTableName) {
3008  {
3009  cat_write_lock write_lock(this);
3010  cat_sqlite_lock sqlite_lock(this);
3011  // rename all corresponding physical tables if this is a logical table
3012  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
3013  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
3014  const auto physicalTables = physicalTableIt->second;
3015  CHECK(!physicalTables.empty());
3016  for (size_t i = 0; i < physicalTables.size(); i++) {
3017  int32_t physical_tb_id = physicalTables[i];
3018  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
3019  CHECK(phys_td);
3020  std::string newPhysTableName =
3021  generatePhysicalTableName(newTableName, static_cast<int32_t>(i + 1));
3022  renamePhysicalTable(phys_td, newPhysTableName);
3023  }
3024  }
3025  renamePhysicalTable(td, newTableName);
3026  }
3027  {
3028  DBObject object(newTableName, TableDBObjectType);
3029  // update table name in direct and effective priv map
3030  DBObjectKey key;
3031  key.dbId = currentDB_.dbId;
3032  key.objectId = td->tableId;
3033  key.permissionType = static_cast<int>(DBObjectType::TableDBObjectType);
3034  object.setObjectKey(key);
3035  auto objdescs = SysCatalog::instance().getMetadataForObject(
3036  currentDB_.dbId, static_cast<int>(DBObjectType::TableDBObjectType), td->tableId);
3037  for (auto obj : objdescs) {
3038  Grantee* grnt = SysCatalog::instance().getGrantee(obj->roleName);
3039  if (grnt) {
3040  grnt->renameDbObject(object);
3041  }
3042  }
3044  }
3045 }
3046 
3048  const ColumnDescriptor* cd,
3049  const string& newColumnName) {
3050  cat_write_lock write_lock(this);
3051  cat_sqlite_lock sqlite_lock(this);
3052  sqliteConnector_.query("BEGIN TRANSACTION");
3053  try {
3054  for (int i = 0; i <= cd->columnType.get_physical_cols(); ++i) {
3055  auto cdx = getMetadataForColumn(td->tableId, cd->columnId + i);
3056  CHECK(cdx);
3057  std::string new_column_name = cdx->columnName;
3058  new_column_name.replace(0, cd->columnName.size(), newColumnName);
3060  "UPDATE mapd_columns SET name = ? WHERE tableid = ? AND columnid = ?",
3061  std::vector<std::string>{new_column_name,
3062  std::to_string(td->tableId),
3063  std::to_string(cdx->columnId)});
3064  }
3065  } catch (std::exception& e) {
3066  sqliteConnector_.query("ROLLBACK TRANSACTION");
3067  throw;
3068  }
3069  sqliteConnector_.query("END TRANSACTION");
3070  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
3071  for (int i = 0; i <= cd->columnType.get_physical_cols(); ++i) {
3072  auto cdx = getMetadataForColumn(td->tableId, cd->columnId + i);
3073  CHECK(cdx);
3074  ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.find(
3075  std::make_tuple(td->tableId, to_upper(cdx->columnName)));
3076  CHECK(columnDescIt != columnDescriptorMap_.end());
3077  ColumnDescriptor* changeCd = columnDescIt->second;
3078  changeCd->columnName.replace(0, cd->columnName.size(), newColumnName);
3079  columnDescriptorMap_.erase(columnDescIt); // erase entry under old name
3080  columnDescriptorMap_[std::make_tuple(td->tableId, to_upper(changeCd->columnName))] =
3081  changeCd;
3082  }
3083  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
3084 }
3085 
3087  {
3088  cat_write_lock write_lock(this);
3089  cat_sqlite_lock sqlite_lock(this);
3090  sqliteConnector_.query("BEGIN TRANSACTION");
3091  try {
3092  // TODO(andrew): this should be an upsert
3094  "SELECT id FROM mapd_dashboards WHERE name = ? and userid = ?",
3095  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
3096  if (sqliteConnector_.getNumRows() > 0) {
3098  "UPDATE mapd_dashboards SET state = ?, image_hash = ?, metadata = ?, "
3099  "update_time = "
3100  "datetime('now') where name = ? "
3101  "and userid = ?",
3102  std::vector<std::string>{vd.dashboardState,
3103  vd.imageHash,
3104  vd.dashboardMetadata,
3105  vd.dashboardName,
3106  std::to_string(vd.userId)});
3107  } else {
3109  "INSERT INTO mapd_dashboards (name, state, image_hash, metadata, "
3110  "update_time, "
3111  "userid) "
3112  "VALUES "
3113  "(?,?,?,?, "
3114  "datetime('now'), ?)",
3115  std::vector<std::string>{vd.dashboardName,
3116  vd.dashboardState,
3117  vd.imageHash,
3118  vd.dashboardMetadata,
3119  std::to_string(vd.userId)});
3120  }
3121  } catch (std::exception& e) {
3122  sqliteConnector_.query("ROLLBACK TRANSACTION");
3123  throw;
3124  }
3125  sqliteConnector_.query("END TRANSACTION");
3126 
3127  // now get the auto generated dashboardId
3128  try {
3130  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_dashboards "
3131  "WHERE name = ? and userid = ?",
3132  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
3133  vd.dashboardId = sqliteConnector_.getData<int>(0, 0);
3134  vd.updateTime = sqliteConnector_.getData<std::string>(0, 1);
3135  } catch (std::exception& e) {
3136  throw;
3137  }
3141  }
3142  // NOTE(wamsi): Transactionally unsafe
3145  return vd.dashboardId;
3146 }
3147 
3149  cat_write_lock write_lock(this);
3150  cat_sqlite_lock sqlite_lock(this);
3151 
3152  sqliteConnector_.query("BEGIN TRANSACTION");
3153  try {
3155  "SELECT id FROM mapd_dashboards WHERE id = ?",
3156  std::vector<std::string>{std::to_string(vd.dashboardId)});
3157  if (sqliteConnector_.getNumRows() > 0) {
3159  "UPDATE mapd_dashboards SET name = ?, state = ?, image_hash = ?, metadata = ?, "
3160  "update_time = "
3161  "datetime('now') where id = ? ",
3162  std::vector<std::string>{vd.dashboardName,
3163  vd.dashboardState,
3164  vd.imageHash,
3165  vd.dashboardMetadata,
3167  } else {
3168  LOG(ERROR) << "Error replacing dashboard id " << vd.dashboardId
3169  << " does not exist in db";
3170  throw runtime_error("Error replacing dashboard id " +
3171  std::to_string(vd.dashboardId) + " does not exist in db");
3172  }
3173  } catch (std::exception& e) {
3174  sqliteConnector_.query("ROLLBACK TRANSACTION");
3175  throw;
3176  }
3177  sqliteConnector_.query("END TRANSACTION");
3178 
3179  bool found{false};
3180  for (auto descp : dashboardDescriptorMap_) {
3181  auto dash = descp.second.get();
3182  if (dash->dashboardId == vd.dashboardId) {
3183  found = true;
3184  auto viewDescIt = dashboardDescriptorMap_.find(std::to_string(dash->userId) + ":" +
3185  dash->dashboardName);
3186  if (viewDescIt ==
3187  dashboardDescriptorMap_.end()) { // check to make sure view exists
3188  LOG(ERROR) << "No metadata for dashboard for user " << dash->userId
3189  << " dashboard " << dash->dashboardName << " does not exist in map";
3190  throw runtime_error("No metadata for dashboard for user " +
3191  std::to_string(dash->userId) + " dashboard " +
3192  dash->dashboardName + " does not exist in map");
3193  }
3194  dashboardDescriptorMap_.erase(viewDescIt);
3195  break;
3196  }
3197  }
3198  if (!found) {
3199  LOG(ERROR) << "Error replacing dashboard id " << vd.dashboardId
3200  << " does not exist in map";
3201  throw runtime_error("Error replacing dashboard id " + std::to_string(vd.dashboardId) +
3202  " does not exist in map");
3203  }
3204 
3205  // now reload the object
3207  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM "
3208  "mapd_dashboards "
3209  "WHERE id = ?",
3210  std::vector<std::string>{std::to_string(vd.dashboardId)});
3211  vd.updateTime = sqliteConnector_.getData<string>(0, 1);
3215  // NOTE(wamsi): Transactionally unsafe
3218 }
3219 
3220 std::string Catalog::calculateSHA1(const std::string& data) {
3221  boost::uuids::detail::sha1 sha1;
3222  unsigned int digest[5];
3223  sha1.process_bytes(data.c_str(), data.length());
3224  sha1.get_digest(digest);
3225  std::stringstream ss;
3226  for (size_t i = 0; i < 5; i++) {
3227  ss << std::hex << digest[i];
3228  }
3229  return ss.str();
3230 }
3231 
3232 std::string Catalog::createLink(LinkDescriptor& ld, size_t min_length) {
3233  cat_write_lock write_lock(this);
3234  cat_sqlite_lock sqlite_lock(this);
3235  sqliteConnector_.query("BEGIN TRANSACTION");
3236  try {
3238  .substr(0, 8);
3240  "SELECT linkid FROM mapd_links WHERE link = ? and userid = ?",
3241  std::vector<std::string>{ld.link, std::to_string(ld.userId)});
3242  if (sqliteConnector_.getNumRows() > 0) {
3244  "UPDATE mapd_links SET update_time = datetime('now') WHERE userid = ? AND link "
3245  "= ?",
3246  std::vector<std::string>{std::to_string(ld.userId), ld.link});
3247  } else {
3249  "INSERT INTO mapd_links (userid, link, view_state, view_metadata, update_time) "
3250  "VALUES (?,?,?,?, "
3251  "datetime('now'))",
3252  std::vector<std::string>{
3253  std::to_string(ld.userId), ld.link, ld.viewState, ld.viewMetadata});
3254  }
3255  // now get the auto generated dashid
3257  "SELECT linkid, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_links "
3258  "WHERE link = ?",
3259  ld.link);
3260  ld.linkId = sqliteConnector_.getData<int>(0, 0);
3261  ld.updateTime = sqliteConnector_.getData<std::string>(0, 1);
3262  } catch (std::exception& e) {
3263  sqliteConnector_.query("ROLLBACK TRANSACTION");
3264  throw;
3265  }
3266  sqliteConnector_.query("END TRANSACTION");
3267  addLinkToMap(ld);
3268  return ld.link;
3269 }
3270 
3272  const TableDescriptor* td) const {
3273  const auto column_descriptors =
3274  getAllColumnMetadataForTable(td->tableId, false, true, true);
3275 
3276  const ColumnDescriptor* shard_cd{nullptr};
3277  int i = 1;
3278  for (auto cd_itr = column_descriptors.begin(); cd_itr != column_descriptors.end();
3279  ++cd_itr, ++i) {
3280  if (i == td->shardedColumnId) {
3281  shard_cd = *cd_itr;
3282  }
3283  }
3284  return shard_cd;
3285 }
3286 
3287 std::vector<const TableDescriptor*> Catalog::getPhysicalTablesDescriptors(
3288  const TableDescriptor* logicalTableDesc) const {
3289  cat_read_lock read_lock(this);
3290  const auto physicalTableIt =
3291  logicalToPhysicalTableMapById_.find(logicalTableDesc->tableId);
3292  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
3293  return {logicalTableDesc};
3294  }
3295 
3296  const auto physicalTablesIds = physicalTableIt->second;
3297  CHECK(!physicalTablesIds.empty());
3298  std::vector<const TableDescriptor*> physicalTables;
3299  for (size_t i = 0; i < physicalTablesIds.size(); i++) {
3300  physicalTables.push_back(getMetadataForTable(physicalTablesIds[i]));
3301  }
3302 
3303  return physicalTables;
3304 }
3305 
3306 std::vector<std::string> Catalog::getTableNamesForUser(
3307  const UserMetadata& user_metadata,
3308  const GetTablesType get_tables_type) const {
3309  std::vector<std::string> table_names;
3310  const auto tables = getAllTableMetadata();
3311  for (const auto td : tables) {
3312  if (td->shard >= 0) {
3313  // skip shards, they're not standalone tables
3314  continue;
3315  }
3316  switch (get_tables_type) {
3317  case GET_PHYSICAL_TABLES: {
3318  if (td->isView) {
3319  continue;
3320  }
3321  break;
3322  }
3323  case GET_VIEWS: {
3324  if (!td->isView) {
3325  continue;
3326  }
3327  }
3328  default:
3329  break;
3330  }
3332  dbObject.loadKey(*this);
3333  std::vector<DBObject> privObjects = {dbObject};
3334  if (!SysCatalog::instance().hasAnyPrivileges(user_metadata, privObjects)) {
3335  // skip table, as there are no privileges to access it
3336  continue;
3337  }
3338  table_names.push_back(td->tableName);
3339  }
3340  return table_names;
3341 }
3342 
3343 int Catalog::getLogicalTableId(const int physicalTableId) const {
3344  cat_read_lock read_lock(this);
3345  for (const auto& l : logicalToPhysicalTableMapById_) {
3346  if (l.second.end() != std::find_if(l.second.begin(),
3347  l.second.end(),
3348  [&](decltype(*l.second.begin()) tid) -> bool {
3349  return physicalTableId == tid;
3350  })) {
3351  return l.first;
3352  }
3353  }
3354  return physicalTableId;
3355 }
3356 
3357 void Catalog::checkpoint(const int logicalTableId) const {
3358  const auto td = getMetadataForTable(logicalTableId);
3359  const auto shards = getPhysicalTablesDescriptors(td);
3360  for (const auto shard : shards) {
3361  getDataMgr().checkpoint(getCurrentDB().dbId, shard->tableId);
3362  }
3363 }
3364 
3366  cat_write_lock write_lock(this);
3367  // Physically erase all tables and dictionaries from disc and memory
3368  const auto tables = getAllTableMetadata();
3369  for (const auto table : tables) {
3370  eraseTablePhysicalData(table);
3371  }
3372  // Physically erase database metadata
3373  boost::filesystem::remove(basePath_ + "/mapd_catalogs/" + currentDB_.dbName);
3374  calciteMgr_->updateMetadata(currentDB_.dbName, "");
3375 }
3376 
3378  cat_write_lock write_lock(this);
3379  const int tableId = td->tableId;
3380  // must destroy fragmenter before deleteChunks is called.
3381  if (td->fragmenter != nullptr) {
3382  auto tableDescIt = tableDescriptorMapById_.find(tableId);
3383  {
3384  INJECT_TIMER(deleting_fragmenter);
3385  delete td->fragmenter;
3386  }
3387  tableDescIt->second->fragmenter = nullptr; // get around const-ness
3388  }
3389  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
3390  {
3391  INJECT_TIMER(deleteChunksWithPrefix);
3392  // assuming deleteChunksWithPrefix is atomic
3393  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
3394  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
3395  }
3396  if (!td->isView) {
3397  INJECT_TIMER(Remove_Table);
3398  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
3399  }
3400  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
3401  {
3402  INJECT_TIMER(removeTableFromMap_);
3403  removeTableFromMap(td->tableName, tableId);
3404  }
3405 }
3406 
3407 std::string Catalog::generatePhysicalTableName(const std::string& logicalTableName,
3408  const int32_t& shardNumber) {
3409  std::string physicalTableName =
3410  logicalTableName + physicalTableNameTag_ + std::to_string(shardNumber);
3411  return (physicalTableName);
3412 }
3413 
3414 void Catalog::set(const std::string& dbName, std::shared_ptr<Catalog> cat) {
3415  mapd_cat_map_[dbName] = cat;
3416 }
3417 
3418 std::shared_ptr<Catalog> Catalog::get(const std::string& dbName) {
3419  auto cat_it = mapd_cat_map_.find(dbName);
3420  if (cat_it != mapd_cat_map_.end()) {
3421  return cat_it->second;
3422  }
3423  return nullptr;
3424 }
3425 
3426 std::shared_ptr<Catalog> Catalog::get(const string& basePath,
3427  const DBMetadata& curDB,
3428  std::shared_ptr<Data_Namespace::DataMgr> dataMgr,
3429  const std::vector<LeafHostInfo>& string_dict_hosts,
3430  std::shared_ptr<Calcite> calcite,
3431  bool is_new_db) {
3432  auto cat = Catalog::get(curDB.dbName);
3433 
3434  if (cat) {
3435  return cat;
3436  } else {
3437  cat = std::make_shared<Catalog>(
3438  basePath, curDB, dataMgr, string_dict_hosts, calcite, is_new_db);
3439  Catalog::set(curDB.dbName, cat);
3440  return cat;
3441  }
3442 }
3443 
3444 void Catalog::remove(const std::string& dbName) {
3445  mapd_cat_map_.erase(dbName);
3446 }
3447 
3448 void Catalog::vacuumDeletedRows(const int logicalTableId) const {
3449  // shard here to serve request from TableOptimizer and elsewhere
3450  const auto td = getMetadataForTable(logicalTableId);
3451  const auto shards = getPhysicalTablesDescriptors(td);
3452  for (const auto shard : shards) {
3453  vacuumDeletedRows(shard);
3454  }
3455 }
3456 
3458  // "if not a table that supports delete return nullptr, nothing more to do"
3459  const ColumnDescriptor* cd = getDeletedColumn(td);
3460  if (nullptr == cd) {
3461  return;
3462  }
3463  // vacuum chunks which show sign of deleted rows in metadata
3464  ChunkKey chunkKeyPrefix = {currentDB_.dbId, td->tableId, cd->columnId};
3465  std::vector<std::pair<ChunkKey, ChunkMetadata>> chunkMetadataVec;
3466  dataMgr_->getChunkMetadataVecForKeyPrefix(chunkMetadataVec, chunkKeyPrefix);
3467  for (auto cm : chunkMetadataVec) {
3468  // "delete has occured"
3469  if (cm.second.chunkStats.max.tinyintval == 1) {
3470  UpdelRoll updel_roll;
3471  updel_roll.catalog = this;
3472  updel_roll.logicalTableId = getLogicalTableId(td->tableId);
3474  const auto cd = getMetadataForColumn(td->tableId, cm.first[2]);
3475  const auto chunk = Chunk_NS::Chunk::getChunk(cd,
3476  &getDataMgr(),
3477  cm.first,
3478  updel_roll.memoryLevel,
3479  0,
3480  cm.second.numBytes,
3481  cm.second.numElements);
3482  td->fragmenter->compactRows(this,
3483  td,
3484  cm.first[3],
3485  td->fragmenter->getVacuumOffsets(chunk),
3486  updel_roll.memoryLevel,
3487  updel_roll);
3488  updel_roll.commitUpdate();
3489  }
3490  }
3491 }
3492 
3495  "SELECT id, name, data_wrapper_type, options, owner_user_id FROM "
3496  "omnisci_foreign_servers");
3498  for (size_t row = 0; row < num_rows; row++) {
3499  auto foreign_server = std::make_shared<foreign_storage::ForeignServer>(
3500  foreign_storage::DataWrapper{sqliteConnector_.getData<std::string>(row, 2)});
3501  foreign_server->id = sqliteConnector_.getData<int>(row, 0);
3502  foreign_server->name = sqliteConnector_.getData<std::string>(row, 1);
3503  foreign_server->populateOptionsMap(sqliteConnector_.getData<std::string>(row, 3));
3504  foreign_server->user_id = sqliteConnector_.getData<std::int32_t>(row, 4);
3505  foreignServerMap_[foreign_server->name] = foreign_server;
3506  foreignServerMapById_[foreign_server->id] = foreign_server;
3507  }
3508 }
3509 
3512  "SELECT table_id, server_id, options from omnisci_foreign_tables");
3513  auto num_rows = sqliteConnector_.getNumRows();
3514  for (size_t r = 0; r < num_rows; r++) {
3515  const auto table_id = sqliteConnector_.getData<int>(r, 0);
3516  const auto server_id = sqliteConnector_.getData<int>(r, 1);
3517  const auto& options = sqliteConnector_.getData<std::string>(r, 2);
3518 
3519  CHECK(tableDescriptorMapById_.find(table_id) != tableDescriptorMapById_.end());
3520  auto foreign_table =
3521  dynamic_cast<foreign_storage::ForeignTable*>(tableDescriptorMapById_[table_id]);
3522  CHECK(foreign_table);
3523  foreign_table->foreign_server = foreignServerMapById_[server_id].get();
3524  CHECK(foreign_table->foreign_server);
3525  foreign_table->populateOptionsMap(options);
3526  }
3527 }
3528 
3530  auto local_csv_server = std::make_unique<foreign_storage::ForeignServer>(
3532  local_csv_server->name = "omnisci_local_csv";
3533  local_csv_server
3534  ->options[std::string{foreign_storage::ForeignServer::STORAGE_TYPE_KEY}] =
3536  local_csv_server->user_id = OMNISCI_ROOT_USER_ID;
3537  local_csv_server->options[std::string{foreign_storage::ForeignServer::BASE_PATH_KEY}] =
3538  "/";
3539  local_csv_server->validate();
3540  createForeignServerNoLocks(std::move(local_csv_server), true);
3541 
3542  auto local_parquet_server = std::make_unique<foreign_storage::ForeignServer>(
3544  local_parquet_server->name = "omnisci_local_parquet";
3545  local_parquet_server
3546  ->options[std::string{foreign_storage::ForeignServer::STORAGE_TYPE_KEY}] =
3548  local_parquet_server->user_id = OMNISCI_ROOT_USER_ID;
3549  local_parquet_server
3550  ->options[std::string{foreign_storage::ForeignServer::BASE_PATH_KEY}] = "/";
3551  local_parquet_server->validate();
3552  createForeignServerNoLocks(std::move(local_parquet_server), true);
3553 }
3554 
3555 // prepare a fresh file reload on next table access
3556 void Catalog::setForReload(const int32_t tableId) {
3557  cat_read_lock read_lock(this);
3558  const auto td = getMetadataForTable(tableId);
3559  for (const auto shard : getPhysicalTablesDescriptors(td)) {
3560  const auto tableEpoch = getTableEpoch(currentDB_.dbId, shard->tableId);
3561  setTableEpoch(currentDB_.dbId, shard->tableId, tableEpoch);
3562  }
3563 }
3564 
3565 // get a table's data dirs
3566 std::vector<std::string> Catalog::getTableDataDirectories(
3567  const TableDescriptor* td) const {
3568  const auto global_file_mgr = getDataMgr().getGlobalFileMgr();
3569  std::vector<std::string> file_paths;
3570  for (auto shard : getPhysicalTablesDescriptors(td)) {
3571  const auto file_mgr = dynamic_cast<File_Namespace::FileMgr*>(
3572  global_file_mgr->getFileMgr(currentDB_.dbId, shard->tableId));
3573  boost::filesystem::path file_path(file_mgr->getFileMgrBasePath());
3574  file_paths.push_back(file_path.filename().string());
3575  }
3576  return file_paths;
3577 }
3578 
3579 // get a column's dict dir basename
3581  if ((cd->columnType.is_string() || cd->columnType.is_string_array()) &&
3583  cd->columnType.get_comp_param() > 0) {
3584  const auto dictId = cd->columnType.get_comp_param();
3585  const DictRef dictRef(currentDB_.dbId, dictId);
3586  const auto dit = dictDescriptorMapByRef_.find(dictRef);
3587  CHECK(dit != dictDescriptorMapByRef_.end());
3588  CHECK(dit->second);
3589  boost::filesystem::path file_path(dit->second->dictFolderPath);
3590  return file_path.filename().string();
3591  }
3592  return std::string();
3593 }
3594 
3595 // get a table's dict dirs
3596 std::vector<std::string> Catalog::getTableDictDirectories(
3597  const TableDescriptor* td) const {
3598  std::vector<std::string> file_paths;
3599  for (auto cd : getAllColumnMetadataForTable(td->tableId, false, false, true)) {
3600  auto file_base = getColumnDictDirectory(cd);
3601  if (!file_base.empty() &&
3602  file_paths.end() == std::find(file_paths.begin(), file_paths.end(), file_base)) {
3603  file_paths.push_back(file_base);
3604  }
3605  }
3606  return file_paths;
3607 }
3608 
3609 // returns table schema in a string
3610 std::string Catalog::dumpSchema(const TableDescriptor* td) const {
3611  cat_read_lock read_lock(this);
3612 
3613  std::ostringstream os;
3614  os << "CREATE TABLE @T (";
3615  // gather column defines
3616  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
3617  std::string comma;
3618  std::vector<std::string> shared_dicts;
3619  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
3620  for (const auto cd : cds) {
3621  if (!(cd->isSystemCol || cd->isVirtualCol)) {
3622  const auto& ti = cd->columnType;
3623  os << comma << cd->columnName;
3624  // CHAR is perculiar... better dump it as TEXT(32) like \d does
3625  if (ti.get_type() == SQLTypes::kCHAR) {
3626  os << " "
3627  << "TEXT";
3628  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
3629  os << " "
3630  << "TEXT[]";
3631  } else {
3632  os << " " << ti.get_type_name();
3633  }
3634  os << (ti.get_notnull() ? " NOT NULL" : "");
3635  if (ti.is_string()) {
3636  if (ti.get_compression() == kENCODING_DICT) {
3637  // if foreign reference, get referenced tab.col
3638  const auto dict_id = ti.get_comp_param();
3639  const DictRef dict_ref(currentDB_.dbId, dict_id);
3640  const auto dict_it = dictDescriptorMapByRef_.find(dict_ref);
3641  CHECK(dict_it != dictDescriptorMapByRef_.end());
3642  const auto dict_name = dict_it->second->dictName;
3643  // when migrating a table, any foreign dict ref will be dropped
3644  // and the first cd of a dict will become root of the dict
3645  if (dict_root_cds.end() == dict_root_cds.find(dict_name)) {
3646  dict_root_cds[dict_name] = cd;
3647  os << " ENCODING " << ti.get_compression_name() << "(" << (ti.get_size() * 8)
3648  << ")";
3649  } else {
3650  const auto dict_root_cd = dict_root_cds[dict_name];
3651  shared_dicts.push_back("SHARED DICTIONARY (" + cd->columnName +
3652  ") REFERENCES @T(" + dict_root_cd->columnName + ")");
3653  // "... shouldn't specify an encoding, it borrows from the referenced column"
3654  }
3655  } else {
3656  os << " ENCODING NONE";
3657  }
3658  } else if (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size()) {
3659  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
3660  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
3661  }
3662  comma = ", ";
3663  }
3664  }
3665  // gather SHARED DICTIONARYs
3666  if (shared_dicts.size()) {
3667  os << ", " << boost::algorithm::join(shared_dicts, ", ");
3668  }
3669  // gather WITH options ...
3670  std::vector<std::string> with_options;
3671  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
3672  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
3673  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
3674  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
3675  with_options.emplace_back(td->hasDeletedCol ? "VACUUM='DELAYED'"
3676  : "VACUUM='IMMEDIATE'");
3677  if (!td->partitions.empty()) {
3678  with_options.push_back("PARTITIONS='" + td->partitions + "'");
3679  }
3680  if (td->nShards > 0) {
3681  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
3682  CHECK(shard_cd);
3683  os << ", SHARD KEY(" << shard_cd->columnName << ")";
3684  with_options.push_back("SHARD_COUNT=" + std::to_string(td->nShards));
3685  }
3686  if (td->sortedColumnId > 0) {
3687  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
3688  CHECK(sort_cd);
3689  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
3690  }
3691  os << ") WITH (" + boost::algorithm::join(with_options, ", ") + ");";
3692  return os.str();
3693 }
3694 
3695 } // namespace Catalog_Namespace
void serializeTableJsonUnlocked(const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
Definition: Catalog.cpp:2290
void instantiateFragmenter(TableDescriptor *td) const
Definition: Catalog.cpp:1366
const Parser::SharedDictionaryDef compress_reference_path(Parser::SharedDictionaryDef cur_node, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Data_Namespace::MemoryLevel memoryLevel
Definition: UpdelRoll.h:65
std::string virtualExpr
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:249
void set_compression(EncodingType c)
Definition: sqltypes.h:348
void set_size(int s)
Definition: sqltypes.h:346
#define CHECK_EQ(x, y)
Definition: Logger.h:205
void executeDropTableSqliteQueries(const TableDescriptor *td)
Definition: Catalog.cpp:2947
const int MAPD_TEMP_TABLE_START_ID
Definition: Catalog.cpp:99
std::string partitions
std::vector< int > ChunkKey
Definition: types.h:35
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:3377
std::string dictFolderPath
std::tuple< int, std::string > ColumnKey
Definition: Types.h:36
HOST DEVICE int get_size() const
Definition: sqltypes.h:258
void doDropTable(const TableDescriptor *td)
Definition: Catalog.cpp:2939
~Catalog()
Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the ...
Definition: Catalog.cpp:188
void getDictionary(const ColumnDescriptor &cd, std::map< int, StringDictionary * > &stringDicts)
Definition: Catalog.cpp:1776
T getData(const int row, const int col)
void commitUpdate()
static constexpr char const * PARQUET_WRAPPER_NAME
Definition: ForeignServer.h:32
const int8_t const int64_t * num_rows
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:86
SQLTypes
Definition: sqltypes.h:39
#define SPIMAP_MAGIC1
Definition: Catalog.h:73
std::string tableName
void addReferenceToForeignDict(ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def, const bool persist_reference)
Definition: Catalog.cpp:2631
const ColumnDescriptor * getDeletedColumn(const TableDescriptor *td) const
Definition: Catalog.cpp:2539
void setDeletedColumn(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:2605
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:365
static const AccessPrivileges ALL_DATABASE
Definition: DBObject.h:151
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:2080
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:407
EncodingType
Definition: encodetypes.h:22
virtual void query_with_text_params(std::string const &query_only)
DBObjectType
Definition: DBObject.h:42
void createForeignServer(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2384
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:183
void addColumn(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:1804
void checkpoint(const int db_id, const int tb_id)
Definition: DataMgr.cpp:467
void roll(const bool forward)
Definition: Catalog.cpp:1892
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:1692
#define LOG(tag)
Definition: Logger.h:188
SqliteConnector sqliteConnector_
Definition: Catalog.h:374
static std::shared_ptr< Catalog > get(const std::string &dbName)
Definition: Catalog.cpp:3418
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:378
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logicalTableDesc) const
Definition: Catalog.cpp:3287
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:367
HOST DEVICE int get_scale() const
Definition: sqltypes.h:253
std::string storageType
void revokeDBObjectPrivilegesFromAll(DBObject object, Catalog *catalog)
void revokeDBObjectPrivileges(const std::string &grantee, const DBObject &object, const Catalog_Namespace::Catalog &catalog)
std::string join(T const &container, std::string const &delim)
Catalog(const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
Constructor - takes basePath to already extant data directory for writing.
Definition: Catalog.cpp:158
#define DEFAULT_MAX_CHUNK_SIZE
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:376
const LinkDescriptor * getMetadataForLink(const std::string &link) const
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:339
void populateOptionsMap(const rapidjson::Value &ddl_options)
const std::string & get_foreign_table() const
Definition: ParserNode.h:897
virtual void query(const std::string &queryString)
#define CHECK_GE(x, y)
Definition: Logger.h:210
void setObjectKey(const DBObjectKey &objectKey)
Definition: DBObject.h:213
void dropTableFromJsonUnlocked(const std::string &table_name) const
Definition: Catalog.cpp:2352
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:150
Grantee * getGrantee(const std::string &name) const
std::string fragments
void setForReload(const int32_t tableId)
Definition: Catalog.cpp:3556
int32_t objectId
Definition: DBObject.h:57
int32_t getTableEpoch(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:2485
void replaceDashboard(DashboardDescriptor &vd)
Definition: Catalog.cpp:3148
const int MAPD_TEMP_DICT_START_ID
Definition: Catalog.cpp:101
Definition: Grantee.h:76
std::vector< std::string > getTableDataDirectories(const TableDescriptor *td) const
Definition: Catalog.cpp:3566
std::string generatePhysicalTableName(const std::string &logicalTableName, const int32_t &shardNumber)
Definition: Catalog.cpp:3407
The InsertOrderFragmenter is a child class of AbstractFragmenter, and fragments data in insert order...
void dropColumn(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:1861
void checkDateInDaysColumnMigration()
Definition: Catalog.cpp:811
void setPrivileges(const AccessPrivileges &privs)
Definition: DBObject.h:215
void addFrontendViewToMap(DashboardDescriptor &vd)
Definition: Catalog.cpp:1276
void deleteMetadataForDashboard(const std::string &userId, const std::string &dashName)
Definition: Catalog.cpp:1541
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:248
static const AccessPrivileges SELECT_FROM_TABLE
Definition: DBObject.h:160
const ColumnDescriptor * getShardColumnMetadataForTable(const TableDescriptor *td) const
Definition: Catalog.cpp:3271
std::vector< int > columnIdBySpi_
#define CHECK_GT(x, y)
Definition: Logger.h:209
void renameColumn(const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
Definition: Catalog.cpp:3047
DeletedColumnPerTableMap deletedColumnPerTable_
Definition: Catalog.h:397
const ColumnDescriptor * get_foreign_col(const Catalog &cat, const Parser::SharedDictionaryDef &shared_dict_def)
Definition: Catalog.cpp:2619
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:366
DBObject * findDbObject(const DBObjectKey &objectKey, bool only_direct) const
Definition: Grantee.cpp:72
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:368
std::string to_string(char const *&&v)
std::shared_ptr< std::mutex > mutex_
std::vector< std::string > getTableNamesForUser(const UserMetadata &user, const GetTablesType get_tables_type) const
Definition: Catalog.cpp:3306
void addTableToMap(const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
Definition: Catalog.cpp:1134
static constexpr char const * CSV_WRAPPER_NAME
Definition: ForeignServer.h:31
static const AccessPrivileges ALL_VIEW
Definition: DBObject.h:177
std::string chunks
void grantRoleBatch(const std::vector< std::string > &roles, const std::vector< std::string > &grantees)
foreign_storage::ForeignServer * getForeignServerSkipCache(const std::string &server_name)
Definition: Catalog.cpp:2434
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:683
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:117
#define SPIMAP_MAGIC2
Definition: Catalog.h:74
const bool checkMetadataForDeletedRecs(const TableDescriptor *td, int column_id) const
Definition: Catalog.cpp:2545
std::string getColumnDictDirectory(const ColumnDescriptor *cd) const
Definition: Catalog.cpp:3580
static constexpr std::string_view STORAGE_TYPE_KEY
Definition: ForeignServer.h:50
std::string name() const
Definition: Catalog.h:236
void createRole(const std::string &roleName, const bool &userPrivateRole=false)
std::string dashboardSystemRoleName
This file contains the class specification and related data structures for Catalog.
std::list< const DashboardDescriptor * > getAllDashboardsMetadata() const
Definition: Catalog.cpp:1701
virtual void compactRows(const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const int fragmentId, const std::vector< uint64_t > &fragOffsets, const Data_Namespace::MemoryLevel memoryLevel, UpdelRoll &updelRoll)=0
virtual const std::vector< uint64_t > getVacuumOffsets(const std::shared_ptr< Chunk_NS::Chunk > &chunk)=0
int get_physical_cols() const
Definition: sqltypes.h:269
static SysCatalog & instance()
Definition: SysCatalog.h:256
This file contains the class specification and related data structures for SysCatalog.
CHECK(cgen_state)
Classes representing a parse tree.
const Catalog_Namespace::Catalog * catalog
Definition: UpdelRoll.h:63
std::string calculateSHA1(const std::string &data)
Definition: Catalog.cpp:3220
std::string getUserFromId(const int32_t id)
Definition: Catalog.cpp:913
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:182
void addLinkToMap(LinkDescriptor &ld)
Definition: Catalog.cpp:1358
const ColumnDescriptor * getDeletedColumnIfRowsDeleted(const TableDescriptor *td) const
Definition: Catalog.cpp:2571
#define INJECT_TIMER(DESC)
Definition: measure.h:91
void delDictionary(const ColumnDescriptor &cd)
Definition: Catalog.cpp:1733
GetTablesType
Definition: Catalog.h:61
void renameObjectsInDescriptorMap(DBObject &object, const Catalog_Namespace::Catalog &cat)
void removeChunks(const int table_id)
Definition: Catalog.cpp:2887
void set_scale(int s)
Definition: sqltypes.h:343
void dropForeignServer(const std::string &server_name)
Definition: Catalog.cpp:2457
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:2610
const std::string & get_foreign_column() const
Definition: ParserNode.h:899
void dropTable(const TableDescriptor *td)
Definition: Catalog.cpp:2906
std::vector< std::string > parse_underlying_dashboard_objects(const std::string &meta)
static void migrateDateInDaysMetadata(const Catalog_Namespace::TableDescriptorMapById &table_descriptors_by_id, const int database_id, const Catalog_Namespace::Catalog *cat, SqliteConnector &sqlite)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
static const int32_t MAPD_VERSION
Definition: release.h:33
Role * getRoleGrantee(const std::string &name) const
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const
static const AccessPrivileges ALL_SERVER
Definition: DBObject.h:187
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:364
void setOwner(int32_t userId)
Definition: DBObject.h:223
static std::shared_ptr< Chunk > getChunk(const ColumnDescriptor *cd, DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId, const size_t num_bytes, const size_t num_elems)
Definition: Chunk.cpp:28
void grantDBObjectPrivilegesBatch(const std::vector< std::string > &grantees, const std::vector< DBObject > &objects, const Catalog_Namespace::Catalog &catalog)
int getLogicalTableId(const int physicalTableId) const
Definition: Catalog.cpp:3343
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1444
specifies the content in-memory of a row in the column metadata table
int32_t dictId
Definition: DictRef.h:10
static void registerTable(Catalog_Namespace::Catalog *catalog, const TableDescriptor &td, const std::list< ColumnDescriptor > &cols)
ids are created
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:1950
const std::vector< LeafHostInfo > & getStringDictionaryHosts() const
Definition: Catalog.cpp:1480
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1317
const ColumnDescriptor * getMetadataForColumnBySpi(const int tableId, const size_t spi) const
Definition: Catalog.cpp:1531
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:372
std::string keyMetainfo
int g_test_against_columnId_gap
Definition: Catalog.cpp:88
bool g_serialize_temp_tables
Definition: Catalog.cpp:94
struct dict_ref_t DictRef
Definition: DictRef.h:37
void createShardedTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:2763
ForeignServerMap foreignServerMap_
Definition: Catalog.h:371
std::string to_upper(const std::string &str)
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1675
const DBObjectMap * getDbObjects(bool only_direct) const
Definition: Grantee.h:56
void checkpoint(const int logicalTableId) const
Definition: Catalog.cpp:3357
void set_comp_param(int p)
Definition: sqltypes.h:349
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2392
void loadKey()
Definition: DBObject.cpp:177
const int DEFAULT_INITIAL_VERSION
Definition: Catalog.cpp:98
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:379
static const std::string physicalTableNameTag_
Definition: Catalog.h:383
bool isNull(const int row, const int col) const
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1281
static void remove(const std::string &dbName)
Definition: Catalog.cpp:3444
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:256
static const AccessPrivileges SELECT_FROM_VIEW
Definition: DBObject.h:180
bool table_is_temporary(const TableDescriptor *const td)
void setTableEpoch(const int db_id, const int table_id, const int new_epoch)
Definition: Catalog.cpp:2515
static void set(const std::string &dbName, std::shared_ptr< Catalog > cat)
Definition: Catalog.cpp:3414
std::vector< std::string > getTableDictDirectories(const TableDescriptor *td) const
Definition: Catalog.cpp:3596
static constexpr std::string_view LOCAL_FILE_STORAGE_TYPE
Definition: ForeignServer.h:52
void set_dimension(int d)
Definition: sqltypes.h:340
const int OMNISCI_ROOT_USER_ID
Definition: SysCatalog.h:60
void vacuumDeletedRows(const TableDescriptor *td) const
Definition: Catalog.cpp:3457
Fragmenter_Namespace::FragmenterType fragType
int32_t dbId
Definition: DBObject.h:56
Data_Namespace::MemoryLevel persistenceLevel
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:250
int32_t createDashboard(DashboardDescriptor &vd)
Definition: Catalog.cpp:3086
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1194
static const AccessPrivileges ALL_DASHBOARD
Definition: DBObject.h:169
Definition: sqltypes.h:42
static const AccessPrivileges ALL_TABLE
Definition: DBObject.h:157
int logicalTableId
Definition: UpdelRoll.h:64
bool g_cache_string_hash
Definition: Execute.cpp:89
const int getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const
Definition: Catalog.cpp:1509
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:257
bool setColumnSharedDictionary(ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:2653
void set_notnull(bool n)
Definition: sqltypes.h:345
static constexpr std::string_view BASE_PATH_KEY
Definition: ForeignServer.h:51
bool is_geometry() const
Definition: sqltypes.h:411
const TableDescriptor * getMetadataForTableImpl(int tableId, const bool populateFragmenter) const
Definition: Catalog.cpp:1424
std::string createLink(LinkDescriptor &ld, size_t min_length)
Definition: Catalog.cpp:3232
void createFsiSchemasAndDefaultServers()
Definition: Catalog.cpp:608
Descriptor for a dictionary for a string columne.
void updateLogicalToPhysicalTableLinkSchema()
Definition: Catalog.cpp:547
FileBuffer Chunk
A Chunk is the fundamental unit of execution in Map-D.
Definition: FileMgr.h:67
std::unordered_map< std::string, std::vector< std::string > > getGranteesOfSharedDashboards(const std::vector< std::string > &dashboard_ids)
int32_t permissionType
Definition: DBObject.h:55
void populateRoleDbObjects(const std::vector< DBObject > &objects)
std::string viewSQL
Definition: sqltypes.h:46
SQLTypeInfo columnType
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
DictRef addDictionary(ColumnDescriptor &cd)
Definition: Catalog.cpp:1709
bool is_string() const
Definition: sqltypes.h:399
specifies the content in-memory of a row in the table metadata table
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:255
void renameTable(const TableDescriptor *td, const std::string &newTableName)
Definition: Catalog.cpp:3007
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:182
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:370
static std::map< std::string, std::shared_ptr< Catalog > > mapd_cat_map_
Definition: Catalog.h:396
std::string dumpSchema(const TableDescriptor *td) const
Definition: Catalog.cpp:3610
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:2722
static constexpr char const * FOREIGN_TABLE
bool is_string_array() const
Definition: sqltypes.h:400
Fragmenter_Namespace::AbstractFragmenter * fragmenter
bool g_enable_fsi
Definition: Catalog.cpp:89
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:393
void doTruncateTable(const TableDescriptor *td)
Definition: Catalog.cpp:2817
void renamePhysicalTable(const TableDescriptor *td, const std::string &newTableName)
Definition: Catalog.cpp:2981
std::string columnName
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
std::vector< DBObject > parseDashboardObjects(const std::string &view_meta, const int &user_id)
Definition: Catalog.cpp:1287
foreign_storage::ForeignServer * getForeignServer(const std::string &server_name) const
Definition: Catalog.cpp:2424
#define IS_GEO(T)
Definition: sqltypes.h:163
std::vector< ObjectRoleDescriptor * > getMetadataForObject(int32_t dbId, int32_t dbType, int32_t objectId) const
virtual void renameDbObject(const DBObject &object)
Definition: Grantee.cpp:108
virtual size_t getNumRows() const
A selection of helper methods for File I/O.
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:381
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:363
static thread_local bool thread_holds_read_lock
Definition: Catalog.h:421
bool is_array() const
Definition: sqltypes.h:407
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:369
#define VLOG(n)
Definition: Logger.h:291
void CheckAndExecuteMigrationsPostBuildMaps()
Definition: Catalog.cpp:907
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
Definition: DataMgr.cpp:498
const int getColumnIdBySpi(const int tableId, const size_t spi) const
Definition: Catalog.cpp:1526
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)
void createDefaultServersIfNotExists()
Definition: Catalog.cpp:3529
std::tuple< int, int > ColumnIdKey
Definition: Types.h:38
static void prepareTable(const int db_id, TableDescriptor &td, std::list< ColumnDescriptor > &cols)
prepare table options and modify columns
void truncateTable(const TableDescriptor *td)
Definition: Catalog.cpp:2799
void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id)
Definition: Catalog.cpp:561
HOST DEVICE void set_type(SQLTypes t)
Definition: sqltypes.h:338