OmniSciDB  c07336695a
Catalog.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
25 #include "Catalog.h"
26 #include <algorithm>
27 #include <cassert>
28 #include <exception>
29 #include <list>
30 #include <memory>
31 #include <random>
32 #include <sstream>
33 #include "SysCatalog.h"
34 
35 #include "LockMgr/LockMgr.h"
37 
38 #include <boost/algorithm/string/predicate.hpp>
39 #include <boost/filesystem.hpp>
40 #include <boost/range/adaptor/map.hpp>
41 #include <boost/version.hpp>
42 #if BOOST_VERSION >= 106600
43 #include <boost/uuid/detail/sha1.hpp>
44 #else
45 #include <boost/uuid/sha1.hpp>
46 #endif
47 
48 #include "../QueryEngine/Execute.h"
49 #include "../QueryEngine/TableOptimizer.h"
50 
51 #include "../Fragmenter/Fragmenter.h"
52 #include "../Fragmenter/SortedOrderFragmenter.h"
53 #include "../Parser/ParserNode.h"
54 #include "../Shared/File.h"
55 #include "../Shared/StringTransform.h"
56 #include "../Shared/measure.h"
57 #include "../StringDictionary/StringDictionaryClient.h"
58 #include "MapDRelease.h"
59 #include "RWLocks.h"
61 
62 using Chunk_NS::Chunk;
65 using std::list;
66 using std::map;
67 using std::pair;
68 using std::runtime_error;
69 using std::string;
70 using std::vector;
71 
73 extern bool g_cache_string_hash;
74 
75 namespace Catalog_Namespace {
76 
77 const int DEFAULT_INITIAL_VERSION = 1; // start at version 1
79  1073741824; // 2^30, give room for over a billion non-temp tables
81  1073741824; // 2^30, give room for over a billion non-temp dictionaries
82 
83 const std::string Catalog::physicalTableNameTag_("_shard_#");
84 std::map<std::string, std::shared_ptr<Catalog>> Catalog::mapd_cat_map_;
85 
86 thread_local bool Catalog::thread_holds_read_lock = false;
87 
91 
92 // migration will be done as two step process this release
93 // will create and use new table
94 // next release will remove old table, doing this to have fall back path
95 // incase of migration failure
98  sqliteConnector_.query("BEGIN TRANSACTION");
99  try {
101  "SELECT name FROM sqlite_master WHERE type='table' AND name='mapd_dashboards'");
102  if (sqliteConnector_.getNumRows() != 0) {
103  // already done
104  sqliteConnector_.query("END TRANSACTION");
105  return;
106  }
108  "CREATE TABLE mapd_dashboards (id integer primary key autoincrement, name text , "
109  "userid integer references mapd_users, state text, image_hash text, update_time "
110  "timestamp, "
111  "metadata text, UNIQUE(userid, name) )");
112  // now copy content from old table to new table
114  "insert into mapd_dashboards (id, name , "
115  "userid, state, image_hash, update_time , "
116  "metadata) "
117  "SELECT viewid , name , userid, view_state, image_hash, update_time, "
118  "view_metadata "
119  "from mapd_frontend_views");
120  } catch (const std::exception& e) {
121  sqliteConnector_.query("ROLLBACK TRANSACTION");
122  throw;
123  }
124  sqliteConnector_.query("END TRANSACTION");
125 }
126 
127 Catalog::Catalog(const string& basePath,
128  const DBMetadata& curDB,
129  std::shared_ptr<Data_Namespace::DataMgr> dataMgr,
130  const std::vector<LeafHostInfo>& string_dict_hosts,
131  std::shared_ptr<Calcite> calcite,
132  bool is_new_db)
133  : basePath_(basePath)
134  , sqliteConnector_(curDB.dbName, basePath + "/mapd_catalogs/")
135  , currentDB_(curDB)
136  , dataMgr_(dataMgr)
137  , string_dict_hosts_(string_dict_hosts)
138  , calciteMgr_(calcite)
139  , nextTempTableId_(MAPD_TEMP_TABLE_START_ID)
140  , nextTempDictId_(MAPD_TEMP_DICT_START_ID)
141  , sqliteMutex_()
142  , sharedMutex_()
145  if (!is_new_db) {
147  }
148  buildMaps();
149  if (!is_new_db) {
151  }
152 }
153 
156  // must clean up heap-allocated TableDescriptor and ColumnDescriptor structs
157  for (TableDescriptorMap::iterator tableDescIt = tableDescriptorMap_.begin();
158  tableDescIt != tableDescriptorMap_.end();
159  ++tableDescIt) {
160  if (tableDescIt->second->fragmenter != nullptr) {
161  delete tableDescIt->second->fragmenter;
162  }
163  delete tableDescIt->second;
164  }
165 
166  // TableDescriptorMapById points to the same descriptors. No need to delete
167 
168  for (ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.begin();
169  columnDescIt != columnDescriptorMap_.end();
170  ++columnDescIt) {
171  delete columnDescIt->second;
172  }
173 
174  // ColumnDescriptorMapById points to the same descriptors. No need to delete
175 }
176 
179  sqliteConnector_.query("BEGIN TRANSACTION");
180  try {
181  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_tables)");
182  std::vector<std::string> cols;
183  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
184  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
185  }
186  if (std::find(cols.begin(), cols.end(), std::string("max_chunk_size")) ==
187  cols.end()) {
188  string queryString("ALTER TABLE mapd_tables ADD max_chunk_size BIGINT DEFAULT " +
190  sqliteConnector_.query(queryString);
191  }
192  if (std::find(cols.begin(), cols.end(), std::string("shard_column_id")) ==
193  cols.end()) {
194  string queryString("ALTER TABLE mapd_tables ADD shard_column_id BIGINT DEFAULT " +
195  std::to_string(0));
196  sqliteConnector_.query(queryString);
197  }
198  if (std::find(cols.begin(), cols.end(), std::string("shard")) == cols.end()) {
199  string queryString("ALTER TABLE mapd_tables ADD shard BIGINT DEFAULT " +
200  std::to_string(-1));
201  sqliteConnector_.query(queryString);
202  }
203  if (std::find(cols.begin(), cols.end(), std::string("num_shards")) == cols.end()) {
204  string queryString("ALTER TABLE mapd_tables ADD num_shards BIGINT DEFAULT " +
205  std::to_string(0));
206  sqliteConnector_.query(queryString);
207  }
208  if (std::find(cols.begin(), cols.end(), std::string("key_metainfo")) == cols.end()) {
209  string queryString("ALTER TABLE mapd_tables ADD key_metainfo TEXT DEFAULT '[]'");
210  sqliteConnector_.query(queryString);
211  }
212  if (std::find(cols.begin(), cols.end(), std::string("userid")) == cols.end()) {
213  string queryString("ALTER TABLE mapd_tables ADD userid integer DEFAULT " +
215  sqliteConnector_.query(queryString);
216  }
217  if (std::find(cols.begin(), cols.end(), std::string("sort_column_id")) ==
218  cols.end()) {
220  "ALTER TABLE mapd_tables ADD sort_column_id INTEGER DEFAULT 0");
221  }
222  } catch (std::exception& e) {
223  sqliteConnector_.query("ROLLBACK TRANSACTION");
224  throw;
225  }
226  sqliteConnector_.query("END TRANSACTION");
227 }
228 
231  sqliteConnector_.query("BEGIN TRANSACTION");
232  try {
234  "select name from sqlite_master WHERE type='table' AND "
235  "name='mapd_version_history'");
236  if (sqliteConnector_.getNumRows() == 0) {
238  "CREATE TABLE mapd_version_history(version integer, migration_history text "
239  "unique)");
240  } else {
242  "select * from mapd_version_history where migration_history = "
243  "'notnull_fixlen_arrays'");
244  if (sqliteConnector_.getNumRows() != 0) {
245  // legacy fixlen arrays had migrated
246  // no need for further execution
247  sqliteConnector_.query("END TRANSACTION");
248  return;
249  }
250  }
251  // Insert check for migration
253  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
254  std::vector<std::string>{std::to_string(MAPD_VERSION), "notnull_fixlen_arrays"});
255  LOG(INFO) << "Updating mapd_columns, legacy fixlen arrays";
256  // Upating all fixlen array columns
257  string queryString("UPDATE mapd_columns SET is_notnull=1 WHERE coltype=" +
258  std::to_string(kARRAY) + " AND size>0;");
259  sqliteConnector_.query(queryString);
260  } catch (std::exception& e) {
261  sqliteConnector_.query("ROLLBACK TRANSACTION");
262  throw;
263  }
264  sqliteConnector_.query("END TRANSACTION");
265 }
266 
269  sqliteConnector_.query("BEGIN TRANSACTION");
270  try {
271  // check table still exists
273  "SELECT name FROM sqlite_master WHERE type='table' AND "
274  "name='mapd_frontend_views'");
275  if (sqliteConnector_.getNumRows() == 0) {
276  // table does not exists
277  // no need to migrate
278  sqliteConnector_.query("END TRANSACTION");
279  return;
280  }
281  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_frontend_views)");
282  std::vector<std::string> cols;
283  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
284  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
285  }
286  if (std::find(cols.begin(), cols.end(), std::string("image_hash")) == cols.end()) {
287  sqliteConnector_.query("ALTER TABLE mapd_frontend_views ADD image_hash text");
288  }
289  if (std::find(cols.begin(), cols.end(), std::string("update_time")) == cols.end()) {
290  sqliteConnector_.query("ALTER TABLE mapd_frontend_views ADD update_time timestamp");
291  }
292  if (std::find(cols.begin(), cols.end(), std::string("view_metadata")) == cols.end()) {
293  sqliteConnector_.query("ALTER TABLE mapd_frontend_views ADD view_metadata text");
294  }
295  } catch (std::exception& e) {
296  sqliteConnector_.query("ROLLBACK TRANSACTION");
297  throw;
298  }
299  sqliteConnector_.query("END TRANSACTION");
300 }
301 
304  sqliteConnector_.query("BEGIN TRANSACTION");
305  try {
307  "CREATE TABLE IF NOT EXISTS mapd_links (linkid integer primary key, userid "
308  "integer references mapd_users, "
309  "link text unique, view_state text, update_time timestamp, view_metadata text)");
310  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_links)");
311  std::vector<std::string> cols;
312  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
313  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
314  }
315  if (std::find(cols.begin(), cols.end(), std::string("view_metadata")) == cols.end()) {
316  sqliteConnector_.query("ALTER TABLE mapd_links ADD view_metadata text");
317  }
318  } catch (const std::exception& e) {
319  sqliteConnector_.query("ROLLBACK TRANSACTION");
320  throw;
321  }
322  sqliteConnector_.query("END TRANSACTION");
323 }
324 
327  sqliteConnector_.query("BEGIN TRANSACTION");
328  try {
329  sqliteConnector_.query("UPDATE mapd_links SET userid = 0 WHERE userid IS NULL");
330  // check table still exists
332  "SELECT name FROM sqlite_master WHERE type='table' AND "
333  "name='mapd_frontend_views'");
334  if (sqliteConnector_.getNumRows() == 0) {
335  // table does not exists
336  // no need to migrate
337  sqliteConnector_.query("END TRANSACTION");
338  return;
339  }
341  "UPDATE mapd_frontend_views SET userid = 0 WHERE userid IS NULL");
342  } catch (const std::exception& e) {
343  sqliteConnector_.query("ROLLBACK TRANSACTION");
344  throw;
345  }
346  sqliteConnector_.query("END TRANSACTION");
347 }
348 
349 // introduce DB version into the tables table
350 // if the DB does not have a version reset all pagesizes to 2097152 to be compatible with
351 // old value
352 
355  if (currentDB_.dbName.length() == 0) {
356  // updateDictionaryNames dbName length is zero nothing to do here
357  return;
358  }
359  sqliteConnector_.query("BEGIN TRANSACTION");
360  try {
361  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_tables)");
362  std::vector<std::string> cols;
363  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
364  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
365  }
366  if (std::find(cols.begin(), cols.end(), std::string("version_num")) == cols.end()) {
367  LOG(INFO) << "Updating mapd_tables updatePageSize";
368  // No version number
369  // need to update the defaul tpagesize to old correct value
370  sqliteConnector_.query("UPDATE mapd_tables SET frag_page_size = 2097152 ");
371  // need to add new version info
372  string queryString("ALTER TABLE mapd_tables ADD version_num BIGINT DEFAULT " +
373  std::to_string(DEFAULT_INITIAL_VERSION));
374  sqliteConnector_.query(queryString);
375  }
376  } catch (std::exception& e) {
377  sqliteConnector_.query("ROLLBACK TRANSACTION");
378  throw;
379  }
380  sqliteConnector_.query("END TRANSACTION");
381 }
382 
385  sqliteConnector_.query("BEGIN TRANSACTION");
386  try {
387  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_columns)");
388  std::vector<std::string> cols;
389  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
390  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
391  }
392  if (std::find(cols.begin(), cols.end(), std::string("version_num")) == cols.end()) {
393  LOG(INFO) << "Updating mapd_columns updateDeletedColumnIndicator";
394  // need to add new version info
395  string queryString("ALTER TABLE mapd_columns ADD version_num BIGINT DEFAULT " +
396  std::to_string(DEFAULT_INITIAL_VERSION));
397  sqliteConnector_.query(queryString);
398  // need to add new column to table defintion to indicate deleted column, column used
399  // as bitmap for deleted rows.
401  "ALTER TABLE mapd_columns ADD is_deletedcol boolean default 0 ");
402  }
403  } catch (std::exception& e) {
404  sqliteConnector_.query("ROLLBACK TRANSACTION");
405  throw;
406  }
407  sqliteConnector_.query("END TRANSACTION");
408 }
409 
410 // introduce DB version into the dictionary tables
411 // if the DB does not have a version rename all dictionary tables
412 
415  if (currentDB_.dbName.length() == 0) {
416  // updateDictionaryNames dbName length is zero nothing to do here
417  return;
418  }
419  sqliteConnector_.query("BEGIN TRANSACTION");
420  try {
421  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_dictionaries)");
422  std::vector<std::string> cols;
423  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
424  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
425  }
426  if (std::find(cols.begin(), cols.end(), std::string("version_num")) == cols.end()) {
427  // No version number
428  // need to rename dictionaries
429  string dictQuery("SELECT dictid, name from mapd_dictionaries");
430  sqliteConnector_.query(dictQuery);
431  size_t numRows = sqliteConnector_.getNumRows();
432  for (size_t r = 0; r < numRows; ++r) {
433  int dictId = sqliteConnector_.getData<int>(r, 0);
434  std::string dictName = sqliteConnector_.getData<string>(r, 1);
435 
436  std::string oldName =
437  basePath_ + "/mapd_data/" + currentDB_.dbName + "_" + dictName;
438  std::string newName = basePath_ + "/mapd_data/DB_" +
439  std::to_string(currentDB_.dbId) + "_DICT_" +
440  std::to_string(dictId);
441 
442  int result = rename(oldName.c_str(), newName.c_str());
443 
444  if (result == 0) {
445  LOG(INFO) << "Dictionary upgrade: successfully renamed " << oldName << " to "
446  << newName;
447  } else {
448  LOG(ERROR) << "Failed to rename old dictionary directory " << oldName << " to "
449  << newName + " dbname '" << currentDB_.dbName << "' error code "
450  << std::to_string(result);
451  }
452  }
453  // need to add new version info
454  string queryString("ALTER TABLE mapd_dictionaries ADD version_num BIGINT DEFAULT " +
455  std::to_string(DEFAULT_INITIAL_VERSION));
456  sqliteConnector_.query(queryString);
457  }
458  } catch (std::exception& e) {
459  sqliteConnector_.query("ROLLBACK TRANSACTION");
460  throw;
461  }
462  sqliteConnector_.query("END TRANSACTION");
463 }
464 
467  sqliteConnector_.query("BEGIN TRANSACTION");
468  try {
470  "CREATE TABLE IF NOT EXISTS mapd_logical_to_physical("
471  "logical_table_id integer, physical_table_id integer)");
472  } catch (const std::exception& e) {
473  sqliteConnector_.query("ROLLBACK TRANSACTION");
474  throw;
475  }
476  sqliteConnector_.query("END TRANSACTION");
477 }
478 
479 void Catalog::updateLogicalToPhysicalTableMap(const int32_t logical_tb_id) {
480  /* this proc inserts/updates all pairs of (logical_tb_id, physical_tb_id) in
481  * sqlite mapd_logical_to_physical table for given logical_tb_id as needed
482  */
483 
485  sqliteConnector_.query("BEGIN TRANSACTION");
486  try {
487  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
488  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
489  const auto physicalTables = physicalTableIt->second;
490  CHECK(!physicalTables.empty());
491  for (size_t i = 0; i < physicalTables.size(); i++) {
492  int32_t physical_tb_id = physicalTables[i];
494  "INSERT OR REPLACE INTO mapd_logical_to_physical (logical_table_id, "
495  "physical_table_id) VALUES (?1, ?2)",
496  std::vector<std::string>{std::to_string(logical_tb_id),
497  std::to_string(physical_tb_id)});
498  }
499  }
500  } catch (std::exception& e) {
501  sqliteConnector_.query("ROLLBACK TRANSACTION");
502  throw;
503  }
504  sqliteConnector_.query("END TRANSACTION");
505 }
506 
509  sqliteConnector_.query("BEGIN TRANSACTION");
510  try {
511  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_dictionaries)");
512  std::vector<std::string> cols;
513  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
514  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
515  }
516  if (std::find(cols.begin(), cols.end(), std::string("refcount")) == cols.end()) {
517  sqliteConnector_.query("ALTER TABLE mapd_dictionaries ADD refcount DEFAULT 1");
518  }
519  } catch (std::exception& e) {
520  sqliteConnector_.query("ROLLBACK TRANSACTION");
521  throw;
522  }
523  sqliteConnector_.query("END TRANSACTION");
524 }
525 
528  sqliteConnector_.query("BEGIN TRANSACTION");
529  std::vector<DBObject> objects;
530  try {
532  "SELECT name FROM sqlite_master WHERE type='table' AND "
533  "name='mapd_record_ownership_marker'");
534  // check if mapd catalog - marker exists
535  if (sqliteConnector_.getNumRows() != 0 && currentDB_.dbId == 1) {
536  // already done
537  sqliteConnector_.query("END TRANSACTION");
538  return;
539  }
540  // check if different catalog - marker exists
541  else if (sqliteConnector_.getNumRows() != 0 && currentDB_.dbId != 1) {
542  sqliteConnector_.query("SELECT dummy FROM mapd_record_ownership_marker");
543  // Check if migration is being performed on existing non mapd catalogs
544  // Older non mapd dbs will have table but no record in them
545  if (sqliteConnector_.getNumRows() != 0) {
546  // already done
547  sqliteConnector_.query("END TRANSACTION");
548  return;
549  }
550  }
551  // marker not exists - create one
552  else {
553  sqliteConnector_.query("CREATE TABLE mapd_record_ownership_marker (dummy integer)");
554  }
555 
556  DBMetadata db;
557  CHECK(SysCatalog::instance().getMetadataForDB(currentDB_.dbName, db));
558  // place dbId as a refernce for migration being performed
560  "INSERT INTO mapd_record_ownership_marker (dummy) VALUES (?1)",
561  std::vector<std::string>{std::to_string(db.dbOwner)});
562 
563  static const std::map<const DBObjectType, const AccessPrivileges>
564  object_level_all_privs_lookup{
569 
570  // grant owner all permissions on DB
571  DBObjectKey key;
572  key.dbId = currentDB_.dbId;
573  auto _key_place = [&key](auto type) {
574  key.permissionType = type;
575  return key;
576  };
577  for (auto& it : object_level_all_privs_lookup) {
578  objects.emplace_back(_key_place(it.first), it.second, db.dbOwner);
579  objects.back().setName(currentDB_.dbName);
580  }
581 
582  {
583  // other users tables and views
584  string tableQuery(
585  "SELECT tableid, name, userid, isview FROM mapd_tables WHERE userid > 0");
586  sqliteConnector_.query(tableQuery);
587  size_t numRows = sqliteConnector_.getNumRows();
588  for (size_t r = 0; r < numRows; ++r) {
589  int32_t tableid = sqliteConnector_.getData<int>(r, 0);
590  std::string tableName = sqliteConnector_.getData<string>(r, 1);
591  int32_t ownerid = sqliteConnector_.getData<int>(r, 2);
592  bool isview = sqliteConnector_.getData<bool>(r, 3);
593 
596  DBObjectKey key;
597  key.dbId = currentDB_.dbId;
598  key.objectId = tableid;
599  key.permissionType = type;
600 
601  DBObject obj(tableName, type);
602  obj.setObjectKey(key);
603  obj.setOwner(ownerid);
604  obj.setPrivileges(isview ? AccessPrivileges::ALL_VIEW
606 
607  objects.push_back(obj);
608  }
609  }
610 
611  {
612  // other users dashboards
613  string tableQuery("SELECT id, name, userid FROM mapd_dashboards WHERE userid > 0");
614  sqliteConnector_.query(tableQuery);
615  size_t numRows = sqliteConnector_.getNumRows();
616  for (size_t r = 0; r < numRows; ++r) {
617  int32_t dashId = sqliteConnector_.getData<int>(r, 0);
618  std::string dashName = sqliteConnector_.getData<string>(r, 1);
619  int32_t ownerid = sqliteConnector_.getData<int>(r, 2);
620 
622  DBObjectKey key;
623  key.dbId = currentDB_.dbId;
624  key.objectId = dashId;
625  key.permissionType = type;
626 
627  DBObject obj(dashName, type);
628  obj.setObjectKey(key);
629  obj.setOwner(ownerid);
631 
632  objects.push_back(obj);
633  }
634  }
635  } catch (const std::exception& e) {
636  sqliteConnector_.query("ROLLBACK TRANSACTION");
637  throw;
638  }
639  sqliteConnector_.query("END TRANSACTION");
640 
641  // now apply the objects to the syscat to track the permisisons
642  // moved outside transaction to avoid lock in sqlite
643  try {
645  } catch (const std::exception& e) {
646  LOG(ERROR) << " Issue during migration of DB " << name() << " issue was " << e.what();
647  throw std::runtime_error(" Issue during migration of DB " + name() + " issue was " +
648  e.what());
649  // will need to remove the mapd_record_ownership_marker table and retry
650  }
651 }
652 
655  std::vector<int> tables_migrated = {};
656  std::unordered_map<int, std::vector<std::string>> tables_to_migrate;
657  sqliteConnector_.query("BEGIN TRANSACTION");
658  try {
660  "select name from sqlite_master WHERE type='table' AND "
661  "name='mapd_version_history'");
662  if (sqliteConnector_.getNumRows() == 0) {
664  "CREATE TABLE mapd_version_history(version integer, migration_history text "
665  "unique)");
667  "CREATE TABLE mapd_date_in_days_column_migration_tmp(table_id integer primary "
668  "key)");
669  } else {
671  "select * from mapd_version_history where migration_history = "
672  "'date_in_days_column'");
673  if (sqliteConnector_.getNumRows() != 0) {
674  // no need for further execution
675  sqliteConnector_.query("END TRANSACTION");
676  return;
677  }
678  LOG(INFO) << "Performing Date in days columns migration.";
680  "select name from sqlite_master where type='table' AND "
681  "name='mapd_date_in_days_column_migration_tmp'");
682  if (sqliteConnector_.getNumRows() != 0) {
684  "select table_id from mapd_date_in_days_column_migration_tmp");
685  if (sqliteConnector_.getNumRows() != 0) {
686  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
687  tables_migrated.push_back(sqliteConnector_.getData<int>(i, 0));
688  }
689  }
690  } else {
692  "CREATE TABLE mapd_date_in_days_column_migration_tmp(table_id integer "
693  "primary key)");
694  }
695  }
697  "SELECT tables.tableid, tables.name, columns.name FROM mapd_tables tables, "
698  "mapd_columns columns where tables.tableid = columns.tableid AND "
699  "columns.coltype = ?1 AND columns.compression = ?2",
700  std::vector<std::string>{
701  std::to_string(static_cast<int>(SQLTypes::kDATE)),
703  if (sqliteConnector_.getNumRows() != 0) {
704  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
705  tables_to_migrate[sqliteConnector_.getData<int>(i, 0)] = {
706  sqliteConnector_.getData<std::string>(i, 1),
707  sqliteConnector_.getData<std::string>(i, 2)};
708  }
709  }
710  } catch (const std::exception& e) {
711  LOG(ERROR) << "Failed to complete migraion on date in days column: " << e.what();
712  sqliteConnector_.query("ROLLBACK");
713  throw;
714  }
715  sqliteConnector_.query("END TRANSACTION");
716 
717  for (auto& id_names : tables_to_migrate) {
718  if (std::find(tables_migrated.begin(), tables_migrated.end(), id_names.first) ==
719  tables_migrated.end()) {
720  sqliteConnector_.query("BEGIN TRANSACTION");
721  try {
722  LOG(INFO) << "Table: " << id_names.second[0]
723  << " may suffer from issues with DATE column: " << id_names.second[1]
724  << ". Running an OPTIMIZE command to solve any issues with metadata.";
725 
726  auto executor = Executor::getExecutor(getCurrentDB().dbId);
727  TableDescriptorMapById::iterator tableDescIt =
728  tableDescriptorMapById_.find(id_names.first);
729  if (tableDescIt == tableDescriptorMapById_.end()) {
730  throw runtime_error("Table descriptor does not exist for table " +
731  id_names.second[0] + " does not exist.");
732  }
733  auto td = tableDescIt->second;
734  TableOptimizer optimizer(td, executor.get(), *this);
735  optimizer.recomputeMetadata();
736 
738  "INSERT INTO mapd_date_in_days_column_migration_tmp VALUES(?)",
739  std::vector<std::string>{std::to_string(id_names.first)});
740  } catch (const std::exception& e) {
741  LOG(ERROR) << "Failed to complete migraion on date in days column: " << e.what();
742  sqliteConnector_.query("ROLLBACK");
743  throw;
744  }
745  sqliteConnector_.query("COMMIT");
746  }
747  }
748 
749  sqliteConnector_.query("BEGIN TRANSACTION");
750  try {
751  sqliteConnector_.query("DROP TABLE mapd_date_in_days_column_migration_tmp");
753  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
754  std::vector<std::string>{std::to_string(MAPD_VERSION), "date_in_days_column"});
755  } catch (const std::exception& e) {
756  LOG(ERROR) << "Failed to complete migraion on date in days column: " << e.what();
757  sqliteConnector_.query("ROLLBACK");
758  throw;
759  }
760  sqliteConnector_.query("END TRANSACTION");
761  LOG(INFO) << "Migration successfull on Date in days columns";
762 }
763 
765  std::unordered_map<std::string, std::pair<int, std::string>> dashboards;
766  std::vector<std::string> dashboard_ids;
767  static const std::string migration_name{"dashboard_roles_migration"};
768  {
770  sqliteConnector_.query("BEGIN TRANSACTION");
771  try {
772  // migration_history should be present in all catalogs by now
773  // if not then would be created before this migration
775  "select * from mapd_version_history where migration_history = '" +
776  migration_name + "'");
777  if (sqliteConnector_.getNumRows() != 0) {
778  // no need for further execution
779  sqliteConnector_.query("END TRANSACTION");
780  return;
781  }
782  LOG(INFO) << "Performing dashboard internal roles Migration.";
783  sqliteConnector_.query("select id, userid, metadata from mapd_dashboards");
784  for (size_t i = 0; i < sqliteConnector_.getNumRows(); ++i) {
787  sqliteConnector_.getData<string>(i, 0)))) {
788  // Successfully created roles during previous migration/crash
789  // No need to include them
790  continue;
791  }
792  dashboards[sqliteConnector_.getData<string>(i, 0)] = std::make_pair(
793  sqliteConnector_.getData<int>(i, 1), sqliteConnector_.getData<string>(i, 2));
794  dashboard_ids.push_back(sqliteConnector_.getData<string>(i, 0));
795  }
796  } catch (const std::exception& e) {
797  sqliteConnector_.query("ROLLBACK TRANSACTION");
798  throw;
799  }
800  sqliteConnector_.query("END TRANSACTION");
801  }
802  // All current grantees with shared dashboards.
803  const auto active_grantees =
805 
806  try {
807  // NOTE(wamsi): Transactionally unsafe
808  for (auto dash : dashboards) {
809  createOrUpdateDashboardSystemRole(dash.second.second,
810  dash.second.first,
812  std::to_string(currentDB_.dbId), dash.first));
813  auto result = active_grantees.find(dash.first);
814  if (result != active_grantees.end()) {
817  dash.first)},
818  result->second);
819  }
820  }
822  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
823  std::vector<std::string>{std::to_string(MAPD_VERSION), migration_name});
824  } catch (const std::exception& e) {
825  LOG(ERROR) << "Failed to create dashboard system roles during migration: "
826  << e.what();
827  throw;
828  }
829  LOG(INFO) << "Successfully created dashboard system roles during migration.";
830 }
831 
841  updatePageSize();
845 }
846 
850 }
851 
852 namespace {
853 std::string getUserFromId(const int32_t id) {
855  if (SysCatalog::instance().getMetadataForUserById(id, user)) {
856  return user.userName;
857  }
858  // a user could be deleted and a dashboard still exist?
859  return "Unknown";
860 }
861 } // namespace
862 
866 
867  string dictQuery(
868  "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
869  sqliteConnector_.query(dictQuery);
870  size_t numRows = sqliteConnector_.getNumRows();
871  for (size_t r = 0; r < numRows; ++r) {
872  int dictId = sqliteConnector_.getData<int>(r, 0);
873  std::string dictName = sqliteConnector_.getData<string>(r, 1);
874  int dictNBits = sqliteConnector_.getData<int>(r, 2);
875  bool is_shared = sqliteConnector_.getData<bool>(r, 3);
876  int refcount = sqliteConnector_.getData<int>(r, 4);
877  std::string fname = basePath_ + "/mapd_data/DB_" + std::to_string(currentDB_.dbId) +
878  "_DICT_" + std::to_string(dictId);
879  DictRef dict_ref(currentDB_.dbId, dictId);
880  DictDescriptor* dd = new DictDescriptor(
881  dict_ref, dictName, dictNBits, is_shared, refcount, fname, false);
882  dictDescriptorMapByRef_[dict_ref].reset(dd);
883  }
884 
885  string tableQuery(
886  "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
887  "max_chunk_size, frag_page_size, "
888  "max_rows, partitions, shard_column_id, shard, num_shards, key_metainfo, userid, "
889  "sort_column_id "
890  "from mapd_tables");
891  sqliteConnector_.query(tableQuery);
892  numRows = sqliteConnector_.getNumRows();
893  for (size_t r = 0; r < numRows; ++r) {
894  TableDescriptor* td = new TableDescriptor();
895  td->tableId = sqliteConnector_.getData<int>(r, 0);
896  td->tableName = sqliteConnector_.getData<string>(r, 1);
897  td->nColumns = sqliteConnector_.getData<int>(r, 2);
898  td->isView = sqliteConnector_.getData<bool>(r, 3);
899  td->fragments = sqliteConnector_.getData<string>(r, 4);
900  td->fragType =
902  td->maxFragRows = sqliteConnector_.getData<int>(r, 6);
903  td->maxChunkSize = sqliteConnector_.getData<int>(r, 7);
904  td->fragPageSize = sqliteConnector_.getData<int>(r, 8);
905  td->maxRows = sqliteConnector_.getData<int64_t>(r, 9);
906  td->partitions = sqliteConnector_.getData<string>(r, 10);
907  td->shardedColumnId = sqliteConnector_.getData<int>(r, 11);
908  td->shard = sqliteConnector_.getData<int>(r, 12);
909  td->nShards = sqliteConnector_.getData<int>(r, 13);
910  td->keyMetainfo = sqliteConnector_.getData<string>(r, 14);
911  td->userId = sqliteConnector_.getData<int>(r, 15);
912  td->sortedColumnId =
913  sqliteConnector_.isNull(r, 16) ? 0 : sqliteConnector_.getData<int>(r, 16);
914  if (!td->isView) {
915  td->fragmenter = nullptr;
916  }
917  td->hasDeletedCol = false;
920  }
921  string columnQuery(
922  "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
923  "is_notnull, compression, comp_param, "
924  "size, chunks, is_systemcol, is_virtualcol, virtual_expr, is_deletedcol from "
925  "mapd_columns ORDER BY tableid, "
926  "columnid");
927  sqliteConnector_.query(columnQuery);
928  numRows = sqliteConnector_.getNumRows();
929  int32_t skip_physical_cols = 0;
930  for (size_t r = 0; r < numRows; ++r) {
932  cd->tableId = sqliteConnector_.getData<int>(r, 0);
933  cd->columnId = sqliteConnector_.getData<int>(r, 1);
934  cd->columnName = sqliteConnector_.getData<string>(r, 2);
942  cd->columnType.set_size(sqliteConnector_.getData<int>(r, 10));
943  cd->chunks = sqliteConnector_.getData<string>(r, 11);
944  cd->isSystemCol = sqliteConnector_.getData<bool>(r, 12);
945  cd->isVirtualCol = sqliteConnector_.getData<bool>(r, 13);
946  cd->virtualExpr = sqliteConnector_.getData<string>(r, 14);
947  cd->isDeletedCol = sqliteConnector_.getData<bool>(r, 15);
948  cd->isGeoPhyCol = skip_physical_cols > 0;
949  ColumnKey columnKey(cd->tableId, to_upper(cd->columnName));
950  columnDescriptorMap_[columnKey] = cd;
951  ColumnIdKey columnIdKey(cd->tableId, cd->columnId);
952  columnDescriptorMapById_[columnIdKey] = cd;
953 
954  if (skip_physical_cols <= 0) {
955  skip_physical_cols = cd->columnType.get_physical_cols();
956  }
957 
958  auto td_itr = tableDescriptorMapById_.find(cd->tableId);
959  CHECK(td_itr != tableDescriptorMapById_.end());
960 
961  if (cd->isDeletedCol) {
962  td_itr->second->hasDeletedCol = true;
963  setDeletedColumnUnlocked(td_itr->second, cd);
964  } else if (cd->columnType.is_geometry() || skip_physical_cols-- <= 0) {
965  tableDescriptorMapById_[cd->tableId]->columnIdBySpi_.push_back(cd->columnId);
966  }
967  }
968  // sort columnIdBySpi_ based on columnId
969  for (auto& tit : tableDescriptorMapById_) {
970  std::sort(tit.second->columnIdBySpi_.begin(),
971  tit.second->columnIdBySpi_.end(),
972  [](const size_t a, const size_t b) -> bool { return a < b; });
973  }
974 
975  string viewQuery("SELECT tableid, sql FROM mapd_views");
976  sqliteConnector_.query(viewQuery);
977  numRows = sqliteConnector_.getNumRows();
978  for (size_t r = 0; r < numRows; ++r) {
979  int32_t tableId = sqliteConnector_.getData<int>(r, 0);
980  TableDescriptor* td = tableDescriptorMapById_[tableId];
981  td->viewSQL = sqliteConnector_.getData<string>(r, 1);
982  td->fragmenter = nullptr;
983  }
984 
985  string frontendViewQuery(
986  "SELECT id, state, name, image_hash, strftime('%Y-%m-%dT%H:%M:%SZ', update_time), "
987  "userid, "
988  "metadata "
989  "FROM mapd_dashboards");
990  sqliteConnector_.query(frontendViewQuery);
991  numRows = sqliteConnector_.getNumRows();
992  for (size_t r = 0; r < numRows; ++r) {
993  std::shared_ptr<DashboardDescriptor> vd = std::make_shared<DashboardDescriptor>();
994  vd->dashboardId = sqliteConnector_.getData<int>(r, 0);
995  vd->dashboardState = sqliteConnector_.getData<string>(r, 1);
996  vd->dashboardName = sqliteConnector_.getData<string>(r, 2);
997  vd->imageHash = sqliteConnector_.getData<string>(r, 3);
998  vd->updateTime = sqliteConnector_.getData<string>(r, 4);
999  vd->userId = sqliteConnector_.getData<int>(r, 5);
1000  vd->dashboardMetadata = sqliteConnector_.getData<string>(r, 6);
1001  vd->user = getUserFromId(vd->userId);
1002  vd->dashboardSystemRoleName = generate_dashboard_system_rolename(
1004  dashboardDescriptorMap_[std::to_string(vd->userId) + ":" + vd->dashboardName] = vd;
1005  }
1006 
1007  string linkQuery(
1008  "SELECT linkid, userid, link, view_state, strftime('%Y-%m-%dT%H:%M:%SZ', "
1009  "update_time), view_metadata "
1010  "FROM mapd_links");
1011  sqliteConnector_.query(linkQuery);
1012  numRows = sqliteConnector_.getNumRows();
1013  for (size_t r = 0; r < numRows; ++r) {
1014  LinkDescriptor* ld = new LinkDescriptor();
1015  ld->linkId = sqliteConnector_.getData<int>(r, 0);
1016  ld->userId = sqliteConnector_.getData<int>(r, 1);
1017  ld->link = sqliteConnector_.getData<string>(r, 2);
1018  ld->viewState = sqliteConnector_.getData<string>(r, 3);
1019  ld->updateTime = sqliteConnector_.getData<string>(r, 4);
1020  ld->viewMetadata = sqliteConnector_.getData<string>(r, 5);
1022  linkDescriptorMapById_[ld->linkId] = ld;
1023  }
1024 
1025  /* rebuild map linking logical tables to corresponding physical ones */
1026  string logicalToPhysicalTableMapQuery(
1027  "SELECT logical_table_id, physical_table_id "
1028  "FROM mapd_logical_to_physical");
1029  sqliteConnector_.query(logicalToPhysicalTableMapQuery);
1030  numRows = sqliteConnector_.getNumRows();
1031  for (size_t r = 0; r < numRows; ++r) {
1032  int32_t logical_tb_id = sqliteConnector_.getData<int>(r, 0);
1033  int32_t physical_tb_id = sqliteConnector_.getData<int>(r, 1);
1034  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
1035  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
1036  /* add new entity to the map logicalToPhysicalTableMapById_ */
1037  std::vector<int32_t> physicalTables;
1038  physicalTables.push_back(physical_tb_id);
1039  const auto it_ok =
1040  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
1041  CHECK(it_ok.second);
1042  } else {
1043  /* update map logicalToPhysicalTableMapById_ */
1044  physicalTableIt->second.push_back(physical_tb_id);
1045  }
1046  }
1047 }
1048 
1050  const list<ColumnDescriptor>& columns,
1051  const list<DictDescriptor>& dicts) {
1052  cat_write_lock write_lock(this);
1053  TableDescriptor* new_td = new TableDescriptor();
1054  *new_td = td;
1055  new_td->mutex_ = std::make_shared<std::mutex>();
1056  tableDescriptorMap_[to_upper(td.tableName)] = new_td;
1057  tableDescriptorMapById_[td.tableId] = new_td;
1058  for (auto cd : columns) {
1059  ColumnDescriptor* new_cd = new ColumnDescriptor();
1060  *new_cd = cd;
1061  ColumnKey columnKey(new_cd->tableId, to_upper(new_cd->columnName));
1062  columnDescriptorMap_[columnKey] = new_cd;
1063  ColumnIdKey columnIdKey(new_cd->tableId, new_cd->columnId);
1064  columnDescriptorMapById_[columnIdKey] = new_cd;
1065 
1066  // Add deleted column to the map
1067  if (cd.isDeletedCol) {
1068  CHECK(new_td->hasDeletedCol);
1069  setDeletedColumnUnlocked(new_td, new_cd);
1070  }
1071  }
1072 
1073  std::sort(new_td->columnIdBySpi_.begin(),
1074  new_td->columnIdBySpi_.end(),
1075  [](const size_t a, const size_t b) -> bool { return a < b; });
1076 
1077  std::unique_ptr<StringDictionaryClient> client;
1078  DictRef dict_ref(currentDB_.dbId, -1);
1079  if (!string_dict_hosts_.empty()) {
1080  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1081  }
1082  for (auto dd : dicts) {
1083  if (!dd.dictRef.dictId) {
1084  // Dummy entry created for a shard of a logical table, nothing to do.
1085  continue;
1086  }
1087  dict_ref.dictId = dd.dictRef.dictId;
1088  if (client) {
1089  client->create(dict_ref, dd.dictIsTemp);
1090  }
1091  DictDescriptor* new_dd = new DictDescriptor(dd);
1092  dictDescriptorMapByRef_[dict_ref].reset(new_dd);
1093  if (!dd.dictIsTemp) {
1094  boost::filesystem::create_directory(new_dd->dictFolderPath);
1095  }
1096  }
1097 }
1098 
1099 void Catalog::removeTableFromMap(const string& tableName,
1100  const int tableId,
1101  const bool is_on_error) {
1102  cat_write_lock write_lock(this);
1103  TableDescriptorMapById::iterator tableDescIt = tableDescriptorMapById_.find(tableId);
1104  if (tableDescIt == tableDescriptorMapById_.end()) {
1105  throw runtime_error("Table " + tableName + " does not exist.");
1106  }
1107 
1108  TableDescriptor* td = tableDescIt->second;
1109 
1110  if (td->hasDeletedCol) {
1111  const auto ret = deletedColumnPerTable_.erase(td);
1112  CHECK_EQ(ret, size_t(1));
1113  }
1114 
1115  tableDescriptorMapById_.erase(tableDescIt);
1116  tableDescriptorMap_.erase(to_upper(tableName));
1117  if (td->fragmenter != nullptr) {
1118  delete td->fragmenter;
1119  }
1121  delete td;
1122 
1123  std::unique_ptr<StringDictionaryClient> client;
1124  if (SysCatalog::instance().isAggregator()) {
1125  CHECK(!string_dict_hosts_.empty());
1126  DictRef dict_ref(currentDB_.dbId, -1);
1127  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1128  }
1129 
1130  // delete all column descriptors for the table
1131  // no more link columnIds to sequential indexes!
1132  for (auto cit = columnDescriptorMapById_.begin();
1133  cit != columnDescriptorMapById_.end();) {
1134  if (tableId != std::get<0>(cit->first)) {
1135  ++cit;
1136  } else {
1137  int i = std::get<1>(cit++->first);
1138  ColumnIdKey cidKey(tableId, i);
1139  ColumnDescriptorMapById::iterator colDescIt = columnDescriptorMapById_.find(cidKey);
1140  ColumnDescriptor* cd = colDescIt->second;
1141  columnDescriptorMapById_.erase(colDescIt);
1142  ColumnKey cnameKey(tableId, to_upper(cd->columnName));
1143  columnDescriptorMap_.erase(cnameKey);
1144  const int dictId = cd->columnType.get_comp_param();
1145  // Dummy dictionaries created for a shard of a logical table have the id set to
1146  // zero.
1147  if (cd->columnType.get_compression() == kENCODING_DICT && dictId) {
1148  INJECT_TIMER(removingDicts);
1149  DictRef dict_ref(currentDB_.dbId, dictId);
1150  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
1151  // If we're removing this table due to an error, it is possible that the string
1152  // dictionary reference was never populated. Don't crash, just continue cleaning
1153  // up the TableDescriptor and ColumnDescriptors
1154  if (!is_on_error) {
1155  CHECK(dictIt != dictDescriptorMapByRef_.end());
1156  } else {
1157  if (dictIt == dictDescriptorMapByRef_.end()) {
1158  continue;
1159  }
1160  }
1161  const auto& dd = dictIt->second;
1162  CHECK_GE(dd->refcount, 1);
1163  --dd->refcount;
1164  if (!dd->refcount) {
1165  dd->stringDict.reset();
1166  if (!isTemp) {
1167  File_Namespace::renameForDelete(dd->dictFolderPath);
1168  }
1169  if (client) {
1170  client->drop(dict_ref);
1171  }
1172  dictDescriptorMapByRef_.erase(dictIt);
1173  }
1174  }
1175 
1176  delete cd;
1177  }
1178  }
1179 }
1180 
1182  cat_write_lock write_lock(this);
1184 }
1185 
1187  cat_write_lock write_lock(this);
1189  std::make_shared<DashboardDescriptor>(vd);
1190 }
1191 
1192 std::vector<DBObject> Catalog::parseDashboardObjects(const std::string& view_meta,
1193  const int& user_id) {
1194  std::vector<DBObject> objects;
1195  DBObjectKey key;
1196  key.dbId = currentDB_.dbId;
1197  auto _key_place = [&key](auto type, auto id) {
1198  key.permissionType = type;
1199  key.objectId = id;
1200  return key;
1201  };
1202  for (auto object_name : parse_underlying_dashboard_objects(view_meta)) {
1203  auto td = getMetadataForTable(object_name);
1204  if (!td) {
1205  // Parsed object source is not present in current database
1206  // LOG the info and ignore
1207  LOG(INFO) << "Ignoring dashboard source Table/View: " << object_name
1208  << " no longer exists in current DB.";
1209  continue;
1210  }
1211  // Dashboard source can be Table or View
1212  const auto object_type = td->isView ? ViewDBObjectType : TableDBObjectType;
1213  const auto priv = td->isView ? AccessPrivileges::SELECT_FROM_VIEW
1215  objects.emplace_back(_key_place(object_type, td->tableId), priv, user_id);
1216  objects.back().setObjectType(td->isView ? ViewDBObjectType : TableDBObjectType);
1217  objects.back().setName(td->tableName);
1218  }
1219  return objects;
1220 }
1221 
1222 void Catalog::createOrUpdateDashboardSystemRole(const std::string& view_meta,
1223  const int32_t& user_id,
1224  const std::string& dash_role_name) {
1225  auto objects = parseDashboardObjects(view_meta, user_id);
1226  Role* rl = SysCatalog::instance().getRoleGrantee(dash_role_name);
1227  if (!rl) {
1228  // Dashboard role does not exist
1229  // create role and grant privileges
1230  // NOTE(wamsi): Transactionally unsafe
1231  SysCatalog::instance().createRole(dash_role_name, false);
1232  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1233  } else {
1234  // Dashboard system role already exists
1235  // Add/remove privileges on objects
1236  auto ex_objects = rl->getDbObjects(true);
1237  for (auto key : *ex_objects | boost::adaptors::map_keys) {
1238  if (key.permissionType != TableDBObjectType &&
1239  key.permissionType != ViewDBObjectType) {
1240  continue;
1241  }
1242  bool found = false;
1243  for (auto obj : objects) {
1244  found = key == obj.getObjectKey() ? true : false;
1245  if (found) {
1246  break;
1247  }
1248  }
1249  if (!found) {
1250  // revoke privs on object since the object is no
1251  // longer used by the dashboard as source
1252  // NOTE(wamsi): Transactionally unsafe
1254  dash_role_name, *rl->findDbObject(key, true), *this);
1255  }
1256  }
1257  // Update privileges on remaining objects
1258  // NOTE(wamsi): Transactionally unsafe
1259  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1260  }
1261 }
1262 
1264  cat_write_lock write_lock(this);
1265  LinkDescriptor* new_ld = new LinkDescriptor();
1266  *new_ld = ld;
1268  linkDescriptorMapById_[ld.linkId] = new_ld;
1269 }
1270 
1272  auto time_ms = measure<>::execution([&]() {
1273  // instanciate table fragmenter upon first use
1274  // assume only insert order fragmenter is supported
1276  vector<Chunk> chunkVec;
1277  list<const ColumnDescriptor*> columnDescs;
1278  getAllColumnMetadataForTable(td, columnDescs, true, false, true);
1279  Chunk::translateColumnDescriptorsToChunkVec(columnDescs, chunkVec);
1280  ChunkKey chunkKeyPrefix = {currentDB_.dbId, td->tableId};
1281  if (td->sortedColumnId > 0) {
1282  td->fragmenter = new SortedOrderFragmenter(chunkKeyPrefix,
1283  chunkVec,
1284  dataMgr_.get(),
1285  const_cast<Catalog*>(this),
1286  td->tableId,
1287  td->shard,
1288  td->maxFragRows,
1289  td->maxChunkSize,
1290  td->fragPageSize,
1291  td->maxRows,
1292  td->persistenceLevel);
1293  } else {
1294  td->fragmenter = new InsertOrderFragmenter(chunkKeyPrefix,
1295  chunkVec,
1296  dataMgr_.get(),
1297  const_cast<Catalog*>(this),
1298  td->tableId,
1299  td->shard,
1300  td->maxFragRows,
1301  td->maxChunkSize,
1302  td->fragPageSize,
1303  td->maxRows,
1304  td->persistenceLevel);
1305  }
1306  });
1307  LOG(INFO) << "Instantiating Fragmenter for table " << td->tableName << " took "
1308  << time_ms << "ms";
1309 }
1310 
1311 const TableDescriptor* Catalog::getMetadataForTable(const string& tableName,
1312  const bool populateFragmenter) const {
1313  // we give option not to populate fragmenter (default true/yes) as it can be heavy for
1314  // pure metadata calls
1315  cat_read_lock read_lock(this);
1316  auto tableDescIt = tableDescriptorMap_.find(to_upper(tableName));
1317  if (tableDescIt == tableDescriptorMap_.end()) { // check to make sure table exists
1318  return nullptr;
1319  }
1320  TableDescriptor* td = tableDescIt->second;
1321  std::unique_lock<std::mutex> td_lock(*td->mutex_.get());
1322  if (populateFragmenter && td->fragmenter == nullptr && !td->isView) {
1324  }
1325  return td; // returns pointer to table descriptor
1326 }
1327 
1329  int tableId,
1330  const bool populateFragmenter) const {
1331  cat_read_lock read_lock(this);
1332  auto tableDescIt = tableDescriptorMapById_.find(tableId);
1333  if (tableDescIt == tableDescriptorMapById_.end()) { // check to make sure table exists
1334  return nullptr;
1335  }
1336  TableDescriptor* td = tableDescIt->second;
1337  std::unique_lock<std::mutex> td_lock(*td->mutex_.get());
1338  if (populateFragmenter && td->fragmenter == nullptr && !td->isView) {
1340  }
1341  return td; // returns pointer to table descriptor
1342 }
1343 
1345  return getMetadataForTableImpl(tableId, true);
1346 }
1347 
1349  const bool loadDict) const {
1350  const DictRef dictRef(currentDB_.dbId, dictId);
1351  cat_read_lock read_lock(this);
1352  auto dictDescIt = dictDescriptorMapByRef_.find(dictRef);
1353  if (dictDescIt ==
1354  dictDescriptorMapByRef_.end()) { // check to make sure dictionary exists
1355  return nullptr;
1356  }
1357  auto& dd = dictDescIt->second;
1358 
1359  if (loadDict) {
1361  if (!dd->stringDict) {
1362  auto time_ms = measure<>::execution([&]() {
1363  if (string_dict_hosts_.empty()) {
1364  if (dd->dictIsTemp) {
1365  dd->stringDict = std::make_shared<StringDictionary>(
1366  dd->dictFolderPath, true, true, g_cache_string_hash);
1367  } else {
1368  dd->stringDict = std::make_shared<StringDictionary>(
1369  dd->dictFolderPath, false, true, g_cache_string_hash);
1370  }
1371  } else {
1372  dd->stringDict =
1373  std::make_shared<StringDictionary>(string_dict_hosts_.front(), dd->dictRef);
1374  }
1375  });
1376  LOG(INFO) << "Time to load Dictionary " << dd->dictRef.dbId << "_"
1377  << dd->dictRef.dictId << " was " << time_ms << "ms";
1378  }
1379  }
1380 
1381  return dd.get();
1382 }
1383 
1384 const std::vector<LeafHostInfo>& Catalog::getStringDictionaryHosts() const {
1385  return string_dict_hosts_;
1386 }
1387 
1389  const string& columnName) const {
1390  cat_read_lock read_lock(this);
1391 
1392  ColumnKey columnKey(tableId, to_upper(columnName));
1393  auto colDescIt = columnDescriptorMap_.find(columnKey);
1394  if (colDescIt ==
1395  columnDescriptorMap_.end()) { // need to check to make sure column exists for table
1396  return nullptr;
1397  }
1398  return colDescIt->second;
1399 }
1400 
1401 const ColumnDescriptor* Catalog::getMetadataForColumn(int tableId, int columnId) const {
1402  cat_read_lock read_lock(this);
1403 
1404  ColumnIdKey columnIdKey(tableId, columnId);
1405  auto colDescIt = columnDescriptorMapById_.find(columnIdKey);
1406  if (colDescIt == columnDescriptorMapById_
1407  .end()) { // need to check to make sure column exists for table
1408  return nullptr;
1409  }
1410  return colDescIt->second;
1411 }
1412 
1413 const int Catalog::getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const {
1414  const auto tabDescIt = tableDescriptorMapById_.find(table_id);
1415  CHECK(tableDescriptorMapById_.end() != tabDescIt);
1416  const auto& columnIdBySpi = tabDescIt->second->columnIdBySpi_;
1417 
1418  auto spx = spi;
1419  int phi = 0;
1420  if (spx >= SPIMAP_MAGIC1) // see Catalog.h
1421  {
1422  phi = (spx - SPIMAP_MAGIC1) % SPIMAP_MAGIC2;
1423  spx = (spx - SPIMAP_MAGIC1) / SPIMAP_MAGIC2;
1424  }
1425 
1426  CHECK(0 < spx && spx <= columnIdBySpi.size());
1427  return columnIdBySpi[spx - 1] + phi;
1428 }
1429 
1430 const int Catalog::getColumnIdBySpi(const int table_id, const size_t spi) const {
1431  cat_read_lock read_lock(this);
1432  return getColumnIdBySpiUnlocked(table_id, spi);
1433 }
1434 
1436  const size_t spi) const {
1437  cat_read_lock read_lock(this);
1438 
1439  const auto columnId = getColumnIdBySpiUnlocked(tableId, spi);
1440  ColumnIdKey columnIdKey(tableId, columnId);
1441  const auto colDescIt = columnDescriptorMapById_.find(columnIdKey);
1442  return columnDescriptorMapById_.end() == colDescIt ? nullptr : colDescIt->second;
1443 }
1444 
1445 void Catalog::deleteMetadataForDashboard(const std::string& userId,
1446  const std::string& dashName) {
1447  cat_write_lock write_lock(this);
1448 
1449  auto viewDescIt = dashboardDescriptorMap_.find(userId + ":" + dashName);
1450  if (viewDescIt == dashboardDescriptorMap_.end()) { // check to make sure view exists
1451  LOG(ERROR) << "No metadata for dashboard for user " << userId << " dashboard "
1452  << dashName << " does not exist in map";
1453  throw runtime_error("No metadata for dashboard for user " + userId + " dashboard " +
1454  dashName + " does not exist in map");
1455  }
1456  // found view in Map now remove it
1457  dashboardDescriptorMap_.erase(viewDescIt);
1458  // remove from DB
1460  sqliteConnector_.query("BEGIN TRANSACTION");
1461  try {
1463  "DELETE FROM mapd_dashboards WHERE name = ? and userid = ?",
1464  std::vector<std::string>{dashName, userId});
1465  } catch (std::exception& e) {
1466  sqliteConnector_.query("ROLLBACK TRANSACTION");
1467  throw;
1468  }
1469  sqliteConnector_.query("END TRANSACTION");
1470 }
1471 
1473  const string& userId,
1474  const string& dashName) const {
1475  cat_read_lock read_lock(this);
1476 
1477  auto viewDescIt = dashboardDescriptorMap_.find(userId + ":" + dashName);
1478  if (viewDescIt == dashboardDescriptorMap_.end()) { // check to make sure view exists
1479  return nullptr;
1480  }
1481  return viewDescIt->second.get(); // returns pointer to view descriptor
1482 }
1483 
1485  cat_read_lock read_lock(this);
1486  std::string userId;
1487  std::string name;
1488  bool found{false};
1489  {
1490  for (auto descp : dashboardDescriptorMap_) {
1491  auto dash = descp.second.get();
1492  if (dash->dashboardId == id) {
1493  userId = std::to_string(dash->userId);
1494  name = dash->dashboardName;
1495  found = true;
1496  break;
1497  }
1498  }
1499  }
1500  if (found) {
1501  return getMetadataForDashboard(userId, name);
1502  }
1503  return nullptr;
1504 }
1505 
1506 void Catalog::deleteMetadataForDashboard(const int32_t id) {
1507  std::string userId;
1508  std::string name;
1509  bool found{false};
1510  {
1511  cat_read_lock read_lock(this);
1512  for (auto descp : dashboardDescriptorMap_) {
1513  auto dash = descp.second.get();
1514  if (dash->dashboardId == id) {
1515  userId = std::to_string(dash->userId);
1516  name = dash->dashboardName;
1517  found = true;
1518  break;
1519  }
1520  }
1521  }
1522  if (found) {
1523  // TODO: transactionally unsafe
1525  DBObject(id, DashboardDBObjectType), this);
1526  deleteMetadataForDashboard(userId, name);
1527  }
1528 }
1529 
1530 const LinkDescriptor* Catalog::getMetadataForLink(const string& link) const {
1531  cat_read_lock read_lock(this);
1532  auto linkDescIt = linkDescriptorMap_.find(link);
1533  if (linkDescIt == linkDescriptorMap_.end()) { // check to make sure view exists
1534  return nullptr;
1535  }
1536  return linkDescIt->second; // returns pointer to view descriptor
1537 }
1538 
1540  cat_read_lock read_lock(this);
1541  auto linkDescIt = linkDescriptorMapById_.find(linkId);
1542  if (linkDescIt == linkDescriptorMapById_.end()) { // check to make sure view exists
1543  return nullptr;
1544  }
1545  return linkDescIt->second;
1546 }
1547 
1549  const TableDescriptor* td,
1550  list<const ColumnDescriptor*>& columnDescriptors,
1551  const bool fetchSystemColumns,
1552  const bool fetchVirtualColumns,
1553  const bool fetchPhysicalColumns) const {
1554  cat_read_lock read_lock(this);
1555  int32_t skip_physical_cols = 0;
1556  for (const auto& columnDescriptor : columnDescriptorMapById_) {
1557  if (!fetchPhysicalColumns && skip_physical_cols > 0) {
1558  --skip_physical_cols;
1559  continue;
1560  }
1561  auto cd = columnDescriptor.second;
1562  if (cd->tableId != td->tableId) {
1563  continue;
1564  }
1565  if (!fetchSystemColumns && cd->isSystemCol) {
1566  continue;
1567  }
1568  if (!fetchVirtualColumns && cd->isVirtualCol) {
1569  continue;
1570  }
1571  if (!fetchPhysicalColumns) {
1572  const auto& col_ti = cd->columnType;
1573  skip_physical_cols = col_ti.get_physical_cols();
1574  }
1575  columnDescriptors.push_back(cd);
1576  }
1577 }
1578 
1579 list<const ColumnDescriptor*> Catalog::getAllColumnMetadataForTable(
1580  const int tableId,
1581  const bool fetchSystemColumns,
1582  const bool fetchVirtualColumns,
1583  const bool fetchPhysicalColumns) const {
1584  cat_read_lock read_lock(this);
1585  list<const ColumnDescriptor*> columnDescriptors;
1586  const TableDescriptor* td =
1587  getMetadataForTableImpl(tableId, false); // dont instantiate fragmenter
1589  columnDescriptors,
1590  fetchSystemColumns,
1591  fetchVirtualColumns,
1592  fetchPhysicalColumns);
1593  return columnDescriptors;
1594 }
1595 
1596 list<const TableDescriptor*> Catalog::getAllTableMetadata() const {
1597  cat_read_lock read_lock(this);
1598  list<const TableDescriptor*> table_list;
1599  for (auto p : tableDescriptorMapById_) {
1600  table_list.push_back(p.second);
1601  }
1602  return table_list;
1603 }
1604 
1605 list<const DashboardDescriptor*> Catalog::getAllDashboardsMetadata() const {
1606  list<const DashboardDescriptor*> view_list;
1607  for (auto p : dashboardDescriptorMap_) {
1608  view_list.push_back(p.second.get());
1609  }
1610  return view_list;
1611 }
1612 
1614  const auto& td = *tableDescriptorMapById_[cd.tableId];
1615  list<DictDescriptor> dds;
1616  setColumnDictionary(cd, dds, td, true);
1617  auto& dd = dds.back();
1618  CHECK(dd.dictRef.dictId);
1619 
1620  std::unique_ptr<StringDictionaryClient> client;
1621  if (!string_dict_hosts_.empty()) {
1622  client.reset(new StringDictionaryClient(
1623  string_dict_hosts_.front(), DictRef(currentDB_.dbId, -1), true));
1624  }
1625  if (client) {
1626  client->create(dd.dictRef, dd.dictIsTemp);
1627  }
1628 
1629  DictDescriptor* new_dd = new DictDescriptor(dd);
1630  dictDescriptorMapByRef_[dd.dictRef].reset(new_dd);
1631  if (!dd.dictIsTemp) {
1632  boost::filesystem::create_directory(new_dd->dictFolderPath);
1633  }
1634  return dd.dictRef;
1635 }
1636 
1637 // TODO this all looks incorrect there is no reference count being checked here
1638 // this code needs to be fixed
1640  if (!(cd.columnType.is_string() || cd.columnType.is_string_array())) {
1641  return;
1642  }
1643  if (!(cd.columnType.get_compression() == kENCODING_DICT)) {
1644  return;
1645  }
1646  if (!(cd.columnType.get_comp_param() > 0)) {
1647  return;
1648  }
1649 
1650  const auto& td = *tableDescriptorMapById_[cd.tableId];
1651  const auto dictId = cd.columnType.get_comp_param();
1652  const DictRef dictRef(currentDB_.dbId, dictId);
1653  const auto dictName =
1654  td.tableName + "_" + cd.columnName + "_dict" + std::to_string(dictId);
1655  sqliteConnector_.query_with_text_param("DELETE FROM mapd_dictionaries WHERE name = ?",
1656  dictName);
1657  File_Namespace::renameForDelete(basePath_ + "/mapd_data/DB_" +
1658  std::to_string(currentDB_.dbId) + "_DICT_" +
1659  std::to_string(dictId));
1660 
1661  std::unique_ptr<StringDictionaryClient> client;
1662  if (!string_dict_hosts_.empty()) {
1663  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dictRef, true));
1664  }
1665  if (client) {
1666  client->drop(dictRef);
1667  }
1668 
1669  dictDescriptorMapByRef_.erase(dictRef);
1670 }
1671 
1673  std::map<int, StringDictionary*>& stringDicts) {
1674  // learn 'committed' ColumnDescriptor of this column
1675  auto cit = columnDescriptorMap_.find(ColumnKey(cd.tableId, to_upper(cd.columnName)));
1676  CHECK(cit != columnDescriptorMap_.end());
1677  auto& ccd = *cit->second;
1678 
1679  if (!(ccd.columnType.is_string() || ccd.columnType.is_string_array())) {
1680  return;
1681  }
1682  if (!(ccd.columnType.get_compression() == kENCODING_DICT)) {
1683  return;
1684  }
1685  if (!(ccd.columnType.get_comp_param() > 0)) {
1686  return;
1687  }
1688 
1689  auto dictId = ccd.columnType.get_comp_param();
1690  getMetadataForDict(dictId);
1691 
1692  const DictRef dictRef(currentDB_.dbId, dictId);
1693  auto dit = dictDescriptorMapByRef_.find(dictRef);
1694  CHECK(dit != dictDescriptorMapByRef_.end());
1695  CHECK(dit->second);
1696  CHECK(dit->second.get()->stringDict);
1697  stringDicts[ccd.columnId] = dit->second.get()->stringDict.get();
1698 }
1699 
1701  // caller must handle sqlite/chunk transaction TOGETHER
1702  cd.tableId = td.tableId;
1704  addDictionary(cd);
1705  }
1706 
1708  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
1709  "colscale, is_notnull, "
1710  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
1711  "is_deletedcol) "
1712  "VALUES (?, "
1713  "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
1714  "?, ?, ?, "
1715  "?, "
1716  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1717  std::vector<std::string>{std::to_string(td.tableId),
1718  std::to_string(td.tableId),
1719  cd.columnName,
1728  "",
1731  cd.virtualExpr,
1733 
1735  "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
1736  std::vector<std::string>{std::to_string(td.tableId)});
1737 
1739  "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
1740  std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
1741  cd.columnId = sqliteConnector_.getData<int>(0, 0);
1742 
1743  ++tableDescriptorMapById_[td.tableId]->nColumns;
1744  auto ncd = new ColumnDescriptor(cd);
1747  columnDescriptorsForRoll.emplace_back(nullptr, ncd);
1748 }
1749 
1750 void Catalog::roll(const bool forward) {
1751  std::set<const TableDescriptor*> tds;
1752 
1753  for (const auto& cdr : columnDescriptorsForRoll) {
1754  auto ocd = cdr.first;
1755  auto ncd = cdr.second;
1756  CHECK(ocd || ncd);
1757  auto tabDescIt = tableDescriptorMapById_.find((ncd ? ncd : ocd)->tableId);
1758  CHECK(tableDescriptorMapById_.end() != tabDescIt);
1759  auto td = tabDescIt->second;
1760  auto& vc = td->columnIdBySpi_;
1761  if (forward) {
1762  if (ocd) {
1763  if (nullptr == ncd ||
1764  ncd->columnType.get_comp_param() != ocd->columnType.get_comp_param()) {
1765  delDictionary(*ocd);
1766  }
1767 
1768  vc.erase(std::remove(vc.begin(), vc.end(), ocd->columnId), vc.end());
1769 
1770  delete ocd;
1771  }
1772  if (ncd) {
1773  // append columnId if its new and not phy geo
1774  if (vc.end() == std::find(vc.begin(), vc.end(), ncd->columnId)) {
1775  if (!ncd->isGeoPhyCol) {
1776  vc.push_back(ncd->columnId);
1777  }
1778  }
1779  }
1780  tds.insert(td);
1781  } else {
1782  if (ocd) {
1783  columnDescriptorMap_[ColumnKey(ocd->tableId, to_upper(ocd->columnName))] = ocd;
1784  columnDescriptorMapById_[ColumnIdKey(ocd->tableId, ocd->columnId)] = ocd;
1785  }
1786  // roll back the dict of new column
1787  if (ncd) {
1788  columnDescriptorMap_.erase(ColumnKey(ncd->tableId, to_upper(ncd->columnName)));
1789  columnDescriptorMapById_.erase(ColumnIdKey(ncd->tableId, ncd->columnId));
1790  if (nullptr == ocd ||
1791  ocd->columnType.get_comp_param() != ncd->columnType.get_comp_param()) {
1792  delDictionary(*ncd);
1793  }
1794  delete ncd;
1795  }
1796  }
1797  }
1798  columnDescriptorsForRoll.clear();
1799 
1800  if (forward) {
1801  for (const auto td : tds) {
1802  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
1803  }
1804  }
1805 }
1806 
1808  list<ColumnDescriptor>& columns) {
1809  const auto& col_ti = cd.columnType;
1810  if (IS_GEO(col_ti.get_type())) {
1811  switch (col_ti.get_type()) {
1812  case kPOINT: {
1813  ColumnDescriptor physical_cd_coords(true);
1814  physical_cd_coords.columnName = cd.columnName + "_coords";
1815  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, true);
1816  // Raw data: compressed/uncompressed coords
1817  coords_ti.set_subtype(kTINYINT);
1818  size_t unit_size;
1819  if (col_ti.get_compression() == kENCODING_GEOINT &&
1820  col_ti.get_comp_param() == 32) {
1821  unit_size = 4 * sizeof(int8_t);
1822  } else {
1823  CHECK(col_ti.get_compression() == kENCODING_NONE);
1824  unit_size = 8 * sizeof(int8_t);
1825  }
1826  coords_ti.set_size(2 * unit_size);
1827  physical_cd_coords.columnType = coords_ti;
1828  columns.push_back(physical_cd_coords);
1829 
1830  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1831 
1832  break;
1833  }
1834  case kLINESTRING: {
1835  ColumnDescriptor physical_cd_coords(true);
1836  physical_cd_coords.columnName = cd.columnName + "_coords";
1837  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, true);
1838  // Raw data: compressed/uncompressed coords
1839  coords_ti.set_subtype(kTINYINT);
1840  physical_cd_coords.columnType = coords_ti;
1841  columns.push_back(physical_cd_coords);
1842 
1843  ColumnDescriptor physical_cd_bounds(true);
1844  physical_cd_bounds.columnName = cd.columnName + "_bounds";
1845  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, true);
1846  bounds_ti.set_subtype(kDOUBLE);
1847  bounds_ti.set_size(4 * sizeof(double));
1848  physical_cd_bounds.columnType = bounds_ti;
1849  columns.push_back(physical_cd_bounds);
1850 
1851  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1852 
1853  break;
1854  }
1855  case kPOLYGON: {
1856  ColumnDescriptor physical_cd_coords(true);
1857  physical_cd_coords.columnName = cd.columnName + "_coords";
1858  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, true);
1859  // Raw data: compressed/uncompressed coords
1860  coords_ti.set_subtype(kTINYINT);
1861  physical_cd_coords.columnType = coords_ti;
1862  columns.push_back(physical_cd_coords);
1863 
1864  ColumnDescriptor physical_cd_ring_sizes(true);
1865  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
1866  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, true);
1867  ring_sizes_ti.set_subtype(kINT);
1868  physical_cd_ring_sizes.columnType = ring_sizes_ti;
1869  columns.push_back(physical_cd_ring_sizes);
1870 
1871  ColumnDescriptor physical_cd_bounds(true);
1872  physical_cd_bounds.columnName = cd.columnName + "_bounds";
1873  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, true);
1874  bounds_ti.set_subtype(kDOUBLE);
1875  bounds_ti.set_size(4 * sizeof(double));
1876  physical_cd_bounds.columnType = bounds_ti;
1877  columns.push_back(physical_cd_bounds);
1878 
1879  ColumnDescriptor physical_cd_render_group(true);
1880  physical_cd_render_group.columnName = cd.columnName + "_render_group";
1881  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, true);
1882  physical_cd_render_group.columnType = render_group_ti;
1883  columns.push_back(physical_cd_render_group);
1884 
1885  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1886 
1887  break;
1888  }
1889  case kMULTIPOLYGON: {
1890  ColumnDescriptor physical_cd_coords(true);
1891  physical_cd_coords.columnName = cd.columnName + "_coords";
1892  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, true);
1893  // Raw data: compressed/uncompressed coords
1894  coords_ti.set_subtype(kTINYINT);
1895  physical_cd_coords.columnType = coords_ti;
1896  columns.push_back(physical_cd_coords);
1897 
1898  ColumnDescriptor physical_cd_ring_sizes(true);
1899  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
1900  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, true);
1901  ring_sizes_ti.set_subtype(kINT);
1902  physical_cd_ring_sizes.columnType = ring_sizes_ti;
1903  columns.push_back(physical_cd_ring_sizes);
1904 
1905  ColumnDescriptor physical_cd_poly_rings(true);
1906  physical_cd_poly_rings.columnName = cd.columnName + "_poly_rings";
1907  SQLTypeInfo poly_rings_ti = SQLTypeInfo(kARRAY, true);
1908  poly_rings_ti.set_subtype(kINT);
1909  physical_cd_poly_rings.columnType = poly_rings_ti;
1910  columns.push_back(physical_cd_poly_rings);
1911 
1912  ColumnDescriptor physical_cd_bounds(true);
1913  physical_cd_bounds.columnName = cd.columnName + "_bounds";
1914  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, true);
1915  bounds_ti.set_subtype(kDOUBLE);
1916  bounds_ti.set_size(4 * sizeof(double));
1917  physical_cd_bounds.columnType = bounds_ti;
1918  columns.push_back(physical_cd_bounds);
1919 
1920  ColumnDescriptor physical_cd_render_group(true);
1921  physical_cd_render_group.columnName = cd.columnName + "_render_group";
1922  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, true);
1923  physical_cd_render_group.columnType = render_group_ti;
1924  columns.push_back(physical_cd_render_group);
1925 
1926  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1927 
1928  break;
1929  }
1930  default:
1931  throw runtime_error("Unrecognized geometry type.");
1932  break;
1933  }
1934  }
1935 }
1936 
1938  TableDescriptor& td,
1939  const list<ColumnDescriptor>& cols,
1940  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs,
1941  bool isLogicalTable) {
1942  cat_write_lock write_lock(this);
1943  list<ColumnDescriptor> cds;
1944  list<DictDescriptor> dds;
1945  std::set<std::string> toplevel_column_names;
1946  list<ColumnDescriptor> columns;
1947  for (auto cd : cols) {
1948  if (cd.columnName == "rowid") {
1949  throw std::runtime_error(
1950  "Cannot create column with name rowid. rowid is a system defined column.");
1951  }
1952  columns.push_back(cd);
1953  toplevel_column_names.insert(cd.columnName);
1954  if (cd.columnType.is_geometry()) {
1955  expandGeoColumn(cd, columns);
1956  }
1957  }
1958 
1959  ColumnDescriptor cd;
1960  // add row_id column -- Must be last column in the table
1961  cd.columnName = "rowid";
1962  cd.isSystemCol = true;
1963  cd.columnType = SQLTypeInfo(kBIGINT, true);
1964 #ifdef MATERIALIZED_ROWID
1965  cd.isVirtualCol = false;
1966 #else
1967  cd.isVirtualCol = true;
1968  cd.virtualExpr = "MAPD_FRAG_ID * MAPD_ROWS_PER_FRAG + MAPD_FRAG_ROW_ID";
1969 #endif
1970  columns.push_back(cd);
1971  toplevel_column_names.insert(cd.columnName);
1972 
1973  if (td.hasDeletedCol) {
1974  ColumnDescriptor cd_del;
1975  cd_del.columnName = "$deleted$";
1976  cd_del.isSystemCol = true;
1977  cd_del.isVirtualCol = false;
1978  cd_del.columnType = SQLTypeInfo(kBOOLEAN, true);
1979  cd_del.isDeletedCol = true;
1980 
1981  columns.push_back(cd_del);
1982  }
1983 
1984  td.nColumns = columns.size();
1986  sqliteConnector_.query("BEGIN TRANSACTION");
1988  try {
1990  "INSERT INTO mapd_tables (name, userid, ncolumns, isview, fragments, "
1991  "frag_type, max_frag_rows, "
1992  "max_chunk_size, "
1993  "frag_page_size, max_rows, partitions, shard_column_id, shard, num_shards, "
1994  "sort_column_id, "
1995  "key_metainfo) VALUES (?, ?, ?, "
1996  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1997 
1998  std::vector<std::string>{td.tableName,
1999  std::to_string(td.userId),
2001  std::to_string(td.isView),
2002  "",
2007  std::to_string(td.maxRows),
2008  td.partitions,
2010  std::to_string(td.shard),
2011  std::to_string(td.nShards),
2013  td.keyMetainfo});
2014 
2015  // now get the auto generated tableid
2017  "SELECT tableid FROM mapd_tables WHERE name = ?", td.tableName);
2018  td.tableId = sqliteConnector_.getData<int>(0, 0);
2019  int colId = 1;
2020  for (auto cd : columns) {
2022  const bool is_foreign_col =
2023  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2024  if (!is_foreign_col) {
2025  setColumnDictionary(cd, dds, td, isLogicalTable);
2026  }
2027  }
2028 
2029  if (toplevel_column_names.count(cd.columnName)) {
2030  // make up colId gap for sanity test (begin with 1 bc much code depends on it!)
2031  if (colId > 1) {
2032  colId += g_test_against_columnId_gap;
2033  }
2034  if (!cd.isGeoPhyCol) {
2035  td.columnIdBySpi_.push_back(colId);
2036  }
2037  }
2038 
2040  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, "
2041  "coldim, colscale, is_notnull, "
2042  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
2043  "virtual_expr, is_deletedcol) "
2044  "VALUES (?, ?, ?, ?, ?, "
2045  "?, "
2046  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2047  std::vector<std::string>{std::to_string(td.tableId),
2048  std::to_string(colId),
2049  cd.columnName,
2058  "",
2061  cd.virtualExpr,
2063  cd.tableId = td.tableId;
2064  cd.columnId = colId++;
2065  cds.push_back(cd);
2066  }
2067  if (td.isView) {
2069  "INSERT INTO mapd_views (tableid, sql) VALUES (?,?)",
2070  std::vector<std::string>{std::to_string(td.tableId), td.viewSQL});
2071  }
2072  } catch (std::exception& e) {
2073  sqliteConnector_.query("ROLLBACK TRANSACTION");
2074  throw;
2075  }
2076 
2077  } else { // Temporary table
2078  td.tableId = nextTempTableId_++;
2079  int colId = 1;
2080  for (auto cd : columns) {
2081  auto col_ti = cd.columnType;
2082  if (IS_GEO(col_ti.get_type())) {
2083  throw runtime_error("Geometry types in temporary tables are not supported.");
2084  }
2085 
2087  // TODO(vraj) : create shared dictionary for temp table if needed
2088  std::string fileName("");
2089  std::string folderPath("");
2091  nextTempDictId_++;
2092  DictDescriptor dd(dict_ref,
2093  fileName,
2095  false,
2096  1,
2097  folderPath,
2098  true); // Is dictName (2nd argument) used?
2099  dds.push_back(dd);
2100  if (!cd.columnType.is_array()) {
2102  }
2103  cd.columnType.set_comp_param(dict_ref.dictId);
2104  }
2105  cd.tableId = td.tableId;
2106  cd.columnId = colId++;
2107  cds.push_back(cd);
2108  }
2109  }
2110  try {
2111  addTableToMap(td, cds, dds);
2112  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
2113  } catch (std::exception& e) {
2114  sqliteConnector_.query("ROLLBACK TRANSACTION");
2115  removeTableFromMap(td.tableName, td.tableId, true);
2116  throw;
2117  }
2118 
2119  sqliteConnector_.query("END TRANSACTION");
2120 }
2121 
2122 // returns the table epoch or -1 if there is something wrong with the shared epoch
2123 int32_t Catalog::getTableEpoch(const int32_t db_id, const int32_t table_id) const {
2124  cat_read_lock read_lock(this);
2125  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(table_id);
2126  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2127  // check all shards have same checkpoint
2128  const auto physicalTables = physicalTableIt->second;
2129  CHECK(!physicalTables.empty());
2130  size_t curr_epoch = 0;
2131  for (size_t i = 0; i < physicalTables.size(); i++) {
2132  int32_t physical_tb_id = physicalTables[i];
2133  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2134  CHECK(phys_td);
2135  if (i == 0) {
2136  curr_epoch = dataMgr_->getTableEpoch(db_id, physical_tb_id);
2137  } else {
2138  if (curr_epoch != dataMgr_->getTableEpoch(db_id, physical_tb_id)) {
2139  // oh dear the leaves do not agree on the epoch for this table
2140  LOG(ERROR) << "Epochs on shards do not all agree on table id " << table_id
2141  << " db id " << db_id << " epoch " << curr_epoch << " leaf_epoch "
2142  << dataMgr_->getTableEpoch(db_id, physical_tb_id);
2143  return -1;
2144  }
2145  }
2146  }
2147  return curr_epoch;
2148  } else {
2149  return dataMgr_->getTableEpoch(db_id, table_id);
2150  }
2151 }
2152 
2153 void Catalog::setTableEpoch(const int db_id, const int table_id, int new_epoch) {
2154  cat_read_lock read_lock(this);
2155  LOG(INFO) << "Set table epoch db:" << db_id << " Table ID " << table_id
2156  << " back to new epoch " << new_epoch;
2157  removeChunks(table_id);
2158  dataMgr_->setTableEpoch(db_id, table_id, new_epoch);
2159 
2160  // check if sharded
2161  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(table_id);
2162  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2163  const auto physicalTables = physicalTableIt->second;
2164  CHECK(!physicalTables.empty());
2165  for (size_t i = 0; i < physicalTables.size(); i++) {
2166  int32_t physical_tb_id = physicalTables[i];
2167  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2168  CHECK(phys_td);
2169  LOG(INFO) << "Set sharded table epoch db:" << db_id << " Table ID "
2170  << physical_tb_id << " back to new epoch " << new_epoch;
2171  removeChunks(physical_tb_id);
2172  dataMgr_->setTableEpoch(db_id, physical_tb_id, new_epoch);
2173  }
2174  }
2175 }
2176 
2178  cat_read_lock read_lock(this);
2179  const auto it = deletedColumnPerTable_.find(td);
2180  return it != deletedColumnPerTable_.end() ? it->second : nullptr;
2181 }
2182 
2184  int tableId,
2185  int columnId) const {
2186  // check if there are rows deleted by examining metadata for the deletedColumn metadata
2187  ChunkKey chunkKeyPrefix = {dbId, tableId, columnId};
2188  std::vector<std::pair<ChunkKey, ChunkMetadata>> chunkMetadataVec;
2189  dataMgr_->getChunkMetadataVecForKeyPrefix(chunkMetadataVec, chunkKeyPrefix);
2190  int64_t chunk_max{0};
2191 
2192  for (auto cm : chunkMetadataVec) {
2193  chunk_max = cm.second.chunkStats.max.tinyintval;
2194  // delete has occured
2195  if (chunk_max == 1) {
2196  return true;
2197  }
2198  }
2199  return false;
2200 }
2201 
2203  const TableDescriptor* td) const {
2204  cat_read_lock read_lock(this);
2205 
2206  const auto it = deletedColumnPerTable_.find(td);
2207  // if not a table that supports delete return nullptr, nothing more to do
2208  if (it == deletedColumnPerTable_.end()) {
2209  return nullptr;
2210  }
2211  const ColumnDescriptor* cd = it->second;
2212 
2213  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
2214 
2215  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2216  // check all shards
2217  const auto physicalTables = physicalTableIt->second;
2218  CHECK(!physicalTables.empty());
2219  for (size_t i = 0; i < physicalTables.size(); i++) {
2220  int32_t physical_tb_id = physicalTables[i];
2221  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2222  CHECK(phys_td);
2224  return cd;
2225  }
2226  }
2227  } else {
2229  return cd;
2230  }
2231  }
2232  // no deletes so far recorded in metadata
2233  return nullptr;
2234 }
2235 
2237  cat_write_lock write_lock(this);
2238  setDeletedColumnUnlocked(td, cd);
2239 }
2240 
2242  const ColumnDescriptor* cd) {
2243  cat_write_lock write_lock(this);
2244  const auto it_ok = deletedColumnPerTable_.emplace(td, cd);
2245  CHECK(it_ok.second);
2246 }
2247 
2248 namespace {
2249 
2251  const Catalog& cat,
2252  const Parser::SharedDictionaryDef& shared_dict_def) {
2253  const auto& table_name = shared_dict_def.get_foreign_table();
2254  const auto td = cat.getMetadataForTable(table_name);
2255  CHECK(td);
2256  const auto& foreign_col_name = shared_dict_def.get_foreign_column();
2257  return cat.getMetadataForColumn(td->tableId, foreign_col_name);
2258 }
2259 
2260 } // namespace
2261 
2263  Parser::SharedDictionaryDef shared_dict_def) {
2264  cat_write_lock write_lock(this);
2265  const auto foreign_ref_col = get_foreign_col(*this, shared_dict_def);
2266  CHECK(foreign_ref_col);
2267  referencing_column.columnType = foreign_ref_col->columnType;
2268  const int dict_id = referencing_column.columnType.get_comp_param();
2269  const DictRef dict_ref(currentDB_.dbId, dict_id);
2270  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
2271  CHECK(dictIt != dictDescriptorMapByRef_.end());
2272  const auto& dd = dictIt->second;
2273  CHECK_GE(dd->refcount, 1);
2274  ++dd->refcount;
2277  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
2278  {std::to_string(dict_id)});
2279 }
2280 
2282  ColumnDescriptor& cd,
2283  std::list<ColumnDescriptor>& cdd,
2284  std::list<DictDescriptor>& dds,
2285  const TableDescriptor td,
2286  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs) {
2287  cat_write_lock write_lock(this);
2289 
2290  if (shared_dict_defs.empty()) {
2291  return false;
2292  }
2293  for (const auto& shared_dict_def : shared_dict_defs) {
2294  // check if the current column is a referencing column
2295  const auto& column = shared_dict_def.get_column();
2296  if (cd.columnName == column) {
2297  if (!shared_dict_def.get_foreign_table().compare(td.tableName)) {
2298  // Dictionaries are being shared in table to be created
2299  const auto& ref_column = shared_dict_def.get_foreign_column();
2300  auto colIt =
2301  std::find_if(cdd.begin(), cdd.end(), [ref_column](const ColumnDescriptor it) {
2302  return !ref_column.compare(it.columnName);
2303  });
2304  CHECK(colIt != cdd.end());
2305  cd.columnType = colIt->columnType;
2306 
2308  "SELECT dictid FROM mapd_dictionaries WHERE dictid in (select comp_param "
2309  "from "
2310  "mapd_columns "
2311  "where compression = ? and tableid = ? and columnid = ?)",
2312  std::vector<std::string>{std::to_string(kENCODING_DICT),
2313  std::to_string(td.tableId),
2314  std::to_string(colIt->columnId)});
2315  const auto dict_id = sqliteConnector_.getData<int>(0, 0);
2316  auto dictIt = std::find_if(
2317  dds.begin(), dds.end(), [this, dict_id](const DictDescriptor it) {
2318  return it.dictRef.dbId == this->currentDB_.dbId &&
2319  it.dictRef.dictId == dict_id;
2320  });
2321  if (dictIt != dds.end()) {
2322  // There exists dictionary definition of a dictionary column
2323  CHECK_GE(dictIt->refcount, 1);
2324  ++dictIt->refcount;
2326  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
2327  {std::to_string(dict_id)});
2328  } else {
2329  // The dictionary is referencing a column which is referencing a column in
2330  // diffrent table
2331  auto root_dict_def = compress_reference_path(shared_dict_def, shared_dict_defs);
2332  addReferenceToForeignDict(cd, root_dict_def);
2333  }
2334  } else {
2335  addReferenceToForeignDict(cd, shared_dict_def);
2336  }
2337  return true;
2338  }
2339  }
2340  return false;
2341 }
2342 
2344  std::list<DictDescriptor>& dds,
2345  const TableDescriptor& td,
2346  const bool isLogicalTable) {
2347  cat_write_lock write_lock(this);
2348 
2349  std::string dictName{"Initial_key"};
2350  int dictId{0};
2351  std::string folderPath;
2352  if (isLogicalTable) {
2354 
2356  "INSERT INTO mapd_dictionaries (name, nbits, is_shared, refcount) VALUES (?, ?, "
2357  "?, 1)",
2358  std::vector<std::string>{
2359  dictName, std::to_string(cd.columnType.get_comp_param()), "0"});
2361  "SELECT dictid FROM mapd_dictionaries WHERE name = ?", dictName);
2362  dictId = sqliteConnector_.getData<int>(0, 0);
2363  dictName = td.tableName + "_" + cd.columnName + "_dict" + std::to_string(dictId);
2365  "UPDATE mapd_dictionaries SET name = ? WHERE name = 'Initial_key'", dictName);
2366  folderPath = basePath_ + "/mapd_data/DB_" + std::to_string(currentDB_.dbId) +
2367  "_DICT_" + std::to_string(dictId);
2368  }
2370  dictId,
2371  dictName,
2373  false,
2374  1,
2375  folderPath,
2376  false);
2377  dds.push_back(dd);
2378  if (!cd.columnType.is_array()) {
2380  }
2381  cd.columnType.set_comp_param(dictId);
2382 }
2383 
2385  TableDescriptor& td,
2386  const list<ColumnDescriptor>& cols,
2387  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs) {
2388  cat_write_lock write_lock(this);
2389 
2390  /* create logical table */
2391  TableDescriptor tdl(td);
2392  createTable(tdl, cols, shared_dict_defs, true); // create logical table
2393  int32_t logical_tb_id = tdl.tableId;
2394 
2395  /* create physical tables and link them to the logical table */
2396  std::vector<int32_t> physicalTables;
2397  for (int32_t i = 1; i <= td.nShards; i++) {
2398  TableDescriptor tdp(td);
2400  tdp.shard = i - 1;
2401  createTable(tdp, cols, shared_dict_defs, false); // create physical table
2402  int32_t physical_tb_id = tdp.tableId;
2403 
2404  /* add physical table to the vector of physical tables */
2405  physicalTables.push_back(physical_tb_id);
2406  }
2407 
2408  if (!physicalTables.empty()) {
2409  /* add logical to physical tables correspondence to the map */
2410  const auto it_ok =
2411  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
2412  CHECK(it_ok.second);
2413  /* update sqlite mapd_logical_to_physical in sqlite database */
2414  updateLogicalToPhysicalTableMap(logical_tb_id);
2415  }
2416 }
2417 
2419  cat_write_lock write_lock(this);
2420 
2421  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
2422  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2423  // truncate all corresponding physical tables if this is a logical table
2424  const auto physicalTables = physicalTableIt->second;
2425  CHECK(!physicalTables.empty());
2426  for (size_t i = 0; i < physicalTables.size(); i++) {
2427  int32_t physical_tb_id = physicalTables[i];
2428  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2429  CHECK(phys_td);
2430  doTruncateTable(phys_td);
2431  }
2432  }
2433  doTruncateTable(td);
2434 }
2435 
2437  cat_write_lock write_lock(this);
2438 
2439  const int tableId = td->tableId;
2440  // must destroy fragmenter before deleteChunks is called.
2441  if (td->fragmenter != nullptr) {
2442  auto tableDescIt = tableDescriptorMapById_.find(tableId);
2443  delete td->fragmenter;
2444  tableDescIt->second->fragmenter = nullptr; // get around const-ness
2445  }
2446  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
2447  // assuming deleteChunksWithPrefix is atomic
2448  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
2449  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
2450 
2451  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
2452 
2453  std::unique_ptr<StringDictionaryClient> client;
2454  if (SysCatalog::instance().isAggregator()) {
2455  CHECK(!string_dict_hosts_.empty());
2456  DictRef dict_ref(currentDB_.dbId, -1);
2457  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
2458  }
2459  // clean up any dictionaries
2460  // delete all column descriptors for the table
2461  for (const auto& columnDescriptor : columnDescriptorMapById_) {
2462  auto cd = columnDescriptor.second;
2463  if (cd->tableId != td->tableId) {
2464  continue;
2465  }
2466  const int dict_id = cd->columnType.get_comp_param();
2467  // Dummy dictionaries created for a shard of a logical table have the id set to zero.
2468  if (cd->columnType.get_compression() == kENCODING_DICT && dict_id) {
2469  const DictRef dict_ref(currentDB_.dbId, dict_id);
2470  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
2471  CHECK(dictIt != dictDescriptorMapByRef_.end());
2472  const auto& dd = dictIt->second;
2473  CHECK_GE(dd->refcount, 1);
2474  // if this is the only table using this dict reset the dict
2475  if (dd->refcount == 1) {
2476  // close the dictionary
2477  dd->stringDict.reset();
2478  File_Namespace::renameForDelete(dd->dictFolderPath);
2479  if (client) {
2480  client->drop(dd->dictRef);
2481  }
2482  if (!dd->dictIsTemp) {
2483  boost::filesystem::create_directory(dd->dictFolderPath);
2484  }
2485  }
2486 
2487  DictDescriptor* new_dd = new DictDescriptor(dd->dictRef,
2488  dd->dictName,
2489  dd->dictNBits,
2490  dd->dictIsShared,
2491  dd->refcount,
2492  dd->dictFolderPath,
2493  dd->dictIsTemp);
2494  dictDescriptorMapByRef_.erase(dictIt);
2495  // now create new Dict -- need to figure out what to do here for temp tables
2496  if (client) {
2497  client->create(new_dd->dictRef, new_dd->dictIsTemp);
2498  }
2499  dictDescriptorMapByRef_[new_dd->dictRef].reset(new_dd);
2501  }
2502  }
2503 }
2504 
2505 // used by rollback_table_epoch to clean up in memory artifacts after a rollback
2506 void Catalog::removeChunks(const int table_id) {
2507  auto td = getMetadataForTable(table_id);
2508 
2509  if (td->fragmenter != nullptr) {
2511  if (td->fragmenter != nullptr) {
2512  auto tableDescIt = tableDescriptorMapById_.find(table_id);
2513  delete td->fragmenter;
2514  tableDescIt->second->fragmenter = nullptr; // get around const-ness
2515  }
2516  }
2517 
2518  // remove the chunks from in memory structures
2519  ChunkKey chunkKey = {currentDB_.dbId, table_id};
2520 
2521  dataMgr_->deleteChunksWithPrefix(chunkKey, MemoryLevel::CPU_LEVEL);
2522  dataMgr_->deleteChunksWithPrefix(chunkKey, MemoryLevel::GPU_LEVEL);
2523 }
2524 
2528  cat_write_lock write_lock(this);
2530  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
2531  sqliteConnector_.query("BEGIN TRANSACTION");
2532  try {
2533  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2534  // remove all corresponding physical tables if this is a logical table
2535  const auto physicalTables = physicalTableIt->second;
2536  CHECK(!physicalTables.empty());
2537  for (size_t i = 0; i < physicalTables.size(); i++) {
2538  int32_t physical_tb_id = physicalTables[i];
2539  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2540  CHECK(phys_td);
2541  doDropTable(phys_td);
2542  }
2543 
2544  // remove corresponding record from the logicalToPhysicalTableMap in sqlite database
2546  "DELETE FROM mapd_logical_to_physical WHERE logical_table_id = ?",
2547  std::to_string(td->tableId));
2549  }
2550  doDropTable(td);
2551  } catch (std::exception& e) {
2552  sqliteConnector_.query("ROLLBACK TRANSACTION");
2553  throw;
2554  }
2555  sqliteConnector_.query("END TRANSACTION");
2556 }
2557 
2559  const int tableId = td->tableId;
2560  sqliteConnector_.query_with_text_param("DELETE FROM mapd_tables WHERE tableid = ?",
2561  std::to_string(tableId));
2563  "select comp_param from mapd_columns where compression = ? and tableid = ?",
2564  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
2565  int numRows = sqliteConnector_.getNumRows();
2566  std::vector<int> dict_id_list;
2567  for (int r = 0; r < numRows; ++r) {
2568  dict_id_list.push_back(sqliteConnector_.getData<int>(r, 0));
2569  }
2570  for (auto dict_id : dict_id_list) {
2572  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
2573  std::vector<std::string>{std::to_string(dict_id)});
2574  }
2576  "DELETE FROM mapd_dictionaries WHERE dictid in (select comp_param from "
2577  "mapd_columns where compression = ? "
2578  "and tableid = ?) and refcount = 0",
2579  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
2580  sqliteConnector_.query_with_text_param("DELETE FROM mapd_columns WHERE tableid = ?",
2581  std::to_string(tableId));
2582  if (td->isView) {
2583  sqliteConnector_.query_with_text_param("DELETE FROM mapd_views WHERE tableid = ?",
2584  std::to_string(tableId));
2585  }
2587 }
2588 
2589 void Catalog::renamePhysicalTable(const TableDescriptor* td, const string& newTableName) {
2590  cat_write_lock write_lock(this);
2592 
2593  sqliteConnector_.query("BEGIN TRANSACTION");
2594  try {
2596  "UPDATE mapd_tables SET name = ? WHERE tableid = ?",
2597  std::vector<std::string>{newTableName, std::to_string(td->tableId)});
2598  } catch (std::exception& e) {
2599  sqliteConnector_.query("ROLLBACK TRANSACTION");
2600  throw;
2601  }
2602  sqliteConnector_.query("END TRANSACTION");
2603  TableDescriptorMap::iterator tableDescIt =
2605  CHECK(tableDescIt != tableDescriptorMap_.end());
2606  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
2607  // Get table descriptor to change it
2608  TableDescriptor* changeTd = tableDescIt->second;
2609  changeTd->tableName = newTableName;
2610  tableDescriptorMap_.erase(tableDescIt); // erase entry under old name
2611  tableDescriptorMap_[to_upper(newTableName)] = changeTd;
2612  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
2613 }
2614 
2615 void Catalog::renameTable(const TableDescriptor* td, const string& newTableName) {
2616  {
2617  cat_write_lock write_lock(this);
2619  // rename all corresponding physical tables if this is a logical table
2620  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
2621  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2622  const auto physicalTables = physicalTableIt->second;
2623  CHECK(!physicalTables.empty());
2624  for (size_t i = 0; i < physicalTables.size(); i++) {
2625  int32_t physical_tb_id = physicalTables[i];
2626  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2627  CHECK(phys_td);
2628  std::string newPhysTableName =
2629  generatePhysicalTableName(newTableName, static_cast<int32_t>(i + 1));
2630  renamePhysicalTable(phys_td, newPhysTableName);
2631  }
2632  }
2633  renamePhysicalTable(td, newTableName);
2634  }
2635  {
2636  DBObject object(newTableName, TableDBObjectType);
2637  // update table name in direct and effective priv map
2638  DBObjectKey key;
2639  key.dbId = currentDB_.dbId;
2640  key.objectId = td->tableId;
2641  key.permissionType = static_cast<int>(DBObjectType::TableDBObjectType);
2642  object.setObjectKey(key);
2643  auto objdescs = SysCatalog::instance().getMetadataForObject(
2644  currentDB_.dbId, static_cast<int>(DBObjectType::TableDBObjectType), td->tableId);
2645  for (auto obj : objdescs) {
2646  Grantee* grnt = SysCatalog::instance().getGrantee(obj->roleName);
2647  if (grnt) {
2648  grnt->renameDbObject(object);
2649  }
2650  }
2652  }
2653 }
2654 
2656  const ColumnDescriptor* cd,
2657  const string& newColumnName) {
2658  cat_write_lock write_lock(this);
2660  sqliteConnector_.query("BEGIN TRANSACTION");
2661  try {
2662  for (int i = 0; i <= cd->columnType.get_physical_cols(); ++i) {
2663  auto cdx = getMetadataForColumn(td->tableId, cd->columnId + i);
2664  CHECK(cdx);
2665  std::string new_column_name = cdx->columnName;
2666  new_column_name.replace(0, cd->columnName.size(), newColumnName);
2668  "UPDATE mapd_columns SET name = ? WHERE tableid = ? AND columnid = ?",
2669  std::vector<std::string>{new_column_name,
2670  std::to_string(td->tableId),
2671  std::to_string(cdx->columnId)});
2672  }
2673  } catch (std::exception& e) {
2674  sqliteConnector_.query("ROLLBACK TRANSACTION");
2675  throw;
2676  }
2677  sqliteConnector_.query("END TRANSACTION");
2678  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
2679  for (int i = 0; i <= cd->columnType.get_physical_cols(); ++i) {
2680  auto cdx = getMetadataForColumn(td->tableId, cd->columnId + i);
2681  CHECK(cdx);
2682  ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.find(
2683  std::make_tuple(td->tableId, to_upper(cdx->columnName)));
2684  CHECK(columnDescIt != columnDescriptorMap_.end());
2685  ColumnDescriptor* changeCd = columnDescIt->second;
2686  changeCd->columnName.replace(0, cd->columnName.size(), newColumnName);
2687  columnDescriptorMap_.erase(columnDescIt); // erase entry under old name
2688  columnDescriptorMap_[std::make_tuple(td->tableId, to_upper(changeCd->columnName))] =
2689  changeCd;
2690  }
2691  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
2692 }
2693 
2695  {
2696  cat_write_lock write_lock(this);
2698  sqliteConnector_.query("BEGIN TRANSACTION");
2699  try {
2700  // TODO(andrew): this should be an upsert
2702  "SELECT id FROM mapd_dashboards WHERE name = ? and userid = ?",
2703  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
2704  if (sqliteConnector_.getNumRows() > 0) {
2706  "UPDATE mapd_dashboards SET state = ?, image_hash = ?, metadata = ?, "
2707  "update_time = "
2708  "datetime('now') where name = ? "
2709  "and userid = ?",
2710  std::vector<std::string>{vd.dashboardState,
2711  vd.imageHash,
2712  vd.dashboardMetadata,
2713  vd.dashboardName,
2714  std::to_string(vd.userId)});
2715  } else {
2717  "INSERT INTO mapd_dashboards (name, state, image_hash, metadata, "
2718  "update_time, "
2719  "userid) "
2720  "VALUES "
2721  "(?,?,?,?, "
2722  "datetime('now'), ?)",
2723  std::vector<std::string>{vd.dashboardName,
2724  vd.dashboardState,
2725  vd.imageHash,
2726  vd.dashboardMetadata,
2727  std::to_string(vd.userId)});
2728  }
2729  } catch (std::exception& e) {
2730  sqliteConnector_.query("ROLLBACK TRANSACTION");
2731  throw;
2732  }
2733  sqliteConnector_.query("END TRANSACTION");
2734 
2735  // now get the auto generated dashboardId
2736  try {
2738  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_dashboards "
2739  "WHERE name = ? and userid = ?",
2740  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
2741  vd.dashboardId = sqliteConnector_.getData<int>(0, 0);
2742  vd.updateTime = sqliteConnector_.getData<std::string>(0, 1);
2743  } catch (std::exception& e) {
2744  throw;
2745  }
2749  }
2750  // NOTE(wamsi): Transactionally unsafe
2753  return vd.dashboardId;
2754 }
2755 
2757  cat_write_lock write_lock(this);
2759 
2760  sqliteConnector_.query("BEGIN TRANSACTION");
2761  try {
2763  "SELECT id FROM mapd_dashboards WHERE id = ?",
2764  std::vector<std::string>{std::to_string(vd.dashboardId)});
2765  if (sqliteConnector_.getNumRows() > 0) {
2767  "UPDATE mapd_dashboards SET name = ?, state = ?, image_hash = ?, metadata = ?, "
2768  "update_time = "
2769  "datetime('now') where id = ? ",
2770  std::vector<std::string>{vd.dashboardName,
2771  vd.dashboardState,
2772  vd.imageHash,
2773  vd.dashboardMetadata,
2775  } else {
2776  LOG(ERROR) << "Error replacing dashboard id " << vd.dashboardId
2777  << " does not exist in db";
2778  throw runtime_error("Error replacing dashboard id " +
2779  std::to_string(vd.dashboardId) + " does not exist in db");
2780  }
2781  } catch (std::exception& e) {
2782  sqliteConnector_.query("ROLLBACK TRANSACTION");
2783  throw;
2784  }
2785  sqliteConnector_.query("END TRANSACTION");
2786 
2787  bool found{false};
2788  for (auto descp : dashboardDescriptorMap_) {
2789  auto dash = descp.second.get();
2790  if (dash->dashboardId == vd.dashboardId) {
2791  found = true;
2792  auto viewDescIt = dashboardDescriptorMap_.find(std::to_string(dash->userId) + ":" +
2793  dash->dashboardName);
2794  if (viewDescIt ==
2795  dashboardDescriptorMap_.end()) { // check to make sure view exists
2796  LOG(ERROR) << "No metadata for dashboard for user " << dash->userId
2797  << " dashboard " << dash->dashboardName << " does not exist in map";
2798  throw runtime_error("No metadata for dashboard for user " +
2799  std::to_string(dash->userId) + " dashboard " +
2800  dash->dashboardName + " does not exist in map");
2801  }
2802  dashboardDescriptorMap_.erase(viewDescIt);
2803  break;
2804  }
2805  }
2806  if (!found) {
2807  LOG(ERROR) << "Error replacing dashboard id " << vd.dashboardId
2808  << " does not exist in map";
2809  throw runtime_error("Error replacing dashboard id " + std::to_string(vd.dashboardId) +
2810  " does not exist in map");
2811  }
2812 
2813  // now reload the object
2815  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM "
2816  "mapd_dashboards "
2817  "WHERE id = ?",
2818  std::vector<std::string>{std::to_string(vd.dashboardId)});
2819  vd.updateTime = sqliteConnector_.getData<string>(0, 1);
2823  // NOTE(wamsi): Transactionally unsafe
2826 }
2827 
2828 std::string Catalog::calculateSHA1(const std::string& data) {
2829  boost::uuids::detail::sha1 sha1;
2830  unsigned int digest[5];
2831  sha1.process_bytes(data.c_str(), data.length());
2832  sha1.get_digest(digest);
2833  std::stringstream ss;
2834  for (size_t i = 0; i < 5; i++) {
2835  ss << std::hex << digest[i];
2836  }
2837  return ss.str();
2838 }
2839 
2840 std::string Catalog::createLink(LinkDescriptor& ld, size_t min_length) {
2841  cat_write_lock write_lock(this);
2843  sqliteConnector_.query("BEGIN TRANSACTION");
2844  try {
2846  .substr(0, 8);
2848  "SELECT linkid FROM mapd_links WHERE link = ? and userid = ?",
2849  std::vector<std::string>{ld.link, std::to_string(ld.userId)});
2850  if (sqliteConnector_.getNumRows() > 0) {
2852  "UPDATE mapd_links SET update_time = datetime('now') WHERE userid = ? AND link "
2853  "= ?",
2854  std::vector<std::string>{std::to_string(ld.userId), ld.link});
2855  } else {
2857  "INSERT INTO mapd_links (userid, link, view_state, view_metadata, update_time) "
2858  "VALUES (?,?,?,?, "
2859  "datetime('now'))",
2860  std::vector<std::string>{
2861  std::to_string(ld.userId), ld.link, ld.viewState, ld.viewMetadata});
2862  }
2863  // now get the auto generated dashid
2865  "SELECT linkid, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_links "
2866  "WHERE link = ?",
2867  ld.link);
2868  ld.linkId = sqliteConnector_.getData<int>(0, 0);
2869  ld.updateTime = sqliteConnector_.getData<std::string>(0, 1);
2870  } catch (std::exception& e) {
2871  sqliteConnector_.query("ROLLBACK TRANSACTION");
2872  throw;
2873  }
2874  sqliteConnector_.query("END TRANSACTION");
2875  addLinkToMap(ld);
2876  return ld.link;
2877 }
2878 
2880  const TableDescriptor* td) const {
2881  const auto column_descriptors =
2882  getAllColumnMetadataForTable(td->tableId, false, true, true);
2883 
2884  const ColumnDescriptor* shard_cd{nullptr};
2885  int i = 1;
2886  for (auto cd_itr = column_descriptors.begin(); cd_itr != column_descriptors.end();
2887  ++cd_itr, ++i) {
2888  if (i == td->shardedColumnId) {
2889  shard_cd = *cd_itr;
2890  }
2891  }
2892  return shard_cd;
2893 }
2894 
2895 std::vector<const TableDescriptor*> Catalog::getPhysicalTablesDescriptors(
2896  const TableDescriptor* logicalTableDesc) const {
2897  cat_read_lock read_lock(this);
2898  const auto physicalTableIt =
2899  logicalToPhysicalTableMapById_.find(logicalTableDesc->tableId);
2900  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
2901  return {logicalTableDesc};
2902  }
2903 
2904  const auto physicalTablesIds = physicalTableIt->second;
2905  CHECK(!physicalTablesIds.empty());
2906  std::vector<const TableDescriptor*> physicalTables;
2907  for (size_t i = 0; i < physicalTablesIds.size(); i++) {
2908  physicalTables.push_back(getMetadataForTable(physicalTablesIds[i]));
2909  }
2910 
2911  return physicalTables;
2912 }
2913 
2914 int Catalog::getLogicalTableId(const int physicalTableId) const {
2915  for (const auto& l : logicalToPhysicalTableMapById_) {
2916  if (l.second.end() != std::find_if(l.second.begin(),
2917  l.second.end(),
2918  [&](decltype(*l.second.begin()) tid) -> bool {
2919  return physicalTableId == tid;
2920  })) {
2921  return l.first;
2922  }
2923  }
2924  return physicalTableId;
2925 }
2926 
2927 void Catalog::checkpoint(const int logicalTableId) const {
2928  const auto td = getMetadataForTable(logicalTableId);
2929  const auto shards = getPhysicalTablesDescriptors(td);
2930  for (const auto shard : shards) {
2931  getDataMgr().checkpoint(getCurrentDB().dbId, shard->tableId);
2932  }
2933 }
2934 
2936  cat_write_lock write_lock(this);
2937  // Physically erase all tables and dictionaries from disc and memory
2938  const auto tables = getAllTableMetadata();
2939  for (const auto table : tables) {
2940  eraseTablePhysicalData(table);
2941  }
2942  // Physically erase database metadata
2943  boost::filesystem::remove(basePath_ + "/mapd_catalogs/" + currentDB_.dbName);
2944  calciteMgr_->updateMetadata(currentDB_.dbName, "");
2945 }
2946 
2948  cat_write_lock write_lock(this);
2949  const int tableId = td->tableId;
2950  // must destroy fragmenter before deleteChunks is called.
2951  if (td->fragmenter != nullptr) {
2952  auto tableDescIt = tableDescriptorMapById_.find(tableId);
2953  {
2954  INJECT_TIMER(deleting_fragmenter);
2955  delete td->fragmenter;
2956  }
2957  tableDescIt->second->fragmenter = nullptr; // get around const-ness
2958  }
2959  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
2960  {
2961  INJECT_TIMER(deleteChunksWithPrefix);
2962  // assuming deleteChunksWithPrefix is atomic
2963  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
2964  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
2965  }
2966  if (!td->isView) {
2967  INJECT_TIMER(Remove_Table);
2968  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
2969  }
2970  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
2971  {
2972  INJECT_TIMER(removeTableFromMap_);
2973  removeTableFromMap(td->tableName, tableId);
2974  }
2975 }
2976 
2977 std::string Catalog::generatePhysicalTableName(const std::string& logicalTableName,
2978  const int32_t& shardNumber) {
2979  std::string physicalTableName =
2980  logicalTableName + physicalTableNameTag_ + std::to_string(shardNumber);
2981  return (physicalTableName);
2982 }
2983 
2984 void Catalog::set(const std::string& dbName, std::shared_ptr<Catalog> cat) {
2985  mapd_cat_map_[dbName] = cat;
2986 }
2987 
2988 std::shared_ptr<Catalog> Catalog::get(const std::string& dbName) {
2989  auto cat_it = mapd_cat_map_.find(dbName);
2990  if (cat_it != mapd_cat_map_.end()) {
2991  return cat_it->second;
2992  }
2993  return nullptr;
2994 }
2995 
2996 std::shared_ptr<Catalog> Catalog::get(const string& basePath,
2997  const DBMetadata& curDB,
2998  std::shared_ptr<Data_Namespace::DataMgr> dataMgr,
2999  const std::vector<LeafHostInfo>& string_dict_hosts,
3000  std::shared_ptr<Calcite> calcite,
3001  bool is_new_db) {
3002  auto cat = Catalog::get(curDB.dbName);
3003 
3004  if (cat) {
3005  return cat;
3006  } else {
3007  cat = std::make_shared<Catalog>(
3008  basePath, curDB, dataMgr, string_dict_hosts, calcite, is_new_db);
3009  Catalog::set(curDB.dbName, cat);
3010  return cat;
3011  }
3012 }
3013 
3014 void Catalog::remove(const std::string& dbName) {
3015  mapd_cat_map_.erase(dbName);
3016 }
3017 
3018 void Catalog::vacuumDeletedRows(const int logicalTableId) const {
3019  // shard here to serve request from TableOptimizer and elsewhere
3020  cat_read_lock read_lock(this);
3021  const auto td = getMetadataForTable(logicalTableId);
3022  const auto shards = getPhysicalTablesDescriptors(td);
3023  for (const auto shard : shards) {
3024  vacuumDeletedRows(shard);
3025  }
3026 }
3027 
3029  cat_read_lock read_lock(this);
3030  // "if not a table that supports delete return nullptr, nothing more to do"
3031  const ColumnDescriptor* cd = getDeletedColumn(td);
3032  if (nullptr == cd) {
3033  return;
3034  }
3035  // vacuum chunks which show sign of deleted rows in metadata
3036  ChunkKey chunkKeyPrefix = {currentDB_.dbId, td->tableId, cd->columnId};
3037  std::vector<std::pair<ChunkKey, ChunkMetadata>> chunkMetadataVec;
3038  dataMgr_->getChunkMetadataVecForKeyPrefix(chunkMetadataVec, chunkKeyPrefix);
3039  for (auto cm : chunkMetadataVec) {
3040  // "delete has occured"
3041  if (cm.second.chunkStats.max.tinyintval == 1) {
3042  UpdelRoll updel_roll;
3043  updel_roll.catalog = this;
3044  updel_roll.logicalTableId = getLogicalTableId(td->tableId);
3046  const auto cd = getMetadataForColumn(td->tableId, cm.first[2]);
3047  const auto chunk = Chunk_NS::Chunk::getChunk(cd,
3048  &getDataMgr(),
3049  cm.first,
3050  updel_roll.memoryLevel,
3051  0,
3052  cm.second.numBytes,
3053  cm.second.numElements);
3054  td->fragmenter->compactRows(this,
3055  td,
3056  cm.first[3],
3057  td->fragmenter->getVacuumOffsets(chunk),
3058  updel_roll.memoryLevel,
3059  updel_roll);
3060  updel_roll.commitUpdate();
3061  }
3062  }
3063 }
3064 
3065 } // namespace Catalog_Namespace
const Parser::SharedDictionaryDef compress_reference_path(Parser::SharedDictionaryDef cur_node, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Data_Namespace::MemoryLevel memoryLevel
Definition: UpdelRoll.h:65
std::string virtualExpr
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:1596
#define CHECK_EQ(x, y)
Definition: Logger.h:195
const int MAPD_TEMP_TABLE_START_ID
Definition: Catalog.cpp:78
std::string partitions
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:2947
std::string dictFolderPath
void doDropTable(const TableDescriptor *td)
Definition: Catalog.cpp:2558
bool is_string_array() const
Definition: sqltypes.h:447
virtual ~Catalog()
Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the ...
Definition: Catalog.cpp:154
HOST DEVICE int get_size() const
Definition: sqltypes.h:329
static std::shared_ptr< Executor > getExecutor(const int db_id, const std::string &debug_dir="", const std::string &debug_file="", const MapDParameters mapd_parameters=MapDParameters(), ::QueryRenderer::QueryRenderManager *render_manager=nullptr)
Definition: Execute.cpp:122
T getData(const int row, const int col)
mapd_shared_mutex sharedMutex_
Definition: Catalog.h:336
void commitUpdate()
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:321
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:81
SQLTypes
Definition: sqltypes.h:40
const std::vector< LeafHostInfo > & getStringDictionaryHosts() const
Definition: Catalog.cpp:1384
#define SPIMAP_MAGIC1
Definition: Catalog.h:68
std::string tableName
void setDeletedColumn(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:2236
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:303
static const AccessPrivileges ALL_DATABASE
Definition: DBObject.h:144
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:1937
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:325
EncodingType
Definition: encodetypes.h:22
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
DBObjectType
Definition: DBObject.h:42
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:177
void addColumn(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:1700
void checkpoint(const int db_id, const int tb_id)
Definition: DataMgr.cpp:406
void roll(const bool forward)
Definition: Catalog.cpp:1750
#define LOG(tag)
Definition: Logger.h:182
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:326
SqliteConnector sqliteConnector_
Definition: Catalog.h:309
static std::shared_ptr< Catalog > get(const std::string &dbName)
Definition: Catalog.cpp:2988
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:313
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:305
void set_size(int s)
Definition: sqltypes.h:417
void query_with_text_params(std::string const &query_only)
void revokeDBObjectPrivilegesFromAll(DBObject object, Catalog *catalog)
const ColumnDescriptor * getDeletedColumnIfRowsDeleted(const TableDescriptor *td) const
Definition: Catalog.cpp:2202
HOST DEVICE int get_scale() const
Definition: sqltypes.h:324
void revokeDBObjectPrivileges(const std::string &grantee, const DBObject &object, const Catalog_Namespace::Catalog &catalog)
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:319
Catalog(const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
Constructor - takes basePath to already extant data directory for writing.
Definition: Catalog.cpp:127
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:311
DBObject * findDbObject(const DBObjectKey &objectKey, bool only_direct) const
Definition: Grantee.cpp:72
void query(const std::string &queryString)
#define CHECK_GE(x, y)
Definition: Logger.h:200
void setObjectKey(const DBObjectKey &objectKey)
Definition: DBObject.h:200
Driver for running cleanup processes on a table. TableOptimizer provides functions for various cleanu...
int32_t getTableEpoch(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:2123
std::string fragments
int32_t objectId
Definition: DBObject.h:56
HOST DEVICE void set_type(SQLTypes t)
Definition: sqltypes.h:409
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
void replaceDashboard(DashboardDescriptor &vd)
Definition: Catalog.cpp:2756
const int MAPD_TEMP_DICT_START_ID
Definition: Catalog.cpp:80
Definition: Grantee.h:76
std::string generatePhysicalTableName(const std::string &logicalTableName, const int32_t &shardNumber)
Definition: Catalog.cpp:2977
The InsertOrderFragmenter is a child class of AbstractFragmenter, and fragments data in insert order...
void addTableToMap(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
Definition: Catalog.cpp:1049
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logicalTableDesc) const
Definition: Catalog.cpp:2895
void checkDateInDaysColumnMigration()
Definition: Catalog.cpp:653
void set_dimension(int d)
Definition: sqltypes.h:411
void setPrivileges(const AccessPrivileges &privs)
Definition: DBObject.h:202
void addFrontendViewToMap(DashboardDescriptor &vd)
Definition: Catalog.cpp:1181
void deleteMetadataForDashboard(const std::string &userId, const std::string &dashName)
Definition: Catalog.cpp:1445
void vacuumDeletedRows(const TableDescriptor *td) const
Definition: Catalog.cpp:3028
static const AccessPrivileges SELECT_FROM_TABLE
Definition: DBObject.h:153
void set_scale(int s)
Definition: sqltypes.h:414
std::vector< int > columnIdBySpi_
const bool checkMetadataForDeletedRecs(int dbId, int tableId, int columnId) const
Definition: Catalog.cpp:2183
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:327
void set_compression(EncodingType c)
Definition: sqltypes.h:419
Grantee * getGrantee(const std::string &name) const
std::atomic< std::thread::id > thread_holding_sqlite_lock
Definition: Catalog.h:337
void renameColumn(const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
Definition: Catalog.cpp:2655
DeletedColumnPerTableMap deletedColumnPerTable_
Definition: Catalog.h:332
const ColumnDescriptor * get_foreign_col(const Catalog &cat, const Parser::SharedDictionaryDef &shared_dict_def)
Definition: Catalog.cpp:2250
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:304
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:306
std::string to_string(char const *&&v)
void set_notnull(bool n)
Definition: sqltypes.h:416
std::shared_ptr< std::mutex > mutex_
static const AccessPrivileges ALL_VIEW
Definition: DBObject.h:170
std::string chunks
void grantRoleBatch(const std::vector< std::string > &roles, const std::vector< std::string > &grantees)
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:526
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:96
#define SPIMAP_MAGIC2
Definition: Catalog.h:69
size_t getNumRows() const
const ColumnDescriptor * getShardColumnMetadataForTable(const TableDescriptor *td) const
Definition: Catalog.cpp:2879
void createRole(const std::string &roleName, const bool &userPrivateRole=false)
std::string dashboardSystemRoleName
This file contains the class specification and related data structures for Catalog.
virtual void compactRows(const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const int fragmentId, const std::vector< uint64_t > &fragOffsets, const Data_Namespace::MemoryLevel memoryLevel, UpdelRoll &updelRoll)=0
int getLogicalTableId(const int physicalTableId) const
Definition: Catalog.cpp:2914
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:410
bool is_array() const
Definition: sqltypes.h:454
void checkpoint(const int logicalTableId) const
Definition: Catalog.cpp:2927
virtual const std::vector< uint64_t > getVacuumOffsets(const std::shared_ptr< Chunk_NS::Chunk > &chunk)=0
static SysCatalog & instance()
Definition: SysCatalog.h:240
This file contains the class specification and related data structures for SysCatalog.
const Catalog_Namespace::Catalog * catalog
Definition: UpdelRoll.h:63
std::string calculateSHA1(const std::string &data)
Definition: Catalog.cpp:2828
std::string getUserFromId(const int32_t id)
Definition: Catalog.cpp:853
void addLinkToMap(LinkDescriptor &ld)
Definition: Catalog.cpp:1263
#define INJECT_TIMER(DESC)
Definition: measure.h:91
void delDictionary(const ColumnDescriptor &cd)
Definition: Catalog.cpp:1639
const std::string & get_foreign_column() const
Definition: ParserNode.h:912
const TableDescriptor * getMetadataForTableImpl(int tableId, const bool populateFragmenter) const
Definition: Catalog.cpp:1328
void renameObjectsInDescriptorMap(DBObject &object, const Catalog_Namespace::Catalog &cat)
void removeChunks(const int table_id)
Definition: Catalog.cpp:2506
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:320
mapd::shared_ptr< MapDClient > client
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:2241
void dropTable(const TableDescriptor *td)
Definition: Catalog.cpp:2525
std::tuple< int, int > ColumnIdKey
Definition: Catalog.h:231
std::vector< std::string > parse_underlying_dashboard_objects(const std::string &meta)
static const int32_t MAPD_VERSION
Definition: release.h:33
std::tuple< int, std::string > ColumnKey
Definition: Catalog.h:229
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:302
void setOwner(int32_t userId)
Definition: DBObject.h:210
static std::shared_ptr< Chunk > getChunk(const ColumnDescriptor *cd, DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId, const size_t num_bytes, const size_t num_elems)
Definition: Chunk.cpp:28
void grantDBObjectPrivilegesBatch(const std::vector< std::string > &grantees, const std::vector< DBObject > &objects, const Catalog_Namespace::Catalog &catalog)
specifies the content in-memory of a row in the column metadata table
int32_t dictId
Definition: DictRef.h:10
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:176
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:1807
SQLTypeInfoCore< ArrayContextTypeSizer, ExecutorTypePackaging, DateTimeFacilities > SQLTypeInfo
Definition: sqltypes.h:819
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1222
std::string keyMetainfo
int g_test_against_columnId_gap
Definition: Catalog.cpp:72
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1348
struct dict_ref_t DictRef
Definition: DictRef.h:37
void createShardedTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:2384
std::string to_upper(const std::string &str)
const int DEFAULT_INITIAL_VERSION
Definition: Catalog.cpp:77
Definition: sqltypes.h:55
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:314
static const std::string physicalTableNameTag_
Definition: Catalog.h:318
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1186
static void remove(const std::string &dbName)
Definition: Catalog.cpp:3014
static const AccessPrivileges SELECT_FROM_VIEW
Definition: DBObject.h:173
void setTableEpoch(const int db_id, const int table_id, const int new_epoch)
Definition: Catalog.cpp:2153
static void set(const std::string &dbName, std::shared_ptr< Catalog > cat)
Definition: Catalog.cpp:2984
std::vector< ObjectRoleDescriptor * > getMetadataForObject(int32_t dbId, int32_t dbType, int32_t objectId) const
const DBObjectMap * getDbObjects(bool only_direct) const
Definition: Grantee.h:56
void recomputeMetadata() const
Recomputes per-chunk metadata for each fragment in the table. Updates and deletes can cause chunk met...
const int OMNISCI_ROOT_USER_ID
Definition: SysCatalog.h:60
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1579
Fragmenter_Namespace::FragmenterType fragType
int32_t dbId
Definition: DBObject.h:55
Data_Namespace::MemoryLevel persistenceLevel
std::string name() const
Definition: Catalog.h:219
const std::string & get_foreign_table() const
Definition: ParserNode.h:910
int32_t createDashboard(DashboardDescriptor &vd)
Definition: Catalog.cpp:2694
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1099
static const AccessPrivileges ALL_DASHBOARD
Definition: DBObject.h:162
void addReferenceToForeignDict(ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def)
Definition: Catalog.cpp:2262
static const AccessPrivileges ALL_TABLE
Definition: DBObject.h:150
int logicalTableId
Definition: UpdelRoll.h:64
bool g_cache_string_hash
Definition: Execute.cpp:86
bool isNull(const int row, const int col) const
const ColumnDescriptor * getDeletedColumn(const TableDescriptor *td) const
Definition: Catalog.cpp:2177
void set_comp_param(int p)
Definition: sqltypes.h:420
const ColumnDescriptor * getMetadataForColumnBySpi(const int tableId, const size_t spi) const
Definition: Catalog.cpp:1435
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:328
const int getColumnIdBySpi(const int tableId, const size_t spi) const
Definition: Catalog.cpp:1430
bool setColumnSharedDictionary(ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:2281
int get_physical_cols() const
Definition: sqltypes.h:340
#define CHECK(condition)
Definition: Logger.h:187
std::vector< int > ChunkKey
Definition: types.h:35
void getDictionary(const ColumnDescriptor &cd, std::map< int, StringDictionary *> &stringDicts)
Definition: Catalog.cpp:1672
std::string createLink(LinkDescriptor &ld, size_t min_length)
Definition: Catalog.cpp:2840
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const
Descriptor for a dictionary for a string columne.
void updateLogicalToPhysicalTableLinkSchema()
Definition: Catalog.cpp:465
bool is_geometry() const
Definition: sqltypes.h:458
FileBuffer Chunk
A Chunk is the fundamental unit of execution in Map-D.
Definition: FileMgr.h:68
const int getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const
Definition: Catalog.cpp:1413
std::unordered_map< std::string, std::vector< std::string > > getGranteesOfSharedDashboards(const std::vector< std::string > &dashboard_ids)
std::atomic< std::thread::id > thread_holding_write_lock
Definition: Catalog.h:338
static TimeT::rep execution(F func, Args &&... args)
Definition: sample.cpp:29
int32_t permissionType
Definition: DBObject.h:54
void populateRoleDbObjects(const std::vector< DBObject > &objects)
#define DEFAULT_MAX_CHUNK_SIZE
std::string viewSQL
Definition: sqltypes.h:47
SQLTypeInfo columnType
Role * getRoleGrantee(const std::string &name) const
DictRef addDictionary(ColumnDescriptor &cd)
Definition: Catalog.cpp:1613
specifies the content in-memory of a row in the table metadata table
void renameTable(const TableDescriptor *td, const std::string &newTableName)
Definition: Catalog.cpp:2615
const LinkDescriptor * getMetadataForLink(const std::string &link) const
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_<EPOCH>_<oldname>.
Definition: File.cpp:183
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:308
static std::map< std::string, std::shared_ptr< Catalog > > mapd_cat_map_
Definition: Catalog.h:331
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:2343
Fragmenter_Namespace::AbstractFragmenter * fragmenter
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:328
std::list< const DashboardDescriptor * > getAllDashboardsMetadata() const
Definition: Catalog.cpp:1605
void doTruncateTable(const TableDescriptor *td)
Definition: Catalog.cpp:2436
void renamePhysicalTable(const TableDescriptor *td, const std::string &newTableName)
Definition: Catalog.cpp:2589
std::string columnName
void query_with_text_param(const std::string &queryString, const std::string &text_param)
std::vector< DBObject > parseDashboardObjects(const std::string &view_meta, const int &user_id)
Definition: Catalog.cpp:1192
bool is_string() const
Definition: sqltypes.h:446
#define IS_GEO(T)
Definition: sqltypes.h:165
virtual void renameDbObject(const DBObject &object)
Definition: Grantee.cpp:108
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:316
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:301
static thread_local bool thread_holds_read_lock
Definition: Catalog.h:340
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:307
void CheckAndExecuteMigrationsPostBuildMaps()
Definition: Catalog.cpp:847
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)
void truncateTable(const TableDescriptor *td)
Definition: Catalog.cpp:2418
void instantiateFragmenter(TableDescriptor *td) const
Definition: Catalog.cpp:1271
void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id)
Definition: Catalog.cpp:479