OmniSciDB  1dac507f6e
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Catalog.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
24 #include <sys/wait.h>
25 
26 #include <algorithm>
27 #include <cassert>
28 #include <cerrno>
29 #include <cstdio>
30 #include <cstring>
31 #include <exception>
32 #include <list>
33 #include <memory>
34 #include <random>
35 #include <regex>
36 #include <sstream>
37 #include "Catalog.h"
38 #include "SysCatalog.h"
39 
40 #include <boost/algorithm/string/predicate.hpp>
41 #include <boost/filesystem.hpp>
42 #include <boost/range/adaptor/map.hpp>
43 #include <boost/version.hpp>
44 #if BOOST_VERSION >= 106600
45 #include <boost/uuid/detail/sha1.hpp>
46 #else
47 #include <boost/uuid/sha1.hpp>
48 #endif
49 
50 #include "../Fragmenter/Fragmenter.h"
51 #include "../Fragmenter/SortedOrderFragmenter.h"
52 #include "../LockMgr/TableLockMgr.h"
53 #include "../Parser/ParserNode.h"
54 #include "../QueryEngine/Execute.h"
55 #include "../QueryEngine/TableOptimizer.h"
56 #include "../Shared/File.h"
57 #include "../Shared/StringTransform.h"
58 #include "../Shared/measure.h"
59 #include "../StringDictionary/StringDictionaryClient.h"
60 #include "MapDRelease.h"
61 #include "RWLocks.h"
63 
64 using Chunk_NS::Chunk;
67 using std::list;
68 using std::map;
69 using std::pair;
70 using std::runtime_error;
71 using std::string;
72 using std::vector;
73 
75 extern bool g_cache_string_hash;
76 
77 namespace Catalog_Namespace {
78 
79 const int DEFAULT_INITIAL_VERSION = 1; // start at version 1
81  1073741824; // 2^30, give room for over a billion non-temp tables
83  1073741824; // 2^30, give room for over a billion non-temp dictionaries
84 
85 const std::string Catalog::physicalTableNameTag_("_shard_#");
86 std::map<std::string, std::shared_ptr<Catalog>> Catalog::mapd_cat_map_;
87 
88 thread_local bool Catalog::thread_holds_read_lock = false;
89 
93 
94 // migration will be done as two step process this release
95 // will create and use new table
96 // next release will remove old table, doing this to have fall back path
97 // incase of migration failure
100  sqliteConnector_.query("BEGIN TRANSACTION");
101  try {
103  "SELECT name FROM sqlite_master WHERE type='table' AND name='mapd_dashboards'");
104  if (sqliteConnector_.getNumRows() != 0) {
105  // already done
106  sqliteConnector_.query("END TRANSACTION");
107  return;
108  }
110  "CREATE TABLE mapd_dashboards (id integer primary key autoincrement, name text , "
111  "userid integer references mapd_users, state text, image_hash text, update_time "
112  "timestamp, "
113  "metadata text, UNIQUE(userid, name) )");
114  // now copy content from old table to new table
116  "insert into mapd_dashboards (id, name , "
117  "userid, state, image_hash, update_time , "
118  "metadata) "
119  "SELECT viewid , name , userid, view_state, image_hash, update_time, "
120  "view_metadata "
121  "from mapd_frontend_views");
122  } catch (const std::exception& e) {
123  sqliteConnector_.query("ROLLBACK TRANSACTION");
124  throw;
125  }
126  sqliteConnector_.query("END TRANSACTION");
127 }
128 
129 Catalog::Catalog(const string& basePath,
130  const DBMetadata& curDB,
131  std::shared_ptr<Data_Namespace::DataMgr> dataMgr,
132  const std::vector<LeafHostInfo>& string_dict_hosts,
133  std::shared_ptr<Calcite> calcite,
134  bool is_new_db)
135  : basePath_(basePath)
136  , sqliteConnector_(curDB.dbName, basePath + "/mapd_catalogs/")
137  , currentDB_(curDB)
138  , dataMgr_(dataMgr)
139  , string_dict_hosts_(string_dict_hosts)
140  , calciteMgr_(calcite)
141  , nextTempTableId_(MAPD_TEMP_TABLE_START_ID)
142  , nextTempDictId_(MAPD_TEMP_DICT_START_ID)
143  , sqliteMutex_()
144  , sharedMutex_()
145  , thread_holding_sqlite_lock()
146  , thread_holding_write_lock() {
147  if (!is_new_db) {
149  }
150  buildMaps();
151  if (!is_new_db) {
153  }
154 }
155 
158  // must clean up heap-allocated TableDescriptor and ColumnDescriptor structs
159  for (TableDescriptorMap::iterator tableDescIt = tableDescriptorMap_.begin();
160  tableDescIt != tableDescriptorMap_.end();
161  ++tableDescIt) {
162  if (tableDescIt->second->fragmenter != nullptr) {
163  delete tableDescIt->second->fragmenter;
164  }
165  delete tableDescIt->second;
166  }
167 
168  // TableDescriptorMapById points to the same descriptors. No need to delete
169 
170  for (ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.begin();
171  columnDescIt != columnDescriptorMap_.end();
172  ++columnDescIt) {
173  delete columnDescIt->second;
174  }
175 
176  // ColumnDescriptorMapById points to the same descriptors. No need to delete
177 }
178 
181  sqliteConnector_.query("BEGIN TRANSACTION");
182  try {
183  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_tables)");
184  std::vector<std::string> cols;
185  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
186  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
187  }
188  if (std::find(cols.begin(), cols.end(), std::string("max_chunk_size")) ==
189  cols.end()) {
190  string queryString("ALTER TABLE mapd_tables ADD max_chunk_size BIGINT DEFAULT " +
192  sqliteConnector_.query(queryString);
193  }
194  if (std::find(cols.begin(), cols.end(), std::string("shard_column_id")) ==
195  cols.end()) {
196  string queryString("ALTER TABLE mapd_tables ADD shard_column_id BIGINT DEFAULT " +
197  std::to_string(0));
198  sqliteConnector_.query(queryString);
199  }
200  if (std::find(cols.begin(), cols.end(), std::string("shard")) == cols.end()) {
201  string queryString("ALTER TABLE mapd_tables ADD shard BIGINT DEFAULT " +
202  std::to_string(-1));
203  sqliteConnector_.query(queryString);
204  }
205  if (std::find(cols.begin(), cols.end(), std::string("num_shards")) == cols.end()) {
206  string queryString("ALTER TABLE mapd_tables ADD num_shards BIGINT DEFAULT " +
207  std::to_string(0));
208  sqliteConnector_.query(queryString);
209  }
210  if (std::find(cols.begin(), cols.end(), std::string("key_metainfo")) == cols.end()) {
211  string queryString("ALTER TABLE mapd_tables ADD key_metainfo TEXT DEFAULT '[]'");
212  sqliteConnector_.query(queryString);
213  }
214  if (std::find(cols.begin(), cols.end(), std::string("userid")) == cols.end()) {
215  string queryString("ALTER TABLE mapd_tables ADD userid integer DEFAULT " +
217  sqliteConnector_.query(queryString);
218  }
219  if (std::find(cols.begin(), cols.end(), std::string("sort_column_id")) ==
220  cols.end()) {
222  "ALTER TABLE mapd_tables ADD sort_column_id INTEGER DEFAULT 0");
223  }
224  } catch (std::exception& e) {
225  sqliteConnector_.query("ROLLBACK TRANSACTION");
226  throw;
227  }
228  sqliteConnector_.query("END TRANSACTION");
229 }
230 
233  sqliteConnector_.query("BEGIN TRANSACTION");
234  try {
236  "select name from sqlite_master WHERE type='table' AND "
237  "name='mapd_version_history'");
238  if (sqliteConnector_.getNumRows() == 0) {
240  "CREATE TABLE mapd_version_history(version integer, migration_history text "
241  "unique)");
242  } else {
244  "select * from mapd_version_history where migration_history = "
245  "'notnull_fixlen_arrays'");
246  if (sqliteConnector_.getNumRows() != 0) {
247  // legacy fixlen arrays had migrated
248  // no need for further execution
249  sqliteConnector_.query("END TRANSACTION");
250  return;
251  }
252  }
253  // Insert check for migration
255  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
256  std::vector<std::string>{std::to_string(MAPD_VERSION), "notnull_fixlen_arrays"});
257  LOG(INFO) << "Updating mapd_columns, legacy fixlen arrays";
258  // Upating all fixlen array columns
259  string queryString("UPDATE mapd_columns SET is_notnull=1 WHERE coltype=" +
260  std::to_string(kARRAY) + " AND size>0;");
261  sqliteConnector_.query(queryString);
262  } catch (std::exception& e) {
263  sqliteConnector_.query("ROLLBACK TRANSACTION");
264  throw;
265  }
266  sqliteConnector_.query("END TRANSACTION");
267 }
268 
271  sqliteConnector_.query("BEGIN TRANSACTION");
272  try {
273  // check table still exists
275  "SELECT name FROM sqlite_master WHERE type='table' AND "
276  "name='mapd_frontend_views'");
277  if (sqliteConnector_.getNumRows() == 0) {
278  // table does not exists
279  // no need to migrate
280  sqliteConnector_.query("END TRANSACTION");
281  return;
282  }
283  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_frontend_views)");
284  std::vector<std::string> cols;
285  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
286  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
287  }
288  if (std::find(cols.begin(), cols.end(), std::string("image_hash")) == cols.end()) {
289  sqliteConnector_.query("ALTER TABLE mapd_frontend_views ADD image_hash text");
290  }
291  if (std::find(cols.begin(), cols.end(), std::string("update_time")) == cols.end()) {
292  sqliteConnector_.query("ALTER TABLE mapd_frontend_views ADD update_time timestamp");
293  }
294  if (std::find(cols.begin(), cols.end(), std::string("view_metadata")) == cols.end()) {
295  sqliteConnector_.query("ALTER TABLE mapd_frontend_views ADD view_metadata text");
296  }
297  } catch (std::exception& e) {
298  sqliteConnector_.query("ROLLBACK TRANSACTION");
299  throw;
300  }
301  sqliteConnector_.query("END TRANSACTION");
302 }
303 
306  sqliteConnector_.query("BEGIN TRANSACTION");
307  try {
309  "CREATE TABLE IF NOT EXISTS mapd_links (linkid integer primary key, userid "
310  "integer references mapd_users, "
311  "link text unique, view_state text, update_time timestamp, view_metadata text)");
312  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_links)");
313  std::vector<std::string> cols;
314  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
315  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
316  }
317  if (std::find(cols.begin(), cols.end(), std::string("view_metadata")) == cols.end()) {
318  sqliteConnector_.query("ALTER TABLE mapd_links ADD view_metadata text");
319  }
320  } catch (const std::exception& e) {
321  sqliteConnector_.query("ROLLBACK TRANSACTION");
322  throw;
323  }
324  sqliteConnector_.query("END TRANSACTION");
325 }
326 
329  sqliteConnector_.query("BEGIN TRANSACTION");
330  try {
331  sqliteConnector_.query("UPDATE mapd_links SET userid = 0 WHERE userid IS NULL");
332  // check table still exists
334  "SELECT name FROM sqlite_master WHERE type='table' AND "
335  "name='mapd_frontend_views'");
336  if (sqliteConnector_.getNumRows() == 0) {
337  // table does not exists
338  // no need to migrate
339  sqliteConnector_.query("END TRANSACTION");
340  return;
341  }
343  "UPDATE mapd_frontend_views SET userid = 0 WHERE userid IS NULL");
344  } catch (const std::exception& e) {
345  sqliteConnector_.query("ROLLBACK TRANSACTION");
346  throw;
347  }
348  sqliteConnector_.query("END TRANSACTION");
349 }
350 
351 // introduce DB version into the tables table
352 // if the DB does not have a version reset all pagesizes to 2097152 to be compatible with
353 // old value
354 
357  if (currentDB_.dbName.length() == 0) {
358  // updateDictionaryNames dbName length is zero nothing to do here
359  return;
360  }
361  sqliteConnector_.query("BEGIN TRANSACTION");
362  try {
363  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_tables)");
364  std::vector<std::string> cols;
365  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
366  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
367  }
368  if (std::find(cols.begin(), cols.end(), std::string("version_num")) == cols.end()) {
369  LOG(INFO) << "Updating mapd_tables updatePageSize";
370  // No version number
371  // need to update the defaul tpagesize to old correct value
372  sqliteConnector_.query("UPDATE mapd_tables SET frag_page_size = 2097152 ");
373  // need to add new version info
374  string queryString("ALTER TABLE mapd_tables ADD version_num BIGINT DEFAULT " +
376  sqliteConnector_.query(queryString);
377  }
378  } catch (std::exception& e) {
379  sqliteConnector_.query("ROLLBACK TRANSACTION");
380  throw;
381  }
382  sqliteConnector_.query("END TRANSACTION");
383 }
384 
387  sqliteConnector_.query("BEGIN TRANSACTION");
388  try {
389  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_columns)");
390  std::vector<std::string> cols;
391  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
392  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
393  }
394  if (std::find(cols.begin(), cols.end(), std::string("version_num")) == cols.end()) {
395  LOG(INFO) << "Updating mapd_columns updateDeletedColumnIndicator";
396  // need to add new version info
397  string queryString("ALTER TABLE mapd_columns ADD version_num BIGINT DEFAULT " +
399  sqliteConnector_.query(queryString);
400  // need to add new column to table defintion to indicate deleted column, column used
401  // as bitmap for deleted rows.
403  "ALTER TABLE mapd_columns ADD is_deletedcol boolean default 0 ");
404  }
405  } catch (std::exception& e) {
406  sqliteConnector_.query("ROLLBACK TRANSACTION");
407  throw;
408  }
409  sqliteConnector_.query("END TRANSACTION");
410 }
411 
412 // introduce DB version into the dictionary tables
413 // if the DB does not have a version rename all dictionary tables
414 
417  if (currentDB_.dbName.length() == 0) {
418  // updateDictionaryNames dbName length is zero nothing to do here
419  return;
420  }
421  sqliteConnector_.query("BEGIN TRANSACTION");
422  try {
423  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_dictionaries)");
424  std::vector<std::string> cols;
425  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
426  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
427  }
428  if (std::find(cols.begin(), cols.end(), std::string("version_num")) == cols.end()) {
429  // No version number
430  // need to rename dictionaries
431  string dictQuery("SELECT dictid, name from mapd_dictionaries");
432  sqliteConnector_.query(dictQuery);
433  size_t numRows = sqliteConnector_.getNumRows();
434  for (size_t r = 0; r < numRows; ++r) {
435  int dictId = sqliteConnector_.getData<int>(r, 0);
436  std::string dictName = sqliteConnector_.getData<string>(r, 1);
437 
438  std::string oldName =
439  basePath_ + "/mapd_data/" + currentDB_.dbName + "_" + dictName;
440  std::string newName = basePath_ + "/mapd_data/DB_" +
441  std::to_string(currentDB_.dbId) + "_DICT_" +
442  std::to_string(dictId);
443 
444  int result = rename(oldName.c_str(), newName.c_str());
445 
446  if (result == 0) {
447  LOG(INFO) << "Dictionary upgrade: successfully renamed " << oldName << " to "
448  << newName;
449  } else {
450  LOG(ERROR) << "Failed to rename old dictionary directory " << oldName << " to "
451  << newName + " dbname '" << currentDB_.dbName << "' error code "
452  << std::to_string(result);
453  }
454  }
455  // need to add new version info
456  string queryString("ALTER TABLE mapd_dictionaries ADD version_num BIGINT DEFAULT " +
458  sqliteConnector_.query(queryString);
459  }
460  } catch (std::exception& e) {
461  sqliteConnector_.query("ROLLBACK TRANSACTION");
462  throw;
463  }
464  sqliteConnector_.query("END TRANSACTION");
465 }
466 
469  sqliteConnector_.query("BEGIN TRANSACTION");
470  try {
472  "CREATE TABLE IF NOT EXISTS mapd_logical_to_physical("
473  "logical_table_id integer, physical_table_id integer)");
474  } catch (const std::exception& e) {
475  sqliteConnector_.query("ROLLBACK TRANSACTION");
476  throw;
477  }
478  sqliteConnector_.query("END TRANSACTION");
479 }
480 
481 void Catalog::updateLogicalToPhysicalTableMap(const int32_t logical_tb_id) {
482  /* this proc inserts/updates all pairs of (logical_tb_id, physical_tb_id) in
483  * sqlite mapd_logical_to_physical table for given logical_tb_id as needed
484  */
485 
487  sqliteConnector_.query("BEGIN TRANSACTION");
488  try {
489  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
490  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
491  const auto physicalTables = physicalTableIt->second;
492  CHECK(!physicalTables.empty());
493  for (size_t i = 0; i < physicalTables.size(); i++) {
494  int32_t physical_tb_id = physicalTables[i];
496  "INSERT OR REPLACE INTO mapd_logical_to_physical (logical_table_id, "
497  "physical_table_id) VALUES (?1, ?2)",
498  std::vector<std::string>{std::to_string(logical_tb_id),
499  std::to_string(physical_tb_id)});
500  }
501  }
502  } catch (std::exception& e) {
503  sqliteConnector_.query("ROLLBACK TRANSACTION");
504  throw;
505  }
506  sqliteConnector_.query("END TRANSACTION");
507 }
508 
511  sqliteConnector_.query("BEGIN TRANSACTION");
512  try {
513  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_dictionaries)");
514  std::vector<std::string> cols;
515  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
516  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
517  }
518  if (std::find(cols.begin(), cols.end(), std::string("refcount")) == cols.end()) {
519  sqliteConnector_.query("ALTER TABLE mapd_dictionaries ADD refcount DEFAULT 1");
520  }
521  } catch (std::exception& e) {
522  sqliteConnector_.query("ROLLBACK TRANSACTION");
523  throw;
524  }
525  sqliteConnector_.query("END TRANSACTION");
526 }
527 
530  sqliteConnector_.query("BEGIN TRANSACTION");
531  std::vector<DBObject> objects;
532  try {
534  "SELECT name FROM sqlite_master WHERE type='table' AND "
535  "name='mapd_record_ownership_marker'");
536  // check if mapd catalog - marker exists
537  if (sqliteConnector_.getNumRows() != 0 && currentDB_.dbId == 1) {
538  // already done
539  sqliteConnector_.query("END TRANSACTION");
540  return;
541  }
542  // check if different catalog - marker exists
543  else if (sqliteConnector_.getNumRows() != 0 && currentDB_.dbId != 1) {
544  sqliteConnector_.query("SELECT dummy FROM mapd_record_ownership_marker");
545  // Check if migration is being performed on existing non mapd catalogs
546  // Older non mapd dbs will have table but no record in them
547  if (sqliteConnector_.getNumRows() != 0) {
548  // already done
549  sqliteConnector_.query("END TRANSACTION");
550  return;
551  }
552  }
553  // marker not exists - create one
554  else {
555  sqliteConnector_.query("CREATE TABLE mapd_record_ownership_marker (dummy integer)");
556  }
557 
558  DBMetadata db;
559  CHECK(SysCatalog::instance().getMetadataForDB(currentDB_.dbName, db));
560  // place dbId as a refernce for migration being performed
562  "INSERT INTO mapd_record_ownership_marker (dummy) VALUES (?1)",
563  std::vector<std::string>{std::to_string(db.dbOwner)});
564 
565  static const std::map<const DBObjectType, const AccessPrivileges>
566  object_level_all_privs_lookup{
571 
572  // grant owner all permissions on DB
573  DBObjectKey key;
574  key.dbId = currentDB_.dbId;
575  auto _key_place = [&key](auto type) {
576  key.permissionType = type;
577  return key;
578  };
579  for (auto& it : object_level_all_privs_lookup) {
580  objects.emplace_back(_key_place(it.first), it.second, db.dbOwner);
581  objects.back().setName(currentDB_.dbName);
582  }
583 
584  {
585  // other users tables and views
586  string tableQuery(
587  "SELECT tableid, name, userid, isview FROM mapd_tables WHERE userid > 0");
588  sqliteConnector_.query(tableQuery);
589  size_t numRows = sqliteConnector_.getNumRows();
590  for (size_t r = 0; r < numRows; ++r) {
591  int32_t tableid = sqliteConnector_.getData<int>(r, 0);
592  std::string tableName = sqliteConnector_.getData<string>(r, 1);
593  int32_t ownerid = sqliteConnector_.getData<int>(r, 2);
594  bool isview = sqliteConnector_.getData<bool>(r, 3);
595 
598  DBObjectKey key;
599  key.dbId = currentDB_.dbId;
600  key.objectId = tableid;
601  key.permissionType = type;
602 
603  DBObject obj(tableName, type);
604  obj.setObjectKey(key);
605  obj.setOwner(ownerid);
608 
609  objects.push_back(obj);
610  }
611  }
612 
613  {
614  // other users dashboards
615  string tableQuery("SELECT id, name, userid FROM mapd_dashboards WHERE userid > 0");
616  sqliteConnector_.query(tableQuery);
617  size_t numRows = sqliteConnector_.getNumRows();
618  for (size_t r = 0; r < numRows; ++r) {
619  int32_t dashId = sqliteConnector_.getData<int>(r, 0);
620  std::string dashName = sqliteConnector_.getData<string>(r, 1);
621  int32_t ownerid = sqliteConnector_.getData<int>(r, 2);
622 
624  DBObjectKey key;
625  key.dbId = currentDB_.dbId;
626  key.objectId = dashId;
627  key.permissionType = type;
628 
629  DBObject obj(dashName, type);
630  obj.setObjectKey(key);
631  obj.setOwner(ownerid);
633 
634  objects.push_back(obj);
635  }
636  }
637  } catch (const std::exception& e) {
638  sqliteConnector_.query("ROLLBACK TRANSACTION");
639  throw;
640  }
641  sqliteConnector_.query("END TRANSACTION");
642 
643  // now apply the objects to the syscat to track the permisisons
644  // moved outside transaction to avoid lock in sqlite
645  try {
647  } catch (const std::exception& e) {
648  LOG(ERROR) << " Issue during migration of DB " << name() << " issue was " << e.what();
649  throw std::runtime_error(" Issue during migration of DB " + name() + " issue was " +
650  e.what());
651  // will need to remove the mapd_record_ownership_marker table and retry
652  }
653 }
654 
657  std::vector<int> tables_migrated = {};
658  std::unordered_map<int, std::vector<std::string>> tables_to_migrate;
659  sqliteConnector_.query("BEGIN TRANSACTION");
660  try {
662  "select name from sqlite_master WHERE type='table' AND "
663  "name='mapd_version_history'");
664  if (sqliteConnector_.getNumRows() == 0) {
666  "CREATE TABLE mapd_version_history(version integer, migration_history text "
667  "unique)");
669  "CREATE TABLE mapd_date_in_days_column_migration_tmp(table_id integer primary "
670  "key)");
671  } else {
673  "select * from mapd_version_history where migration_history = "
674  "'date_in_days_column'");
675  if (sqliteConnector_.getNumRows() != 0) {
676  // no need for further execution
677  sqliteConnector_.query("END TRANSACTION");
678  return;
679  }
680  LOG(INFO) << "Performing Date in days columns migration.";
682  "select name from sqlite_master where type='table' AND "
683  "name='mapd_date_in_days_column_migration_tmp'");
684  if (sqliteConnector_.getNumRows() != 0) {
686  "select table_id from mapd_date_in_days_column_migration_tmp");
687  if (sqliteConnector_.getNumRows() != 0) {
688  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
689  tables_migrated.push_back(sqliteConnector_.getData<int>(i, 0));
690  }
691  }
692  } else {
694  "CREATE TABLE mapd_date_in_days_column_migration_tmp(table_id integer "
695  "primary key)");
696  }
697  }
699  "SELECT tables.tableid, tables.name, columns.name FROM mapd_tables tables, "
700  "mapd_columns columns where tables.tableid = columns.tableid AND "
701  "columns.coltype = ?1 AND columns.compression = ?2",
702  std::vector<std::string>{
703  std::to_string(static_cast<int>(SQLTypes::kDATE)),
705  if (sqliteConnector_.getNumRows() != 0) {
706  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
707  tables_to_migrate[sqliteConnector_.getData<int>(i, 0)] = {
708  sqliteConnector_.getData<std::string>(i, 1),
709  sqliteConnector_.getData<std::string>(i, 2)};
710  }
711  }
712  } catch (const std::exception& e) {
713  LOG(ERROR) << "Failed to complete migraion on date in days column: " << e.what();
714  sqliteConnector_.query("ROLLBACK");
715  throw;
716  }
717  sqliteConnector_.query("END TRANSACTION");
718 
719  for (auto& id_names : tables_to_migrate) {
720  if (std::find(tables_migrated.begin(), tables_migrated.end(), id_names.first) ==
721  tables_migrated.end()) {
722  sqliteConnector_.query("BEGIN TRANSACTION");
723  try {
724  LOG(INFO) << "Table: " << id_names.second[0]
725  << " may suffer from issues with DATE column: " << id_names.second[1]
726  << ". Running an OPTIMIZE command to solve any issues with metadata.";
727 
728  auto executor = Executor::getExecutor(getCurrentDB().dbId);
729  TableDescriptorMapById::iterator tableDescIt =
730  tableDescriptorMapById_.find(id_names.first);
731  if (tableDescIt == tableDescriptorMapById_.end()) {
732  throw runtime_error("Table descriptor does not exist for table " +
733  id_names.second[0] + " does not exist.");
734  }
735  auto td = tableDescIt->second;
736  TableOptimizer optimizer(td, executor.get(), *this);
737  optimizer.recomputeMetadata();
738 
740  "INSERT INTO mapd_date_in_days_column_migration_tmp VALUES(?)",
741  std::vector<std::string>{std::to_string(id_names.first)});
742  } catch (const std::exception& e) {
743  LOG(ERROR) << "Failed to complete migraion on date in days column: " << e.what();
744  sqliteConnector_.query("ROLLBACK");
745  throw;
746  }
747  sqliteConnector_.query("COMMIT");
748  }
749  }
750 
751  sqliteConnector_.query("BEGIN TRANSACTION");
752  try {
753  sqliteConnector_.query("DROP TABLE mapd_date_in_days_column_migration_tmp");
755  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
756  std::vector<std::string>{std::to_string(MAPD_VERSION), "date_in_days_column"});
757  } catch (const std::exception& e) {
758  LOG(ERROR) << "Failed to complete migraion on date in days column: " << e.what();
759  sqliteConnector_.query("ROLLBACK");
760  throw;
761  }
762  sqliteConnector_.query("END TRANSACTION");
763  LOG(INFO) << "Migration successfull on Date in days columns";
764 }
765 
767  std::unordered_map<std::string, std::pair<int, std::string>> dashboards;
768  std::vector<std::string> dashboard_ids;
769  static const std::string migration_name{"dashboard_roles_migration"};
770  {
772  sqliteConnector_.query("BEGIN TRANSACTION");
773  try {
774  // migration_history should be present in all catalogs by now
775  // if not then would be created before this migration
777  "select * from mapd_version_history where migration_history = '" +
778  migration_name + "'");
779  if (sqliteConnector_.getNumRows() != 0) {
780  // no need for further execution
781  sqliteConnector_.query("END TRANSACTION");
782  return;
783  }
784  LOG(INFO) << "Performing dashboard internal roles Migration.";
785  sqliteConnector_.query("select id, userid, metadata from mapd_dashboards");
786  for (size_t i = 0; i < sqliteConnector_.getNumRows(); ++i) {
789  sqliteConnector_.getData<string>(i, 0)))) {
790  // Successfully created roles during previous migration/crash
791  // No need to include them
792  continue;
793  }
794  dashboards[sqliteConnector_.getData<string>(i, 0)] = std::make_pair(
795  sqliteConnector_.getData<int>(i, 1), sqliteConnector_.getData<string>(i, 2));
796  dashboard_ids.push_back(sqliteConnector_.getData<string>(i, 0));
797  }
798  } catch (const std::exception& e) {
799  sqliteConnector_.query("ROLLBACK TRANSACTION");
800  throw;
801  }
802  sqliteConnector_.query("END TRANSACTION");
803  }
804  // All current grantees with shared dashboards.
805  const auto active_grantees =
807 
808  try {
809  // NOTE(wamsi): Transactionally unsafe
810  for (auto dash : dashboards) {
811  createOrUpdateDashboardSystemRole(dash.second.second,
812  dash.second.first,
814  std::to_string(currentDB_.dbId), dash.first));
815  auto result = active_grantees.find(dash.first);
816  if (result != active_grantees.end()) {
819  dash.first)},
820  result->second);
821  }
822  }
824  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
825  std::vector<std::string>{std::to_string(MAPD_VERSION), migration_name});
826  } catch (const std::exception& e) {
827  LOG(ERROR) << "Failed to create dashboard system roles during migration: "
828  << e.what();
829  throw;
830  }
831  LOG(INFO) << "Successfully created dashboard system roles during migration.";
832 }
833 
843  updatePageSize();
847 }
848 
852 }
853 
854 namespace {
855 std::string getUserFromId(const int32_t id) {
856  UserMetadata user;
857  if (SysCatalog::instance().getMetadataForUserById(id, user)) {
858  return user.userName;
859  }
860  // a user could be deleted and a dashboard still exist?
861  return "Unknown";
862 }
863 } // namespace
864 
868 
869  string dictQuery(
870  "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
871  sqliteConnector_.query(dictQuery);
872  size_t numRows = sqliteConnector_.getNumRows();
873  for (size_t r = 0; r < numRows; ++r) {
874  int dictId = sqliteConnector_.getData<int>(r, 0);
875  std::string dictName = sqliteConnector_.getData<string>(r, 1);
876  int dictNBits = sqliteConnector_.getData<int>(r, 2);
877  bool is_shared = sqliteConnector_.getData<bool>(r, 3);
878  int refcount = sqliteConnector_.getData<int>(r, 4);
879  std::string fname = basePath_ + "/mapd_data/DB_" + std::to_string(currentDB_.dbId) +
880  "_DICT_" + std::to_string(dictId);
881  DictRef dict_ref(currentDB_.dbId, dictId);
882  DictDescriptor* dd = new DictDescriptor(
883  dict_ref, dictName, dictNBits, is_shared, refcount, fname, false);
884  dictDescriptorMapByRef_[dict_ref].reset(dd);
885  }
886 
887  string tableQuery(
888  "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
889  "max_chunk_size, frag_page_size, "
890  "max_rows, partitions, shard_column_id, shard, num_shards, key_metainfo, userid, "
891  "sort_column_id "
892  "from mapd_tables");
893  sqliteConnector_.query(tableQuery);
894  numRows = sqliteConnector_.getNumRows();
895  for (size_t r = 0; r < numRows; ++r) {
896  TableDescriptor* td = new TableDescriptor();
897  td->tableId = sqliteConnector_.getData<int>(r, 0);
898  td->tableName = sqliteConnector_.getData<string>(r, 1);
899  td->nColumns = sqliteConnector_.getData<int>(r, 2);
900  td->isView = sqliteConnector_.getData<bool>(r, 3);
901  td->fragments = sqliteConnector_.getData<string>(r, 4);
902  td->fragType =
904  td->maxFragRows = sqliteConnector_.getData<int>(r, 6);
905  td->maxChunkSize = sqliteConnector_.getData<int>(r, 7);
906  td->fragPageSize = sqliteConnector_.getData<int>(r, 8);
907  td->maxRows = sqliteConnector_.getData<int64_t>(r, 9);
908  td->partitions = sqliteConnector_.getData<string>(r, 10);
909  td->shardedColumnId = sqliteConnector_.getData<int>(r, 11);
910  td->shard = sqliteConnector_.getData<int>(r, 12);
911  td->nShards = sqliteConnector_.getData<int>(r, 13);
912  td->keyMetainfo = sqliteConnector_.getData<string>(r, 14);
913  td->userId = sqliteConnector_.getData<int>(r, 15);
914  td->sortedColumnId =
915  sqliteConnector_.isNull(r, 16) ? 0 : sqliteConnector_.getData<int>(r, 16);
916  if (!td->isView) {
917  td->fragmenter = nullptr;
918  }
919  td->hasDeletedCol = false;
922  }
923  string columnQuery(
924  "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
925  "is_notnull, compression, comp_param, "
926  "size, chunks, is_systemcol, is_virtualcol, virtual_expr, is_deletedcol from "
927  "mapd_columns ORDER BY tableid, "
928  "columnid");
929  sqliteConnector_.query(columnQuery);
930  numRows = sqliteConnector_.getNumRows();
931  int32_t skip_physical_cols = 0;
932  for (size_t r = 0; r < numRows; ++r) {
934  cd->tableId = sqliteConnector_.getData<int>(r, 0);
935  cd->columnId = sqliteConnector_.getData<int>(r, 1);
936  cd->columnName = sqliteConnector_.getData<string>(r, 2);
944  cd->columnType.set_size(sqliteConnector_.getData<int>(r, 10));
945  cd->chunks = sqliteConnector_.getData<string>(r, 11);
946  cd->isSystemCol = sqliteConnector_.getData<bool>(r, 12);
947  cd->isVirtualCol = sqliteConnector_.getData<bool>(r, 13);
948  cd->virtualExpr = sqliteConnector_.getData<string>(r, 14);
949  cd->isDeletedCol = sqliteConnector_.getData<bool>(r, 15);
950  cd->isGeoPhyCol = skip_physical_cols > 0;
951  ColumnKey columnKey(cd->tableId, to_upper(cd->columnName));
952  columnDescriptorMap_[columnKey] = cd;
953  ColumnIdKey columnIdKey(cd->tableId, cd->columnId);
954  columnDescriptorMapById_[columnIdKey] = cd;
955 
956  if (skip_physical_cols <= 0) {
957  skip_physical_cols = cd->columnType.get_physical_cols();
958  }
959 
960  auto td_itr = tableDescriptorMapById_.find(cd->tableId);
961  CHECK(td_itr != tableDescriptorMapById_.end());
962 
963  if (cd->isDeletedCol) {
964  td_itr->second->hasDeletedCol = true;
965  setDeletedColumnUnlocked(td_itr->second, cd);
966  } else if (cd->columnType.is_geometry() || skip_physical_cols-- <= 0) {
967  tableDescriptorMapById_[cd->tableId]->columnIdBySpi_.push_back(cd->columnId);
968  }
969  }
970  // sort columnIdBySpi_ based on columnId
971  for (auto& tit : tableDescriptorMapById_) {
972  std::sort(tit.second->columnIdBySpi_.begin(),
973  tit.second->columnIdBySpi_.end(),
974  [](const size_t a, const size_t b) -> bool { return a < b; });
975  }
976 
977  string viewQuery("SELECT tableid, sql FROM mapd_views");
978  sqliteConnector_.query(viewQuery);
979  numRows = sqliteConnector_.getNumRows();
980  for (size_t r = 0; r < numRows; ++r) {
981  int32_t tableId = sqliteConnector_.getData<int>(r, 0);
982  TableDescriptor* td = tableDescriptorMapById_[tableId];
983  td->viewSQL = sqliteConnector_.getData<string>(r, 1);
984  td->fragmenter = nullptr;
985  }
986 
987  string frontendViewQuery(
988  "SELECT id, state, name, image_hash, strftime('%Y-%m-%dT%H:%M:%SZ', update_time), "
989  "userid, "
990  "metadata "
991  "FROM mapd_dashboards");
992  sqliteConnector_.query(frontendViewQuery);
993  numRows = sqliteConnector_.getNumRows();
994  for (size_t r = 0; r < numRows; ++r) {
995  std::shared_ptr<DashboardDescriptor> vd = std::make_shared<DashboardDescriptor>();
996  vd->dashboardId = sqliteConnector_.getData<int>(r, 0);
997  vd->dashboardState = sqliteConnector_.getData<string>(r, 1);
998  vd->dashboardName = sqliteConnector_.getData<string>(r, 2);
999  vd->imageHash = sqliteConnector_.getData<string>(r, 3);
1000  vd->updateTime = sqliteConnector_.getData<string>(r, 4);
1001  vd->userId = sqliteConnector_.getData<int>(r, 5);
1002  vd->dashboardMetadata = sqliteConnector_.getData<string>(r, 6);
1003  vd->user = getUserFromId(vd->userId);
1004  vd->dashboardSystemRoleName = generate_dashboard_system_rolename(
1006  dashboardDescriptorMap_[std::to_string(vd->userId) + ":" + vd->dashboardName] = vd;
1007  }
1008 
1009  string linkQuery(
1010  "SELECT linkid, userid, link, view_state, strftime('%Y-%m-%dT%H:%M:%SZ', "
1011  "update_time), view_metadata "
1012  "FROM mapd_links");
1013  sqliteConnector_.query(linkQuery);
1014  numRows = sqliteConnector_.getNumRows();
1015  for (size_t r = 0; r < numRows; ++r) {
1016  LinkDescriptor* ld = new LinkDescriptor();
1017  ld->linkId = sqliteConnector_.getData<int>(r, 0);
1018  ld->userId = sqliteConnector_.getData<int>(r, 1);
1019  ld->link = sqliteConnector_.getData<string>(r, 2);
1020  ld->viewState = sqliteConnector_.getData<string>(r, 3);
1021  ld->updateTime = sqliteConnector_.getData<string>(r, 4);
1022  ld->viewMetadata = sqliteConnector_.getData<string>(r, 5);
1024  linkDescriptorMapById_[ld->linkId] = ld;
1025  }
1026 
1027  /* rebuild map linking logical tables to corresponding physical ones */
1028  string logicalToPhysicalTableMapQuery(
1029  "SELECT logical_table_id, physical_table_id "
1030  "FROM mapd_logical_to_physical");
1031  sqliteConnector_.query(logicalToPhysicalTableMapQuery);
1032  numRows = sqliteConnector_.getNumRows();
1033  for (size_t r = 0; r < numRows; ++r) {
1034  int32_t logical_tb_id = sqliteConnector_.getData<int>(r, 0);
1035  int32_t physical_tb_id = sqliteConnector_.getData<int>(r, 1);
1036  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
1037  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
1038  /* add new entity to the map logicalToPhysicalTableMapById_ */
1039  std::vector<int32_t> physicalTables;
1040  physicalTables.push_back(physical_tb_id);
1041  const auto it_ok =
1042  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
1043  CHECK(it_ok.second);
1044  } else {
1045  /* update map logicalToPhysicalTableMapById_ */
1046  physicalTableIt->second.push_back(physical_tb_id);
1047  }
1048  }
1049 }
1050 
1052  const list<ColumnDescriptor>& columns,
1053  const list<DictDescriptor>& dicts) {
1054  cat_write_lock write_lock(this);
1055  TableDescriptor* new_td = new TableDescriptor();
1056  *new_td = td;
1057  new_td->mutex_ = std::make_shared<std::mutex>();
1058  tableDescriptorMap_[to_upper(td.tableName)] = new_td;
1059  tableDescriptorMapById_[td.tableId] = new_td;
1060  for (auto cd : columns) {
1061  ColumnDescriptor* new_cd = new ColumnDescriptor();
1062  *new_cd = cd;
1063  ColumnKey columnKey(new_cd->tableId, to_upper(new_cd->columnName));
1064  columnDescriptorMap_[columnKey] = new_cd;
1065  ColumnIdKey columnIdKey(new_cd->tableId, new_cd->columnId);
1066  columnDescriptorMapById_[columnIdKey] = new_cd;
1067 
1068  // Add deleted column to the map
1069  if (cd.isDeletedCol) {
1070  CHECK(new_td->hasDeletedCol);
1071  setDeletedColumnUnlocked(new_td, new_cd);
1072  }
1073  }
1074 
1075  std::sort(new_td->columnIdBySpi_.begin(),
1076  new_td->columnIdBySpi_.end(),
1077  [](const size_t a, const size_t b) -> bool { return a < b; });
1078 
1079  std::unique_ptr<StringDictionaryClient> client;
1080  DictRef dict_ref(currentDB_.dbId, -1);
1081  if (!string_dict_hosts_.empty()) {
1082  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1083  }
1084  for (auto dd : dicts) {
1085  if (!dd.dictRef.dictId) {
1086  // Dummy entry created for a shard of a logical table, nothing to do.
1087  continue;
1088  }
1089  dict_ref.dictId = dd.dictRef.dictId;
1090  if (client) {
1091  client->create(dict_ref, dd.dictIsTemp);
1092  }
1093  DictDescriptor* new_dd = new DictDescriptor(dd);
1094  dictDescriptorMapByRef_[dict_ref].reset(new_dd);
1095  if (!dd.dictIsTemp) {
1096  boost::filesystem::create_directory(new_dd->dictFolderPath);
1097  }
1098  }
1099 }
1100 
1101 void Catalog::removeTableFromMap(const string& tableName,
1102  const int tableId,
1103  const bool is_on_error) {
1104  cat_write_lock write_lock(this);
1105  TableDescriptorMapById::iterator tableDescIt = tableDescriptorMapById_.find(tableId);
1106  if (tableDescIt == tableDescriptorMapById_.end()) {
1107  throw runtime_error("Table " + tableName + " does not exist.");
1108  }
1109 
1110  TableDescriptor* td = tableDescIt->second;
1111 
1112  if (td->hasDeletedCol) {
1113  const auto ret = deletedColumnPerTable_.erase(td);
1114  CHECK_EQ(ret, size_t(1));
1115  }
1116 
1117  tableDescriptorMapById_.erase(tableDescIt);
1118  tableDescriptorMap_.erase(to_upper(tableName));
1119  if (td->fragmenter != nullptr) {
1120  delete td->fragmenter;
1121  }
1123  delete td;
1124 
1125  std::unique_ptr<StringDictionaryClient> client;
1126  if (SysCatalog::instance().isAggregator()) {
1127  CHECK(!string_dict_hosts_.empty());
1128  DictRef dict_ref(currentDB_.dbId, -1);
1129  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1130  }
1131 
1132  // delete all column descriptors for the table
1133  // no more link columnIds to sequential indexes!
1134  for (auto cit = columnDescriptorMapById_.begin();
1135  cit != columnDescriptorMapById_.end();) {
1136  if (tableId != std::get<0>(cit->first)) {
1137  ++cit;
1138  } else {
1139  int i = std::get<1>(cit++->first);
1140  ColumnIdKey cidKey(tableId, i);
1141  ColumnDescriptorMapById::iterator colDescIt = columnDescriptorMapById_.find(cidKey);
1142  ColumnDescriptor* cd = colDescIt->second;
1143  columnDescriptorMapById_.erase(colDescIt);
1144  ColumnKey cnameKey(tableId, to_upper(cd->columnName));
1145  columnDescriptorMap_.erase(cnameKey);
1146  const int dictId = cd->columnType.get_comp_param();
1147  // Dummy dictionaries created for a shard of a logical table have the id set to
1148  // zero.
1149  if (cd->columnType.get_compression() == kENCODING_DICT && dictId) {
1150  INJECT_TIMER(removingDicts);
1151  DictRef dict_ref(currentDB_.dbId, dictId);
1152  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
1153  // If we're removing this table due to an error, it is possible that the string
1154  // dictionary reference was never populated. Don't crash, just continue cleaning
1155  // up the TableDescriptor and ColumnDescriptors
1156  if (!is_on_error) {
1157  CHECK(dictIt != dictDescriptorMapByRef_.end());
1158  } else {
1159  if (dictIt == dictDescriptorMapByRef_.end()) {
1160  continue;
1161  }
1162  }
1163  const auto& dd = dictIt->second;
1164  CHECK_GE(dd->refcount, 1);
1165  --dd->refcount;
1166  if (!dd->refcount) {
1167  dd->stringDict.reset();
1168  if (!isTemp) {
1169  File_Namespace::renameForDelete(dd->dictFolderPath);
1170  }
1171  if (client) {
1172  client->drop(dict_ref);
1173  }
1174  dictDescriptorMapByRef_.erase(dictIt);
1175  }
1176  }
1177 
1178  delete cd;
1179  }
1180  }
1181 }
1182 
1184  cat_write_lock write_lock(this);
1186 }
1187 
1189  cat_write_lock write_lock(this);
1191  std::make_shared<DashboardDescriptor>(vd);
1192 }
1193 
1194 std::vector<DBObject> Catalog::parseDashboardObjects(const std::string& view_meta,
1195  const int& user_id) {
1196  std::vector<DBObject> objects;
1197  DBObjectKey key;
1198  key.dbId = currentDB_.dbId;
1199  auto _key_place = [&key](auto type, auto id) {
1200  key.permissionType = type;
1201  key.objectId = id;
1202  return key;
1203  };
1204  for (auto object_name : parse_underlying_dashboard_objects(view_meta)) {
1205  auto td = getMetadataForTable(object_name);
1206  if (!td) {
1207  // Parsed object source is not present in current database
1208  // LOG the info and ignore
1209  LOG(INFO) << "Ignoring dashboard source Table/View: " << object_name
1210  << " no longer exists in current DB.";
1211  continue;
1212  }
1213  // Dashboard source can be Table or View
1214  const auto object_type = td->isView ? ViewDBObjectType : TableDBObjectType;
1215  const auto priv = td->isView ? AccessPrivileges::SELECT_FROM_VIEW
1217  objects.emplace_back(_key_place(object_type, td->tableId), priv, user_id);
1218  objects.back().setObjectType(td->isView ? ViewDBObjectType : TableDBObjectType);
1219  objects.back().setName(td->tableName);
1220  }
1221  return objects;
1222 }
1223 
1224 void Catalog::createOrUpdateDashboardSystemRole(const std::string& view_meta,
1225  const int32_t& user_id,
1226  const std::string& dash_role_name) {
1227  auto objects = parseDashboardObjects(view_meta, user_id);
1228  Role* rl = SysCatalog::instance().getRoleGrantee(dash_role_name);
1229  if (!rl) {
1230  // Dashboard role does not exist
1231  // create role and grant privileges
1232  // NOTE(wamsi): Transactionally unsafe
1233  SysCatalog::instance().createRole(dash_role_name, false);
1234  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1235  } else {
1236  // Dashboard system role already exists
1237  // Add/remove privileges on objects
1238  auto ex_objects = rl->getDbObjects(true);
1239  for (auto key : *ex_objects | boost::adaptors::map_keys) {
1240  if (key.permissionType != TableDBObjectType &&
1241  key.permissionType != ViewDBObjectType) {
1242  continue;
1243  }
1244  bool found = false;
1245  for (auto obj : objects) {
1246  found = key == obj.getObjectKey() ? true : false;
1247  if (found) {
1248  break;
1249  }
1250  }
1251  if (!found) {
1252  // revoke privs on object since the object is no
1253  // longer used by the dashboard as source
1254  // NOTE(wamsi): Transactionally unsafe
1256  dash_role_name, *rl->findDbObject(key, true), *this);
1257  }
1258  }
1259  // Update privileges on remaining objects
1260  // NOTE(wamsi): Transactionally unsafe
1261  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1262  }
1263 }
1264 
1266  cat_write_lock write_lock(this);
1267  LinkDescriptor* new_ld = new LinkDescriptor();
1268  *new_ld = ld;
1270  linkDescriptorMapById_[ld.linkId] = new_ld;
1271 }
1272 
1274  auto time_ms = measure<>::execution([&]() {
1275  // instanciate table fragmenter upon first use
1276  // assume only insert order fragmenter is supported
1278  vector<Chunk> chunkVec;
1279  list<const ColumnDescriptor*> columnDescs;
1280  getAllColumnMetadataForTable(td, columnDescs, true, false, true);
1281  Chunk::translateColumnDescriptorsToChunkVec(columnDescs, chunkVec);
1282  ChunkKey chunkKeyPrefix = {currentDB_.dbId, td->tableId};
1283  if (td->sortedColumnId > 0) {
1284  td->fragmenter = new SortedOrderFragmenter(chunkKeyPrefix,
1285  chunkVec,
1286  dataMgr_.get(),
1287  const_cast<Catalog*>(this),
1288  td->tableId,
1289  td->shard,
1290  td->maxFragRows,
1291  td->maxChunkSize,
1292  td->fragPageSize,
1293  td->maxRows,
1294  td->persistenceLevel);
1295  } else {
1296  td->fragmenter = new InsertOrderFragmenter(chunkKeyPrefix,
1297  chunkVec,
1298  dataMgr_.get(),
1299  const_cast<Catalog*>(this),
1300  td->tableId,
1301  td->shard,
1302  td->maxFragRows,
1303  td->maxChunkSize,
1304  td->fragPageSize,
1305  td->maxRows,
1306  td->persistenceLevel);
1307  }
1308  });
1309  LOG(INFO) << "Instantiating Fragmenter for table " << td->tableName << " took "
1310  << time_ms << "ms";
1311 }
1312 
1313 const TableDescriptor* Catalog::getMetadataForTable(const string& tableName,
1314  const bool populateFragmenter) const {
1315  // we give option not to populate fragmenter (default true/yes) as it can be heavy for
1316  // pure metadata calls
1317  cat_read_lock read_lock(this);
1318  auto tableDescIt = tableDescriptorMap_.find(to_upper(tableName));
1319  if (tableDescIt == tableDescriptorMap_.end()) { // check to make sure table exists
1320  return nullptr;
1321  }
1322  TableDescriptor* td = tableDescIt->second;
1323  std::unique_lock<std::mutex> td_lock(*td->mutex_.get());
1324  if (populateFragmenter && td->fragmenter == nullptr && !td->isView) {
1326  }
1327  return td; // returns pointer to table descriptor
1328 }
1329 
1331  int tableId,
1332  const bool populateFragmenter) const {
1333  cat_read_lock read_lock(this);
1334  auto tableDescIt = tableDescriptorMapById_.find(tableId);
1335  if (tableDescIt == tableDescriptorMapById_.end()) { // check to make sure table exists
1336  return nullptr;
1337  }
1338  TableDescriptor* td = tableDescIt->second;
1339  std::unique_lock<std::mutex> td_lock(*td->mutex_.get());
1340  if (populateFragmenter && td->fragmenter == nullptr && !td->isView) {
1342  }
1343  return td; // returns pointer to table descriptor
1344 }
1345 
1347  return getMetadataForTableImpl(tableId, true);
1348 }
1349 
1351  const bool loadDict) const {
1352  const DictRef dictRef(currentDB_.dbId, dictId);
1353  cat_read_lock read_lock(this);
1354  auto dictDescIt = dictDescriptorMapByRef_.find(dictRef);
1355  if (dictDescIt ==
1356  dictDescriptorMapByRef_.end()) { // check to make sure dictionary exists
1357  return nullptr;
1358  }
1359  auto& dd = dictDescIt->second;
1360 
1361  if (loadDict) {
1363  if (!dd->stringDict) {
1364  auto time_ms = measure<>::execution([&]() {
1365  if (string_dict_hosts_.empty()) {
1366  if (dd->dictIsTemp) {
1367  dd->stringDict = std::make_shared<StringDictionary>(
1368  dd->dictFolderPath, true, true, g_cache_string_hash);
1369  } else {
1370  dd->stringDict = std::make_shared<StringDictionary>(
1371  dd->dictFolderPath, false, true, g_cache_string_hash);
1372  }
1373  } else {
1374  dd->stringDict =
1375  std::make_shared<StringDictionary>(string_dict_hosts_.front(), dd->dictRef);
1376  }
1377  });
1378  LOG(INFO) << "Time to load Dictionary " << dd->dictRef.dbId << "_"
1379  << dd->dictRef.dictId << " was " << time_ms << "ms";
1380  }
1381  }
1382 
1383  return dd.get();
1384 }
1385 
1386 const std::vector<LeafHostInfo>& Catalog::getStringDictionaryHosts() const {
1387  return string_dict_hosts_;
1388 }
1389 
1391  const string& columnName) const {
1392  cat_read_lock read_lock(this);
1393 
1394  ColumnKey columnKey(tableId, to_upper(columnName));
1395  auto colDescIt = columnDescriptorMap_.find(columnKey);
1396  if (colDescIt ==
1397  columnDescriptorMap_.end()) { // need to check to make sure column exists for table
1398  return nullptr;
1399  }
1400  return colDescIt->second;
1401 }
1402 
1403 const ColumnDescriptor* Catalog::getMetadataForColumn(int tableId, int columnId) const {
1404  cat_read_lock read_lock(this);
1405 
1406  ColumnIdKey columnIdKey(tableId, columnId);
1407  auto colDescIt = columnDescriptorMapById_.find(columnIdKey);
1408  if (colDescIt == columnDescriptorMapById_
1409  .end()) { // need to check to make sure column exists for table
1410  return nullptr;
1411  }
1412  return colDescIt->second;
1413 }
1414 
1415 const int Catalog::getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const {
1416  const auto tabDescIt = tableDescriptorMapById_.find(table_id);
1417  CHECK(tableDescriptorMapById_.end() != tabDescIt);
1418  const auto& columnIdBySpi = tabDescIt->second->columnIdBySpi_;
1419 
1420  auto spx = spi;
1421  int phi = 0;
1422  if (spx >= SPIMAP_MAGIC1) // see Catalog.h
1423  {
1424  phi = (spx - SPIMAP_MAGIC1) % SPIMAP_MAGIC2;
1425  spx = (spx - SPIMAP_MAGIC1) / SPIMAP_MAGIC2;
1426  }
1427 
1428  CHECK(0 < spx && spx <= columnIdBySpi.size());
1429  return columnIdBySpi[spx - 1] + phi;
1430 }
1431 
1432 const int Catalog::getColumnIdBySpi(const int table_id, const size_t spi) const {
1433  cat_read_lock read_lock(this);
1434  return getColumnIdBySpiUnlocked(table_id, spi);
1435 }
1436 
1438  const size_t spi) const {
1439  cat_read_lock read_lock(this);
1440 
1441  const auto columnId = getColumnIdBySpiUnlocked(tableId, spi);
1442  ColumnIdKey columnIdKey(tableId, columnId);
1443  const auto colDescIt = columnDescriptorMapById_.find(columnIdKey);
1444  return columnDescriptorMapById_.end() == colDescIt ? nullptr : colDescIt->second;
1445 }
1446 
1447 void Catalog::deleteMetadataForDashboard(const std::string& userId,
1448  const std::string& dashName) {
1449  cat_write_lock write_lock(this);
1450 
1451  auto viewDescIt = dashboardDescriptorMap_.find(userId + ":" + dashName);
1452  if (viewDescIt == dashboardDescriptorMap_.end()) { // check to make sure view exists
1453  LOG(ERROR) << "No metadata for dashboard for user " << userId << " dashboard "
1454  << dashName << " does not exist in map";
1455  throw runtime_error("No metadata for dashboard for user " + userId + " dashboard " +
1456  dashName + " does not exist in map");
1457  }
1458  // found view in Map now remove it
1459  dashboardDescriptorMap_.erase(viewDescIt);
1460  // remove from DB
1462  sqliteConnector_.query("BEGIN TRANSACTION");
1463  try {
1465  "DELETE FROM mapd_dashboards WHERE name = ? and userid = ?",
1466  std::vector<std::string>{dashName, userId});
1467  } catch (std::exception& e) {
1468  sqliteConnector_.query("ROLLBACK TRANSACTION");
1469  throw;
1470  }
1471  sqliteConnector_.query("END TRANSACTION");
1472 }
1473 
1475  const string& userId,
1476  const string& dashName) const {
1477  cat_read_lock read_lock(this);
1478 
1479  auto viewDescIt = dashboardDescriptorMap_.find(userId + ":" + dashName);
1480  if (viewDescIt == dashboardDescriptorMap_.end()) { // check to make sure view exists
1481  return nullptr;
1482  }
1483  return viewDescIt->second.get(); // returns pointer to view descriptor
1484 }
1485 
1487  cat_read_lock read_lock(this);
1488  std::string userId;
1489  std::string name;
1490  bool found{false};
1491  {
1492  for (auto descp : dashboardDescriptorMap_) {
1493  auto dash = descp.second.get();
1494  if (dash->dashboardId == id) {
1495  userId = std::to_string(dash->userId);
1496  name = dash->dashboardName;
1497  found = true;
1498  break;
1499  }
1500  }
1501  }
1502  if (found) {
1503  return getMetadataForDashboard(userId, name);
1504  }
1505  return nullptr;
1506 }
1507 
1508 void Catalog::deleteMetadataForDashboard(const int32_t id) {
1509  std::string userId;
1510  std::string name;
1511  bool found{false};
1512  {
1513  cat_read_lock read_lock(this);
1514  for (auto descp : dashboardDescriptorMap_) {
1515  auto dash = descp.second.get();
1516  if (dash->dashboardId == id) {
1517  userId = std::to_string(dash->userId);
1518  name = dash->dashboardName;
1519  found = true;
1520  break;
1521  }
1522  }
1523  }
1524  if (found) {
1525  // TODO: transactionally unsafe
1527  DBObject(id, DashboardDBObjectType), this);
1528  deleteMetadataForDashboard(userId, name);
1529  }
1530 }
1531 
1532 const LinkDescriptor* Catalog::getMetadataForLink(const string& link) const {
1533  cat_read_lock read_lock(this);
1534  auto linkDescIt = linkDescriptorMap_.find(link);
1535  if (linkDescIt == linkDescriptorMap_.end()) { // check to make sure view exists
1536  return nullptr;
1537  }
1538  return linkDescIt->second; // returns pointer to view descriptor
1539 }
1540 
1542  cat_read_lock read_lock(this);
1543  auto linkDescIt = linkDescriptorMapById_.find(linkId);
1544  if (linkDescIt == linkDescriptorMapById_.end()) { // check to make sure view exists
1545  return nullptr;
1546  }
1547  return linkDescIt->second;
1548 }
1549 
1551  const TableDescriptor* td,
1552  list<const ColumnDescriptor*>& columnDescriptors,
1553  const bool fetchSystemColumns,
1554  const bool fetchVirtualColumns,
1555  const bool fetchPhysicalColumns) const {
1556  cat_read_lock read_lock(this);
1557  int32_t skip_physical_cols = 0;
1558  for (const auto& columnDescriptor : columnDescriptorMapById_) {
1559  if (!fetchPhysicalColumns && skip_physical_cols > 0) {
1560  --skip_physical_cols;
1561  continue;
1562  }
1563  auto cd = columnDescriptor.second;
1564  if (cd->tableId != td->tableId) {
1565  continue;
1566  }
1567  if (!fetchSystemColumns && cd->isSystemCol) {
1568  continue;
1569  }
1570  if (!fetchVirtualColumns && cd->isVirtualCol) {
1571  continue;
1572  }
1573  if (!fetchPhysicalColumns) {
1574  const auto& col_ti = cd->columnType;
1575  skip_physical_cols = col_ti.get_physical_cols();
1576  }
1577  columnDescriptors.push_back(cd);
1578  }
1579 }
1580 
1581 list<const ColumnDescriptor*> Catalog::getAllColumnMetadataForTable(
1582  const int tableId,
1583  const bool fetchSystemColumns,
1584  const bool fetchVirtualColumns,
1585  const bool fetchPhysicalColumns) const {
1586  cat_read_lock read_lock(this);
1587  list<const ColumnDescriptor*> columnDescriptors;
1588  const TableDescriptor* td =
1589  getMetadataForTableImpl(tableId, false); // dont instantiate fragmenter
1591  columnDescriptors,
1592  fetchSystemColumns,
1593  fetchVirtualColumns,
1594  fetchPhysicalColumns);
1595  return columnDescriptors;
1596 }
1597 
1598 list<const TableDescriptor*> Catalog::getAllTableMetadata() const {
1599  cat_read_lock read_lock(this);
1600  list<const TableDescriptor*> table_list;
1601  for (auto p : tableDescriptorMapById_) {
1602  table_list.push_back(p.second);
1603  }
1604  return table_list;
1605 }
1606 
1607 list<const DashboardDescriptor*> Catalog::getAllDashboardsMetadata() const {
1608  list<const DashboardDescriptor*> view_list;
1609  for (auto p : dashboardDescriptorMap_) {
1610  view_list.push_back(p.second.get());
1611  }
1612  return view_list;
1613 }
1614 
1616  const auto& td = *tableDescriptorMapById_[cd.tableId];
1617  list<DictDescriptor> dds;
1618  setColumnDictionary(cd, dds, td, true);
1619  auto& dd = dds.back();
1620  CHECK(dd.dictRef.dictId);
1621 
1622  std::unique_ptr<StringDictionaryClient> client;
1623  if (!string_dict_hosts_.empty()) {
1624  client.reset(new StringDictionaryClient(
1625  string_dict_hosts_.front(), DictRef(currentDB_.dbId, -1), true));
1626  }
1627  if (client) {
1628  client->create(dd.dictRef, dd.dictIsTemp);
1629  }
1630 
1631  DictDescriptor* new_dd = new DictDescriptor(dd);
1632  dictDescriptorMapByRef_[dd.dictRef].reset(new_dd);
1633  if (!dd.dictIsTemp) {
1634  boost::filesystem::create_directory(new_dd->dictFolderPath);
1635  }
1636  return dd.dictRef;
1637 }
1638 
1639 // TODO this all looks incorrect there is no reference count being checked here
1640 // this code needs to be fixed
1642  if (!(cd.columnType.is_string() || cd.columnType.is_string_array())) {
1643  return;
1644  }
1645  if (!(cd.columnType.get_compression() == kENCODING_DICT)) {
1646  return;
1647  }
1648  if (!(cd.columnType.get_comp_param() > 0)) {
1649  return;
1650  }
1651 
1652  const auto& td = *tableDescriptorMapById_[cd.tableId];
1653  const auto dictId = cd.columnType.get_comp_param();
1654  const DictRef dictRef(currentDB_.dbId, dictId);
1655  const auto dictName =
1656  td.tableName + "_" + cd.columnName + "_dict" + std::to_string(dictId);
1657  sqliteConnector_.query_with_text_param("DELETE FROM mapd_dictionaries WHERE name = ?",
1658  dictName);
1659  File_Namespace::renameForDelete(basePath_ + "/mapd_data/DB_" +
1660  std::to_string(currentDB_.dbId) + "_DICT_" +
1661  std::to_string(dictId));
1662 
1663  std::unique_ptr<StringDictionaryClient> client;
1664  if (!string_dict_hosts_.empty()) {
1665  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dictRef, true));
1666  }
1667  if (client) {
1668  client->drop(dictRef);
1669  }
1670 
1671  dictDescriptorMapByRef_.erase(dictRef);
1672 }
1673 
1675  std::map<int, StringDictionary*>& stringDicts) {
1676  // learn 'committed' ColumnDescriptor of this column
1677  auto cit = columnDescriptorMap_.find(ColumnKey(cd.tableId, to_upper(cd.columnName)));
1678  CHECK(cit != columnDescriptorMap_.end());
1679  auto& ccd = *cit->second;
1680 
1681  if (!(ccd.columnType.is_string() || ccd.columnType.is_string_array())) {
1682  return;
1683  }
1684  if (!(ccd.columnType.get_compression() == kENCODING_DICT)) {
1685  return;
1686  }
1687  if (!(ccd.columnType.get_comp_param() > 0)) {
1688  return;
1689  }
1690 
1691  auto dictId = ccd.columnType.get_comp_param();
1692  getMetadataForDict(dictId);
1693 
1694  const DictRef dictRef(currentDB_.dbId, dictId);
1695  auto dit = dictDescriptorMapByRef_.find(dictRef);
1696  CHECK(dit != dictDescriptorMapByRef_.end());
1697  CHECK(dit->second);
1698  CHECK(dit->second.get()->stringDict);
1699  stringDicts[ccd.columnId] = dit->second.get()->stringDict.get();
1700 }
1701 
1703  // caller must handle sqlite/chunk transaction TOGETHER
1704  cd.tableId = td.tableId;
1706  addDictionary(cd);
1707  }
1708 
1710  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
1711  "colscale, is_notnull, "
1712  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
1713  "is_deletedcol) "
1714  "VALUES (?, "
1715  "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
1716  "?, ?, ?, "
1717  "?, "
1718  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1719  std::vector<std::string>{std::to_string(td.tableId),
1720  std::to_string(td.tableId),
1721  cd.columnName,
1730  "",
1733  cd.virtualExpr,
1735 
1737  "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
1738  std::vector<std::string>{std::to_string(td.tableId)});
1739 
1741  "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
1742  std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
1743  cd.columnId = sqliteConnector_.getData<int>(0, 0);
1744 
1745  ++tableDescriptorMapById_[td.tableId]->nColumns;
1746  auto ncd = new ColumnDescriptor(cd);
1748  columnDescriptorMapById_[ColumnIdKey(cd.tableId, cd.columnId)] = ncd;
1749  columnDescriptorsForRoll.emplace_back(nullptr, ncd);
1750 }
1751 
1752 void Catalog::roll(const bool forward) {
1753  std::set<const TableDescriptor*> tds;
1754 
1755  for (const auto& cdr : columnDescriptorsForRoll) {
1756  auto ocd = cdr.first;
1757  auto ncd = cdr.second;
1758  CHECK(ocd || ncd);
1759  auto tabDescIt = tableDescriptorMapById_.find((ncd ? ncd : ocd)->tableId);
1760  CHECK(tableDescriptorMapById_.end() != tabDescIt);
1761  auto td = tabDescIt->second;
1762  auto& vc = td->columnIdBySpi_;
1763  if (forward) {
1764  if (ocd) {
1765  if (nullptr == ncd ||
1766  ncd->columnType.get_comp_param() != ocd->columnType.get_comp_param()) {
1767  delDictionary(*ocd);
1768  }
1769 
1770  vc.erase(std::remove(vc.begin(), vc.end(), ocd->columnId), vc.end());
1771 
1772  delete ocd;
1773  }
1774  if (ncd) {
1775  // append columnId if its new and not phy geo
1776  if (vc.end() == std::find(vc.begin(), vc.end(), ncd->columnId)) {
1777  if (!ncd->isGeoPhyCol) {
1778  vc.push_back(ncd->columnId);
1779  }
1780  }
1781  }
1782  tds.insert(td);
1783  } else {
1784  if (ocd) {
1785  columnDescriptorMap_[ColumnKey(ocd->tableId, to_upper(ocd->columnName))] = ocd;
1786  columnDescriptorMapById_[ColumnIdKey(ocd->tableId, ocd->columnId)] = ocd;
1787  }
1788  // roll back the dict of new column
1789  if (ncd) {
1790  columnDescriptorMap_.erase(ColumnKey(ncd->tableId, to_upper(ncd->columnName)));
1791  columnDescriptorMapById_.erase(ColumnIdKey(ncd->tableId, ncd->columnId));
1792  if (nullptr == ocd ||
1793  ocd->columnType.get_comp_param() != ncd->columnType.get_comp_param()) {
1794  delDictionary(*ncd);
1795  }
1796  delete ncd;
1797  }
1798  }
1799  }
1800  columnDescriptorsForRoll.clear();
1801 
1802  if (forward) {
1803  for (const auto td : tds) {
1804  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
1805  }
1806  }
1807 }
1808 
1810  list<ColumnDescriptor>& columns) {
1811  const auto& col_ti = cd.columnType;
1812  if (IS_GEO(col_ti.get_type())) {
1813  switch (col_ti.get_type()) {
1814  case kPOINT: {
1815  ColumnDescriptor physical_cd_coords(true);
1816  physical_cd_coords.columnName = cd.columnName + "_coords";
1817  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, true);
1818  // Raw data: compressed/uncompressed coords
1819  coords_ti.set_subtype(kTINYINT);
1820  size_t unit_size;
1821  if (col_ti.get_compression() == kENCODING_GEOINT &&
1822  col_ti.get_comp_param() == 32) {
1823  unit_size = 4 * sizeof(int8_t);
1824  } else {
1825  CHECK(col_ti.get_compression() == kENCODING_NONE);
1826  unit_size = 8 * sizeof(int8_t);
1827  }
1828  coords_ti.set_size(2 * unit_size);
1829  physical_cd_coords.columnType = coords_ti;
1830  columns.push_back(physical_cd_coords);
1831 
1832  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1833 
1834  break;
1835  }
1836  case kLINESTRING: {
1837  ColumnDescriptor physical_cd_coords(true);
1838  physical_cd_coords.columnName = cd.columnName + "_coords";
1839  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, true);
1840  // Raw data: compressed/uncompressed coords
1841  coords_ti.set_subtype(kTINYINT);
1842  physical_cd_coords.columnType = coords_ti;
1843  columns.push_back(physical_cd_coords);
1844 
1845  ColumnDescriptor physical_cd_bounds(true);
1846  physical_cd_bounds.columnName = cd.columnName + "_bounds";
1847  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, true);
1848  bounds_ti.set_subtype(kDOUBLE);
1849  bounds_ti.set_size(4 * sizeof(double));
1850  physical_cd_bounds.columnType = bounds_ti;
1851  columns.push_back(physical_cd_bounds);
1852 
1853  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1854 
1855  break;
1856  }
1857  case kPOLYGON: {
1858  ColumnDescriptor physical_cd_coords(true);
1859  physical_cd_coords.columnName = cd.columnName + "_coords";
1860  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, true);
1861  // Raw data: compressed/uncompressed coords
1862  coords_ti.set_subtype(kTINYINT);
1863  physical_cd_coords.columnType = coords_ti;
1864  columns.push_back(physical_cd_coords);
1865 
1866  ColumnDescriptor physical_cd_ring_sizes(true);
1867  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
1868  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, true);
1869  ring_sizes_ti.set_subtype(kINT);
1870  physical_cd_ring_sizes.columnType = ring_sizes_ti;
1871  columns.push_back(physical_cd_ring_sizes);
1872 
1873  ColumnDescriptor physical_cd_bounds(true);
1874  physical_cd_bounds.columnName = cd.columnName + "_bounds";
1875  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, true);
1876  bounds_ti.set_subtype(kDOUBLE);
1877  bounds_ti.set_size(4 * sizeof(double));
1878  physical_cd_bounds.columnType = bounds_ti;
1879  columns.push_back(physical_cd_bounds);
1880 
1881  ColumnDescriptor physical_cd_render_group(true);
1882  physical_cd_render_group.columnName = cd.columnName + "_render_group";
1883  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, true);
1884  physical_cd_render_group.columnType = render_group_ti;
1885  columns.push_back(physical_cd_render_group);
1886 
1887  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1888 
1889  break;
1890  }
1891  case kMULTIPOLYGON: {
1892  ColumnDescriptor physical_cd_coords(true);
1893  physical_cd_coords.columnName = cd.columnName + "_coords";
1894  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, true);
1895  // Raw data: compressed/uncompressed coords
1896  coords_ti.set_subtype(kTINYINT);
1897  physical_cd_coords.columnType = coords_ti;
1898  columns.push_back(physical_cd_coords);
1899 
1900  ColumnDescriptor physical_cd_ring_sizes(true);
1901  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
1902  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, true);
1903  ring_sizes_ti.set_subtype(kINT);
1904  physical_cd_ring_sizes.columnType = ring_sizes_ti;
1905  columns.push_back(physical_cd_ring_sizes);
1906 
1907  ColumnDescriptor physical_cd_poly_rings(true);
1908  physical_cd_poly_rings.columnName = cd.columnName + "_poly_rings";
1909  SQLTypeInfo poly_rings_ti = SQLTypeInfo(kARRAY, true);
1910  poly_rings_ti.set_subtype(kINT);
1911  physical_cd_poly_rings.columnType = poly_rings_ti;
1912  columns.push_back(physical_cd_poly_rings);
1913 
1914  ColumnDescriptor physical_cd_bounds(true);
1915  physical_cd_bounds.columnName = cd.columnName + "_bounds";
1916  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, true);
1917  bounds_ti.set_subtype(kDOUBLE);
1918  bounds_ti.set_size(4 * sizeof(double));
1919  physical_cd_bounds.columnType = bounds_ti;
1920  columns.push_back(physical_cd_bounds);
1921 
1922  ColumnDescriptor physical_cd_render_group(true);
1923  physical_cd_render_group.columnName = cd.columnName + "_render_group";
1924  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, true);
1925  physical_cd_render_group.columnType = render_group_ti;
1926  columns.push_back(physical_cd_render_group);
1927 
1928  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1929 
1930  break;
1931  }
1932  default:
1933  throw runtime_error("Unrecognized geometry type.");
1934  break;
1935  }
1936  }
1937 }
1938 
1940  TableDescriptor& td,
1941  const list<ColumnDescriptor>& cols,
1942  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs,
1943  bool isLogicalTable) {
1944  cat_write_lock write_lock(this);
1945  list<ColumnDescriptor> cds;
1946  list<DictDescriptor> dds;
1947  std::set<std::string> toplevel_column_names;
1948  list<ColumnDescriptor> columns;
1949  for (auto cd : cols) {
1950  if (cd.columnName == "rowid") {
1951  throw std::runtime_error(
1952  "Cannot create column with name rowid. rowid is a system defined column.");
1953  }
1954  columns.push_back(cd);
1955  toplevel_column_names.insert(cd.columnName);
1956  if (cd.columnType.is_geometry()) {
1957  expandGeoColumn(cd, columns);
1958  }
1959  }
1960 
1961  ColumnDescriptor cd;
1962  // add row_id column -- Must be last column in the table
1963  cd.columnName = "rowid";
1964  cd.isSystemCol = true;
1965  cd.columnType = SQLTypeInfo(kBIGINT, true);
1966 #ifdef MATERIALIZED_ROWID
1967  cd.isVirtualCol = false;
1968 #else
1969  cd.isVirtualCol = true;
1970  cd.virtualExpr = "MAPD_FRAG_ID * MAPD_ROWS_PER_FRAG + MAPD_FRAG_ROW_ID";
1971 #endif
1972  columns.push_back(cd);
1973  toplevel_column_names.insert(cd.columnName);
1974 
1975  if (td.hasDeletedCol) {
1976  ColumnDescriptor cd_del;
1977  cd_del.columnName = "$deleted$";
1978  cd_del.isSystemCol = true;
1979  cd_del.isVirtualCol = false;
1980  cd_del.columnType = SQLTypeInfo(kBOOLEAN, true);
1981  cd_del.isDeletedCol = true;
1982 
1983  columns.push_back(cd_del);
1984  }
1985 
1986  td.nColumns = columns.size();
1988  sqliteConnector_.query("BEGIN TRANSACTION");
1990  try {
1992  "INSERT INTO mapd_tables (name, userid, ncolumns, isview, fragments, "
1993  "frag_type, max_frag_rows, "
1994  "max_chunk_size, "
1995  "frag_page_size, max_rows, partitions, shard_column_id, shard, num_shards, "
1996  "sort_column_id, "
1997  "key_metainfo) VALUES (?, ?, ?, "
1998  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1999 
2000  std::vector<std::string>{td.tableName,
2001  std::to_string(td.userId),
2003  std::to_string(td.isView),
2004  "",
2009  std::to_string(td.maxRows),
2010  td.partitions,
2012  std::to_string(td.shard),
2013  std::to_string(td.nShards),
2015  td.keyMetainfo});
2016 
2017  // now get the auto generated tableid
2019  "SELECT tableid FROM mapd_tables WHERE name = ?", td.tableName);
2020  td.tableId = sqliteConnector_.getData<int>(0, 0);
2021  int colId = 1;
2022  for (auto cd : columns) {
2024  const bool is_foreign_col =
2025  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2026  if (!is_foreign_col) {
2027  setColumnDictionary(cd, dds, td, isLogicalTable);
2028  }
2029  }
2030 
2031  if (toplevel_column_names.count(cd.columnName)) {
2032  // make up colId gap for sanity test (begin with 1 bc much code depends on it!)
2033  if (colId > 1) {
2034  colId += g_test_against_columnId_gap;
2035  }
2036  if (!cd.isGeoPhyCol) {
2037  td.columnIdBySpi_.push_back(colId);
2038  }
2039  }
2040 
2042  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, "
2043  "coldim, colscale, is_notnull, "
2044  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
2045  "virtual_expr, is_deletedcol) "
2046  "VALUES (?, ?, ?, ?, ?, "
2047  "?, "
2048  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2049  std::vector<std::string>{std::to_string(td.tableId),
2050  std::to_string(colId),
2051  cd.columnName,
2060  "",
2063  cd.virtualExpr,
2065  cd.tableId = td.tableId;
2066  cd.columnId = colId++;
2067  cds.push_back(cd);
2068  }
2069  if (td.isView) {
2071  "INSERT INTO mapd_views (tableid, sql) VALUES (?,?)",
2072  std::vector<std::string>{std::to_string(td.tableId), td.viewSQL});
2073  }
2074  } catch (std::exception& e) {
2075  sqliteConnector_.query("ROLLBACK TRANSACTION");
2076  throw;
2077  }
2078 
2079  } else { // Temporary table
2080  td.tableId = nextTempTableId_++;
2081  int colId = 1;
2082  for (auto cd : columns) {
2083  auto col_ti = cd.columnType;
2084  if (IS_GEO(col_ti.get_type())) {
2085  throw runtime_error("Geometry types in temporary tables are not supported.");
2086  }
2087 
2089  // TODO(vraj) : create shared dictionary for temp table if needed
2090  std::string fileName("");
2091  std::string folderPath("");
2093  nextTempDictId_++;
2094  DictDescriptor dd(dict_ref,
2095  fileName,
2097  false,
2098  1,
2099  folderPath,
2100  true); // Is dictName (2nd argument) used?
2101  dds.push_back(dd);
2102  if (!cd.columnType.is_array()) {
2104  }
2105  cd.columnType.set_comp_param(dict_ref.dictId);
2106  }
2107  cd.tableId = td.tableId;
2108  cd.columnId = colId++;
2109  cds.push_back(cd);
2110  }
2111  }
2112  try {
2113  addTableToMap(td, cds, dds);
2114  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
2115  } catch (std::exception& e) {
2116  sqliteConnector_.query("ROLLBACK TRANSACTION");
2117  removeTableFromMap(td.tableName, td.tableId, true);
2118  throw;
2119  }
2120 
2121  sqliteConnector_.query("END TRANSACTION");
2122 }
2123 
2124 // returns the table epoch or -1 if there is something wrong with the shared epoch
2125 int32_t Catalog::getTableEpoch(const int32_t db_id, const int32_t table_id) const {
2126  cat_read_lock read_lock(this);
2127  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(table_id);
2128  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2129  // check all shards have same checkpoint
2130  const auto physicalTables = physicalTableIt->second;
2131  CHECK(!physicalTables.empty());
2132  size_t curr_epoch = 0;
2133  for (size_t i = 0; i < physicalTables.size(); i++) {
2134  int32_t physical_tb_id = physicalTables[i];
2135  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2136  CHECK(phys_td);
2137  if (i == 0) {
2138  curr_epoch = dataMgr_->getTableEpoch(db_id, physical_tb_id);
2139  } else {
2140  if (curr_epoch != dataMgr_->getTableEpoch(db_id, physical_tb_id)) {
2141  // oh dear the leaves do not agree on the epoch for this table
2142  LOG(ERROR) << "Epochs on shards do not all agree on table id " << table_id
2143  << " db id " << db_id << " epoch " << curr_epoch << " leaf_epoch "
2144  << dataMgr_->getTableEpoch(db_id, physical_tb_id);
2145  return -1;
2146  }
2147  }
2148  }
2149  return curr_epoch;
2150  } else {
2151  return dataMgr_->getTableEpoch(db_id, table_id);
2152  }
2153 }
2154 
2155 void Catalog::setTableEpoch(const int db_id, const int table_id, int new_epoch) {
2156  cat_read_lock read_lock(this);
2157  LOG(INFO) << "Set table epoch db:" << db_id << " Table ID " << table_id
2158  << " back to new epoch " << new_epoch;
2159  removeChunks(table_id);
2160  dataMgr_->setTableEpoch(db_id, table_id, new_epoch);
2161 
2162  // check if sharded
2163  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(table_id);
2164  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2165  const auto physicalTables = physicalTableIt->second;
2166  CHECK(!physicalTables.empty());
2167  for (size_t i = 0; i < physicalTables.size(); i++) {
2168  int32_t physical_tb_id = physicalTables[i];
2169  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2170  CHECK(phys_td);
2171  LOG(INFO) << "Set sharded table epoch db:" << db_id << " Table ID "
2172  << physical_tb_id << " back to new epoch " << new_epoch;
2173  removeChunks(physical_tb_id);
2174  dataMgr_->setTableEpoch(db_id, physical_tb_id, new_epoch);
2175  }
2176  }
2177 }
2178 
2180  cat_read_lock read_lock(this);
2181  const auto it = deletedColumnPerTable_.find(td);
2182  return it != deletedColumnPerTable_.end() ? it->second : nullptr;
2183 }
2184 
2186  int tableId,
2187  int columnId) const {
2188  // check if there are rows deleted by examining metadata for the deletedColumn metadata
2189  ChunkKey chunkKeyPrefix = {dbId, tableId, columnId};
2190  std::vector<std::pair<ChunkKey, ChunkMetadata>> chunkMetadataVec;
2191  dataMgr_->getChunkMetadataVecForKeyPrefix(chunkMetadataVec, chunkKeyPrefix);
2192  int64_t chunk_max{0};
2193 
2194  for (auto cm : chunkMetadataVec) {
2195  chunk_max = cm.second.chunkStats.max.tinyintval;
2196  // delete has occured
2197  if (chunk_max == 1) {
2198  return true;
2199  }
2200  }
2201  return false;
2202 }
2203 
2205  const TableDescriptor* td) const {
2206  cat_read_lock read_lock(this);
2207 
2208  const auto it = deletedColumnPerTable_.find(td);
2209  // if not a table that supports delete return nullptr, nothing more to do
2210  if (it == deletedColumnPerTable_.end()) {
2211  return nullptr;
2212  }
2213  const ColumnDescriptor* cd = it->second;
2214 
2215  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
2216 
2217  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2218  // check all shards
2219  const auto physicalTables = physicalTableIt->second;
2220  CHECK(!physicalTables.empty());
2221  for (size_t i = 0; i < physicalTables.size(); i++) {
2222  int32_t physical_tb_id = physicalTables[i];
2223  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2224  CHECK(phys_td);
2226  return cd;
2227  }
2228  }
2229  } else {
2231  return cd;
2232  }
2233  }
2234  // no deletes so far recorded in metadata
2235  return nullptr;
2236 }
2237 
2239  cat_write_lock write_lock(this);
2240  setDeletedColumnUnlocked(td, cd);
2241 }
2242 
2244  const ColumnDescriptor* cd) {
2245  cat_write_lock write_lock(this);
2246  const auto it_ok = deletedColumnPerTable_.emplace(td, cd);
2247  CHECK(it_ok.second);
2248 }
2249 
2250 namespace {
2251 
2253  const Catalog& cat,
2254  const Parser::SharedDictionaryDef& shared_dict_def) {
2255  const auto& table_name = shared_dict_def.get_foreign_table();
2256  const auto td = cat.getMetadataForTable(table_name);
2257  CHECK(td);
2258  const auto& foreign_col_name = shared_dict_def.get_foreign_column();
2259  return cat.getMetadataForColumn(td->tableId, foreign_col_name);
2260 }
2261 
2262 } // namespace
2263 
2265  Parser::SharedDictionaryDef shared_dict_def) {
2266  cat_write_lock write_lock(this);
2267  const auto foreign_ref_col = get_foreign_col(*this, shared_dict_def);
2268  CHECK(foreign_ref_col);
2269  referencing_column.columnType = foreign_ref_col->columnType;
2270  const int dict_id = referencing_column.columnType.get_comp_param();
2271  const DictRef dict_ref(currentDB_.dbId, dict_id);
2272  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
2273  CHECK(dictIt != dictDescriptorMapByRef_.end());
2274  const auto& dd = dictIt->second;
2275  CHECK_GE(dd->refcount, 1);
2276  ++dd->refcount;
2279  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
2280  {std::to_string(dict_id)});
2281 }
2282 
2284  ColumnDescriptor& cd,
2285  std::list<ColumnDescriptor>& cdd,
2286  std::list<DictDescriptor>& dds,
2287  const TableDescriptor td,
2288  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs) {
2289  cat_write_lock write_lock(this);
2291 
2292  if (shared_dict_defs.empty()) {
2293  return false;
2294  }
2295  for (const auto& shared_dict_def : shared_dict_defs) {
2296  // check if the current column is a referencing column
2297  const auto& column = shared_dict_def.get_column();
2298  if (cd.columnName == column) {
2299  if (!shared_dict_def.get_foreign_table().compare(td.tableName)) {
2300  // Dictionaries are being shared in table to be created
2301  const auto& ref_column = shared_dict_def.get_foreign_column();
2302  auto colIt =
2303  std::find_if(cdd.begin(), cdd.end(), [ref_column](const ColumnDescriptor it) {
2304  return !ref_column.compare(it.columnName);
2305  });
2306  CHECK(colIt != cdd.end());
2307  cd.columnType = colIt->columnType;
2308 
2310  "SELECT dictid FROM mapd_dictionaries WHERE dictid in (select comp_param "
2311  "from "
2312  "mapd_columns "
2313  "where compression = ? and tableid = ? and columnid = ?)",
2314  std::vector<std::string>{std::to_string(kENCODING_DICT),
2315  std::to_string(td.tableId),
2316  std::to_string(colIt->columnId)});
2317  const auto dict_id = sqliteConnector_.getData<int>(0, 0);
2318  auto dictIt = std::find_if(
2319  dds.begin(), dds.end(), [this, dict_id](const DictDescriptor it) {
2320  return it.dictRef.dbId == this->currentDB_.dbId &&
2321  it.dictRef.dictId == dict_id;
2322  });
2323  if (dictIt != dds.end()) {
2324  // There exists dictionary definition of a dictionary column
2325  CHECK_GE(dictIt->refcount, 1);
2326  ++dictIt->refcount;
2328  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
2329  {std::to_string(dict_id)});
2330  } else {
2331  // The dictionary is referencing a column which is referencing a column in
2332  // diffrent table
2333  auto root_dict_def = compress_reference_path(shared_dict_def, shared_dict_defs);
2334  addReferenceToForeignDict(cd, root_dict_def);
2335  }
2336  } else {
2337  addReferenceToForeignDict(cd, shared_dict_def);
2338  }
2339  return true;
2340  }
2341  }
2342  return false;
2343 }
2344 
2346  std::list<DictDescriptor>& dds,
2347  const TableDescriptor& td,
2348  const bool isLogicalTable) {
2349  cat_write_lock write_lock(this);
2350 
2351  std::string dictName{"Initial_key"};
2352  int dictId{0};
2353  std::string folderPath;
2354  if (isLogicalTable) {
2356 
2358  "INSERT INTO mapd_dictionaries (name, nbits, is_shared, refcount) VALUES (?, ?, "
2359  "?, 1)",
2360  std::vector<std::string>{
2361  dictName, std::to_string(cd.columnType.get_comp_param()), "0"});
2363  "SELECT dictid FROM mapd_dictionaries WHERE name = ?", dictName);
2364  dictId = sqliteConnector_.getData<int>(0, 0);
2365  dictName = td.tableName + "_" + cd.columnName + "_dict" + std::to_string(dictId);
2367  "UPDATE mapd_dictionaries SET name = ? WHERE name = 'Initial_key'", dictName);
2368  folderPath = basePath_ + "/mapd_data/DB_" + std::to_string(currentDB_.dbId) +
2369  "_DICT_" + std::to_string(dictId);
2370  }
2372  dictId,
2373  dictName,
2375  false,
2376  1,
2377  folderPath,
2378  false);
2379  dds.push_back(dd);
2380  if (!cd.columnType.is_array()) {
2382  }
2383  cd.columnType.set_comp_param(dictId);
2384 }
2385 
2387  TableDescriptor& td,
2388  const list<ColumnDescriptor>& cols,
2389  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs) {
2390  cat_write_lock write_lock(this);
2391 
2392  /* create logical table */
2393  TableDescriptor tdl(td);
2394  createTable(tdl, cols, shared_dict_defs, true); // create logical table
2395  int32_t logical_tb_id = tdl.tableId;
2396 
2397  /* create physical tables and link them to the logical table */
2398  std::vector<int32_t> physicalTables;
2399  for (int32_t i = 1; i <= td.nShards; i++) {
2400  TableDescriptor tdp(td);
2402  tdp.shard = i - 1;
2403  createTable(tdp, cols, shared_dict_defs, false); // create physical table
2404  int32_t physical_tb_id = tdp.tableId;
2405 
2406  /* add physical table to the vector of physical tables */
2407  physicalTables.push_back(physical_tb_id);
2408  }
2409 
2410  if (!physicalTables.empty()) {
2411  /* add logical to physical tables correspondence to the map */
2412  const auto it_ok =
2413  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
2414  CHECK(it_ok.second);
2415  /* update sqlite mapd_logical_to_physical in sqlite database */
2416  updateLogicalToPhysicalTableMap(logical_tb_id);
2417  }
2418 }
2419 
2421  cat_write_lock write_lock(this);
2422 
2423  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
2424  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2425  // truncate all corresponding physical tables if this is a logical table
2426  const auto physicalTables = physicalTableIt->second;
2427  CHECK(!physicalTables.empty());
2428  for (size_t i = 0; i < physicalTables.size(); i++) {
2429  int32_t physical_tb_id = physicalTables[i];
2430  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2431  CHECK(phys_td);
2432  doTruncateTable(phys_td);
2433  }
2434  }
2435  doTruncateTable(td);
2436 }
2437 
2439  cat_write_lock write_lock(this);
2440 
2441  const int tableId = td->tableId;
2442  // must destroy fragmenter before deleteChunks is called.
2443  if (td->fragmenter != nullptr) {
2444  auto tableDescIt = tableDescriptorMapById_.find(tableId);
2445  delete td->fragmenter;
2446  tableDescIt->second->fragmenter = nullptr; // get around const-ness
2447  }
2448  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
2449  // assuming deleteChunksWithPrefix is atomic
2450  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
2451  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
2452 
2453  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
2454 
2455  std::unique_ptr<StringDictionaryClient> client;
2456  if (SysCatalog::instance().isAggregator()) {
2457  CHECK(!string_dict_hosts_.empty());
2458  DictRef dict_ref(currentDB_.dbId, -1);
2459  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
2460  }
2461  // clean up any dictionaries
2462  // delete all column descriptors for the table
2463  for (const auto& columnDescriptor : columnDescriptorMapById_) {
2464  auto cd = columnDescriptor.second;
2465  if (cd->tableId != td->tableId) {
2466  continue;
2467  }
2468  const int dict_id = cd->columnType.get_comp_param();
2469  // Dummy dictionaries created for a shard of a logical table have the id set to zero.
2470  if (cd->columnType.get_compression() == kENCODING_DICT && dict_id) {
2471  const DictRef dict_ref(currentDB_.dbId, dict_id);
2472  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
2473  CHECK(dictIt != dictDescriptorMapByRef_.end());
2474  const auto& dd = dictIt->second;
2475  CHECK_GE(dd->refcount, 1);
2476  // if this is the only table using this dict reset the dict
2477  if (dd->refcount == 1) {
2478  // close the dictionary
2479  dd->stringDict.reset();
2480  File_Namespace::renameForDelete(dd->dictFolderPath);
2481  if (client) {
2482  client->drop(dd->dictRef);
2483  }
2484  if (!dd->dictIsTemp) {
2485  boost::filesystem::create_directory(dd->dictFolderPath);
2486  }
2487  }
2488 
2489  DictDescriptor* new_dd = new DictDescriptor(dd->dictRef,
2490  dd->dictName,
2491  dd->dictNBits,
2492  dd->dictIsShared,
2493  dd->refcount,
2494  dd->dictFolderPath,
2495  dd->dictIsTemp);
2496  dictDescriptorMapByRef_.erase(dictIt);
2497  // now create new Dict -- need to figure out what to do here for temp tables
2498  if (client) {
2499  client->create(new_dd->dictRef, new_dd->dictIsTemp);
2500  }
2501  dictDescriptorMapByRef_[new_dd->dictRef].reset(new_dd);
2503  }
2504  }
2505 }
2506 
2507 // used by rollback_table_epoch to clean up in memory artifacts after a rollback
2508 void Catalog::removeChunks(const int table_id) {
2509  auto td = getMetadataForTable(table_id);
2510 
2511  if (td->fragmenter != nullptr) {
2513  if (td->fragmenter != nullptr) {
2514  auto tableDescIt = tableDescriptorMapById_.find(table_id);
2515  delete td->fragmenter;
2516  tableDescIt->second->fragmenter = nullptr; // get around const-ness
2517  }
2518  }
2519 
2520  // remove the chunks from in memory structures
2521  ChunkKey chunkKey = {currentDB_.dbId, table_id};
2522 
2523  dataMgr_->deleteChunksWithPrefix(chunkKey, MemoryLevel::CPU_LEVEL);
2524  dataMgr_->deleteChunksWithPrefix(chunkKey, MemoryLevel::GPU_LEVEL);
2525 }
2526 
2530  cat_write_lock write_lock(this);
2532  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
2533  sqliteConnector_.query("BEGIN TRANSACTION");
2534  try {
2535  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2536  // remove all corresponding physical tables if this is a logical table
2537  const auto physicalTables = physicalTableIt->second;
2538  CHECK(!physicalTables.empty());
2539  for (size_t i = 0; i < physicalTables.size(); i++) {
2540  int32_t physical_tb_id = physicalTables[i];
2541  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2542  CHECK(phys_td);
2543  doDropTable(phys_td);
2544  }
2545 
2546  // remove corresponding record from the logicalToPhysicalTableMap in sqlite database
2548  "DELETE FROM mapd_logical_to_physical WHERE logical_table_id = ?",
2549  std::to_string(td->tableId));
2551  }
2552  doDropTable(td);
2553  } catch (std::exception& e) {
2554  sqliteConnector_.query("ROLLBACK TRANSACTION");
2555  throw;
2556  }
2557  sqliteConnector_.query("END TRANSACTION");
2558 }
2559 
2561  const int tableId = td->tableId;
2562  sqliteConnector_.query_with_text_param("DELETE FROM mapd_tables WHERE tableid = ?",
2563  std::to_string(tableId));
2565  "select comp_param from mapd_columns where compression = ? and tableid = ?",
2566  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
2567  int numRows = sqliteConnector_.getNumRows();
2568  std::vector<int> dict_id_list;
2569  for (int r = 0; r < numRows; ++r) {
2570  dict_id_list.push_back(sqliteConnector_.getData<int>(r, 0));
2571  }
2572  for (auto dict_id : dict_id_list) {
2574  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
2575  std::vector<std::string>{std::to_string(dict_id)});
2576  }
2578  "DELETE FROM mapd_dictionaries WHERE dictid in (select comp_param from "
2579  "mapd_columns where compression = ? "
2580  "and tableid = ?) and refcount = 0",
2581  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
2582  sqliteConnector_.query_with_text_param("DELETE FROM mapd_columns WHERE tableid = ?",
2583  std::to_string(tableId));
2584  if (td->isView) {
2585  sqliteConnector_.query_with_text_param("DELETE FROM mapd_views WHERE tableid = ?",
2586  std::to_string(tableId));
2587  }
2589 }
2590 
2591 void Catalog::renamePhysicalTable(const TableDescriptor* td, const string& newTableName) {
2592  cat_write_lock write_lock(this);
2594 
2595  sqliteConnector_.query("BEGIN TRANSACTION");
2596  try {
2598  "UPDATE mapd_tables SET name = ? WHERE tableid = ?",
2599  std::vector<std::string>{newTableName, std::to_string(td->tableId)});
2600  } catch (std::exception& e) {
2601  sqliteConnector_.query("ROLLBACK TRANSACTION");
2602  throw;
2603  }
2604  sqliteConnector_.query("END TRANSACTION");
2605  TableDescriptorMap::iterator tableDescIt =
2607  CHECK(tableDescIt != tableDescriptorMap_.end());
2608  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
2609  // Get table descriptor to change it
2610  TableDescriptor* changeTd = tableDescIt->second;
2611  changeTd->tableName = newTableName;
2612  tableDescriptorMap_.erase(tableDescIt); // erase entry under old name
2613  tableDescriptorMap_[to_upper(newTableName)] = changeTd;
2614  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
2615 }
2616 
2617 void Catalog::renameTable(const TableDescriptor* td, const string& newTableName) {
2618  {
2619  cat_write_lock write_lock(this);
2621  // rename all corresponding physical tables if this is a logical table
2622  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
2623  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2624  const auto physicalTables = physicalTableIt->second;
2625  CHECK(!physicalTables.empty());
2626  for (size_t i = 0; i < physicalTables.size(); i++) {
2627  int32_t physical_tb_id = physicalTables[i];
2628  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2629  CHECK(phys_td);
2630  std::string newPhysTableName =
2631  generatePhysicalTableName(newTableName, static_cast<int32_t>(i + 1));
2632  renamePhysicalTable(phys_td, newPhysTableName);
2633  }
2634  }
2635  renamePhysicalTable(td, newTableName);
2636  }
2637  {
2638  DBObject object(newTableName, TableDBObjectType);
2639  // update table name in direct and effective priv map
2640  DBObjectKey key;
2641  key.dbId = currentDB_.dbId;
2642  key.objectId = td->tableId;
2643  key.permissionType = static_cast<int>(DBObjectType::TableDBObjectType);
2644  object.setObjectKey(key);
2645  auto objdescs = SysCatalog::instance().getMetadataForObject(
2646  currentDB_.dbId, static_cast<int>(DBObjectType::TableDBObjectType), td->tableId);
2647  for (auto obj : objdescs) {
2648  Grantee* grnt = SysCatalog::instance().getGrantee(obj->roleName);
2649  if (grnt) {
2650  grnt->renameDbObject(object);
2651  }
2652  }
2654  }
2655 }
2656 
2658  const ColumnDescriptor* cd,
2659  const string& newColumnName) {
2660  cat_write_lock write_lock(this);
2662  sqliteConnector_.query("BEGIN TRANSACTION");
2663  try {
2664  for (int i = 0; i <= cd->columnType.get_physical_cols(); ++i) {
2665  auto cdx = getMetadataForColumn(td->tableId, cd->columnId + i);
2666  CHECK(cdx);
2667  std::string new_column_name = cdx->columnName;
2668  new_column_name.replace(0, cd->columnName.size(), newColumnName);
2670  "UPDATE mapd_columns SET name = ? WHERE tableid = ? AND columnid = ?",
2671  std::vector<std::string>{new_column_name,
2672  std::to_string(td->tableId),
2673  std::to_string(cdx->columnId)});
2674  }
2675  } catch (std::exception& e) {
2676  sqliteConnector_.query("ROLLBACK TRANSACTION");
2677  throw;
2678  }
2679  sqliteConnector_.query("END TRANSACTION");
2680  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
2681  for (int i = 0; i <= cd->columnType.get_physical_cols(); ++i) {
2682  auto cdx = getMetadataForColumn(td->tableId, cd->columnId + i);
2683  CHECK(cdx);
2684  ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.find(
2685  std::make_tuple(td->tableId, to_upper(cdx->columnName)));
2686  CHECK(columnDescIt != columnDescriptorMap_.end());
2687  ColumnDescriptor* changeCd = columnDescIt->second;
2688  changeCd->columnName.replace(0, cd->columnName.size(), newColumnName);
2689  columnDescriptorMap_.erase(columnDescIt); // erase entry under old name
2690  columnDescriptorMap_[std::make_tuple(td->tableId, to_upper(changeCd->columnName))] =
2691  changeCd;
2692  }
2693  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
2694 }
2695 
2697  {
2698  cat_write_lock write_lock(this);
2700  sqliteConnector_.query("BEGIN TRANSACTION");
2701  try {
2702  // TODO(andrew): this should be an upsert
2704  "SELECT id FROM mapd_dashboards WHERE name = ? and userid = ?",
2705  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
2706  if (sqliteConnector_.getNumRows() > 0) {
2708  "UPDATE mapd_dashboards SET state = ?, image_hash = ?, metadata = ?, "
2709  "update_time = "
2710  "datetime('now') where name = ? "
2711  "and userid = ?",
2712  std::vector<std::string>{vd.dashboardState,
2713  vd.imageHash,
2714  vd.dashboardMetadata,
2715  vd.dashboardName,
2716  std::to_string(vd.userId)});
2717  } else {
2719  "INSERT INTO mapd_dashboards (name, state, image_hash, metadata, "
2720  "update_time, "
2721  "userid) "
2722  "VALUES "
2723  "(?,?,?,?, "
2724  "datetime('now'), ?)",
2725  std::vector<std::string>{vd.dashboardName,
2726  vd.dashboardState,
2727  vd.imageHash,
2728  vd.dashboardMetadata,
2729  std::to_string(vd.userId)});
2730  }
2731  } catch (std::exception& e) {
2732  sqliteConnector_.query("ROLLBACK TRANSACTION");
2733  throw;
2734  }
2735  sqliteConnector_.query("END TRANSACTION");
2736 
2737  // now get the auto generated dashboardId
2738  try {
2740  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_dashboards "
2741  "WHERE name = ? and userid = ?",
2742  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
2743  vd.dashboardId = sqliteConnector_.getData<int>(0, 0);
2744  vd.updateTime = sqliteConnector_.getData<std::string>(0, 1);
2745  } catch (std::exception& e) {
2746  throw;
2747  }
2751  }
2752  // NOTE(wamsi): Transactionally unsafe
2755  return vd.dashboardId;
2756 }
2757 
2759  cat_write_lock write_lock(this);
2761 
2762  sqliteConnector_.query("BEGIN TRANSACTION");
2763  try {
2765  "SELECT id FROM mapd_dashboards WHERE id = ?",
2766  std::vector<std::string>{std::to_string(vd.dashboardId)});
2767  if (sqliteConnector_.getNumRows() > 0) {
2769  "UPDATE mapd_dashboards SET name = ?, state = ?, image_hash = ?, metadata = ?, "
2770  "update_time = "
2771  "datetime('now') where id = ? ",
2772  std::vector<std::string>{vd.dashboardName,
2773  vd.dashboardState,
2774  vd.imageHash,
2775  vd.dashboardMetadata,
2777  } else {
2778  LOG(ERROR) << "Error replacing dashboard id " << vd.dashboardId
2779  << " does not exist in db";
2780  throw runtime_error("Error replacing dashboard id " +
2781  std::to_string(vd.dashboardId) + " does not exist in db");
2782  }
2783  } catch (std::exception& e) {
2784  sqliteConnector_.query("ROLLBACK TRANSACTION");
2785  throw;
2786  }
2787  sqliteConnector_.query("END TRANSACTION");
2788 
2789  bool found{false};
2790  for (auto descp : dashboardDescriptorMap_) {
2791  auto dash = descp.second.get();
2792  if (dash->dashboardId == vd.dashboardId) {
2793  found = true;
2794  auto viewDescIt = dashboardDescriptorMap_.find(std::to_string(dash->userId) + ":" +
2795  dash->dashboardName);
2796  if (viewDescIt ==
2797  dashboardDescriptorMap_.end()) { // check to make sure view exists
2798  LOG(ERROR) << "No metadata for dashboard for user " << dash->userId
2799  << " dashboard " << dash->dashboardName << " does not exist in map";
2800  throw runtime_error("No metadata for dashboard for user " +
2801  std::to_string(dash->userId) + " dashboard " +
2802  dash->dashboardName + " does not exist in map");
2803  }
2804  dashboardDescriptorMap_.erase(viewDescIt);
2805  break;
2806  }
2807  }
2808  if (!found) {
2809  LOG(ERROR) << "Error replacing dashboard id " << vd.dashboardId
2810  << " does not exist in map";
2811  throw runtime_error("Error replacing dashboard id " + std::to_string(vd.dashboardId) +
2812  " does not exist in map");
2813  }
2814 
2815  // now reload the object
2817  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM "
2818  "mapd_dashboards "
2819  "WHERE id = ?",
2820  std::vector<std::string>{std::to_string(vd.dashboardId)});
2821  vd.updateTime = sqliteConnector_.getData<string>(0, 1);
2825  // NOTE(wamsi): Transactionally unsafe
2828 }
2829 
2830 std::string Catalog::calculateSHA1(const std::string& data) {
2831  boost::uuids::detail::sha1 sha1;
2832  unsigned int digest[5];
2833  sha1.process_bytes(data.c_str(), data.length());
2834  sha1.get_digest(digest);
2835  std::stringstream ss;
2836  for (size_t i = 0; i < 5; i++) {
2837  ss << std::hex << digest[i];
2838  }
2839  return ss.str();
2840 }
2841 
2842 std::string Catalog::createLink(LinkDescriptor& ld, size_t min_length) {
2843  cat_write_lock write_lock(this);
2845  sqliteConnector_.query("BEGIN TRANSACTION");
2846  try {
2848  .substr(0, 8);
2850  "SELECT linkid FROM mapd_links WHERE link = ? and userid = ?",
2851  std::vector<std::string>{ld.link, std::to_string(ld.userId)});
2852  if (sqliteConnector_.getNumRows() > 0) {
2854  "UPDATE mapd_links SET update_time = datetime('now') WHERE userid = ? AND link "
2855  "= ?",
2856  std::vector<std::string>{std::to_string(ld.userId), ld.link});
2857  } else {
2859  "INSERT INTO mapd_links (userid, link, view_state, view_metadata, update_time) "
2860  "VALUES (?,?,?,?, "
2861  "datetime('now'))",
2862  std::vector<std::string>{
2863  std::to_string(ld.userId), ld.link, ld.viewState, ld.viewMetadata});
2864  }
2865  // now get the auto generated dashid
2867  "SELECT linkid, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_links "
2868  "WHERE link = ?",
2869  ld.link);
2870  ld.linkId = sqliteConnector_.getData<int>(0, 0);
2871  ld.updateTime = sqliteConnector_.getData<std::string>(0, 1);
2872  } catch (std::exception& e) {
2873  sqliteConnector_.query("ROLLBACK TRANSACTION");
2874  throw;
2875  }
2876  sqliteConnector_.query("END TRANSACTION");
2877  addLinkToMap(ld);
2878  return ld.link;
2879 }
2880 
2882  const TableDescriptor* td) const {
2883  const auto column_descriptors =
2884  getAllColumnMetadataForTable(td->tableId, false, true, true);
2885 
2886  const ColumnDescriptor* shard_cd{nullptr};
2887  int i = 1;
2888  for (auto cd_itr = column_descriptors.begin(); cd_itr != column_descriptors.end();
2889  ++cd_itr, ++i) {
2890  if (i == td->shardedColumnId) {
2891  shard_cd = *cd_itr;
2892  }
2893  }
2894  return shard_cd;
2895 }
2896 
2897 std::vector<const TableDescriptor*> Catalog::getPhysicalTablesDescriptors(
2898  const TableDescriptor* logicalTableDesc) const {
2899  cat_read_lock read_lock(this);
2900  const auto physicalTableIt =
2901  logicalToPhysicalTableMapById_.find(logicalTableDesc->tableId);
2902  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
2903  return {logicalTableDesc};
2904  }
2905 
2906  const auto physicalTablesIds = physicalTableIt->second;
2907  CHECK(!physicalTablesIds.empty());
2908  std::vector<const TableDescriptor*> physicalTables;
2909  for (size_t i = 0; i < physicalTablesIds.size(); i++) {
2910  physicalTables.push_back(getMetadataForTable(physicalTablesIds[i]));
2911  }
2912 
2913  return physicalTables;
2914 }
2915 
2916 int Catalog::getLogicalTableId(const int physicalTableId) const {
2917  for (const auto& l : logicalToPhysicalTableMapById_) {
2918  if (l.second.end() != std::find_if(l.second.begin(),
2919  l.second.end(),
2920  [&](decltype(*l.second.begin()) tid) -> bool {
2921  return physicalTableId == tid;
2922  })) {
2923  return l.first;
2924  }
2925  }
2926  return physicalTableId;
2927 }
2928 
2929 void Catalog::checkpoint(const int logicalTableId) const {
2930  const auto td = getMetadataForTable(logicalTableId);
2931  const auto shards = getPhysicalTablesDescriptors(td);
2932  for (const auto shard : shards) {
2933  getDataMgr().checkpoint(getCurrentDB().dbId, shard->tableId);
2934  }
2935 }
2936 
2938  cat_write_lock write_lock(this);
2939  // Physically erase all tables and dictionaries from disc and memory
2940  const auto tables = getAllTableMetadata();
2941  for (const auto table : tables) {
2942  eraseTablePhysicalData(table);
2943  }
2944  // Physically erase database metadata
2945  boost::filesystem::remove(basePath_ + "/mapd_catalogs/" + currentDB_.dbName);
2946  calciteMgr_->updateMetadata(currentDB_.dbName, "");
2947 }
2948 
2950  cat_write_lock write_lock(this);
2951  const int tableId = td->tableId;
2952  // must destroy fragmenter before deleteChunks is called.
2953  if (td->fragmenter != nullptr) {
2954  auto tableDescIt = tableDescriptorMapById_.find(tableId);
2955  {
2956  INJECT_TIMER(deleting_fragmenter);
2957  delete td->fragmenter;
2958  }
2959  tableDescIt->second->fragmenter = nullptr; // get around const-ness
2960  }
2961  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
2962  {
2963  INJECT_TIMER(deleteChunksWithPrefix);
2964  // assuming deleteChunksWithPrefix is atomic
2965  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
2966  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
2967  }
2968  if (!td->isView) {
2969  INJECT_TIMER(Remove_Table);
2970  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
2971  }
2972  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
2973  {
2974  INJECT_TIMER(removeTableFromMap_);
2975  removeTableFromMap(td->tableName, tableId);
2976  }
2977 }
2978 
2979 std::string Catalog::generatePhysicalTableName(const std::string& logicalTableName,
2980  const int32_t& shardNumber) {
2981  std::string physicalTableName =
2982  logicalTableName + physicalTableNameTag_ + std::to_string(shardNumber);
2983  return (physicalTableName);
2984 }
2985 
2986 void Catalog::set(const std::string& dbName, std::shared_ptr<Catalog> cat) {
2987  mapd_cat_map_[dbName] = cat;
2988 }
2989 
2990 std::shared_ptr<Catalog> Catalog::get(const std::string& dbName) {
2991  auto cat_it = mapd_cat_map_.find(dbName);
2992  if (cat_it != mapd_cat_map_.end()) {
2993  return cat_it->second;
2994  }
2995  return nullptr;
2996 }
2997 
2998 std::shared_ptr<Catalog> Catalog::get(const string& basePath,
2999  const DBMetadata& curDB,
3000  std::shared_ptr<Data_Namespace::DataMgr> dataMgr,
3001  const std::vector<LeafHostInfo>& string_dict_hosts,
3002  std::shared_ptr<Calcite> calcite,
3003  bool is_new_db) {
3004  auto cat = Catalog::get(curDB.dbName);
3005 
3006  if (cat) {
3007  return cat;
3008  } else {
3009  cat = std::make_shared<Catalog>(
3010  basePath, curDB, dataMgr, string_dict_hosts, calcite, is_new_db);
3011  Catalog::set(curDB.dbName, cat);
3012  return cat;
3013  }
3014 }
3015 
3016 void Catalog::remove(const std::string& dbName) {
3017  mapd_cat_map_.erase(dbName);
3018 }
3019 
3020 void Catalog::vacuumDeletedRows(const int logicalTableId) const {
3021  // shard here to serve request from TableOptimizer and elsewhere
3022  cat_read_lock read_lock(this);
3023  const auto td = getMetadataForTable(logicalTableId);
3024  const auto shards = getPhysicalTablesDescriptors(td);
3025  for (const auto shard : shards) {
3026  vacuumDeletedRows(shard);
3027  }
3028 }
3029 
3031  cat_read_lock read_lock(this);
3032  // "if not a table that supports delete return nullptr, nothing more to do"
3033  const ColumnDescriptor* cd = getDeletedColumn(td);
3034  if (nullptr == cd) {
3035  return;
3036  }
3037  // vacuum chunks which show sign of deleted rows in metadata
3038  ChunkKey chunkKeyPrefix = {currentDB_.dbId, td->tableId, cd->columnId};
3039  std::vector<std::pair<ChunkKey, ChunkMetadata>> chunkMetadataVec;
3040  dataMgr_->getChunkMetadataVecForKeyPrefix(chunkMetadataVec, chunkKeyPrefix);
3041  for (auto cm : chunkMetadataVec) {
3042  // "delete has occured"
3043  if (cm.second.chunkStats.max.tinyintval == 1) {
3044  UpdelRoll updel_roll;
3045  updel_roll.catalog = this;
3046  updel_roll.logicalTableId = getLogicalTableId(td->tableId);
3048  const auto cd = getMetadataForColumn(td->tableId, cm.first[2]);
3049  const auto chunk = Chunk_NS::Chunk::getChunk(cd,
3050  &getDataMgr(),
3051  cm.first,
3052  updel_roll.memoryLevel,
3053  0,
3054  cm.second.numBytes,
3055  cm.second.numElements);
3056  td->fragmenter->compactRows(this,
3057  td,
3058  cm.first[3],
3059  td->fragmenter->getVacuumOffsets(chunk),
3060  updel_roll.memoryLevel,
3061  updel_roll);
3062  updel_roll.commitUpdate();
3063  }
3064  }
3065 }
3066 
3067 } // namespace Catalog_Namespace
void instantiateFragmenter(TableDescriptor *td) const
Definition: Catalog.cpp:1273
const Parser::SharedDictionaryDef compress_reference_path(Parser::SharedDictionaryDef cur_node, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Data_Namespace::MemoryLevel memoryLevel
Definition: UpdelRoll.h:65
std::string virtualExpr
#define CHECK_EQ(x, y)
Definition: Logger.h:198
const int MAPD_TEMP_TABLE_START_ID
Definition: Catalog.cpp:80
std::string partitions
std::vector< int > ChunkKey
Definition: types.h:35
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:2949
std::string dictFolderPath
void doDropTable(const TableDescriptor *td)
Definition: Catalog.cpp:2560
~Catalog()
Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the ...
Definition: Catalog.cpp:156
void getDictionary(const ColumnDescriptor &cd, std::map< int, StringDictionary * > &stringDicts)
Definition: Catalog.cpp:1674
T getData(const int row, const int col)
void commitUpdate()
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:81
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:334
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:335
SQLTypes
Definition: sqltypes.h:41
static std::shared_ptr< Executor > getExecutor(const int db_id, const std::string &debug_dir="", const std::string &debug_file="", const MapDParameters mapd_parameters=MapDParameters(),::QueryRenderer::QueryRenderManager *render_manager=nullptr)
Definition: Execute.cpp:127
#define SPIMAP_MAGIC1
Definition: Catalog.h:68
std::string tableName
const ColumnDescriptor * getDeletedColumn(const TableDescriptor *td) const
Definition: Catalog.cpp:2179
void setDeletedColumn(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:2238
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:320
static const AccessPrivileges ALL_DATABASE
Definition: DBObject.h:144
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:1939
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:327
EncodingType
Definition: encodetypes.h:22
DBObjectType
Definition: DBObject.h:42
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:177
void addColumn(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:1702
void checkpoint(const int db_id, const int tb_id)
Definition: DataMgr.cpp:406
void roll(const bool forward)
Definition: Catalog.cpp:1752
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:1598
#define LOG(tag)
Definition: Logger.h:185
SqliteConnector sqliteConnector_
Definition: Catalog.h:326
static std::shared_ptr< Catalog > get(const std::string &dbName)
Definition: Catalog.cpp:2990
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:330
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logicalTableDesc) const
Definition: Catalog.cpp:2897
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:322
void set_size(int s)
Definition: sqltypes.h:424
void query_with_text_params(std::string const &query_only)
HOST DEVICE int get_scale() const
Definition: sqltypes.h:331
void revokeDBObjectPrivilegesFromAll(DBObject object, Catalog *catalog)
void revokeDBObjectPrivileges(const std::string &grantee, const DBObject &object, const Catalog_Namespace::Catalog &catalog)
Catalog(const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
Constructor - takes basePath to already extant data directory for writing.
Definition: Catalog.cpp:129
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:328
const LinkDescriptor * getMetadataForLink(const std::string &link) const
HOST DEVICE int get_size() const
Definition: sqltypes.h:336
const std::string & get_foreign_table() const
Definition: ParserNode.h:921
void query(const std::string &queryString)
#define CHECK_GE(x, y)
Definition: Logger.h:203
void setObjectKey(const DBObjectKey &objectKey)
Definition: DBObject.h:201
Driver for running cleanup processes on a table. TableOptimizer provides functions for various cleanu...
Grantee * getGrantee(const std::string &name) const
std::string fragments
int32_t objectId
Definition: DBObject.h:56
int32_t getTableEpoch(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:2125
HOST DEVICE void set_type(SQLTypes t)
Definition: sqltypes.h:416
void replaceDashboard(DashboardDescriptor &vd)
Definition: Catalog.cpp:2758
const int MAPD_TEMP_DICT_START_ID
Definition: Catalog.cpp:82
Definition: Grantee.h:76
std::string generatePhysicalTableName(const std::string &logicalTableName, const int32_t &shardNumber)
Definition: Catalog.cpp:2979
The InsertOrderFragmenter is a child class of AbstractFragmenter, and fragments data in insert order...
void addTableToMap(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
Definition: Catalog.cpp:1051
void checkDateInDaysColumnMigration()
Definition: Catalog.cpp:655
void set_dimension(int d)
Definition: sqltypes.h:418
void setPrivileges(const AccessPrivileges &privs)
Definition: DBObject.h:203
void addFrontendViewToMap(DashboardDescriptor &vd)
Definition: Catalog.cpp:1183
void deleteMetadataForDashboard(const std::string &userId, const std::string &dashName)
Definition: Catalog.cpp:1447
static const AccessPrivileges SELECT_FROM_TABLE
Definition: DBObject.h:153
void set_scale(int s)
Definition: sqltypes.h:421
const ColumnDescriptor * getShardColumnMetadataForTable(const TableDescriptor *td) const
Definition: Catalog.cpp:2881
std::vector< int > columnIdBySpi_
void set_compression(EncodingType c)
Definition: sqltypes.h:426
void renameColumn(const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
Definition: Catalog.cpp:2657
DeletedColumnPerTableMap deletedColumnPerTable_
Definition: Catalog.h:349
const ColumnDescriptor * get_foreign_col(const Catalog &cat, const Parser::SharedDictionaryDef &shared_dict_def)
Definition: Catalog.cpp:2252
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:321
DBObject * findDbObject(const DBObjectKey &objectKey, bool only_direct) const
Definition: Grantee.cpp:72
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:323
std::string to_string(char const *&&v)
void set_notnull(bool n)
Definition: sqltypes.h:423
std::shared_ptr< std::mutex > mutex_
static const AccessPrivileges ALL_VIEW
Definition: DBObject.h:170
std::string chunks
void grantRoleBatch(const std::vector< std::string > &roles, const std::vector< std::string > &grantees)
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:528
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:98
#define SPIMAP_MAGIC2
Definition: Catalog.h:69
std::string name() const
Definition: Catalog.h:219
void createRole(const std::string &roleName, const bool &userPrivateRole=false)
std::string dashboardSystemRoleName
This file contains the class specification and related data structures for Catalog.
std::list< const DashboardDescriptor * > getAllDashboardsMetadata() const
Definition: Catalog.cpp:1607
virtual void compactRows(const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const int fragmentId, const std::vector< uint64_t > &fragOffsets, const Data_Namespace::MemoryLevel memoryLevel, UpdelRoll &updelRoll)=0
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:417
virtual const std::vector< uint64_t > getVacuumOffsets(const std::shared_ptr< Chunk_NS::Chunk > &chunk)=0
static SysCatalog & instance()
Definition: SysCatalog.h:242
This file contains the class specification and related data structures for SysCatalog.
CHECK(cgen_state)
bool is_array() const
Definition: sqltypes.h:485
const Catalog_Namespace::Catalog * catalog
Definition: UpdelRoll.h:63
std::string calculateSHA1(const std::string &data)
Definition: Catalog.cpp:2830
std::string getUserFromId(const int32_t id)
Definition: Catalog.cpp:855
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:176
void addLinkToMap(LinkDescriptor &ld)
Definition: Catalog.cpp:1265
const ColumnDescriptor * getDeletedColumnIfRowsDeleted(const TableDescriptor *td) const
Definition: Catalog.cpp:2204
#define INJECT_TIMER(DESC)
Definition: measure.h:91
void delDictionary(const ColumnDescriptor &cd)
Definition: Catalog.cpp:1641
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:326
void renameObjectsInDescriptorMap(DBObject &object, const Catalog_Namespace::Catalog &cat)
void removeChunks(const int table_id)
Definition: Catalog.cpp:2508
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:2243
const std::string & get_foreign_column() const
Definition: ParserNode.h:923
void dropTable(const TableDescriptor *td)
Definition: Catalog.cpp:2527
std::tuple< int, int > ColumnIdKey
Definition: Catalog.h:248
std::vector< std::string > parse_underlying_dashboard_objects(const std::string &meta)
const bool checkMetadataForDeletedRecs(int dbId, int tableId, int columnId) const
Definition: Catalog.cpp:2185
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
static const int32_t MAPD_VERSION
Definition: release.h:33
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:333
Role * getRoleGrantee(const std::string &name) const
std::tuple< int, std::string > ColumnKey
Definition: Catalog.h:246
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:319
void setOwner(int32_t userId)
Definition: DBObject.h:211
static std::shared_ptr< Chunk > getChunk(const ColumnDescriptor *cd, DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId, const size_t num_bytes, const size_t num_elems)
Definition: Chunk.cpp:28
void grantDBObjectPrivilegesBatch(const std::vector< std::string > &grantees, const std::vector< DBObject > &objects, const Catalog_Namespace::Catalog &catalog)
int getLogicalTableId(const int physicalTableId) const
Definition: Catalog.cpp:2916
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1350
specifies the content in-memory of a row in the column metadata table
int32_t dictId
Definition: DictRef.h:10
size_t getNumRows() const
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:1809
const std::vector< LeafHostInfo > & getStringDictionaryHosts() const
Definition: Catalog.cpp:1386
SQLTypeInfoCore< ArrayContextTypeSizer, ExecutorTypePackaging, DateTimeFacilities > SQLTypeInfo
Definition: sqltypes.h:852
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1224
const ColumnDescriptor * getMetadataForColumnBySpi(const int tableId, const size_t spi) const
Definition: Catalog.cpp:1437
std::string keyMetainfo
int g_test_against_columnId_gap
Definition: Catalog.cpp:74
bool is_geometry() const
Definition: sqltypes.h:489
struct dict_ref_t DictRef
Definition: DictRef.h:37
void createShardedTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:2386
std::string to_upper(const std::string &str)
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1581
const DBObjectMap * getDbObjects(bool only_direct) const
Definition: Grantee.h:56
void checkpoint(const int logicalTableId) const
Definition: Catalog.cpp:2929
const int DEFAULT_INITIAL_VERSION
Definition: Catalog.cpp:79
Definition: sqltypes.h:56
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:331
static const std::string physicalTableNameTag_
Definition: Catalog.h:335
bool isNull(const int row, const int col) const
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1188
static void remove(const std::string &dbName)
Definition: Catalog.cpp:3016
static const AccessPrivileges SELECT_FROM_VIEW
Definition: DBObject.h:173
void setTableEpoch(const int db_id, const int table_id, const int new_epoch)
Definition: Catalog.cpp:2155
static void set(const std::string &dbName, std::shared_ptr< Catalog > cat)
Definition: Catalog.cpp:2986
const int OMNISCI_ROOT_USER_ID
Definition: SysCatalog.h:60
void vacuumDeletedRows(const TableDescriptor *td) const
Definition: Catalog.cpp:3030
Fragmenter_Namespace::FragmenterType fragType
int32_t dbId
Definition: DBObject.h:55
Data_Namespace::MemoryLevel persistenceLevel
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:328
int32_t createDashboard(DashboardDescriptor &vd)
Definition: Catalog.cpp:2696
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1101
static const AccessPrivileges ALL_DASHBOARD
Definition: DBObject.h:162
void addReferenceToForeignDict(ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def)
Definition: Catalog.cpp:2264
static const AccessPrivileges ALL_TABLE
Definition: DBObject.h:150
int logicalTableId
Definition: UpdelRoll.h:64
bool g_cache_string_hash
Definition: Execute.cpp:88
const int getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const
Definition: Catalog.cpp:1415
void set_comp_param(int p)
Definition: sqltypes.h:427
bool setColumnSharedDictionary(ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:2283
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:327
bool is_string() const
Definition: sqltypes.h:477
const TableDescriptor * getMetadataForTableImpl(int tableId, const bool populateFragmenter) const
Definition: Catalog.cpp:1330
std::string createLink(LinkDescriptor &ld, size_t min_length)
Definition: Catalog.cpp:2842
Descriptor for a dictionary for a string columne.
void updateLogicalToPhysicalTableLinkSchema()
Definition: Catalog.cpp:467
int get_physical_cols() const
Definition: sqltypes.h:347
FileBuffer Chunk
A Chunk is the fundamental unit of execution in Map-D.
Definition: FileMgr.h:67
std::unordered_map< std::string, std::vector< std::string > > getGranteesOfSharedDashboards(const std::vector< std::string > &dashboard_ids)
int32_t permissionType
Definition: DBObject.h:54
void populateRoleDbObjects(const std::vector< DBObject > &objects)
#define DEFAULT_MAX_CHUNK_SIZE
std::string viewSQL
Definition: sqltypes.h:48
SQLTypeInfo columnType
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
DictRef addDictionary(ColumnDescriptor &cd)
Definition: Catalog.cpp:1615
specifies the content in-memory of a row in the table metadata table
void renameTable(const TableDescriptor *td, const std::string &newTableName)
Definition: Catalog.cpp:2617
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:183
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:325
static std::map< std::string, std::shared_ptr< Catalog > > mapd_cat_map_
Definition: Catalog.h:348
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:2345
Fragmenter_Namespace::AbstractFragmenter * fragmenter
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:345
void doTruncateTable(const TableDescriptor *td)
Definition: Catalog.cpp:2438
void renamePhysicalTable(const TableDescriptor *td, const std::string &newTableName)
Definition: Catalog.cpp:2591
std::string columnName
void query_with_text_param(const std::string &queryString, const std::string &text_param)
std::vector< DBObject > parseDashboardObjects(const std::string &view_meta, const int &user_id)
Definition: Catalog.cpp:1194
#define IS_GEO(T)
Definition: sqltypes.h:167
std::vector< ObjectRoleDescriptor * > getMetadataForObject(int32_t dbId, int32_t dbType, int32_t objectId) const
virtual void renameDbObject(const DBObject &object)
Definition: Grantee.cpp:108
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:333
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:318
static thread_local bool thread_holds_read_lock
Definition: Catalog.h:363
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:324
void CheckAndExecuteMigrationsPostBuildMaps()
Definition: Catalog.cpp:849
void recomputeMetadata() const
Recomputes per-chunk metadata for each fragment in the table. Updates and deletes can cause chunk met...
const int getColumnIdBySpi(const int tableId, const size_t spi) const
Definition: Catalog.cpp:1432
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)
bool is_string_array() const
Definition: sqltypes.h:478
void truncateTable(const TableDescriptor *td)
Definition: Catalog.cpp:2420
void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id)
Definition: Catalog.cpp:481