OmniSciDB  29e35f4d58
Catalog.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
25 #include "Catalog.h"
26 #include "SysCatalog.h"
27 
28 #include <sys/wait.h>
29 
30 #include <algorithm>
31 #include <boost/algorithm/string/predicate.hpp>
32 #include <boost/filesystem.hpp>
33 #include <boost/range/adaptor/map.hpp>
34 #include <boost/version.hpp>
35 #include <cassert>
36 #include <cerrno>
37 #include <cstdio>
38 #include <cstring>
39 #include <exception>
40 #include <fstream>
41 #include <list>
42 #include <memory>
43 #include <random>
44 #include <regex>
45 #include <sstream>
46 #if BOOST_VERSION >= 106600
47 #include <boost/uuid/detail/sha1.hpp>
48 #else
49 #include <boost/uuid/sha1.hpp>
50 #endif
51 #include <rapidjson/document.h>
52 #include <rapidjson/istreamwrapper.h>
53 #include <rapidjson/ostreamwrapper.h>
54 #include <rapidjson/writer.h>
55 
56 #include "../Fragmenter/Fragmenter.h"
57 #include "../Fragmenter/SortedOrderFragmenter.h"
58 #include "../LockMgr/TableLockMgr.h"
59 #include "../Parser/ParserNode.h"
60 #include "../QueryEngine/Execute.h"
61 #include "../QueryEngine/TableOptimizer.h"
62 #include "../Shared/File.h"
63 #include "../Shared/StringTransform.h"
64 #include "../Shared/measure.h"
65 #include "../StringDictionary/StringDictionaryClient.h"
66 #include "MapDRelease.h"
67 #include "RWLocks.h"
69 
70 using Chunk_NS::Chunk;
73 using std::list;
74 using std::map;
75 using std::pair;
76 using std::runtime_error;
77 using std::string;
78 using std::vector;
79 
81 extern bool g_cache_string_hash;
82 
83 // Serialize temp tables to a json file in the Catalogs directory for Calcite parsing
84 // under unit testing.
86 
87 namespace Catalog_Namespace {
88 
89 const int DEFAULT_INITIAL_VERSION = 1; // start at version 1
91  1073741824; // 2^30, give room for over a billion non-temp tables
93  1073741824; // 2^30, give room for over a billion non-temp dictionaries
94 
95 const std::string Catalog::physicalTableNameTag_("_shard_#");
96 std::map<std::string, std::shared_ptr<Catalog>> Catalog::mapd_cat_map_;
97 
98 thread_local bool Catalog::thread_holds_read_lock = false;
99 
103 
104 // migration will be done as two step process this release
105 // will create and use new table
106 // next release will remove old table, doing this to have fall back path
107 // incase of migration failure
110  sqliteConnector_.query("BEGIN TRANSACTION");
111  try {
113  "SELECT name FROM sqlite_master WHERE type='table' AND name='mapd_dashboards'");
114  if (sqliteConnector_.getNumRows() != 0) {
115  // already done
116  sqliteConnector_.query("END TRANSACTION");
117  return;
118  }
120  "CREATE TABLE mapd_dashboards (id integer primary key autoincrement, name text , "
121  "userid integer references mapd_users, state text, image_hash text, update_time "
122  "timestamp, "
123  "metadata text, UNIQUE(userid, name) )");
124  // now copy content from old table to new table
126  "insert into mapd_dashboards (id, name , "
127  "userid, state, image_hash, update_time , "
128  "metadata) "
129  "SELECT viewid , name , userid, view_state, image_hash, update_time, "
130  "view_metadata "
131  "from mapd_frontend_views");
132  } catch (const std::exception& e) {
133  sqliteConnector_.query("ROLLBACK TRANSACTION");
134  throw;
135  }
136  sqliteConnector_.query("END TRANSACTION");
137 }
138 
139 namespace {
140 
141 inline auto table_json_filepath(const std::string& base_path,
142  const std::string& db_name) {
143  return boost::filesystem::path(base_path + "/mapd_catalogs/" + db_name +
144  "_temp_tables.json");
145 }
146 
147 } // namespace
148 
149 Catalog::Catalog(const string& basePath,
150  const DBMetadata& curDB,
151  std::shared_ptr<Data_Namespace::DataMgr> dataMgr,
152  const std::vector<LeafHostInfo>& string_dict_hosts,
153  std::shared_ptr<Calcite> calcite,
154  bool is_new_db)
155  : basePath_(basePath)
156  , sqliteConnector_(curDB.dbName, basePath + "/mapd_catalogs/")
157  , currentDB_(curDB)
158  , dataMgr_(dataMgr)
159  , string_dict_hosts_(string_dict_hosts)
160  , calciteMgr_(calcite)
161  , nextTempTableId_(MAPD_TEMP_TABLE_START_ID)
162  , nextTempDictId_(MAPD_TEMP_DICT_START_ID)
163  , sqliteMutex_()
164  , sharedMutex_()
167  if (!is_new_db) {
169  }
170  buildMaps();
171  if (!is_new_db) {
173  }
175  boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
176  }
177 }
178 
181  // must clean up heap-allocated TableDescriptor and ColumnDescriptor structs
182  for (TableDescriptorMap::iterator tableDescIt = tableDescriptorMap_.begin();
183  tableDescIt != tableDescriptorMap_.end();
184  ++tableDescIt) {
185  if (tableDescIt->second->fragmenter != nullptr) {
186  delete tableDescIt->second->fragmenter;
187  }
188  delete tableDescIt->second;
189  }
190 
191  // TableDescriptorMapById points to the same descriptors. No need to delete
192 
193  for (ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.begin();
194  columnDescIt != columnDescriptorMap_.end();
195  ++columnDescIt) {
196  delete columnDescIt->second;
197  }
198 
199  // ColumnDescriptorMapById points to the same descriptors. No need to delete
200 
202  boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
203  }
204 }
205 
208  sqliteConnector_.query("BEGIN TRANSACTION");
209  try {
210  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_tables)");
211  std::vector<std::string> cols;
212  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
213  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
214  }
215  if (std::find(cols.begin(), cols.end(), std::string("max_chunk_size")) ==
216  cols.end()) {
217  string queryString("ALTER TABLE mapd_tables ADD max_chunk_size BIGINT DEFAULT " +
219  sqliteConnector_.query(queryString);
220  }
221  if (std::find(cols.begin(), cols.end(), std::string("shard_column_id")) ==
222  cols.end()) {
223  string queryString("ALTER TABLE mapd_tables ADD shard_column_id BIGINT DEFAULT " +
224  std::to_string(0));
225  sqliteConnector_.query(queryString);
226  }
227  if (std::find(cols.begin(), cols.end(), std::string("shard")) == cols.end()) {
228  string queryString("ALTER TABLE mapd_tables ADD shard BIGINT DEFAULT " +
229  std::to_string(-1));
230  sqliteConnector_.query(queryString);
231  }
232  if (std::find(cols.begin(), cols.end(), std::string("num_shards")) == cols.end()) {
233  string queryString("ALTER TABLE mapd_tables ADD num_shards BIGINT DEFAULT " +
234  std::to_string(0));
235  sqliteConnector_.query(queryString);
236  }
237  if (std::find(cols.begin(), cols.end(), std::string("key_metainfo")) == cols.end()) {
238  string queryString("ALTER TABLE mapd_tables ADD key_metainfo TEXT DEFAULT '[]'");
239  sqliteConnector_.query(queryString);
240  }
241  if (std::find(cols.begin(), cols.end(), std::string("userid")) == cols.end()) {
242  string queryString("ALTER TABLE mapd_tables ADD userid integer DEFAULT " +
244  sqliteConnector_.query(queryString);
245  }
246  if (std::find(cols.begin(), cols.end(), std::string("sort_column_id")) ==
247  cols.end()) {
249  "ALTER TABLE mapd_tables ADD sort_column_id INTEGER DEFAULT 0");
250  }
251  } catch (std::exception& e) {
252  sqliteConnector_.query("ROLLBACK TRANSACTION");
253  throw;
254  }
255  sqliteConnector_.query("END TRANSACTION");
256 }
257 
260  sqliteConnector_.query("BEGIN TRANSACTION");
261  try {
263  "select name from sqlite_master WHERE type='table' AND "
264  "name='mapd_version_history'");
265  if (sqliteConnector_.getNumRows() == 0) {
267  "CREATE TABLE mapd_version_history(version integer, migration_history text "
268  "unique)");
269  } else {
271  "select * from mapd_version_history where migration_history = "
272  "'notnull_fixlen_arrays'");
273  if (sqliteConnector_.getNumRows() != 0) {
274  // legacy fixlen arrays had migrated
275  // no need for further execution
276  sqliteConnector_.query("END TRANSACTION");
277  return;
278  }
279  }
280  // Insert check for migration
282  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
283  std::vector<std::string>{std::to_string(MAPD_VERSION), "notnull_fixlen_arrays"});
284  LOG(INFO) << "Updating mapd_columns, legacy fixlen arrays";
285  // Upating all fixlen array columns
286  string queryString("UPDATE mapd_columns SET is_notnull=1 WHERE coltype=" +
287  std::to_string(kARRAY) + " AND size>0;");
288  sqliteConnector_.query(queryString);
289  } catch (std::exception& e) {
290  sqliteConnector_.query("ROLLBACK TRANSACTION");
291  throw;
292  }
293  sqliteConnector_.query("END TRANSACTION");
294 }
295 
298  sqliteConnector_.query("BEGIN TRANSACTION");
299  try {
300  // check table still exists
302  "SELECT name FROM sqlite_master WHERE type='table' AND "
303  "name='mapd_frontend_views'");
304  if (sqliteConnector_.getNumRows() == 0) {
305  // table does not exists
306  // no need to migrate
307  sqliteConnector_.query("END TRANSACTION");
308  return;
309  }
310  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_frontend_views)");
311  std::vector<std::string> cols;
312  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
313  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
314  }
315  if (std::find(cols.begin(), cols.end(), std::string("image_hash")) == cols.end()) {
316  sqliteConnector_.query("ALTER TABLE mapd_frontend_views ADD image_hash text");
317  }
318  if (std::find(cols.begin(), cols.end(), std::string("update_time")) == cols.end()) {
319  sqliteConnector_.query("ALTER TABLE mapd_frontend_views ADD update_time timestamp");
320  }
321  if (std::find(cols.begin(), cols.end(), std::string("view_metadata")) == cols.end()) {
322  sqliteConnector_.query("ALTER TABLE mapd_frontend_views ADD view_metadata text");
323  }
324  } catch (std::exception& e) {
325  sqliteConnector_.query("ROLLBACK TRANSACTION");
326  throw;
327  }
328  sqliteConnector_.query("END TRANSACTION");
329 }
330 
333  sqliteConnector_.query("BEGIN TRANSACTION");
334  try {
336  "CREATE TABLE IF NOT EXISTS mapd_links (linkid integer primary key, userid "
337  "integer references mapd_users, "
338  "link text unique, view_state text, update_time timestamp, view_metadata text)");
339  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_links)");
340  std::vector<std::string> cols;
341  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
342  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
343  }
344  if (std::find(cols.begin(), cols.end(), std::string("view_metadata")) == cols.end()) {
345  sqliteConnector_.query("ALTER TABLE mapd_links ADD view_metadata text");
346  }
347  } catch (const std::exception& e) {
348  sqliteConnector_.query("ROLLBACK TRANSACTION");
349  throw;
350  }
351  sqliteConnector_.query("END TRANSACTION");
352 }
353 
356  sqliteConnector_.query("BEGIN TRANSACTION");
357  try {
358  sqliteConnector_.query("UPDATE mapd_links SET userid = 0 WHERE userid IS NULL");
359  // check table still exists
361  "SELECT name FROM sqlite_master WHERE type='table' AND "
362  "name='mapd_frontend_views'");
363  if (sqliteConnector_.getNumRows() == 0) {
364  // table does not exists
365  // no need to migrate
366  sqliteConnector_.query("END TRANSACTION");
367  return;
368  }
370  "UPDATE mapd_frontend_views SET userid = 0 WHERE userid IS NULL");
371  } catch (const std::exception& e) {
372  sqliteConnector_.query("ROLLBACK TRANSACTION");
373  throw;
374  }
375  sqliteConnector_.query("END TRANSACTION");
376 }
377 
378 // introduce DB version into the tables table
379 // if the DB does not have a version reset all pagesizes to 2097152 to be compatible with
380 // old value
381 
384  if (currentDB_.dbName.length() == 0) {
385  // updateDictionaryNames dbName length is zero nothing to do here
386  return;
387  }
388  sqliteConnector_.query("BEGIN TRANSACTION");
389  try {
390  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_tables)");
391  std::vector<std::string> cols;
392  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
393  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
394  }
395  if (std::find(cols.begin(), cols.end(), std::string("version_num")) == cols.end()) {
396  LOG(INFO) << "Updating mapd_tables updatePageSize";
397  // No version number
398  // need to update the defaul tpagesize to old correct value
399  sqliteConnector_.query("UPDATE mapd_tables SET frag_page_size = 2097152 ");
400  // need to add new version info
401  string queryString("ALTER TABLE mapd_tables ADD version_num BIGINT DEFAULT " +
402  std::to_string(DEFAULT_INITIAL_VERSION));
403  sqliteConnector_.query(queryString);
404  }
405  } catch (std::exception& e) {
406  sqliteConnector_.query("ROLLBACK TRANSACTION");
407  throw;
408  }
409  sqliteConnector_.query("END TRANSACTION");
410 }
411 
414  sqliteConnector_.query("BEGIN TRANSACTION");
415  try {
416  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_columns)");
417  std::vector<std::string> cols;
418  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
419  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
420  }
421  if (std::find(cols.begin(), cols.end(), std::string("version_num")) == cols.end()) {
422  LOG(INFO) << "Updating mapd_columns updateDeletedColumnIndicator";
423  // need to add new version info
424  string queryString("ALTER TABLE mapd_columns ADD version_num BIGINT DEFAULT " +
425  std::to_string(DEFAULT_INITIAL_VERSION));
426  sqliteConnector_.query(queryString);
427  // need to add new column to table defintion to indicate deleted column, column used
428  // as bitmap for deleted rows.
430  "ALTER TABLE mapd_columns ADD is_deletedcol boolean default 0 ");
431  }
432  } catch (std::exception& e) {
433  sqliteConnector_.query("ROLLBACK TRANSACTION");
434  throw;
435  }
436  sqliteConnector_.query("END TRANSACTION");
437 }
438 
439 // introduce DB version into the dictionary tables
440 // if the DB does not have a version rename all dictionary tables
441 
444  if (currentDB_.dbName.length() == 0) {
445  // updateDictionaryNames dbName length is zero nothing to do here
446  return;
447  }
448  sqliteConnector_.query("BEGIN TRANSACTION");
449  try {
450  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_dictionaries)");
451  std::vector<std::string> cols;
452  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
453  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
454  }
455  if (std::find(cols.begin(), cols.end(), std::string("version_num")) == cols.end()) {
456  // No version number
457  // need to rename dictionaries
458  string dictQuery("SELECT dictid, name from mapd_dictionaries");
459  sqliteConnector_.query(dictQuery);
460  size_t numRows = sqliteConnector_.getNumRows();
461  for (size_t r = 0; r < numRows; ++r) {
462  int dictId = sqliteConnector_.getData<int>(r, 0);
463  std::string dictName = sqliteConnector_.getData<string>(r, 1);
464 
465  std::string oldName =
466  basePath_ + "/mapd_data/" + currentDB_.dbName + "_" + dictName;
467  std::string newName = basePath_ + "/mapd_data/DB_" +
468  std::to_string(currentDB_.dbId) + "_DICT_" +
469  std::to_string(dictId);
470 
471  int result = rename(oldName.c_str(), newName.c_str());
472 
473  if (result == 0) {
474  LOG(INFO) << "Dictionary upgrade: successfully renamed " << oldName << " to "
475  << newName;
476  } else {
477  LOG(ERROR) << "Failed to rename old dictionary directory " << oldName << " to "
478  << newName + " dbname '" << currentDB_.dbName << "' error code "
479  << std::to_string(result);
480  }
481  }
482  // need to add new version info
483  string queryString("ALTER TABLE mapd_dictionaries ADD version_num BIGINT DEFAULT " +
484  std::to_string(DEFAULT_INITIAL_VERSION));
485  sqliteConnector_.query(queryString);
486  }
487  } catch (std::exception& e) {
488  sqliteConnector_.query("ROLLBACK TRANSACTION");
489  throw;
490  }
491  sqliteConnector_.query("END TRANSACTION");
492 }
493 
496  sqliteConnector_.query("BEGIN TRANSACTION");
497  try {
499  "CREATE TABLE IF NOT EXISTS mapd_logical_to_physical("
500  "logical_table_id integer, physical_table_id integer)");
501  } catch (const std::exception& e) {
502  sqliteConnector_.query("ROLLBACK TRANSACTION");
503  throw;
504  }
505  sqliteConnector_.query("END TRANSACTION");
506 }
507 
508 void Catalog::updateLogicalToPhysicalTableMap(const int32_t logical_tb_id) {
509  /* this proc inserts/updates all pairs of (logical_tb_id, physical_tb_id) in
510  * sqlite mapd_logical_to_physical table for given logical_tb_id as needed
511  */
512 
514  sqliteConnector_.query("BEGIN TRANSACTION");
515  try {
516  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
517  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
518  const auto physicalTables = physicalTableIt->second;
519  CHECK(!physicalTables.empty());
520  for (size_t i = 0; i < physicalTables.size(); i++) {
521  int32_t physical_tb_id = physicalTables[i];
523  "INSERT OR REPLACE INTO mapd_logical_to_physical (logical_table_id, "
524  "physical_table_id) VALUES (?1, ?2)",
525  std::vector<std::string>{std::to_string(logical_tb_id),
526  std::to_string(physical_tb_id)});
527  }
528  }
529  } catch (std::exception& e) {
530  sqliteConnector_.query("ROLLBACK TRANSACTION");
531  throw;
532  }
533  sqliteConnector_.query("END TRANSACTION");
534 }
535 
538  sqliteConnector_.query("BEGIN TRANSACTION");
539  try {
540  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_dictionaries)");
541  std::vector<std::string> cols;
542  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
543  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
544  }
545  if (std::find(cols.begin(), cols.end(), std::string("refcount")) == cols.end()) {
546  sqliteConnector_.query("ALTER TABLE mapd_dictionaries ADD refcount DEFAULT 1");
547  }
548  } catch (std::exception& e) {
549  sqliteConnector_.query("ROLLBACK TRANSACTION");
550  throw;
551  }
552  sqliteConnector_.query("END TRANSACTION");
553 }
554 
557  sqliteConnector_.query("BEGIN TRANSACTION");
558  std::vector<DBObject> objects;
559  try {
561  "SELECT name FROM sqlite_master WHERE type='table' AND "
562  "name='mapd_record_ownership_marker'");
563  // check if mapd catalog - marker exists
564  if (sqliteConnector_.getNumRows() != 0 && currentDB_.dbId == 1) {
565  // already done
566  sqliteConnector_.query("END TRANSACTION");
567  return;
568  }
569  // check if different catalog - marker exists
570  else if (sqliteConnector_.getNumRows() != 0 && currentDB_.dbId != 1) {
571  sqliteConnector_.query("SELECT dummy FROM mapd_record_ownership_marker");
572  // Check if migration is being performed on existing non mapd catalogs
573  // Older non mapd dbs will have table but no record in them
574  if (sqliteConnector_.getNumRows() != 0) {
575  // already done
576  sqliteConnector_.query("END TRANSACTION");
577  return;
578  }
579  }
580  // marker not exists - create one
581  else {
582  sqliteConnector_.query("CREATE TABLE mapd_record_ownership_marker (dummy integer)");
583  }
584 
585  DBMetadata db;
586  CHECK(SysCatalog::instance().getMetadataForDB(currentDB_.dbName, db));
587  // place dbId as a refernce for migration being performed
589  "INSERT INTO mapd_record_ownership_marker (dummy) VALUES (?1)",
590  std::vector<std::string>{std::to_string(db.dbOwner)});
591 
592  static const std::map<const DBObjectType, const AccessPrivileges>
593  object_level_all_privs_lookup{
598 
599  // grant owner all permissions on DB
600  DBObjectKey key;
601  key.dbId = currentDB_.dbId;
602  auto _key_place = [&key](auto type) {
603  key.permissionType = type;
604  return key;
605  };
606  for (auto& it : object_level_all_privs_lookup) {
607  objects.emplace_back(_key_place(it.first), it.second, db.dbOwner);
608  objects.back().setName(currentDB_.dbName);
609  }
610 
611  {
612  // other users tables and views
613  string tableQuery(
614  "SELECT tableid, name, userid, isview FROM mapd_tables WHERE userid > 0");
615  sqliteConnector_.query(tableQuery);
616  size_t numRows = sqliteConnector_.getNumRows();
617  for (size_t r = 0; r < numRows; ++r) {
618  int32_t tableid = sqliteConnector_.getData<int>(r, 0);
619  std::string tableName = sqliteConnector_.getData<string>(r, 1);
620  int32_t ownerid = sqliteConnector_.getData<int>(r, 2);
621  bool isview = sqliteConnector_.getData<bool>(r, 3);
622 
625  DBObjectKey key;
626  key.dbId = currentDB_.dbId;
627  key.objectId = tableid;
628  key.permissionType = type;
629 
630  DBObject obj(tableName, type);
631  obj.setObjectKey(key);
632  obj.setOwner(ownerid);
633  obj.setPrivileges(isview ? AccessPrivileges::ALL_VIEW
635 
636  objects.push_back(obj);
637  }
638  }
639 
640  {
641  // other users dashboards
642  string tableQuery("SELECT id, name, userid FROM mapd_dashboards WHERE userid > 0");
643  sqliteConnector_.query(tableQuery);
644  size_t numRows = sqliteConnector_.getNumRows();
645  for (size_t r = 0; r < numRows; ++r) {
646  int32_t dashId = sqliteConnector_.getData<int>(r, 0);
647  std::string dashName = sqliteConnector_.getData<string>(r, 1);
648  int32_t ownerid = sqliteConnector_.getData<int>(r, 2);
649 
651  DBObjectKey key;
652  key.dbId = currentDB_.dbId;
653  key.objectId = dashId;
654  key.permissionType = type;
655 
656  DBObject obj(dashName, type);
657  obj.setObjectKey(key);
658  obj.setOwner(ownerid);
660 
661  objects.push_back(obj);
662  }
663  }
664  } catch (const std::exception& e) {
665  sqliteConnector_.query("ROLLBACK TRANSACTION");
666  throw;
667  }
668  sqliteConnector_.query("END TRANSACTION");
669 
670  // now apply the objects to the syscat to track the permisisons
671  // moved outside transaction to avoid lock in sqlite
672  try {
674  } catch (const std::exception& e) {
675  LOG(ERROR) << " Issue during migration of DB " << name() << " issue was " << e.what();
676  throw std::runtime_error(" Issue during migration of DB " + name() + " issue was " +
677  e.what());
678  // will need to remove the mapd_record_ownership_marker table and retry
679  }
680 }
681 
684  std::vector<int> tables_migrated = {};
685  std::unordered_map<int, std::vector<std::string>> tables_to_migrate;
686  sqliteConnector_.query("BEGIN TRANSACTION");
687  try {
689  "select name from sqlite_master WHERE type='table' AND "
690  "name='mapd_version_history'");
691  if (sqliteConnector_.getNumRows() == 0) {
693  "CREATE TABLE mapd_version_history(version integer, migration_history text "
694  "unique)");
696  "CREATE TABLE mapd_date_in_days_column_migration_tmp(table_id integer primary "
697  "key)");
698  } else {
700  "select * from mapd_version_history where migration_history = "
701  "'date_in_days_column'");
702  if (sqliteConnector_.getNumRows() != 0) {
703  // no need for further execution
704  sqliteConnector_.query("END TRANSACTION");
705  return;
706  }
707  LOG(INFO) << "Performing Date in days columns migration.";
709  "select name from sqlite_master where type='table' AND "
710  "name='mapd_date_in_days_column_migration_tmp'");
711  if (sqliteConnector_.getNumRows() != 0) {
713  "select table_id from mapd_date_in_days_column_migration_tmp");
714  if (sqliteConnector_.getNumRows() != 0) {
715  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
716  tables_migrated.push_back(sqliteConnector_.getData<int>(i, 0));
717  }
718  }
719  } else {
721  "CREATE TABLE mapd_date_in_days_column_migration_tmp(table_id integer "
722  "primary key)");
723  }
724  }
726  "SELECT tables.tableid, tables.name, columns.name FROM mapd_tables tables, "
727  "mapd_columns columns where tables.tableid = columns.tableid AND "
728  "columns.coltype = ?1 AND columns.compression = ?2",
729  std::vector<std::string>{
730  std::to_string(static_cast<int>(SQLTypes::kDATE)),
732  if (sqliteConnector_.getNumRows() != 0) {
733  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
734  tables_to_migrate[sqliteConnector_.getData<int>(i, 0)] = {
735  sqliteConnector_.getData<std::string>(i, 1),
736  sqliteConnector_.getData<std::string>(i, 2)};
737  }
738  }
739  } catch (const std::exception& e) {
740  LOG(ERROR) << "Failed to complete migraion on date in days column: " << e.what();
741  sqliteConnector_.query("ROLLBACK");
742  throw;
743  }
744  sqliteConnector_.query("END TRANSACTION");
745 
746  for (auto& id_names : tables_to_migrate) {
747  if (std::find(tables_migrated.begin(), tables_migrated.end(), id_names.first) ==
748  tables_migrated.end()) {
749  sqliteConnector_.query("BEGIN TRANSACTION");
750  try {
751  LOG(INFO) << "Table: " << id_names.second[0]
752  << " may suffer from issues with DATE column: " << id_names.second[1]
753  << ". Running an OPTIMIZE command to solve any issues with metadata.";
754 
755  auto executor = Executor::getExecutor(getCurrentDB().dbId);
756  TableDescriptorMapById::iterator tableDescIt =
757  tableDescriptorMapById_.find(id_names.first);
758  if (tableDescIt == tableDescriptorMapById_.end()) {
759  throw runtime_error("Table descriptor does not exist for table " +
760  id_names.second[0] + " does not exist.");
761  }
762  auto td = tableDescIt->second;
763  TableOptimizer optimizer(td, executor.get(), *this);
764  optimizer.recomputeMetadata();
765 
767  "INSERT INTO mapd_date_in_days_column_migration_tmp VALUES(?)",
768  std::vector<std::string>{std::to_string(id_names.first)});
769  } catch (const std::exception& e) {
770  LOG(ERROR) << "Failed to complete migraion on date in days column: " << e.what();
771  sqliteConnector_.query("ROLLBACK");
772  throw;
773  }
774  sqliteConnector_.query("COMMIT");
775  }
776  }
777 
778  sqliteConnector_.query("BEGIN TRANSACTION");
779  try {
780  sqliteConnector_.query("DROP TABLE mapd_date_in_days_column_migration_tmp");
782  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
783  std::vector<std::string>{std::to_string(MAPD_VERSION), "date_in_days_column"});
784  } catch (const std::exception& e) {
785  LOG(ERROR) << "Failed to complete migraion on date in days column: " << e.what();
786  sqliteConnector_.query("ROLLBACK");
787  throw;
788  }
789  sqliteConnector_.query("END TRANSACTION");
790  LOG(INFO) << "Migration successfull on Date in days columns";
791 }
792 
794  std::unordered_map<std::string, std::pair<int, std::string>> dashboards;
795  std::vector<std::string> dashboard_ids;
796  static const std::string migration_name{"dashboard_roles_migration"};
797  {
799  sqliteConnector_.query("BEGIN TRANSACTION");
800  try {
801  // migration_history should be present in all catalogs by now
802  // if not then would be created before this migration
804  "select * from mapd_version_history where migration_history = '" +
805  migration_name + "'");
806  if (sqliteConnector_.getNumRows() != 0) {
807  // no need for further execution
808  sqliteConnector_.query("END TRANSACTION");
809  return;
810  }
811  LOG(INFO) << "Performing dashboard internal roles Migration.";
812  sqliteConnector_.query("select id, userid, metadata from mapd_dashboards");
813  for (size_t i = 0; i < sqliteConnector_.getNumRows(); ++i) {
816  sqliteConnector_.getData<string>(i, 0)))) {
817  // Successfully created roles during previous migration/crash
818  // No need to include them
819  continue;
820  }
821  dashboards[sqliteConnector_.getData<string>(i, 0)] = std::make_pair(
822  sqliteConnector_.getData<int>(i, 1), sqliteConnector_.getData<string>(i, 2));
823  dashboard_ids.push_back(sqliteConnector_.getData<string>(i, 0));
824  }
825  } catch (const std::exception& e) {
826  sqliteConnector_.query("ROLLBACK TRANSACTION");
827  throw;
828  }
829  sqliteConnector_.query("END TRANSACTION");
830  }
831  // All current grantees with shared dashboards.
832  const auto active_grantees =
834 
835  try {
836  // NOTE(wamsi): Transactionally unsafe
837  for (auto dash : dashboards) {
838  createOrUpdateDashboardSystemRole(dash.second.second,
839  dash.second.first,
841  std::to_string(currentDB_.dbId), dash.first));
842  auto result = active_grantees.find(dash.first);
843  if (result != active_grantees.end()) {
846  dash.first)},
847  result->second);
848  }
849  }
851  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
852  std::vector<std::string>{std::to_string(MAPD_VERSION), migration_name});
853  } catch (const std::exception& e) {
854  LOG(ERROR) << "Failed to create dashboard system roles during migration: "
855  << e.what();
856  throw;
857  }
858  LOG(INFO) << "Successfully created dashboard system roles during migration.";
859 }
860 
870  updatePageSize();
874 }
875 
879 }
880 
881 namespace {
882 std::string getUserFromId(const int32_t id) {
883  UserMetadata user;
884  if (SysCatalog::instance().getMetadataForUserById(id, user)) {
885  return user.userName;
886  }
887  // a user could be deleted and a dashboard still exist?
888  return "Unknown";
889 }
890 } // namespace
891 
895 
896  string dictQuery(
897  "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
898  sqliteConnector_.query(dictQuery);
899  size_t numRows = sqliteConnector_.getNumRows();
900  for (size_t r = 0; r < numRows; ++r) {
901  int dictId = sqliteConnector_.getData<int>(r, 0);
902  std::string dictName = sqliteConnector_.getData<string>(r, 1);
903  int dictNBits = sqliteConnector_.getData<int>(r, 2);
904  bool is_shared = sqliteConnector_.getData<bool>(r, 3);
905  int refcount = sqliteConnector_.getData<int>(r, 4);
906  std::string fname = basePath_ + "/mapd_data/DB_" + std::to_string(currentDB_.dbId) +
907  "_DICT_" + std::to_string(dictId);
908  DictRef dict_ref(currentDB_.dbId, dictId);
909  DictDescriptor* dd = new DictDescriptor(
910  dict_ref, dictName, dictNBits, is_shared, refcount, fname, false);
911  dictDescriptorMapByRef_[dict_ref].reset(dd);
912  }
913 
914  string tableQuery(
915  "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
916  "max_chunk_size, frag_page_size, "
917  "max_rows, partitions, shard_column_id, shard, num_shards, key_metainfo, userid, "
918  "sort_column_id "
919  "from mapd_tables");
920  sqliteConnector_.query(tableQuery);
921  numRows = sqliteConnector_.getNumRows();
922  for (size_t r = 0; r < numRows; ++r) {
923  TableDescriptor* td = new TableDescriptor();
924  td->tableId = sqliteConnector_.getData<int>(r, 0);
925  td->tableName = sqliteConnector_.getData<string>(r, 1);
926  td->nColumns = sqliteConnector_.getData<int>(r, 2);
927  td->isView = sqliteConnector_.getData<bool>(r, 3);
928  td->fragments = sqliteConnector_.getData<string>(r, 4);
929  td->fragType =
931  td->maxFragRows = sqliteConnector_.getData<int>(r, 6);
932  td->maxChunkSize = sqliteConnector_.getData<int>(r, 7);
933  td->fragPageSize = sqliteConnector_.getData<int>(r, 8);
934  td->maxRows = sqliteConnector_.getData<int64_t>(r, 9);
935  td->partitions = sqliteConnector_.getData<string>(r, 10);
936  td->shardedColumnId = sqliteConnector_.getData<int>(r, 11);
937  td->shard = sqliteConnector_.getData<int>(r, 12);
938  td->nShards = sqliteConnector_.getData<int>(r, 13);
939  td->keyMetainfo = sqliteConnector_.getData<string>(r, 14);
940  td->userId = sqliteConnector_.getData<int>(r, 15);
941  td->sortedColumnId =
942  sqliteConnector_.isNull(r, 16) ? 0 : sqliteConnector_.getData<int>(r, 16);
943  if (!td->isView) {
944  td->fragmenter = nullptr;
945  }
946  td->hasDeletedCol = false;
949  }
950  string columnQuery(
951  "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
952  "is_notnull, compression, comp_param, "
953  "size, chunks, is_systemcol, is_virtualcol, virtual_expr, is_deletedcol from "
954  "mapd_columns ORDER BY tableid, "
955  "columnid");
956  sqliteConnector_.query(columnQuery);
957  numRows = sqliteConnector_.getNumRows();
958  int32_t skip_physical_cols = 0;
959  for (size_t r = 0; r < numRows; ++r) {
961  cd->tableId = sqliteConnector_.getData<int>(r, 0);
962  cd->columnId = sqliteConnector_.getData<int>(r, 1);
963  cd->columnName = sqliteConnector_.getData<string>(r, 2);
971  cd->columnType.set_size(sqliteConnector_.getData<int>(r, 10));
972  cd->chunks = sqliteConnector_.getData<string>(r, 11);
973  cd->isSystemCol = sqliteConnector_.getData<bool>(r, 12);
974  cd->isVirtualCol = sqliteConnector_.getData<bool>(r, 13);
975  cd->virtualExpr = sqliteConnector_.getData<string>(r, 14);
976  cd->isDeletedCol = sqliteConnector_.getData<bool>(r, 15);
977  cd->isGeoPhyCol = skip_physical_cols > 0;
978  ColumnKey columnKey(cd->tableId, to_upper(cd->columnName));
979  columnDescriptorMap_[columnKey] = cd;
980  ColumnIdKey columnIdKey(cd->tableId, cd->columnId);
981  columnDescriptorMapById_[columnIdKey] = cd;
982 
983  if (skip_physical_cols <= 0) {
984  skip_physical_cols = cd->columnType.get_physical_cols();
985  }
986 
987  auto td_itr = tableDescriptorMapById_.find(cd->tableId);
988  CHECK(td_itr != tableDescriptorMapById_.end());
989 
990  if (cd->isDeletedCol) {
991  td_itr->second->hasDeletedCol = true;
992  setDeletedColumnUnlocked(td_itr->second, cd);
993  } else if (cd->columnType.is_geometry() || skip_physical_cols-- <= 0) {
994  tableDescriptorMapById_[cd->tableId]->columnIdBySpi_.push_back(cd->columnId);
995  }
996  }
997  // sort columnIdBySpi_ based on columnId
998  for (auto& tit : tableDescriptorMapById_) {
999  std::sort(tit.second->columnIdBySpi_.begin(),
1000  tit.second->columnIdBySpi_.end(),
1001  [](const size_t a, const size_t b) -> bool { return a < b; });
1002  }
1003 
1004  string viewQuery("SELECT tableid, sql FROM mapd_views");
1005  sqliteConnector_.query(viewQuery);
1006  numRows = sqliteConnector_.getNumRows();
1007  for (size_t r = 0; r < numRows; ++r) {
1008  int32_t tableId = sqliteConnector_.getData<int>(r, 0);
1009  TableDescriptor* td = tableDescriptorMapById_[tableId];
1010  td->viewSQL = sqliteConnector_.getData<string>(r, 1);
1011  td->fragmenter = nullptr;
1012  }
1013 
1014  string frontendViewQuery(
1015  "SELECT id, state, name, image_hash, strftime('%Y-%m-%dT%H:%M:%SZ', update_time), "
1016  "userid, "
1017  "metadata "
1018  "FROM mapd_dashboards");
1019  sqliteConnector_.query(frontendViewQuery);
1020  numRows = sqliteConnector_.getNumRows();
1021  for (size_t r = 0; r < numRows; ++r) {
1022  std::shared_ptr<DashboardDescriptor> vd = std::make_shared<DashboardDescriptor>();
1023  vd->dashboardId = sqliteConnector_.getData<int>(r, 0);
1024  vd->dashboardState = sqliteConnector_.getData<string>(r, 1);
1025  vd->dashboardName = sqliteConnector_.getData<string>(r, 2);
1026  vd->imageHash = sqliteConnector_.getData<string>(r, 3);
1027  vd->updateTime = sqliteConnector_.getData<string>(r, 4);
1028  vd->userId = sqliteConnector_.getData<int>(r, 5);
1029  vd->dashboardMetadata = sqliteConnector_.getData<string>(r, 6);
1030  vd->user = getUserFromId(vd->userId);
1031  vd->dashboardSystemRoleName = generate_dashboard_system_rolename(
1033  dashboardDescriptorMap_[std::to_string(vd->userId) + ":" + vd->dashboardName] = vd;
1034  }
1035 
1036  string linkQuery(
1037  "SELECT linkid, userid, link, view_state, strftime('%Y-%m-%dT%H:%M:%SZ', "
1038  "update_time), view_metadata "
1039  "FROM mapd_links");
1040  sqliteConnector_.query(linkQuery);
1041  numRows = sqliteConnector_.getNumRows();
1042  for (size_t r = 0; r < numRows; ++r) {
1043  LinkDescriptor* ld = new LinkDescriptor();
1044  ld->linkId = sqliteConnector_.getData<int>(r, 0);
1045  ld->userId = sqliteConnector_.getData<int>(r, 1);
1046  ld->link = sqliteConnector_.getData<string>(r, 2);
1047  ld->viewState = sqliteConnector_.getData<string>(r, 3);
1048  ld->updateTime = sqliteConnector_.getData<string>(r, 4);
1049  ld->viewMetadata = sqliteConnector_.getData<string>(r, 5);
1051  linkDescriptorMapById_[ld->linkId] = ld;
1052  }
1053 
1054  /* rebuild map linking logical tables to corresponding physical ones */
1055  string logicalToPhysicalTableMapQuery(
1056  "SELECT logical_table_id, physical_table_id "
1057  "FROM mapd_logical_to_physical");
1058  sqliteConnector_.query(logicalToPhysicalTableMapQuery);
1059  numRows = sqliteConnector_.getNumRows();
1060  for (size_t r = 0; r < numRows; ++r) {
1061  int32_t logical_tb_id = sqliteConnector_.getData<int>(r, 0);
1062  int32_t physical_tb_id = sqliteConnector_.getData<int>(r, 1);
1063  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
1064  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
1065  /* add new entity to the map logicalToPhysicalTableMapById_ */
1066  std::vector<int32_t> physicalTables;
1067  physicalTables.push_back(physical_tb_id);
1068  const auto it_ok =
1069  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
1070  CHECK(it_ok.second);
1071  } else {
1072  /* update map logicalToPhysicalTableMapById_ */
1073  physicalTableIt->second.push_back(physical_tb_id);
1074  }
1075  }
1076 }
1077 
1079  const list<ColumnDescriptor>& columns,
1080  const list<DictDescriptor>& dicts) {
1081  cat_write_lock write_lock(this);
1082  TableDescriptor* new_td = new TableDescriptor();
1083  *new_td = td;
1084  new_td->mutex_ = std::make_shared<std::mutex>();
1085  tableDescriptorMap_[to_upper(td.tableName)] = new_td;
1086  tableDescriptorMapById_[td.tableId] = new_td;
1087  for (auto cd : columns) {
1088  ColumnDescriptor* new_cd = new ColumnDescriptor();
1089  *new_cd = cd;
1090  ColumnKey columnKey(new_cd->tableId, to_upper(new_cd->columnName));
1091  columnDescriptorMap_[columnKey] = new_cd;
1092  ColumnIdKey columnIdKey(new_cd->tableId, new_cd->columnId);
1093  columnDescriptorMapById_[columnIdKey] = new_cd;
1094 
1095  // Add deleted column to the map
1096  if (cd.isDeletedCol) {
1097  CHECK(new_td->hasDeletedCol);
1098  setDeletedColumnUnlocked(new_td, new_cd);
1099  }
1100  }
1101 
1102  std::sort(new_td->columnIdBySpi_.begin(),
1103  new_td->columnIdBySpi_.end(),
1104  [](const size_t a, const size_t b) -> bool { return a < b; });
1105 
1106  std::unique_ptr<StringDictionaryClient> client;
1107  DictRef dict_ref(currentDB_.dbId, -1);
1108  if (!string_dict_hosts_.empty()) {
1109  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1110  }
1111  for (auto dd : dicts) {
1112  if (!dd.dictRef.dictId) {
1113  // Dummy entry created for a shard of a logical table, nothing to do.
1114  continue;
1115  }
1116  dict_ref.dictId = dd.dictRef.dictId;
1117  if (client) {
1118  client->create(dict_ref, dd.dictIsTemp);
1119  }
1120  DictDescriptor* new_dd = new DictDescriptor(dd);
1121  dictDescriptorMapByRef_[dict_ref].reset(new_dd);
1122  if (!dd.dictIsTemp) {
1123  boost::filesystem::create_directory(new_dd->dictFolderPath);
1124  }
1125  }
1126 }
1127 
1128 void Catalog::removeTableFromMap(const string& tableName,
1129  const int tableId,
1130  const bool is_on_error) {
1131  cat_write_lock write_lock(this);
1132  TableDescriptorMapById::iterator tableDescIt = tableDescriptorMapById_.find(tableId);
1133  if (tableDescIt == tableDescriptorMapById_.end()) {
1134  throw runtime_error("Table " + tableName + " does not exist.");
1135  }
1136 
1137  TableDescriptor* td = tableDescIt->second;
1138 
1139  if (td->hasDeletedCol) {
1140  const auto ret = deletedColumnPerTable_.erase(td);
1141  CHECK_EQ(ret, size_t(1));
1142  }
1143 
1144  tableDescriptorMapById_.erase(tableDescIt);
1145  tableDescriptorMap_.erase(to_upper(tableName));
1146  if (td->fragmenter != nullptr) {
1147  delete td->fragmenter;
1148  }
1150  delete td;
1151 
1152  std::unique_ptr<StringDictionaryClient> client;
1153  if (SysCatalog::instance().isAggregator()) {
1154  CHECK(!string_dict_hosts_.empty());
1155  DictRef dict_ref(currentDB_.dbId, -1);
1156  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1157  }
1158 
1159  // delete all column descriptors for the table
1160  // no more link columnIds to sequential indexes!
1161  for (auto cit = columnDescriptorMapById_.begin();
1162  cit != columnDescriptorMapById_.end();) {
1163  if (tableId != std::get<0>(cit->first)) {
1164  ++cit;
1165  } else {
1166  int i = std::get<1>(cit++->first);
1167  ColumnIdKey cidKey(tableId, i);
1168  ColumnDescriptorMapById::iterator colDescIt = columnDescriptorMapById_.find(cidKey);
1169  ColumnDescriptor* cd = colDescIt->second;
1170  columnDescriptorMapById_.erase(colDescIt);
1171  ColumnKey cnameKey(tableId, to_upper(cd->columnName));
1172  columnDescriptorMap_.erase(cnameKey);
1173  const int dictId = cd->columnType.get_comp_param();
1174  // Dummy dictionaries created for a shard of a logical table have the id set to
1175  // zero.
1176  if (cd->columnType.get_compression() == kENCODING_DICT && dictId) {
1177  INJECT_TIMER(removingDicts);
1178  DictRef dict_ref(currentDB_.dbId, dictId);
1179  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
1180  // If we're removing this table due to an error, it is possible that the string
1181  // dictionary reference was never populated. Don't crash, just continue cleaning
1182  // up the TableDescriptor and ColumnDescriptors
1183  if (!is_on_error) {
1184  CHECK(dictIt != dictDescriptorMapByRef_.end());
1185  } else {
1186  if (dictIt == dictDescriptorMapByRef_.end()) {
1187  continue;
1188  }
1189  }
1190  const auto& dd = dictIt->second;
1191  CHECK_GE(dd->refcount, 1);
1192  --dd->refcount;
1193  if (!dd->refcount) {
1194  dd->stringDict.reset();
1195  if (!isTemp) {
1196  File_Namespace::renameForDelete(dd->dictFolderPath);
1197  }
1198  if (client) {
1199  client->drop(dict_ref);
1200  }
1201  dictDescriptorMapByRef_.erase(dictIt);
1202  }
1203  }
1204 
1205  delete cd;
1206  }
1207  }
1208 }
1209 
1211  cat_write_lock write_lock(this);
1213 }
1214 
1216  cat_write_lock write_lock(this);
1218  std::make_shared<DashboardDescriptor>(vd);
1219 }
1220 
1221 std::vector<DBObject> Catalog::parseDashboardObjects(const std::string& view_meta,
1222  const int& user_id) {
1223  std::vector<DBObject> objects;
1224  DBObjectKey key;
1225  key.dbId = currentDB_.dbId;
1226  auto _key_place = [&key](auto type, auto id) {
1227  key.permissionType = type;
1228  key.objectId = id;
1229  return key;
1230  };
1231  for (auto object_name : parse_underlying_dashboard_objects(view_meta)) {
1232  auto td = getMetadataForTable(object_name);
1233  if (!td) {
1234  // Parsed object source is not present in current database
1235  // LOG the info and ignore
1236  LOG(INFO) << "Ignoring dashboard source Table/View: " << object_name
1237  << " no longer exists in current DB.";
1238  continue;
1239  }
1240  // Dashboard source can be Table or View
1241  const auto object_type = td->isView ? ViewDBObjectType : TableDBObjectType;
1242  const auto priv = td->isView ? AccessPrivileges::SELECT_FROM_VIEW
1244  objects.emplace_back(_key_place(object_type, td->tableId), priv, user_id);
1245  objects.back().setObjectType(td->isView ? ViewDBObjectType : TableDBObjectType);
1246  objects.back().setName(td->tableName);
1247  }
1248  return objects;
1249 }
1250 
1251 void Catalog::createOrUpdateDashboardSystemRole(const std::string& view_meta,
1252  const int32_t& user_id,
1253  const std::string& dash_role_name) {
1254  auto objects = parseDashboardObjects(view_meta, user_id);
1255  Role* rl = SysCatalog::instance().getRoleGrantee(dash_role_name);
1256  if (!rl) {
1257  // Dashboard role does not exist
1258  // create role and grant privileges
1259  // NOTE(wamsi): Transactionally unsafe
1260  SysCatalog::instance().createRole(dash_role_name, false);
1261  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1262  } else {
1263  // Dashboard system role already exists
1264  // Add/remove privileges on objects
1265  auto ex_objects = rl->getDbObjects(true);
1266  for (auto key : *ex_objects | boost::adaptors::map_keys) {
1267  if (key.permissionType != TableDBObjectType &&
1268  key.permissionType != ViewDBObjectType) {
1269  continue;
1270  }
1271  bool found = false;
1272  for (auto obj : objects) {
1273  found = key == obj.getObjectKey() ? true : false;
1274  if (found) {
1275  break;
1276  }
1277  }
1278  if (!found) {
1279  // revoke privs on object since the object is no
1280  // longer used by the dashboard as source
1281  // NOTE(wamsi): Transactionally unsafe
1283  dash_role_name, *rl->findDbObject(key, true), *this);
1284  }
1285  }
1286  // Update privileges on remaining objects
1287  // NOTE(wamsi): Transactionally unsafe
1288  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1289  }
1290 }
1291 
1293  cat_write_lock write_lock(this);
1294  LinkDescriptor* new_ld = new LinkDescriptor();
1295  *new_ld = ld;
1297  linkDescriptorMapById_[ld.linkId] = new_ld;
1298 }
1299 
1301  auto time_ms = measure<>::execution([&]() {
1302  // instanciate table fragmenter upon first use
1303  // assume only insert order fragmenter is supported
1305  vector<Chunk> chunkVec;
1306  list<const ColumnDescriptor*> columnDescs;
1307  getAllColumnMetadataForTable(td, columnDescs, true, false, true);
1308  Chunk::translateColumnDescriptorsToChunkVec(columnDescs, chunkVec);
1309  ChunkKey chunkKeyPrefix = {currentDB_.dbId, td->tableId};
1310  if (td->sortedColumnId > 0) {
1311  td->fragmenter = new SortedOrderFragmenter(chunkKeyPrefix,
1312  chunkVec,
1313  dataMgr_.get(),
1314  const_cast<Catalog*>(this),
1315  td->tableId,
1316  td->shard,
1317  td->maxFragRows,
1318  td->maxChunkSize,
1319  td->fragPageSize,
1320  td->maxRows,
1321  td->persistenceLevel);
1322  } else {
1323  td->fragmenter = new InsertOrderFragmenter(chunkKeyPrefix,
1324  chunkVec,
1325  dataMgr_.get(),
1326  const_cast<Catalog*>(this),
1327  td->tableId,
1328  td->shard,
1329  td->maxFragRows,
1330  td->maxChunkSize,
1331  td->fragPageSize,
1332  td->maxRows,
1333  td->persistenceLevel);
1334  }
1335  });
1336  LOG(INFO) << "Instantiating Fragmenter for table " << td->tableName << " took "
1337  << time_ms << "ms";
1338 }
1339 
1340 const TableDescriptor* Catalog::getMetadataForTable(const string& tableName,
1341  const bool populateFragmenter) const {
1342  // we give option not to populate fragmenter (default true/yes) as it can be heavy for
1343  // pure metadata calls
1344  cat_read_lock read_lock(this);
1345  auto tableDescIt = tableDescriptorMap_.find(to_upper(tableName));
1346  if (tableDescIt == tableDescriptorMap_.end()) { // check to make sure table exists
1347  return nullptr;
1348  }
1349  TableDescriptor* td = tableDescIt->second;
1350  std::unique_lock<std::mutex> td_lock(*td->mutex_.get());
1351  if (populateFragmenter && td->fragmenter == nullptr && !td->isView) {
1353  }
1354  return td; // returns pointer to table descriptor
1355 }
1356 
1358  int tableId,
1359  const bool populateFragmenter) const {
1360  cat_read_lock read_lock(this);
1361  auto tableDescIt = tableDescriptorMapById_.find(tableId);
1362  if (tableDescIt == tableDescriptorMapById_.end()) { // check to make sure table exists
1363  return nullptr;
1364  }
1365  TableDescriptor* td = tableDescIt->second;
1366  std::unique_lock<std::mutex> td_lock(*td->mutex_.get());
1367  if (populateFragmenter && td->fragmenter == nullptr && !td->isView) {
1369  }
1370  return td; // returns pointer to table descriptor
1371 }
1372 
1374  return getMetadataForTableImpl(tableId, true);
1375 }
1376 
1378  const bool loadDict) const {
1379  const DictRef dictRef(currentDB_.dbId, dictId);
1380  cat_read_lock read_lock(this);
1381  auto dictDescIt = dictDescriptorMapByRef_.find(dictRef);
1382  if (dictDescIt ==
1383  dictDescriptorMapByRef_.end()) { // check to make sure dictionary exists
1384  return nullptr;
1385  }
1386  auto& dd = dictDescIt->second;
1387 
1388  if (loadDict) {
1390  if (!dd->stringDict) {
1391  auto time_ms = measure<>::execution([&]() {
1392  if (string_dict_hosts_.empty()) {
1393  if (dd->dictIsTemp) {
1394  dd->stringDict = std::make_shared<StringDictionary>(
1395  dd->dictFolderPath, true, true, g_cache_string_hash);
1396  } else {
1397  dd->stringDict = std::make_shared<StringDictionary>(
1398  dd->dictFolderPath, false, true, g_cache_string_hash);
1399  }
1400  } else {
1401  dd->stringDict =
1402  std::make_shared<StringDictionary>(string_dict_hosts_.front(), dd->dictRef);
1403  }
1404  });
1405  LOG(INFO) << "Time to load Dictionary " << dd->dictRef.dbId << "_"
1406  << dd->dictRef.dictId << " was " << time_ms << "ms";
1407  }
1408  }
1409 
1410  return dd.get();
1411 }
1412 
1413 const std::vector<LeafHostInfo>& Catalog::getStringDictionaryHosts() const {
1414  return string_dict_hosts_;
1415 }
1416 
1418  const string& columnName) const {
1419  cat_read_lock read_lock(this);
1420 
1421  ColumnKey columnKey(tableId, to_upper(columnName));
1422  auto colDescIt = columnDescriptorMap_.find(columnKey);
1423  if (colDescIt ==
1424  columnDescriptorMap_.end()) { // need to check to make sure column exists for table
1425  return nullptr;
1426  }
1427  return colDescIt->second;
1428 }
1429 
1430 const ColumnDescriptor* Catalog::getMetadataForColumn(int tableId, int columnId) const {
1431  cat_read_lock read_lock(this);
1432 
1433  ColumnIdKey columnIdKey(tableId, columnId);
1434  auto colDescIt = columnDescriptorMapById_.find(columnIdKey);
1435  if (colDescIt == columnDescriptorMapById_
1436  .end()) { // need to check to make sure column exists for table
1437  return nullptr;
1438  }
1439  return colDescIt->second;
1440 }
1441 
1442 const int Catalog::getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const {
1443  const auto tabDescIt = tableDescriptorMapById_.find(table_id);
1444  CHECK(tableDescriptorMapById_.end() != tabDescIt);
1445  const auto& columnIdBySpi = tabDescIt->second->columnIdBySpi_;
1446 
1447  auto spx = spi;
1448  int phi = 0;
1449  if (spx >= SPIMAP_MAGIC1) // see Catalog.h
1450  {
1451  phi = (spx - SPIMAP_MAGIC1) % SPIMAP_MAGIC2;
1452  spx = (spx - SPIMAP_MAGIC1) / SPIMAP_MAGIC2;
1453  }
1454 
1455  CHECK(0 < spx && spx <= columnIdBySpi.size());
1456  return columnIdBySpi[spx - 1] + phi;
1457 }
1458 
1459 const int Catalog::getColumnIdBySpi(const int table_id, const size_t spi) const {
1460  cat_read_lock read_lock(this);
1461  return getColumnIdBySpiUnlocked(table_id, spi);
1462 }
1463 
1465  const size_t spi) const {
1466  cat_read_lock read_lock(this);
1467 
1468  const auto columnId = getColumnIdBySpiUnlocked(tableId, spi);
1469  ColumnIdKey columnIdKey(tableId, columnId);
1470  const auto colDescIt = columnDescriptorMapById_.find(columnIdKey);
1471  return columnDescriptorMapById_.end() == colDescIt ? nullptr : colDescIt->second;
1472 }
1473 
1474 void Catalog::deleteMetadataForDashboard(const std::string& userId,
1475  const std::string& dashName) {
1476  cat_write_lock write_lock(this);
1477 
1478  auto viewDescIt = dashboardDescriptorMap_.find(userId + ":" + dashName);
1479  if (viewDescIt == dashboardDescriptorMap_.end()) { // check to make sure view exists
1480  LOG(ERROR) << "No metadata for dashboard for user " << userId << " dashboard "
1481  << dashName << " does not exist in map";
1482  throw runtime_error("No metadata for dashboard for user " + userId + " dashboard " +
1483  dashName + " does not exist in map");
1484  }
1485  // found view in Map now remove it
1486  dashboardDescriptorMap_.erase(viewDescIt);
1487  // remove from DB
1489  sqliteConnector_.query("BEGIN TRANSACTION");
1490  try {
1492  "DELETE FROM mapd_dashboards WHERE name = ? and userid = ?",
1493  std::vector<std::string>{dashName, userId});
1494  } catch (std::exception& e) {
1495  sqliteConnector_.query("ROLLBACK TRANSACTION");
1496  throw;
1497  }
1498  sqliteConnector_.query("END TRANSACTION");
1499 }
1500 
1502  const string& userId,
1503  const string& dashName) const {
1504  cat_read_lock read_lock(this);
1505 
1506  auto viewDescIt = dashboardDescriptorMap_.find(userId + ":" + dashName);
1507  if (viewDescIt == dashboardDescriptorMap_.end()) { // check to make sure view exists
1508  return nullptr;
1509  }
1510  return viewDescIt->second.get(); // returns pointer to view descriptor
1511 }
1512 
1514  cat_read_lock read_lock(this);
1515  std::string userId;
1516  std::string name;
1517  bool found{false};
1518  {
1519  for (auto descp : dashboardDescriptorMap_) {
1520  auto dash = descp.second.get();
1521  if (dash->dashboardId == id) {
1522  userId = std::to_string(dash->userId);
1523  name = dash->dashboardName;
1524  found = true;
1525  break;
1526  }
1527  }
1528  }
1529  if (found) {
1530  return getMetadataForDashboard(userId, name);
1531  }
1532  return nullptr;
1533 }
1534 
1535 void Catalog::deleteMetadataForDashboard(const int32_t id) {
1536  std::string userId;
1537  std::string name;
1538  bool found{false};
1539  {
1540  cat_read_lock read_lock(this);
1541  for (auto descp : dashboardDescriptorMap_) {
1542  auto dash = descp.second.get();
1543  if (dash->dashboardId == id) {
1544  userId = std::to_string(dash->userId);
1545  name = dash->dashboardName;
1546  found = true;
1547  break;
1548  }
1549  }
1550  }
1551  if (found) {
1552  // TODO: transactionally unsafe
1554  DBObject(id, DashboardDBObjectType), this);
1555  deleteMetadataForDashboard(userId, name);
1556  }
1557 }
1558 
1559 const LinkDescriptor* Catalog::getMetadataForLink(const string& link) const {
1560  cat_read_lock read_lock(this);
1561  auto linkDescIt = linkDescriptorMap_.find(link);
1562  if (linkDescIt == linkDescriptorMap_.end()) { // check to make sure view exists
1563  return nullptr;
1564  }
1565  return linkDescIt->second; // returns pointer to view descriptor
1566 }
1567 
1569  cat_read_lock read_lock(this);
1570  auto linkDescIt = linkDescriptorMapById_.find(linkId);
1571  if (linkDescIt == linkDescriptorMapById_.end()) { // check to make sure view exists
1572  return nullptr;
1573  }
1574  return linkDescIt->second;
1575 }
1576 
1578  const TableDescriptor* td,
1579  list<const ColumnDescriptor*>& columnDescriptors,
1580  const bool fetchSystemColumns,
1581  const bool fetchVirtualColumns,
1582  const bool fetchPhysicalColumns) const {
1583  cat_read_lock read_lock(this);
1584  int32_t skip_physical_cols = 0;
1585  for (const auto& columnDescriptor : columnDescriptorMapById_) {
1586  if (!fetchPhysicalColumns && skip_physical_cols > 0) {
1587  --skip_physical_cols;
1588  continue;
1589  }
1590  auto cd = columnDescriptor.second;
1591  if (cd->tableId != td->tableId) {
1592  continue;
1593  }
1594  if (!fetchSystemColumns && cd->isSystemCol) {
1595  continue;
1596  }
1597  if (!fetchVirtualColumns && cd->isVirtualCol) {
1598  continue;
1599  }
1600  if (!fetchPhysicalColumns) {
1601  const auto& col_ti = cd->columnType;
1602  skip_physical_cols = col_ti.get_physical_cols();
1603  }
1604  columnDescriptors.push_back(cd);
1605  }
1606 }
1607 
1608 list<const ColumnDescriptor*> Catalog::getAllColumnMetadataForTable(
1609  const int tableId,
1610  const bool fetchSystemColumns,
1611  const bool fetchVirtualColumns,
1612  const bool fetchPhysicalColumns) const {
1613  cat_read_lock read_lock(this);
1614  list<const ColumnDescriptor*> columnDescriptors;
1615  const TableDescriptor* td =
1616  getMetadataForTableImpl(tableId, false); // dont instantiate fragmenter
1618  columnDescriptors,
1619  fetchSystemColumns,
1620  fetchVirtualColumns,
1621  fetchPhysicalColumns);
1622  return columnDescriptors;
1623 }
1624 
1625 list<const TableDescriptor*> Catalog::getAllTableMetadata() const {
1626  cat_read_lock read_lock(this);
1627  list<const TableDescriptor*> table_list;
1628  for (auto p : tableDescriptorMapById_) {
1629  table_list.push_back(p.second);
1630  }
1631  return table_list;
1632 }
1633 
1634 list<const DashboardDescriptor*> Catalog::getAllDashboardsMetadata() const {
1635  list<const DashboardDescriptor*> view_list;
1636  for (auto p : dashboardDescriptorMap_) {
1637  view_list.push_back(p.second.get());
1638  }
1639  return view_list;
1640 }
1641 
1643  const auto& td = *tableDescriptorMapById_[cd.tableId];
1644  list<DictDescriptor> dds;
1645  setColumnDictionary(cd, dds, td, true);
1646  auto& dd = dds.back();
1647  CHECK(dd.dictRef.dictId);
1648 
1649  std::unique_ptr<StringDictionaryClient> client;
1650  if (!string_dict_hosts_.empty()) {
1651  client.reset(new StringDictionaryClient(
1652  string_dict_hosts_.front(), DictRef(currentDB_.dbId, -1), true));
1653  }
1654  if (client) {
1655  client->create(dd.dictRef, dd.dictIsTemp);
1656  }
1657 
1658  DictDescriptor* new_dd = new DictDescriptor(dd);
1659  dictDescriptorMapByRef_[dd.dictRef].reset(new_dd);
1660  if (!dd.dictIsTemp) {
1661  boost::filesystem::create_directory(new_dd->dictFolderPath);
1662  }
1663  return dd.dictRef;
1664 }
1665 
1666 // TODO this all looks incorrect there is no reference count being checked here
1667 // this code needs to be fixed
1669  if (!(cd.columnType.is_string() || cd.columnType.is_string_array())) {
1670  return;
1671  }
1672  if (!(cd.columnType.get_compression() == kENCODING_DICT)) {
1673  return;
1674  }
1675  if (!(cd.columnType.get_comp_param() > 0)) {
1676  return;
1677  }
1678 
1679  const auto& td = *tableDescriptorMapById_[cd.tableId];
1680  const auto dictId = cd.columnType.get_comp_param();
1681  const DictRef dictRef(currentDB_.dbId, dictId);
1682  const auto dictName =
1683  td.tableName + "_" + cd.columnName + "_dict" + std::to_string(dictId);
1684  sqliteConnector_.query_with_text_param("DELETE FROM mapd_dictionaries WHERE name = ?",
1685  dictName);
1686  File_Namespace::renameForDelete(basePath_ + "/mapd_data/DB_" +
1687  std::to_string(currentDB_.dbId) + "_DICT_" +
1688  std::to_string(dictId));
1689 
1690  std::unique_ptr<StringDictionaryClient> client;
1691  if (!string_dict_hosts_.empty()) {
1692  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dictRef, true));
1693  }
1694  if (client) {
1695  client->drop(dictRef);
1696  }
1697 
1698  dictDescriptorMapByRef_.erase(dictRef);
1699 }
1700 
1702  std::map<int, StringDictionary*>& stringDicts) {
1703  // learn 'committed' ColumnDescriptor of this column
1704  auto cit = columnDescriptorMap_.find(ColumnKey(cd.tableId, to_upper(cd.columnName)));
1705  CHECK(cit != columnDescriptorMap_.end());
1706  auto& ccd = *cit->second;
1707 
1708  if (!(ccd.columnType.is_string() || ccd.columnType.is_string_array())) {
1709  return;
1710  }
1711  if (!(ccd.columnType.get_compression() == kENCODING_DICT)) {
1712  return;
1713  }
1714  if (!(ccd.columnType.get_comp_param() > 0)) {
1715  return;
1716  }
1717 
1718  auto dictId = ccd.columnType.get_comp_param();
1719  getMetadataForDict(dictId);
1720 
1721  const DictRef dictRef(currentDB_.dbId, dictId);
1722  auto dit = dictDescriptorMapByRef_.find(dictRef);
1723  CHECK(dit != dictDescriptorMapByRef_.end());
1724  CHECK(dit->second);
1725  CHECK(dit->second.get()->stringDict);
1726  stringDicts[ccd.columnId] = dit->second.get()->stringDict.get();
1727 }
1728 
1730  // caller must handle sqlite/chunk transaction TOGETHER
1731  cd.tableId = td.tableId;
1733  addDictionary(cd);
1734  }
1735 
1737  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
1738  "colscale, is_notnull, "
1739  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
1740  "is_deletedcol) "
1741  "VALUES (?, "
1742  "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
1743  "?, ?, ?, "
1744  "?, "
1745  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1746  std::vector<std::string>{std::to_string(td.tableId),
1747  std::to_string(td.tableId),
1748  cd.columnName,
1757  "",
1760  cd.virtualExpr,
1762 
1764  "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
1765  std::vector<std::string>{std::to_string(td.tableId)});
1766 
1768  "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
1769  std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
1770  cd.columnId = sqliteConnector_.getData<int>(0, 0);
1771 
1772  ++tableDescriptorMapById_[td.tableId]->nColumns;
1773  auto ncd = new ColumnDescriptor(cd);
1776  columnDescriptorsForRoll.emplace_back(nullptr, ncd);
1777 }
1778 
1779 void Catalog::roll(const bool forward) {
1780  std::set<const TableDescriptor*> tds;
1781 
1782  for (const auto& cdr : columnDescriptorsForRoll) {
1783  auto ocd = cdr.first;
1784  auto ncd = cdr.second;
1785  CHECK(ocd || ncd);
1786  auto tabDescIt = tableDescriptorMapById_.find((ncd ? ncd : ocd)->tableId);
1787  CHECK(tableDescriptorMapById_.end() != tabDescIt);
1788  auto td = tabDescIt->second;
1789  auto& vc = td->columnIdBySpi_;
1790  if (forward) {
1791  if (ocd) {
1792  if (nullptr == ncd ||
1793  ncd->columnType.get_comp_param() != ocd->columnType.get_comp_param()) {
1794  delDictionary(*ocd);
1795  }
1796 
1797  vc.erase(std::remove(vc.begin(), vc.end(), ocd->columnId), vc.end());
1798 
1799  delete ocd;
1800  }
1801  if (ncd) {
1802  // append columnId if its new and not phy geo
1803  if (vc.end() == std::find(vc.begin(), vc.end(), ncd->columnId)) {
1804  if (!ncd->isGeoPhyCol) {
1805  vc.push_back(ncd->columnId);
1806  }
1807  }
1808  }
1809  tds.insert(td);
1810  } else {
1811  if (ocd) {
1812  columnDescriptorMap_[ColumnKey(ocd->tableId, to_upper(ocd->columnName))] = ocd;
1813  columnDescriptorMapById_[ColumnIdKey(ocd->tableId, ocd->columnId)] = ocd;
1814  }
1815  // roll back the dict of new column
1816  if (ncd) {
1817  columnDescriptorMap_.erase(ColumnKey(ncd->tableId, to_upper(ncd->columnName)));
1818  columnDescriptorMapById_.erase(ColumnIdKey(ncd->tableId, ncd->columnId));
1819  if (nullptr == ocd ||
1820  ocd->columnType.get_comp_param() != ncd->columnType.get_comp_param()) {
1821  delDictionary(*ncd);
1822  }
1823  delete ncd;
1824  }
1825  }
1826  }
1827  columnDescriptorsForRoll.clear();
1828 
1829  if (forward) {
1830  for (const auto td : tds) {
1831  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
1832  }
1833  }
1834 }
1835 
1837  list<ColumnDescriptor>& columns) {
1838  const auto& col_ti = cd.columnType;
1839  if (IS_GEO(col_ti.get_type())) {
1840  switch (col_ti.get_type()) {
1841  case kPOINT: {
1842  ColumnDescriptor physical_cd_coords(true);
1843  physical_cd_coords.columnName = cd.columnName + "_coords";
1844  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, true);
1845  // Raw data: compressed/uncompressed coords
1846  coords_ti.set_subtype(kTINYINT);
1847  size_t unit_size;
1848  if (col_ti.get_compression() == kENCODING_GEOINT &&
1849  col_ti.get_comp_param() == 32) {
1850  unit_size = 4 * sizeof(int8_t);
1851  } else {
1852  CHECK(col_ti.get_compression() == kENCODING_NONE);
1853  unit_size = 8 * sizeof(int8_t);
1854  }
1855  coords_ti.set_size(2 * unit_size);
1856  physical_cd_coords.columnType = coords_ti;
1857  columns.push_back(physical_cd_coords);
1858 
1859  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1860 
1861  break;
1862  }
1863  case kLINESTRING: {
1864  ColumnDescriptor physical_cd_coords(true);
1865  physical_cd_coords.columnName = cd.columnName + "_coords";
1866  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, true);
1867  // Raw data: compressed/uncompressed coords
1868  coords_ti.set_subtype(kTINYINT);
1869  physical_cd_coords.columnType = coords_ti;
1870  columns.push_back(physical_cd_coords);
1871 
1872  ColumnDescriptor physical_cd_bounds(true);
1873  physical_cd_bounds.columnName = cd.columnName + "_bounds";
1874  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, true);
1875  bounds_ti.set_subtype(kDOUBLE);
1876  bounds_ti.set_size(4 * sizeof(double));
1877  physical_cd_bounds.columnType = bounds_ti;
1878  columns.push_back(physical_cd_bounds);
1879 
1880  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1881 
1882  break;
1883  }
1884  case kPOLYGON: {
1885  ColumnDescriptor physical_cd_coords(true);
1886  physical_cd_coords.columnName = cd.columnName + "_coords";
1887  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, true);
1888  // Raw data: compressed/uncompressed coords
1889  coords_ti.set_subtype(kTINYINT);
1890  physical_cd_coords.columnType = coords_ti;
1891  columns.push_back(physical_cd_coords);
1892 
1893  ColumnDescriptor physical_cd_ring_sizes(true);
1894  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
1895  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, true);
1896  ring_sizes_ti.set_subtype(kINT);
1897  physical_cd_ring_sizes.columnType = ring_sizes_ti;
1898  columns.push_back(physical_cd_ring_sizes);
1899 
1900  ColumnDescriptor physical_cd_bounds(true);
1901  physical_cd_bounds.columnName = cd.columnName + "_bounds";
1902  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, true);
1903  bounds_ti.set_subtype(kDOUBLE);
1904  bounds_ti.set_size(4 * sizeof(double));
1905  physical_cd_bounds.columnType = bounds_ti;
1906  columns.push_back(physical_cd_bounds);
1907 
1908  ColumnDescriptor physical_cd_render_group(true);
1909  physical_cd_render_group.columnName = cd.columnName + "_render_group";
1910  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, true);
1911  physical_cd_render_group.columnType = render_group_ti;
1912  columns.push_back(physical_cd_render_group);
1913 
1914  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1915 
1916  break;
1917  }
1918  case kMULTIPOLYGON: {
1919  ColumnDescriptor physical_cd_coords(true);
1920  physical_cd_coords.columnName = cd.columnName + "_coords";
1921  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, true);
1922  // Raw data: compressed/uncompressed coords
1923  coords_ti.set_subtype(kTINYINT);
1924  physical_cd_coords.columnType = coords_ti;
1925  columns.push_back(physical_cd_coords);
1926 
1927  ColumnDescriptor physical_cd_ring_sizes(true);
1928  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
1929  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, true);
1930  ring_sizes_ti.set_subtype(kINT);
1931  physical_cd_ring_sizes.columnType = ring_sizes_ti;
1932  columns.push_back(physical_cd_ring_sizes);
1933 
1934  ColumnDescriptor physical_cd_poly_rings(true);
1935  physical_cd_poly_rings.columnName = cd.columnName + "_poly_rings";
1936  SQLTypeInfo poly_rings_ti = SQLTypeInfo(kARRAY, true);
1937  poly_rings_ti.set_subtype(kINT);
1938  physical_cd_poly_rings.columnType = poly_rings_ti;
1939  columns.push_back(physical_cd_poly_rings);
1940 
1941  ColumnDescriptor physical_cd_bounds(true);
1942  physical_cd_bounds.columnName = cd.columnName + "_bounds";
1943  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, true);
1944  bounds_ti.set_subtype(kDOUBLE);
1945  bounds_ti.set_size(4 * sizeof(double));
1946  physical_cd_bounds.columnType = bounds_ti;
1947  columns.push_back(physical_cd_bounds);
1948 
1949  ColumnDescriptor physical_cd_render_group(true);
1950  physical_cd_render_group.columnName = cd.columnName + "_render_group";
1951  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, true);
1952  physical_cd_render_group.columnType = render_group_ti;
1953  columns.push_back(physical_cd_render_group);
1954 
1955  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1956 
1957  break;
1958  }
1959  default:
1960  throw runtime_error("Unrecognized geometry type.");
1961  break;
1962  }
1963  }
1964 }
1965 
1967  TableDescriptor& td,
1968  const list<ColumnDescriptor>& cols,
1969  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs,
1970  bool isLogicalTable) {
1971  cat_write_lock write_lock(this);
1972  list<ColumnDescriptor> cds;
1973  list<DictDescriptor> dds;
1974  std::set<std::string> toplevel_column_names;
1975  list<ColumnDescriptor> columns;
1976  for (auto cd : cols) {
1977  if (cd.columnName == "rowid") {
1978  throw std::runtime_error(
1979  "Cannot create column with name rowid. rowid is a system defined column.");
1980  }
1981  columns.push_back(cd);
1982  toplevel_column_names.insert(cd.columnName);
1983  if (cd.columnType.is_geometry()) {
1984  expandGeoColumn(cd, columns);
1985  }
1986  }
1987 
1988  ColumnDescriptor cd;
1989  // add row_id column -- Must be last column in the table
1990  cd.columnName = "rowid";
1991  cd.isSystemCol = true;
1992  cd.columnType = SQLTypeInfo(kBIGINT, true);
1993 #ifdef MATERIALIZED_ROWID
1994  cd.isVirtualCol = false;
1995 #else
1996  cd.isVirtualCol = true;
1997  cd.virtualExpr = "MAPD_FRAG_ID * MAPD_ROWS_PER_FRAG + MAPD_FRAG_ROW_ID";
1998 #endif
1999  columns.push_back(cd);
2000  toplevel_column_names.insert(cd.columnName);
2001 
2002  if (td.hasDeletedCol) {
2003  ColumnDescriptor cd_del;
2004  cd_del.columnName = "$deleted$";
2005  cd_del.isSystemCol = true;
2006  cd_del.isVirtualCol = false;
2007  cd_del.columnType = SQLTypeInfo(kBOOLEAN, true);
2008  cd_del.isDeletedCol = true;
2009 
2010  columns.push_back(cd_del);
2011  }
2012 
2013  td.nColumns = columns.size();
2015  sqliteConnector_.query("BEGIN TRANSACTION");
2017  try {
2019  "INSERT INTO mapd_tables (name, userid, ncolumns, isview, fragments, "
2020  "frag_type, max_frag_rows, "
2021  "max_chunk_size, "
2022  "frag_page_size, max_rows, partitions, shard_column_id, shard, num_shards, "
2023  "sort_column_id, "
2024  "key_metainfo) VALUES (?, ?, ?, "
2025  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2026 
2027  std::vector<std::string>{td.tableName,
2028  std::to_string(td.userId),
2030  std::to_string(td.isView),
2031  "",
2036  std::to_string(td.maxRows),
2037  td.partitions,
2039  std::to_string(td.shard),
2040  std::to_string(td.nShards),
2042  td.keyMetainfo});
2043 
2044  // now get the auto generated tableid
2046  "SELECT tableid FROM mapd_tables WHERE name = ?", td.tableName);
2047  td.tableId = sqliteConnector_.getData<int>(0, 0);
2048  int colId = 1;
2049  for (auto cd : columns) {
2051  const bool is_foreign_col =
2052  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2053  if (!is_foreign_col) {
2054  setColumnDictionary(cd, dds, td, isLogicalTable);
2055  }
2056  }
2057 
2058  if (toplevel_column_names.count(cd.columnName)) {
2059  // make up colId gap for sanity test (begin with 1 bc much code depends on it!)
2060  if (colId > 1) {
2061  colId += g_test_against_columnId_gap;
2062  }
2063  if (!cd.isGeoPhyCol) {
2064  td.columnIdBySpi_.push_back(colId);
2065  }
2066  }
2067 
2069  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, "
2070  "coldim, colscale, is_notnull, "
2071  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
2072  "virtual_expr, is_deletedcol) "
2073  "VALUES (?, ?, ?, ?, ?, "
2074  "?, "
2075  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2076  std::vector<std::string>{std::to_string(td.tableId),
2077  std::to_string(colId),
2078  cd.columnName,
2087  "",
2090  cd.virtualExpr,
2092  cd.tableId = td.tableId;
2093  cd.columnId = colId++;
2094  cds.push_back(cd);
2095  }
2096  if (td.isView) {
2098  "INSERT INTO mapd_views (tableid, sql) VALUES (?,?)",
2099  std::vector<std::string>{std::to_string(td.tableId), td.viewSQL});
2100  }
2101  } catch (std::exception& e) {
2102  sqliteConnector_.query("ROLLBACK TRANSACTION");
2103  throw;
2104  }
2105 
2106  } else { // Temporary table
2107  td.tableId = nextTempTableId_++;
2108  int colId = 1;
2109  for (auto cd : columns) {
2110  auto col_ti = cd.columnType;
2111  if (IS_GEO(col_ti.get_type())) {
2112  throw runtime_error("Geometry types in temporary tables are not supported.");
2113  }
2114 
2116  const bool is_foreign_col =
2117  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2118 
2119  if (!is_foreign_col) {
2120  // Create a new temporary dictionary
2121  std::string fileName("");
2122  std::string folderPath("");
2124  nextTempDictId_++;
2125  DictDescriptor dd(dict_ref,
2126  fileName,
2128  false,
2129  1,
2130  folderPath,
2131  true); // Is dictName (2nd argument) used?
2132  dds.push_back(dd);
2133  if (!cd.columnType.is_array()) {
2135  }
2136  cd.columnType.set_comp_param(dict_ref.dictId);
2137  }
2138  }
2139  cd.tableId = td.tableId;
2140  cd.columnId = colId++;
2141  td.columnIdBySpi_.push_back(cd.columnId);
2142  cds.push_back(cd);
2143  }
2144 
2146  serializeTableJsonUnlocked(&td, cds);
2147  }
2148  }
2149 
2150  try {
2151  addTableToMap(td, cds, dds);
2152  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
2153  } catch (std::exception& e) {
2154  sqliteConnector_.query("ROLLBACK TRANSACTION");
2155  removeTableFromMap(td.tableName, td.tableId, true);
2156  throw;
2157  }
2158 
2159  sqliteConnector_.query("END TRANSACTION");
2160 }
2161 
2163  const std::list<ColumnDescriptor>& cds) const {
2164  // relies on the catalog write lock
2165  using namespace rapidjson;
2166 
2167  VLOG(1) << "Serializing temporary table " << td->tableName << " to JSON for Calcite.";
2168 
2169  const auto db_name = currentDB_.dbName;
2170  const auto file_path = table_json_filepath(basePath_, db_name);
2171 
2172  Document d;
2173  if (boost::filesystem::exists(file_path)) {
2174  // look for an existing file for this database
2175  std::ifstream reader(file_path.string());
2176  CHECK(reader.is_open());
2177  IStreamWrapper json_read_wrapper(reader);
2178  d.ParseStream(json_read_wrapper);
2179  } else {
2180  d.SetObject();
2181  }
2182  CHECK(d.IsObject());
2183  CHECK(!d.HasMember(StringRef(td->tableName.c_str())));
2184 
2185  Value table(kObjectType);
2186  table.AddMember(
2187  "name", Value().SetString(StringRef(td->tableName.c_str())), d.GetAllocator());
2188  table.AddMember("id", Value().SetInt(td->tableId), d.GetAllocator());
2189  table.AddMember("columns", Value(kArrayType), d.GetAllocator());
2190 
2191  for (const auto& cd : cds) {
2192  Value column(kObjectType);
2193  column.AddMember(
2194  "name", Value().SetString(StringRef(cd.columnName)), d.GetAllocator());
2195  column.AddMember("coltype",
2196  Value().SetInt(static_cast<int>(cd.columnType.get_type())),
2197  d.GetAllocator());
2198  column.AddMember("colsubtype",
2199  Value().SetInt(static_cast<int>(cd.columnType.get_subtype())),
2200  d.GetAllocator());
2201  column.AddMember(
2202  "coldim", Value().SetInt(cd.columnType.get_dimension()), d.GetAllocator());
2203  column.AddMember(
2204  "colscale", Value().SetInt(cd.columnType.get_scale()), d.GetAllocator());
2205  column.AddMember(
2206  "is_notnull", Value().SetBool(cd.columnType.get_notnull()), d.GetAllocator());
2207  column.AddMember("is_systemcol", Value().SetBool(cd.isSystemCol), d.GetAllocator());
2208  column.AddMember("is_virtualcol", Value().SetBool(cd.isVirtualCol), d.GetAllocator());
2209  column.AddMember("is_deletedcol", Value().SetBool(cd.isDeletedCol), d.GetAllocator());
2210  table["columns"].PushBack(column, d.GetAllocator());
2211  }
2212  d.AddMember(StringRef(td->tableName.c_str()), table, d.GetAllocator());
2213 
2214  // Overwrite the existing file
2215  std::ofstream writer(file_path.string(), writer.trunc | writer.out);
2216  CHECK(writer.is_open());
2217  OStreamWrapper json_wrapper(writer);
2218 
2219  Writer<OStreamWrapper> json_writer(json_wrapper);
2220  d.Accept(json_writer);
2221  writer.close();
2222 }
2223 
2224 void Catalog::dropTableFromJsonUnlocked(const std::string& table_name) const {
2225  // relies on the catalog write lock
2226  using namespace rapidjson;
2227 
2228  VLOG(1) << "Dropping temporary table " << table_name << " to JSON for Calcite.";
2229 
2230  const auto db_name = currentDB_.dbName;
2231  const auto file_path = table_json_filepath(basePath_, db_name);
2232 
2233  CHECK(boost::filesystem::exists(file_path));
2234  Document d;
2235 
2236  std::ifstream reader(file_path.string());
2237  CHECK(reader.is_open());
2238  IStreamWrapper json_read_wrapper(reader);
2239  d.ParseStream(json_read_wrapper);
2240 
2241  CHECK(d.IsObject());
2242  auto table_name_ref = StringRef(table_name.c_str());
2243  CHECK(d.HasMember(table_name_ref));
2244  CHECK(d.RemoveMember(table_name_ref));
2245 
2246  // Overwrite the existing file
2247  std::ofstream writer(file_path.string(), writer.trunc | writer.out);
2248  CHECK(writer.is_open());
2249  OStreamWrapper json_wrapper(writer);
2250 
2251  Writer<OStreamWrapper> json_writer(json_wrapper);
2252  d.Accept(json_writer);
2253  writer.close();
2254 }
2255 
2256 // returns the table epoch or -1 if there is something wrong with the shared epoch
2257 int32_t Catalog::getTableEpoch(const int32_t db_id, const int32_t table_id) const {
2258  cat_read_lock read_lock(this);
2259  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(table_id);
2260  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2261  // check all shards have same checkpoint
2262  const auto physicalTables = physicalTableIt->second;
2263  CHECK(!physicalTables.empty());
2264  size_t curr_epoch = 0;
2265  for (size_t i = 0; i < physicalTables.size(); i++) {
2266  int32_t physical_tb_id = physicalTables[i];
2267  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2268  CHECK(phys_td);
2269  if (i == 0) {
2270  curr_epoch = dataMgr_->getTableEpoch(db_id, physical_tb_id);
2271  } else {
2272  if (curr_epoch != dataMgr_->getTableEpoch(db_id, physical_tb_id)) {
2273  // oh dear the leaves do not agree on the epoch for this table
2274  LOG(ERROR) << "Epochs on shards do not all agree on table id " << table_id
2275  << " db id " << db_id << " epoch " << curr_epoch << " leaf_epoch "
2276  << dataMgr_->getTableEpoch(db_id, physical_tb_id);
2277  return -1;
2278  }
2279  }
2280  }
2281  return curr_epoch;
2282  } else {
2283  return dataMgr_->getTableEpoch(db_id, table_id);
2284  }
2285 }
2286 
2287 void Catalog::setTableEpoch(const int db_id, const int table_id, int new_epoch) {
2288  cat_read_lock read_lock(this);
2289  LOG(INFO) << "Set table epoch db:" << db_id << " Table ID " << table_id
2290  << " back to new epoch " << new_epoch;
2291  removeChunks(table_id);
2292  dataMgr_->setTableEpoch(db_id, table_id, new_epoch);
2293 
2294  // check if sharded
2295  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(table_id);
2296  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2297  const auto physicalTables = physicalTableIt->second;
2298  CHECK(!physicalTables.empty());
2299  for (size_t i = 0; i < physicalTables.size(); i++) {
2300  int32_t physical_tb_id = physicalTables[i];
2301  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2302  CHECK(phys_td);
2303  LOG(INFO) << "Set sharded table epoch db:" << db_id << " Table ID "
2304  << physical_tb_id << " back to new epoch " << new_epoch;
2305  removeChunks(physical_tb_id);
2306  dataMgr_->setTableEpoch(db_id, physical_tb_id, new_epoch);
2307  }
2308  }
2309 }
2310 
2312  cat_read_lock read_lock(this);
2313  const auto it = deletedColumnPerTable_.find(td);
2314  return it != deletedColumnPerTable_.end() ? it->second : nullptr;
2315 }
2316 
2318  int tableId,
2319  int columnId) const {
2320  // check if there are rows deleted by examining metadata for the deletedColumn metadata
2321  ChunkKey chunkKeyPrefix = {dbId, tableId, columnId};
2322  std::vector<std::pair<ChunkKey, ChunkMetadata>> chunkMetadataVec;
2323  dataMgr_->getChunkMetadataVecForKeyPrefix(chunkMetadataVec, chunkKeyPrefix);
2324  int64_t chunk_max{0};
2325 
2326  for (auto cm : chunkMetadataVec) {
2327  chunk_max = cm.second.chunkStats.max.tinyintval;
2328  // delete has occured
2329  if (chunk_max == 1) {
2330  return true;
2331  }
2332  }
2333  return false;
2334 }
2335 
2337  const TableDescriptor* td) const {
2338  cat_read_lock read_lock(this);
2339 
2340  const auto it = deletedColumnPerTable_.find(td);
2341  // if not a table that supports delete return nullptr, nothing more to do
2342  if (it == deletedColumnPerTable_.end()) {
2343  return nullptr;
2344  }
2345  const ColumnDescriptor* cd = it->second;
2346 
2347  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
2348 
2349  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2350  // check all shards
2351  const auto physicalTables = physicalTableIt->second;
2352  CHECK(!physicalTables.empty());
2353  for (size_t i = 0; i < physicalTables.size(); i++) {
2354  int32_t physical_tb_id = physicalTables[i];
2355  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2356  CHECK(phys_td);
2358  return cd;
2359  }
2360  }
2361  } else {
2363  return cd;
2364  }
2365  }
2366  // no deletes so far recorded in metadata
2367  return nullptr;
2368 }
2369 
2371  cat_write_lock write_lock(this);
2372  setDeletedColumnUnlocked(td, cd);
2373 }
2374 
2376  const ColumnDescriptor* cd) {
2377  cat_write_lock write_lock(this);
2378  const auto it_ok = deletedColumnPerTable_.emplace(td, cd);
2379  CHECK(it_ok.second);
2380 }
2381 
2382 namespace {
2383 
2385  const Catalog& cat,
2386  const Parser::SharedDictionaryDef& shared_dict_def) {
2387  const auto& table_name = shared_dict_def.get_foreign_table();
2388  const auto td = cat.getMetadataForTable(table_name);
2389  CHECK(td);
2390  const auto& foreign_col_name = shared_dict_def.get_foreign_column();
2391  return cat.getMetadataForColumn(td->tableId, foreign_col_name);
2392 }
2393 
2394 } // namespace
2395 
2397  Parser::SharedDictionaryDef shared_dict_def,
2398  const bool persist_reference) {
2399  cat_write_lock write_lock(this);
2400  const auto foreign_ref_col = get_foreign_col(*this, shared_dict_def);
2401  CHECK(foreign_ref_col);
2402  referencing_column.columnType = foreign_ref_col->columnType;
2403  const int dict_id = referencing_column.columnType.get_comp_param();
2404  const DictRef dict_ref(currentDB_.dbId, dict_id);
2405  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
2406  CHECK(dictIt != dictDescriptorMapByRef_.end());
2407  const auto& dd = dictIt->second;
2408  CHECK_GE(dd->refcount, 1);
2409  ++dd->refcount;
2410  if (persist_reference) {
2413  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
2414  {std::to_string(dict_id)});
2415  }
2416 }
2417 
2419  ColumnDescriptor& cd,
2420  std::list<ColumnDescriptor>& cdd,
2421  std::list<DictDescriptor>& dds,
2422  const TableDescriptor td,
2423  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs) {
2424  cat_write_lock write_lock(this);
2426 
2427  if (shared_dict_defs.empty()) {
2428  return false;
2429  }
2430  for (const auto& shared_dict_def : shared_dict_defs) {
2431  // check if the current column is a referencing column
2432  const auto& column = shared_dict_def.get_column();
2433  if (cd.columnName == column) {
2434  if (!shared_dict_def.get_foreign_table().compare(td.tableName)) {
2435  // Dictionaries are being shared in table to be created
2436  const auto& ref_column = shared_dict_def.get_foreign_column();
2437  auto colIt =
2438  std::find_if(cdd.begin(), cdd.end(), [ref_column](const ColumnDescriptor it) {
2439  return !ref_column.compare(it.columnName);
2440  });
2441  CHECK(colIt != cdd.end());
2442  cd.columnType = colIt->columnType;
2443 
2444  const int dict_id = colIt->columnType.get_comp_param();
2445  CHECK_GE(dict_id, 1);
2446  auto dictIt = std::find_if(
2447  dds.begin(), dds.end(), [this, dict_id](const DictDescriptor it) {
2448  return it.dictRef.dbId == this->currentDB_.dbId &&
2449  it.dictRef.dictId == dict_id;
2450  });
2451  if (dictIt != dds.end()) {
2452  // There exists dictionary definition of a dictionary column
2453  CHECK_GE(dictIt->refcount, 1);
2454  ++dictIt->refcount;
2455  if (!table_is_temporary(&td)) {
2456  // Persist reference count
2458  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
2459  {std::to_string(dict_id)});
2460  }
2461  } else {
2462  // The dictionary is referencing a column which is referencing a column in
2463  // diffrent table
2464  auto root_dict_def = compress_reference_path(shared_dict_def, shared_dict_defs);
2465  addReferenceToForeignDict(cd, root_dict_def, !table_is_temporary(&td));
2466  }
2467  } else {
2468  const auto& foreign_table_name = shared_dict_def.get_foreign_table();
2469  const auto foreign_td = getMetadataForTable(foreign_table_name, false);
2470  if (table_is_temporary(foreign_td)) {
2471  if (!table_is_temporary(&td)) {
2472  throw std::runtime_error(
2473  "Only temporary tables can share dictionaries with other temporary "
2474  "tables.");
2475  }
2476  addReferenceToForeignDict(cd, shared_dict_def, false);
2477  } else {
2478  addReferenceToForeignDict(cd, shared_dict_def, !table_is_temporary(&td));
2479  }
2480  }
2481  return true;
2482  }
2483  }
2484  return false;
2485 }
2486 
2488  std::list<DictDescriptor>& dds,
2489  const TableDescriptor& td,
2490  const bool isLogicalTable) {
2491  cat_write_lock write_lock(this);
2492 
2493  std::string dictName{"Initial_key"};
2494  int dictId{0};
2495  std::string folderPath;
2496  if (isLogicalTable) {
2498 
2500  "INSERT INTO mapd_dictionaries (name, nbits, is_shared, refcount) VALUES (?, ?, "
2501  "?, 1)",
2502  std::vector<std::string>{
2503  dictName, std::to_string(cd.columnType.get_comp_param()), "0"});
2505  "SELECT dictid FROM mapd_dictionaries WHERE name = ?", dictName);
2506  dictId = sqliteConnector_.getData<int>(0, 0);
2507  dictName = td.tableName + "_" + cd.columnName + "_dict" + std::to_string(dictId);
2509  "UPDATE mapd_dictionaries SET name = ? WHERE name = 'Initial_key'", dictName);
2510  folderPath = basePath_ + "/mapd_data/DB_" + std::to_string(currentDB_.dbId) +
2511  "_DICT_" + std::to_string(dictId);
2512  }
2514  dictId,
2515  dictName,
2517  false,
2518  1,
2519  folderPath,
2520  false);
2521  dds.push_back(dd);
2522  if (!cd.columnType.is_array()) {
2524  }
2525  cd.columnType.set_comp_param(dictId);
2526 }
2527 
2529  TableDescriptor& td,
2530  const list<ColumnDescriptor>& cols,
2531  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs) {
2532  cat_write_lock write_lock(this);
2533 
2534  /* create logical table */
2535  TableDescriptor tdl(td);
2536  createTable(tdl, cols, shared_dict_defs, true); // create logical table
2537  int32_t logical_tb_id = tdl.tableId;
2538 
2539  /* create physical tables and link them to the logical table */
2540  std::vector<int32_t> physicalTables;
2541  for (int32_t i = 1; i <= td.nShards; i++) {
2542  TableDescriptor tdp(td);
2544  tdp.shard = i - 1;
2545  createTable(tdp, cols, shared_dict_defs, false); // create physical table
2546  int32_t physical_tb_id = tdp.tableId;
2547 
2548  /* add physical table to the vector of physical tables */
2549  physicalTables.push_back(physical_tb_id);
2550  }
2551 
2552  if (!physicalTables.empty()) {
2553  /* add logical to physical tables correspondence to the map */
2554  const auto it_ok =
2555  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
2556  CHECK(it_ok.second);
2557  /* update sqlite mapd_logical_to_physical in sqlite database */
2558  updateLogicalToPhysicalTableMap(logical_tb_id);
2559  }
2560 }
2561 
2563  cat_write_lock write_lock(this);
2564 
2565  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
2566  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2567  // truncate all corresponding physical tables if this is a logical table
2568  const auto physicalTables = physicalTableIt->second;
2569  CHECK(!physicalTables.empty());
2570  for (size_t i = 0; i < physicalTables.size(); i++) {
2571  int32_t physical_tb_id = physicalTables[i];
2572  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2573  CHECK(phys_td);
2574  doTruncateTable(phys_td);
2575  }
2576  }
2577  doTruncateTable(td);
2578 }
2579 
2581  cat_write_lock write_lock(this);
2582 
2583  const int tableId = td->tableId;
2584  // must destroy fragmenter before deleteChunks is called.
2585  if (td->fragmenter != nullptr) {
2586  auto tableDescIt = tableDescriptorMapById_.find(tableId);
2587  delete td->fragmenter;
2588  tableDescIt->second->fragmenter = nullptr; // get around const-ness
2589  }
2590  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
2591  // assuming deleteChunksWithPrefix is atomic
2592  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
2593  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
2594 
2595  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
2596 
2597  std::unique_ptr<StringDictionaryClient> client;
2598  if (SysCatalog::instance().isAggregator()) {
2599  CHECK(!string_dict_hosts_.empty());
2600  DictRef dict_ref(currentDB_.dbId, -1);
2601  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
2602  }
2603  // clean up any dictionaries
2604  // delete all column descriptors for the table
2605  for (const auto& columnDescriptor : columnDescriptorMapById_) {
2606  auto cd = columnDescriptor.second;
2607  if (cd->tableId != td->tableId) {
2608  continue;
2609  }
2610  const int dict_id = cd->columnType.get_comp_param();
2611  // Dummy dictionaries created for a shard of a logical table have the id set to zero.
2612  if (cd->columnType.get_compression() == kENCODING_DICT && dict_id) {
2613  const DictRef dict_ref(currentDB_.dbId, dict_id);
2614  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
2615  CHECK(dictIt != dictDescriptorMapByRef_.end());
2616  const auto& dd = dictIt->second;
2617  CHECK_GE(dd->refcount, 1);
2618  // if this is the only table using this dict reset the dict
2619  if (dd->refcount == 1) {
2620  // close the dictionary
2621  dd->stringDict.reset();
2622  File_Namespace::renameForDelete(dd->dictFolderPath);
2623  if (client) {
2624  client->drop(dd->dictRef);
2625  }
2626  if (!dd->dictIsTemp) {
2627  boost::filesystem::create_directory(dd->dictFolderPath);
2628  }
2629  }
2630 
2631  DictDescriptor* new_dd = new DictDescriptor(dd->dictRef,
2632  dd->dictName,
2633  dd->dictNBits,
2634  dd->dictIsShared,
2635  dd->refcount,
2636  dd->dictFolderPath,
2637  dd->dictIsTemp);
2638  dictDescriptorMapByRef_.erase(dictIt);
2639  // now create new Dict -- need to figure out what to do here for temp tables
2640  if (client) {
2641  client->create(new_dd->dictRef, new_dd->dictIsTemp);
2642  }
2643  dictDescriptorMapByRef_[new_dd->dictRef].reset(new_dd);
2645  }
2646  }
2647 }
2648 
2649 // used by rollback_table_epoch to clean up in memory artifacts after a rollback
2650 void Catalog::removeChunks(const int table_id) {
2651  auto td = getMetadataForTable(table_id);
2652 
2653  if (td->fragmenter != nullptr) {
2655  if (td->fragmenter != nullptr) {
2656  auto tableDescIt = tableDescriptorMapById_.find(table_id);
2657  delete td->fragmenter;
2658  tableDescIt->second->fragmenter = nullptr; // get around const-ness
2659  }
2660  }
2661 
2662  // remove the chunks from in memory structures
2663  ChunkKey chunkKey = {currentDB_.dbId, table_id};
2664 
2665  dataMgr_->deleteChunksWithPrefix(chunkKey, MemoryLevel::CPU_LEVEL);
2666  dataMgr_->deleteChunksWithPrefix(chunkKey, MemoryLevel::GPU_LEVEL);
2667 }
2668 
2672  cat_write_lock write_lock(this);
2674  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
2675  sqliteConnector_.query("BEGIN TRANSACTION");
2676  try {
2677  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2678  // remove all corresponding physical tables if this is a logical table
2679  const auto physicalTables = physicalTableIt->second;
2680  CHECK(!physicalTables.empty());
2681  for (size_t i = 0; i < physicalTables.size(); i++) {
2682  int32_t physical_tb_id = physicalTables[i];
2683  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2684  CHECK(phys_td);
2685  doDropTable(phys_td);
2686  }
2687 
2688  // remove corresponding record from the logicalToPhysicalTableMap in sqlite database
2690  "DELETE FROM mapd_logical_to_physical WHERE logical_table_id = ?",
2691  std::to_string(td->tableId));
2693  }
2694  doDropTable(td);
2695  } catch (std::exception& e) {
2696  sqliteConnector_.query("ROLLBACK TRANSACTION");
2697  throw;
2698  }
2699  sqliteConnector_.query("END TRANSACTION");
2700 }
2701 
2703  const int tableId = td->tableId;
2704  sqliteConnector_.query_with_text_param("DELETE FROM mapd_tables WHERE tableid = ?",
2705  std::to_string(tableId));
2707  "select comp_param from mapd_columns where compression = ? and tableid = ?",
2708  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
2709  int numRows = sqliteConnector_.getNumRows();
2710  std::vector<int> dict_id_list;
2711  for (int r = 0; r < numRows; ++r) {
2712  dict_id_list.push_back(sqliteConnector_.getData<int>(r, 0));
2713  }
2714  for (auto dict_id : dict_id_list) {
2716  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
2717  std::vector<std::string>{std::to_string(dict_id)});
2718  }
2720  "DELETE FROM mapd_dictionaries WHERE dictid in (select comp_param from "
2721  "mapd_columns where compression = ? "
2722  "and tableid = ?) and refcount = 0",
2723  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
2724  sqliteConnector_.query_with_text_param("DELETE FROM mapd_columns WHERE tableid = ?",
2725  std::to_string(tableId));
2726  if (td->isView) {
2727  sqliteConnector_.query_with_text_param("DELETE FROM mapd_views WHERE tableid = ?",
2728  std::to_string(tableId));
2729  }
2732  }
2734 }
2735 
2736 void Catalog::renamePhysicalTable(const TableDescriptor* td, const string& newTableName) {
2737  cat_write_lock write_lock(this);
2739 
2740  sqliteConnector_.query("BEGIN TRANSACTION");
2741  try {
2743  "UPDATE mapd_tables SET name = ? WHERE tableid = ?",
2744  std::vector<std::string>{newTableName, std::to_string(td->tableId)});
2745  } catch (std::exception& e) {
2746  sqliteConnector_.query("ROLLBACK TRANSACTION");
2747  throw;
2748  }
2749  sqliteConnector_.query("END TRANSACTION");
2750  TableDescriptorMap::iterator tableDescIt =
2752  CHECK(tableDescIt != tableDescriptorMap_.end());
2753  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
2754  // Get table descriptor to change it
2755  TableDescriptor* changeTd = tableDescIt->second;
2756  changeTd->tableName = newTableName;
2757  tableDescriptorMap_.erase(tableDescIt); // erase entry under old name
2758  tableDescriptorMap_[to_upper(newTableName)] = changeTd;
2759  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
2760 }
2761 
2762 void Catalog::renameTable(const TableDescriptor* td, const string& newTableName) {
2763  {
2764  cat_write_lock write_lock(this);
2766  // rename all corresponding physical tables if this is a logical table
2767  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
2768  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2769  const auto physicalTables = physicalTableIt->second;
2770  CHECK(!physicalTables.empty());
2771  for (size_t i = 0; i < physicalTables.size(); i++) {
2772  int32_t physical_tb_id = physicalTables[i];
2773  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2774  CHECK(phys_td);
2775  std::string newPhysTableName =
2776  generatePhysicalTableName(newTableName, static_cast<int32_t>(i + 1));
2777  renamePhysicalTable(phys_td, newPhysTableName);
2778  }
2779  }
2780  renamePhysicalTable(td, newTableName);
2781  }
2782  {
2783  DBObject object(newTableName, TableDBObjectType);
2784  // update table name in direct and effective priv map
2785  DBObjectKey key;
2786  key.dbId = currentDB_.dbId;
2787  key.objectId = td->tableId;
2788  key.permissionType = static_cast<int>(DBObjectType::TableDBObjectType);
2789  object.setObjectKey(key);
2790  auto objdescs = SysCatalog::instance().getMetadataForObject(
2791  currentDB_.dbId, static_cast<int>(DBObjectType::TableDBObjectType), td->tableId);
2792  for (auto obj : objdescs) {
2793  Grantee* grnt = SysCatalog::instance().getGrantee(obj->roleName);
2794  if (grnt) {
2795  grnt->renameDbObject(object);
2796  }
2797  }
2799  }
2800 }
2801 
2803  const ColumnDescriptor* cd,
2804  const string& newColumnName) {
2805  cat_write_lock write_lock(this);
2807  sqliteConnector_.query("BEGIN TRANSACTION");
2808  try {
2809  for (int i = 0; i <= cd->columnType.get_physical_cols(); ++i) {
2810  auto cdx = getMetadataForColumn(td->tableId, cd->columnId + i);
2811  CHECK(cdx);
2812  std::string new_column_name = cdx->columnName;
2813  new_column_name.replace(0, cd->columnName.size(), newColumnName);
2815  "UPDATE mapd_columns SET name = ? WHERE tableid = ? AND columnid = ?",
2816  std::vector<std::string>{new_column_name,
2817  std::to_string(td->tableId),
2818  std::to_string(cdx->columnId)});
2819  }
2820  } catch (std::exception& e) {
2821  sqliteConnector_.query("ROLLBACK TRANSACTION");
2822  throw;
2823  }
2824  sqliteConnector_.query("END TRANSACTION");
2825  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
2826  for (int i = 0; i <= cd->columnType.get_physical_cols(); ++i) {
2827  auto cdx = getMetadataForColumn(td->tableId, cd->columnId + i);
2828  CHECK(cdx);
2829  ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.find(
2830  std::make_tuple(td->tableId, to_upper(cdx->columnName)));
2831  CHECK(columnDescIt != columnDescriptorMap_.end());
2832  ColumnDescriptor* changeCd = columnDescIt->second;
2833  changeCd->columnName.replace(0, cd->columnName.size(), newColumnName);
2834  columnDescriptorMap_.erase(columnDescIt); // erase entry under old name
2835  columnDescriptorMap_[std::make_tuple(td->tableId, to_upper(changeCd->columnName))] =
2836  changeCd;
2837  }
2838  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
2839 }
2840 
2842  {
2843  cat_write_lock write_lock(this);
2845  sqliteConnector_.query("BEGIN TRANSACTION");
2846  try {
2847  // TODO(andrew): this should be an upsert
2849  "SELECT id FROM mapd_dashboards WHERE name = ? and userid = ?",
2850  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
2851  if (sqliteConnector_.getNumRows() > 0) {
2853  "UPDATE mapd_dashboards SET state = ?, image_hash = ?, metadata = ?, "
2854  "update_time = "
2855  "datetime('now') where name = ? "
2856  "and userid = ?",
2857  std::vector<std::string>{vd.dashboardState,
2858  vd.imageHash,
2859  vd.dashboardMetadata,
2860  vd.dashboardName,
2861  std::to_string(vd.userId)});
2862  } else {
2864  "INSERT INTO mapd_dashboards (name, state, image_hash, metadata, "
2865  "update_time, "
2866  "userid) "
2867  "VALUES "
2868  "(?,?,?,?, "
2869  "datetime('now'), ?)",
2870  std::vector<std::string>{vd.dashboardName,
2871  vd.dashboardState,
2872  vd.imageHash,
2873  vd.dashboardMetadata,
2874  std::to_string(vd.userId)});
2875  }
2876  } catch (std::exception& e) {
2877  sqliteConnector_.query("ROLLBACK TRANSACTION");
2878  throw;
2879  }
2880  sqliteConnector_.query("END TRANSACTION");
2881 
2882  // now get the auto generated dashboardId
2883  try {
2885  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_dashboards "
2886  "WHERE name = ? and userid = ?",
2887  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
2888  vd.dashboardId = sqliteConnector_.getData<int>(0, 0);
2889  vd.updateTime = sqliteConnector_.getData<std::string>(0, 1);
2890  } catch (std::exception& e) {
2891  throw;
2892  }
2896  }
2897  // NOTE(wamsi): Transactionally unsafe
2900  return vd.dashboardId;
2901 }
2902 
2904  cat_write_lock write_lock(this);
2906 
2907  sqliteConnector_.query("BEGIN TRANSACTION");
2908  try {
2910  "SELECT id FROM mapd_dashboards WHERE id = ?",
2911  std::vector<std::string>{std::to_string(vd.dashboardId)});
2912  if (sqliteConnector_.getNumRows() > 0) {
2914  "UPDATE mapd_dashboards SET name = ?, state = ?, image_hash = ?, metadata = ?, "
2915  "update_time = "
2916  "datetime('now') where id = ? ",
2917  std::vector<std::string>{vd.dashboardName,
2918  vd.dashboardState,
2919  vd.imageHash,
2920  vd.dashboardMetadata,
2922  } else {
2923  LOG(ERROR) << "Error replacing dashboard id " << vd.dashboardId
2924  << " does not exist in db";
2925  throw runtime_error("Error replacing dashboard id " +
2926  std::to_string(vd.dashboardId) + " does not exist in db");
2927  }
2928  } catch (std::exception& e) {
2929  sqliteConnector_.query("ROLLBACK TRANSACTION");
2930  throw;
2931  }
2932  sqliteConnector_.query("END TRANSACTION");
2933 
2934  bool found{false};
2935  for (auto descp : dashboardDescriptorMap_) {
2936  auto dash = descp.second.get();
2937  if (dash->dashboardId == vd.dashboardId) {
2938  found = true;
2939  auto viewDescIt = dashboardDescriptorMap_.find(std::to_string(dash->userId) + ":" +
2940  dash->dashboardName);
2941  if (viewDescIt ==
2942  dashboardDescriptorMap_.end()) { // check to make sure view exists
2943  LOG(ERROR) << "No metadata for dashboard for user " << dash->userId
2944  << " dashboard " << dash->dashboardName << " does not exist in map";
2945  throw runtime_error("No metadata for dashboard for user " +
2946  std::to_string(dash->userId) + " dashboard " +
2947  dash->dashboardName + " does not exist in map");
2948  }
2949  dashboardDescriptorMap_.erase(viewDescIt);
2950  break;
2951  }
2952  }
2953  if (!found) {
2954  LOG(ERROR) << "Error replacing dashboard id " << vd.dashboardId
2955  << " does not exist in map";
2956  throw runtime_error("Error replacing dashboard id " + std::to_string(vd.dashboardId) +
2957  " does not exist in map");
2958  }
2959 
2960  // now reload the object
2962  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM "
2963  "mapd_dashboards "
2964  "WHERE id = ?",
2965  std::vector<std::string>{std::to_string(vd.dashboardId)});
2966  vd.updateTime = sqliteConnector_.getData<string>(0, 1);
2970  // NOTE(wamsi): Transactionally unsafe
2973 }
2974 
2975 std::string Catalog::calculateSHA1(const std::string& data) {
2976  boost::uuids::detail::sha1 sha1;
2977  unsigned int digest[5];
2978  sha1.process_bytes(data.c_str(), data.length());
2979  sha1.get_digest(digest);
2980  std::stringstream ss;
2981  for (size_t i = 0; i < 5; i++) {
2982  ss << std::hex << digest[i];
2983  }
2984  return ss.str();
2985 }
2986 
2987 std::string Catalog::createLink(LinkDescriptor& ld, size_t min_length) {
2988  cat_write_lock write_lock(this);
2990  sqliteConnector_.query("BEGIN TRANSACTION");
2991  try {
2993  .substr(0, 8);
2995  "SELECT linkid FROM mapd_links WHERE link = ? and userid = ?",
2996  std::vector<std::string>{ld.link, std::to_string(ld.userId)});
2997  if (sqliteConnector_.getNumRows() > 0) {
2999  "UPDATE mapd_links SET update_time = datetime('now') WHERE userid = ? AND link "
3000  "= ?",
3001  std::vector<std::string>{std::to_string(ld.userId), ld.link});
3002  } else {
3004  "INSERT INTO mapd_links (userid, link, view_state, view_metadata, update_time) "
3005  "VALUES (?,?,?,?, "
3006  "datetime('now'))",
3007  std::vector<std::string>{
3008  std::to_string(ld.userId), ld.link, ld.viewState, ld.viewMetadata});
3009  }
3010  // now get the auto generated dashid
3012  "SELECT linkid, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_links "
3013  "WHERE link = ?",
3014  ld.link);
3015  ld.linkId = sqliteConnector_.getData<int>(0, 0);
3016  ld.updateTime = sqliteConnector_.getData<std::string>(0, 1);
3017  } catch (std::exception& e) {
3018  sqliteConnector_.query("ROLLBACK TRANSACTION");
3019  throw;
3020  }
3021  sqliteConnector_.query("END TRANSACTION");
3022  addLinkToMap(ld);
3023  return ld.link;
3024 }
3025 
3027  const TableDescriptor* td) const {
3028  const auto column_descriptors =
3029  getAllColumnMetadataForTable(td->tableId, false, true, true);
3030 
3031  const ColumnDescriptor* shard_cd{nullptr};
3032  int i = 1;
3033  for (auto cd_itr = column_descriptors.begin(); cd_itr != column_descriptors.end();
3034  ++cd_itr, ++i) {
3035  if (i == td->shardedColumnId) {
3036  shard_cd = *cd_itr;
3037  }
3038  }
3039  return shard_cd;
3040 }
3041 
3042 std::vector<const TableDescriptor*> Catalog::getPhysicalTablesDescriptors(
3043  const TableDescriptor* logicalTableDesc) const {
3044  cat_read_lock read_lock(this);
3045  const auto physicalTableIt =
3046  logicalToPhysicalTableMapById_.find(logicalTableDesc->tableId);
3047  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
3048  return {logicalTableDesc};
3049  }
3050 
3051  const auto physicalTablesIds = physicalTableIt->second;
3052  CHECK(!physicalTablesIds.empty());
3053  std::vector<const TableDescriptor*> physicalTables;
3054  for (size_t i = 0; i < physicalTablesIds.size(); i++) {
3055  physicalTables.push_back(getMetadataForTable(physicalTablesIds[i]));
3056  }
3057 
3058  return physicalTables;
3059 }
3060 
3061 int Catalog::getLogicalTableId(const int physicalTableId) const {
3062  for (const auto& l : logicalToPhysicalTableMapById_) {
3063  if (l.second.end() != std::find_if(l.second.begin(),
3064  l.second.end(),
3065  [&](decltype(*l.second.begin()) tid) -> bool {
3066  return physicalTableId == tid;
3067  })) {
3068  return l.first;
3069  }
3070  }
3071  return physicalTableId;
3072 }
3073 
3074 void Catalog::checkpoint(const int logicalTableId) const {
3075  const auto td = getMetadataForTable(logicalTableId);
3076  const auto shards = getPhysicalTablesDescriptors(td);
3077  for (const auto shard : shards) {
3078  getDataMgr().checkpoint(getCurrentDB().dbId, shard->tableId);
3079  }
3080 }
3081 
3083  cat_write_lock write_lock(this);
3084  // Physically erase all tables and dictionaries from disc and memory
3085  const auto tables = getAllTableMetadata();
3086  for (const auto table : tables) {
3087  eraseTablePhysicalData(table);
3088  }
3089  // Physically erase database metadata
3090  boost::filesystem::remove(basePath_ + "/mapd_catalogs/" + currentDB_.dbName);
3091  calciteMgr_->updateMetadata(currentDB_.dbName, "");
3092 }
3093 
3095  cat_write_lock write_lock(this);
3096  const int tableId = td->tableId;
3097  // must destroy fragmenter before deleteChunks is called.
3098  if (td->fragmenter != nullptr) {
3099  auto tableDescIt = tableDescriptorMapById_.find(tableId);
3100  {
3101  INJECT_TIMER(deleting_fragmenter);
3102  delete td->fragmenter;
3103  }
3104  tableDescIt->second->fragmenter = nullptr; // get around const-ness
3105  }
3106  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
3107  {
3108  INJECT_TIMER(deleteChunksWithPrefix);
3109  // assuming deleteChunksWithPrefix is atomic
3110  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
3111  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
3112  }
3113  if (!td->isView) {
3114  INJECT_TIMER(Remove_Table);
3115  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
3116  }
3117  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
3118  {
3119  INJECT_TIMER(removeTableFromMap_);
3120  removeTableFromMap(td->tableName, tableId);
3121  }
3122 }
3123 
3124 std::string Catalog::generatePhysicalTableName(const std::string& logicalTableName,
3125  const int32_t& shardNumber) {
3126  std::string physicalTableName =
3127  logicalTableName + physicalTableNameTag_ + std::to_string(shardNumber);
3128  return (physicalTableName);
3129 }
3130 
3131 void Catalog::set(const std::string& dbName, std::shared_ptr<Catalog> cat) {
3132  mapd_cat_map_[dbName] = cat;
3133 }
3134 
3135 std::shared_ptr<Catalog> Catalog::get(const std::string& dbName) {
3136  auto cat_it = mapd_cat_map_.find(dbName);
3137  if (cat_it != mapd_cat_map_.end()) {
3138  return cat_it->second;
3139  }
3140  return nullptr;
3141 }
3142 
3143 std::shared_ptr<Catalog> Catalog::get(const string& basePath,
3144  const DBMetadata& curDB,
3145  std::shared_ptr<Data_Namespace::DataMgr> dataMgr,
3146  const std::vector<LeafHostInfo>& string_dict_hosts,
3147  std::shared_ptr<Calcite> calcite,
3148  bool is_new_db) {
3149  auto cat = Catalog::get(curDB.dbName);
3150 
3151  if (cat) {
3152  return cat;
3153  } else {
3154  cat = std::make_shared<Catalog>(
3155  basePath, curDB, dataMgr, string_dict_hosts, calcite, is_new_db);
3156  Catalog::set(curDB.dbName, cat);
3157  return cat;
3158  }
3159 }
3160 
3161 void Catalog::remove(const std::string& dbName) {
3162  mapd_cat_map_.erase(dbName);
3163 }
3164 
3165 void Catalog::vacuumDeletedRows(const int logicalTableId) const {
3166  // shard here to serve request from TableOptimizer and elsewhere
3167  cat_read_lock read_lock(this);
3168  const auto td = getMetadataForTable(logicalTableId);
3169  const auto shards = getPhysicalTablesDescriptors(td);
3170  for (const auto shard : shards) {
3171  vacuumDeletedRows(shard);
3172  }
3173 }
3174 
3176  cat_read_lock read_lock(this);
3177  // "if not a table that supports delete return nullptr, nothing more to do"
3178  const ColumnDescriptor* cd = getDeletedColumn(td);
3179  if (nullptr == cd) {
3180  return;
3181  }
3182  // vacuum chunks which show sign of deleted rows in metadata
3183  ChunkKey chunkKeyPrefix = {currentDB_.dbId, td->tableId, cd->columnId};
3184  std::vector<std::pair<ChunkKey, ChunkMetadata>> chunkMetadataVec;
3185  dataMgr_->getChunkMetadataVecForKeyPrefix(chunkMetadataVec, chunkKeyPrefix);
3186  for (auto cm : chunkMetadataVec) {
3187  // "delete has occured"
3188  if (cm.second.chunkStats.max.tinyintval == 1) {
3189  UpdelRoll updel_roll;
3190  updel_roll.catalog = this;
3191  updel_roll.logicalTableId = getLogicalTableId(td->tableId);
3193  const auto cd = getMetadataForColumn(td->tableId, cm.first[2]);
3194  const auto chunk = Chunk_NS::Chunk::getChunk(cd,
3195  &getDataMgr(),
3196  cm.first,
3197  updel_roll.memoryLevel,
3198  0,
3199  cm.second.numBytes,
3200  cm.second.numElements);
3201  td->fragmenter->compactRows(this,
3202  td,
3203  cm.first[3],
3204  td->fragmenter->getVacuumOffsets(chunk),
3205  updel_roll.memoryLevel,
3206  updel_roll);
3207  updel_roll.commitUpdate();
3208  }
3209  }
3210 }
3211 
3212 } // namespace Catalog_Namespace
void serializeTableJsonUnlocked(const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
Definition: Catalog.cpp:2162
const Parser::SharedDictionaryDef compress_reference_path(Parser::SharedDictionaryDef cur_node, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Data_Namespace::MemoryLevel memoryLevel
Definition: UpdelRoll.h:65
std::string virtualExpr
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:1625
#define CHECK_EQ(x, y)
Definition: Logger.h:201
const int MAPD_TEMP_TABLE_START_ID
Definition: Catalog.cpp:90
std::string partitions
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:3094
std::string dictFolderPath
void doDropTable(const TableDescriptor *td)
Definition: Catalog.cpp:2702
bool is_string_array() const
Definition: sqltypes.h:478
~Catalog()
Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the ...
Definition: Catalog.cpp:179
HOST DEVICE int get_size() const
Definition: sqltypes.h:336
T getData(const int row, const int col)
mapd_shared_mutex sharedMutex_
Definition: Catalog.h:364
void commitUpdate()
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:328
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:81
SQLTypes
Definition: sqltypes.h:41
const std::vector< LeafHostInfo > & getStringDictionaryHosts() const
Definition: Catalog.cpp:1413
#define SPIMAP_MAGIC1
Definition: Catalog.h:68
std::string tableName
void addReferenceToForeignDict(ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def, const bool persist_reference)
Definition: Catalog.cpp:2396
void setDeletedColumn(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:2370
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:325
static const AccessPrivileges ALL_DATABASE
Definition: DBObject.h:144
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:1966
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:354
EncodingType
Definition: encodetypes.h:22
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
DBObjectType
Definition: DBObject.h:42
static std::shared_ptr< Executor > getExecutor(const int db_id, const std::string &debug_dir="", const std::string &debug_file="", const MapDParameters mapd_parameters=MapDParameters())
Definition: Execute.cpp:127
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:177
void addColumn(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:1729
void checkpoint(const int db_id, const int tb_id)
Definition: DataMgr.cpp:406
void roll(const bool forward)
Definition: Catalog.cpp:1779
#define LOG(tag)
Definition: Logger.h:188
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:333
SqliteConnector sqliteConnector_
Definition: Catalog.h:331
static std::shared_ptr< Catalog > get(const std::string &dbName)
Definition: Catalog.cpp:3135
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:335
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:327
void set_size(int s)
Definition: sqltypes.h:424
void query_with_text_params(std::string const &query_only)
void revokeDBObjectPrivilegesFromAll(DBObject object, Catalog *catalog)
const ColumnDescriptor * getDeletedColumnIfRowsDeleted(const TableDescriptor *td) const
Definition: Catalog.cpp:2336
HOST DEVICE int get_scale() const
Definition: sqltypes.h:331
void revokeDBObjectPrivileges(const std::string &grantee, const DBObject &object, const Catalog_Namespace::Catalog &catalog)
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:326
Catalog(const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
Constructor - takes basePath to already extant data directory for writing.
Definition: Catalog.cpp:149
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:333
DBObject * findDbObject(const DBObjectKey &objectKey, bool only_direct) const
Definition: Grantee.cpp:72
void query(const std::string &queryString)
#define CHECK_GE(x, y)
Definition: Logger.h:206
void setObjectKey(const DBObjectKey &objectKey)
Definition: DBObject.h:201
Driver for running cleanup processes on a table. TableOptimizer provides functions for various cleanu...
int32_t getTableEpoch(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:2257
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:141
std::string fragments
int32_t objectId
Definition: DBObject.h:56
HOST DEVICE void set_type(SQLTypes t)
Definition: sqltypes.h:416
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
void replaceDashboard(DashboardDescriptor &vd)
Definition: Catalog.cpp:2903
const int MAPD_TEMP_DICT_START_ID
Definition: Catalog.cpp:92
Definition: Grantee.h:76
std::string generatePhysicalTableName(const std::string &logicalTableName, const int32_t &shardNumber)
Definition: Catalog.cpp:3124
The InsertOrderFragmenter is a child class of AbstractFragmenter, and fragments data in insert order...
void addTableToMap(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
Definition: Catalog.cpp:1078
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logicalTableDesc) const
Definition: Catalog.cpp:3042
void checkDateInDaysColumnMigration()
Definition: Catalog.cpp:682
void set_dimension(int d)
Definition: sqltypes.h:418
void setPrivileges(const AccessPrivileges &privs)
Definition: DBObject.h:203
void addFrontendViewToMap(DashboardDescriptor &vd)
Definition: Catalog.cpp:1210
void deleteMetadataForDashboard(const std::string &userId, const std::string &dashName)
Definition: Catalog.cpp:1474
void vacuumDeletedRows(const TableDescriptor *td) const
Definition: Catalog.cpp:3175
static const AccessPrivileges SELECT_FROM_TABLE
Definition: DBObject.h:153
void set_scale(int s)
Definition: sqltypes.h:421
std::vector< int > columnIdBySpi_
const bool checkMetadataForDeletedRecs(int dbId, int tableId, int columnId) const
Definition: Catalog.cpp:2317
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:334
void set_compression(EncodingType c)
Definition: sqltypes.h:426
Grantee * getGrantee(const std::string &name) const
std::atomic< std::thread::id > thread_holding_sqlite_lock
Definition: Catalog.h:365
void renameColumn(const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
Definition: Catalog.cpp:2802
DeletedColumnPerTableMap deletedColumnPerTable_
Definition: Catalog.h:354
const ColumnDescriptor * get_foreign_col(const Catalog &cat, const Parser::SharedDictionaryDef &shared_dict_def)
Definition: Catalog.cpp:2384
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:326
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:328
std::string to_string(char const *&&v)
void set_notnull(bool n)
Definition: sqltypes.h:423
std::shared_ptr< std::mutex > mutex_
static const AccessPrivileges ALL_VIEW
Definition: DBObject.h:170
std::string chunks
void grantRoleBatch(const std::vector< std::string > &roles, const std::vector< std::string > &grantees)
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:555
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:108
#define SPIMAP_MAGIC2
Definition: Catalog.h:69
size_t getNumRows() const
const ColumnDescriptor * getShardColumnMetadataForTable(const TableDescriptor *td) const
Definition: Catalog.cpp:3026
void createRole(const std::string &roleName, const bool &userPrivateRole=false)
std::string dashboardSystemRoleName
This file contains the class specification and related data structures for Catalog.
virtual void compactRows(const Catalog_Namespace::Catalog *catalog, const TableDescriptor *td, const int fragmentId, const std::vector< uint64_t > &fragOffsets, const Data_Namespace::MemoryLevel memoryLevel, UpdelRoll &updelRoll)=0
int getLogicalTableId(const int physicalTableId) const
Definition: Catalog.cpp:3061
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:417
bool is_array() const
Definition: sqltypes.h:485
void checkpoint(const int logicalTableId) const
Definition: Catalog.cpp:3074
virtual const std::vector< uint64_t > getVacuumOffsets(const std::shared_ptr< Chunk_NS::Chunk > &chunk)=0
static SysCatalog & instance()
Definition: SysCatalog.h:257
This file contains the class specification and related data structures for SysCatalog.
const Catalog_Namespace::Catalog * catalog
Definition: UpdelRoll.h:63
std::string calculateSHA1(const std::string &data)
Definition: Catalog.cpp:2975
std::string getUserFromId(const int32_t id)
Definition: Catalog.cpp:882
void addLinkToMap(LinkDescriptor &ld)
Definition: Catalog.cpp:1292
#define INJECT_TIMER(DESC)
Definition: measure.h:91
void delDictionary(const ColumnDescriptor &cd)
Definition: Catalog.cpp:1668
const std::string & get_foreign_column() const
Definition: ParserNode.h:923
const TableDescriptor * getMetadataForTableImpl(int tableId, const bool populateFragmenter) const
Definition: Catalog.cpp:1357
void renameObjectsInDescriptorMap(DBObject &object, const Catalog_Namespace::Catalog &cat)
void removeChunks(const int table_id)
Definition: Catalog.cpp:2650
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:327
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:2375
void dropTable(const TableDescriptor *td)
Definition: Catalog.cpp:2669
std::tuple< int, int > ColumnIdKey
Definition: Catalog.h:248
std::vector< std::string > parse_underlying_dashboard_objects(const std::string &meta)
static const int32_t MAPD_VERSION
Definition: release.h:33
std::tuple< int, std::string > ColumnKey
Definition: Catalog.h:246
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:324
void setOwner(int32_t userId)
Definition: DBObject.h:211
static std::shared_ptr< Chunk > getChunk(const ColumnDescriptor *cd, DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId, const size_t num_bytes, const size_t num_elems)
Definition: Chunk.cpp:28
bool table_is_temporary(const TableDescriptor *td)
void grantDBObjectPrivilegesBatch(const std::vector< std::string > &grantees, const std::vector< DBObject > &objects, const Catalog_Namespace::Catalog &catalog)
specifies the content in-memory of a row in the column metadata table
int32_t dictId
Definition: DictRef.h:10
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:176
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:1836
SQLTypeInfoCore< ArrayContextTypeSizer, ExecutorTypePackaging, DateTimeFacilities > SQLTypeInfo
Definition: sqltypes.h:852
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1251
std::string keyMetainfo
int g_test_against_columnId_gap
Definition: Catalog.cpp:80
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1377
bool g_serialize_temp_tables
Definition: Catalog.cpp:85
struct dict_ref_t DictRef
Definition: DictRef.h:37
void createShardedTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:2528
std::string to_upper(const std::string &str)
const int DEFAULT_INITIAL_VERSION
Definition: Catalog.cpp:89
Definition: sqltypes.h:56
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:336
static const std::string physicalTableNameTag_
Definition: Catalog.h:340
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1215
static void remove(const std::string &dbName)
Definition: Catalog.cpp:3161
static const AccessPrivileges SELECT_FROM_VIEW
Definition: DBObject.h:173
void setTableEpoch(const int db_id, const int table_id, const int new_epoch)
Definition: Catalog.cpp:2287
static void set(const std::string &dbName, std::shared_ptr< Catalog > cat)
Definition: Catalog.cpp:3131
std::vector< ObjectRoleDescriptor * > getMetadataForObject(int32_t dbId, int32_t dbType, int32_t objectId) const
const DBObjectMap * getDbObjects(bool only_direct) const
Definition: Grantee.h:56
void recomputeMetadata() const
Recomputes per-chunk metadata for each fragment in the table. Updates and deletes can cause chunk met...
const int OMNISCI_ROOT_USER_ID
Definition: SysCatalog.h:61
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1608
Fragmenter_Namespace::FragmenterType fragType
int32_t dbId
Definition: DBObject.h:55
Data_Namespace::MemoryLevel persistenceLevel
void dropTableFromJsonUnlocked(const std::string &table_name) const
Definition: Catalog.cpp:2224
std::string name() const
Definition: Catalog.h:219
const std::string & get_foreign_table() const
Definition: ParserNode.h:921
int32_t createDashboard(DashboardDescriptor &vd)
Definition: Catalog.cpp:2841
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1128
static const AccessPrivileges ALL_DASHBOARD
Definition: DBObject.h:162
static const AccessPrivileges ALL_TABLE
Definition: DBObject.h:150
int logicalTableId
Definition: UpdelRoll.h:64
bool g_cache_string_hash
Definition: Execute.cpp:88
bool isNull(const int row, const int col) const
const ColumnDescriptor * getDeletedColumn(const TableDescriptor *td) const
Definition: Catalog.cpp:2311
void set_comp_param(int p)
Definition: sqltypes.h:427
const ColumnDescriptor * getMetadataForColumnBySpi(const int tableId, const size_t spi) const
Definition: Catalog.cpp:1464
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:335
const int getColumnIdBySpi(const int tableId, const size_t spi) const
Definition: Catalog.cpp:1459
bool setColumnSharedDictionary(ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:2418
int get_physical_cols() const
Definition: sqltypes.h:347
#define CHECK(condition)
Definition: Logger.h:193
std::vector< int > ChunkKey
Definition: types.h:35
void getDictionary(const ColumnDescriptor &cd, std::map< int, StringDictionary *> &stringDicts)
Definition: Catalog.cpp:1701
std::string createLink(LinkDescriptor &ld, size_t min_length)
Definition: Catalog.cpp:2987
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const
Descriptor for a dictionary for a string columne.
void updateLogicalToPhysicalTableLinkSchema()
Definition: Catalog.cpp:494
bool is_geometry() const
Definition: sqltypes.h:489
FileBuffer Chunk
A Chunk is the fundamental unit of execution in Map-D.
Definition: FileMgr.h:67
const int getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const
Definition: Catalog.cpp:1442
std::unordered_map< std::string, std::vector< std::string > > getGranteesOfSharedDashboards(const std::vector< std::string > &dashboard_ids)
std::atomic< std::thread::id > thread_holding_write_lock
Definition: Catalog.h:366
static TimeT::rep execution(F func, Args &&... args)
Definition: sample.cpp:29
int32_t permissionType
Definition: DBObject.h:54
void populateRoleDbObjects(const std::vector< DBObject > &objects)
#define DEFAULT_MAX_CHUNK_SIZE
std::string viewSQL
Definition: sqltypes.h:48
SQLTypeInfo columnType
Role * getRoleGrantee(const std::string &name) const
DictRef addDictionary(ColumnDescriptor &cd)
Definition: Catalog.cpp:1642
specifies the content in-memory of a row in the table metadata table
void renameTable(const TableDescriptor *td, const std::string &newTableName)
Definition: Catalog.cpp:2762
const LinkDescriptor * getMetadataForLink(const std::string &link) const
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_<EPOCH>_<oldname>.
Definition: File.cpp:182
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:330
static std::map< std::string, std::shared_ptr< Catalog > > mapd_cat_map_
Definition: Catalog.h:353
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:2487
Fragmenter_Namespace::AbstractFragmenter * fragmenter
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:350
std::list< const DashboardDescriptor * > getAllDashboardsMetadata() const
Definition: Catalog.cpp:1634
void doTruncateTable(const TableDescriptor *td)
Definition: Catalog.cpp:2580
void renamePhysicalTable(const TableDescriptor *td, const std::string &newTableName)
Definition: Catalog.cpp:2736
std::string columnName
void query_with_text_param(const std::string &queryString, const std::string &text_param)
std::vector< DBObject > parseDashboardObjects(const std::string &view_meta, const int &user_id)
Definition: Catalog.cpp:1221
bool is_string() const
Definition: sqltypes.h:477
#define IS_GEO(T)
Definition: sqltypes.h:167
virtual void renameDbObject(const DBObject &object)
Definition: Grantee.cpp:108
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:338
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:323
static thread_local bool thread_holds_read_lock
Definition: Catalog.h:368
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:329
#define VLOG(n)
Definition: Logger.h:283
void CheckAndExecuteMigrationsPostBuildMaps()
Definition: Catalog.cpp:876
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)
void truncateTable(const TableDescriptor *td)
Definition: Catalog.cpp:2562
void instantiateFragmenter(TableDescriptor *td) const
Definition: Catalog.cpp:1300
void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id)
Definition: Catalog.cpp:508