OmniSciDB  0b528656ed
Catalog.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
25 #include "Catalog.h"
26 #include "SysCatalog.h"
27 
28 #include <sys/wait.h>
29 
30 #include <algorithm>
31 #include <boost/algorithm/string/predicate.hpp>
32 #include <boost/filesystem.hpp>
33 #include <boost/range/adaptor/map.hpp>
34 #include <boost/version.hpp>
35 #include <cassert>
36 #include <cerrno>
37 #include <cstdio>
38 #include <cstring>
39 #include <exception>
40 #include <fstream>
41 #include <list>
42 #include <memory>
43 #include <random>
44 #include <regex>
45 #include <sstream>
46 #if BOOST_VERSION >= 106600
47 #include <boost/uuid/detail/sha1.hpp>
48 #else
49 #include <boost/uuid/sha1.hpp>
50 #endif
51 #include <rapidjson/document.h>
52 #include <rapidjson/istreamwrapper.h>
53 #include <rapidjson/ostreamwrapper.h>
54 #include <rapidjson/writer.h>
55 
56 #include "QueryEngine/Execute.h"
58 
62 #include "Fragmenter/Fragmenter.h"
64 #include "LockMgr/LockMgr.h"
66 #include "Parser/ParserNode.h"
67 #include "QueryEngine/Execute.h"
69 #include "Shared/File.h"
70 #include "Shared/StringTransform.h"
71 #include "Shared/TimeGM.h"
72 #include "Shared/measure.h"
74 
75 #include "MapDRelease.h"
76 #include "RWLocks.h"
78 
79 using Chunk_NS::Chunk;
82 using std::list;
83 using std::map;
84 using std::pair;
85 using std::runtime_error;
86 using std::string;
87 using std::vector;
88 
90 bool g_enable_fsi{false};
91 extern bool g_cache_string_hash;
92 
93 // Serialize temp tables to a json file in the Catalogs directory for Calcite parsing
94 // under unit testing.
96 
97 namespace Catalog_Namespace {
98 
99 const int DEFAULT_INITIAL_VERSION = 1; // start at version 1
101  1073741824; // 2^30, give room for over a billion non-temp tables
103  1073741824; // 2^30, give room for over a billion non-temp dictionaries
104 
105 const std::string Catalog::physicalTableNameTag_("_shard_#");
106 std::map<std::string, std::shared_ptr<Catalog>> Catalog::mapd_cat_map_;
107 
108 thread_local bool Catalog::thread_holds_read_lock = false;
109 
114 
115 // migration will be done as two step process this release
116 // will create and use new table
117 // next release will remove old table, doing this to have fall back path
118 // incase of migration failure
121  sqliteConnector_.query("BEGIN TRANSACTION");
122  try {
124  "SELECT name FROM sqlite_master WHERE type='table' AND name='mapd_dashboards'");
125  if (sqliteConnector_.getNumRows() != 0) {
126  // already done
127  sqliteConnector_.query("END TRANSACTION");
128  return;
129  }
131  "CREATE TABLE mapd_dashboards (id integer primary key autoincrement, name text , "
132  "userid integer references mapd_users, state text, image_hash text, update_time "
133  "timestamp, "
134  "metadata text, UNIQUE(userid, name) )");
135  // now copy content from old table to new table
137  "insert into mapd_dashboards (id, name , "
138  "userid, state, image_hash, update_time , "
139  "metadata) "
140  "SELECT viewid , name , userid, view_state, image_hash, update_time, "
141  "view_metadata "
142  "from mapd_frontend_views");
143  } catch (const std::exception& e) {
144  sqliteConnector_.query("ROLLBACK TRANSACTION");
145  throw;
146  }
147  sqliteConnector_.query("END TRANSACTION");
148 }
149 
150 namespace {
151 
152 inline auto table_json_filepath(const std::string& base_path,
153  const std::string& db_name) {
154  return boost::filesystem::path(base_path + "/mapd_catalogs/" + db_name +
155  "_temp_tables.json");
156 }
157 
158 } // namespace
159 
160 Catalog::Catalog(const string& basePath,
161  const DBMetadata& curDB,
162  std::shared_ptr<Data_Namespace::DataMgr> dataMgr,
163  const std::vector<LeafHostInfo>& string_dict_hosts,
164  std::shared_ptr<Calcite> calcite,
165  bool is_new_db)
166  : basePath_(basePath)
167  , sqliteConnector_(curDB.dbName, basePath + "/mapd_catalogs/")
168  , currentDB_(curDB)
169  , dataMgr_(dataMgr)
170  , string_dict_hosts_(string_dict_hosts)
171  , calciteMgr_(calcite)
172  , nextTempTableId_(MAPD_TEMP_TABLE_START_ID)
173  , nextTempDictId_(MAPD_TEMP_DICT_START_ID)
174  , sqliteMutex_()
175  , sharedMutex_()
178  if (!is_new_db) {
180  }
181  buildMaps();
182  if (!is_new_db) {
184  }
186  boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
187  }
188 }
189 
192  // must clean up heap-allocated TableDescriptor and ColumnDescriptor structs
193  for (TableDescriptorMap::iterator tableDescIt = tableDescriptorMap_.begin();
194  tableDescIt != tableDescriptorMap_.end();
195  ++tableDescIt) {
196  tableDescIt->second->fragmenter = nullptr;
197  delete tableDescIt->second;
198  }
199 
200  // TableDescriptorMapById points to the same descriptors. No need to delete
201 
202  for (ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.begin();
203  columnDescIt != columnDescriptorMap_.end();
204  ++columnDescIt) {
205  delete columnDescIt->second;
206  }
207 
208  // ColumnDescriptorMapById points to the same descriptors. No need to delete
209 
211  boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
212  }
213 }
214 
217  sqliteConnector_.query("BEGIN TRANSACTION");
218  try {
219  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_tables)");
220  std::vector<std::string> cols;
221  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
222  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
223  }
224  if (std::find(cols.begin(), cols.end(), std::string("max_chunk_size")) ==
225  cols.end()) {
226  string queryString("ALTER TABLE mapd_tables ADD max_chunk_size BIGINT DEFAULT " +
228  sqliteConnector_.query(queryString);
229  }
230  if (std::find(cols.begin(), cols.end(), std::string("shard_column_id")) ==
231  cols.end()) {
232  string queryString("ALTER TABLE mapd_tables ADD shard_column_id BIGINT DEFAULT " +
233  std::to_string(0));
234  sqliteConnector_.query(queryString);
235  }
236  if (std::find(cols.begin(), cols.end(), std::string("shard")) == cols.end()) {
237  string queryString("ALTER TABLE mapd_tables ADD shard BIGINT DEFAULT " +
238  std::to_string(-1));
239  sqliteConnector_.query(queryString);
240  }
241  if (std::find(cols.begin(), cols.end(), std::string("num_shards")) == cols.end()) {
242  string queryString("ALTER TABLE mapd_tables ADD num_shards BIGINT DEFAULT " +
243  std::to_string(0));
244  sqliteConnector_.query(queryString);
245  }
246  if (std::find(cols.begin(), cols.end(), std::string("key_metainfo")) == cols.end()) {
247  string queryString("ALTER TABLE mapd_tables ADD key_metainfo TEXT DEFAULT '[]'");
248  sqliteConnector_.query(queryString);
249  }
250  if (std::find(cols.begin(), cols.end(), std::string("userid")) == cols.end()) {
251  string queryString("ALTER TABLE mapd_tables ADD userid integer DEFAULT " +
253  sqliteConnector_.query(queryString);
254  }
255  if (std::find(cols.begin(), cols.end(), std::string("sort_column_id")) ==
256  cols.end()) {
258  "ALTER TABLE mapd_tables ADD sort_column_id INTEGER DEFAULT 0");
259  }
260  if (std::find(cols.begin(), cols.end(), std::string("storage_type")) == cols.end()) {
261  string queryString("ALTER TABLE mapd_tables ADD storage_type TEXT DEFAULT ''");
262  sqliteConnector_.query(queryString);
263  }
264  } catch (std::exception& e) {
265  sqliteConnector_.query("ROLLBACK TRANSACTION");
266  throw;
267  }
268  sqliteConnector_.query("END TRANSACTION");
269 }
270 
273  sqliteConnector_.query("BEGIN TRANSACTION");
274  try {
276  "select name from sqlite_master WHERE type='table' AND "
277  "name='mapd_version_history'");
278  if (sqliteConnector_.getNumRows() == 0) {
280  "CREATE TABLE mapd_version_history(version integer, migration_history text "
281  "unique)");
282  } else {
284  "select * from mapd_version_history where migration_history = "
285  "'notnull_fixlen_arrays'");
286  if (sqliteConnector_.getNumRows() != 0) {
287  // legacy fixlen arrays had migrated
288  // no need for further execution
289  sqliteConnector_.query("END TRANSACTION");
290  return;
291  }
292  }
293  // Insert check for migration
295  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
296  std::vector<std::string>{std::to_string(MAPD_VERSION), "notnull_fixlen_arrays"});
297  LOG(INFO) << "Updating mapd_columns, legacy fixlen arrays";
298  // Upating all fixlen array columns
299  string queryString("UPDATE mapd_columns SET is_notnull=1 WHERE coltype=" +
300  std::to_string(kARRAY) + " AND size>0;");
301  sqliteConnector_.query(queryString);
302  } catch (std::exception& e) {
303  sqliteConnector_.query("ROLLBACK TRANSACTION");
304  throw;
305  }
306  sqliteConnector_.query("END TRANSACTION");
307 }
308 
311  sqliteConnector_.query("BEGIN TRANSACTION");
312  try {
314  "select name from sqlite_master WHERE type='table' AND "
315  "name='mapd_version_history'");
316  if (sqliteConnector_.getNumRows() == 0) {
318  "CREATE TABLE mapd_version_history(version integer, migration_history text "
319  "unique)");
320  } else {
322  "select * from mapd_version_history where migration_history = "
323  "'notnull_geo_columns'");
324  if (sqliteConnector_.getNumRows() != 0) {
325  // legacy geo columns had migrated
326  // no need for further execution
327  sqliteConnector_.query("END TRANSACTION");
328  return;
329  }
330  }
331  // Insert check for migration
333  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
334  std::vector<std::string>{std::to_string(MAPD_VERSION), "notnull_geo_columns"});
335  LOG(INFO) << "Updating mapd_columns, legacy geo columns";
336  // Upating all geo columns
337  string queryString(
338  "UPDATE mapd_columns SET is_notnull=1 WHERE coltype=" + std::to_string(kPOINT) +
339  " OR coltype=" + std::to_string(kLINESTRING) + " OR coltype=" +
340  std::to_string(kPOLYGON) + " OR coltype=" + std::to_string(kMULTIPOLYGON) + ";");
341  sqliteConnector_.query(queryString);
342  } catch (std::exception& e) {
343  sqliteConnector_.query("ROLLBACK TRANSACTION");
344  throw;
345  }
346  sqliteConnector_.query("END TRANSACTION");
347 }
348 
351  sqliteConnector_.query("BEGIN TRANSACTION");
352  try {
353  // check table still exists
355  "SELECT name FROM sqlite_master WHERE type='table' AND "
356  "name='mapd_frontend_views'");
357  if (sqliteConnector_.getNumRows() == 0) {
358  // table does not exists
359  // no need to migrate
360  sqliteConnector_.query("END TRANSACTION");
361  return;
362  }
363  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_frontend_views)");
364  std::vector<std::string> cols;
365  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
366  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
367  }
368  if (std::find(cols.begin(), cols.end(), std::string("image_hash")) == cols.end()) {
369  sqliteConnector_.query("ALTER TABLE mapd_frontend_views ADD image_hash text");
370  }
371  if (std::find(cols.begin(), cols.end(), std::string("update_time")) == cols.end()) {
372  sqliteConnector_.query("ALTER TABLE mapd_frontend_views ADD update_time timestamp");
373  }
374  if (std::find(cols.begin(), cols.end(), std::string("view_metadata")) == cols.end()) {
375  sqliteConnector_.query("ALTER TABLE mapd_frontend_views ADD view_metadata text");
376  }
377  } catch (std::exception& e) {
378  sqliteConnector_.query("ROLLBACK TRANSACTION");
379  throw;
380  }
381  sqliteConnector_.query("END TRANSACTION");
382 }
383 
386  sqliteConnector_.query("BEGIN TRANSACTION");
387  try {
389  "CREATE TABLE IF NOT EXISTS mapd_links (linkid integer primary key, userid "
390  "integer references mapd_users, "
391  "link text unique, view_state text, update_time timestamp, view_metadata text)");
392  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_links)");
393  std::vector<std::string> cols;
394  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
395  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
396  }
397  if (std::find(cols.begin(), cols.end(), std::string("view_metadata")) == cols.end()) {
398  sqliteConnector_.query("ALTER TABLE mapd_links ADD view_metadata text");
399  }
400  } catch (const std::exception& e) {
401  sqliteConnector_.query("ROLLBACK TRANSACTION");
402  throw;
403  }
404  sqliteConnector_.query("END TRANSACTION");
405 }
406 
409  sqliteConnector_.query("BEGIN TRANSACTION");
410  try {
411  sqliteConnector_.query("UPDATE mapd_links SET userid = 0 WHERE userid IS NULL");
412  // check table still exists
414  "SELECT name FROM sqlite_master WHERE type='table' AND "
415  "name='mapd_frontend_views'");
416  if (sqliteConnector_.getNumRows() == 0) {
417  // table does not exists
418  // no need to migrate
419  sqliteConnector_.query("END TRANSACTION");
420  return;
421  }
423  "UPDATE mapd_frontend_views SET userid = 0 WHERE userid IS NULL");
424  } catch (const std::exception& e) {
425  sqliteConnector_.query("ROLLBACK TRANSACTION");
426  throw;
427  }
428  sqliteConnector_.query("END TRANSACTION");
429 }
430 
431 // introduce DB version into the tables table
432 // if the DB does not have a version reset all pagesizes to 2097152 to be compatible with
433 // old value
434 
437  if (currentDB_.dbName.length() == 0) {
438  // updateDictionaryNames dbName length is zero nothing to do here
439  return;
440  }
441  sqliteConnector_.query("BEGIN TRANSACTION");
442  try {
443  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_tables)");
444  std::vector<std::string> cols;
445  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
446  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
447  }
448  if (std::find(cols.begin(), cols.end(), std::string("version_num")) == cols.end()) {
449  LOG(INFO) << "Updating mapd_tables updatePageSize";
450  // No version number
451  // need to update the defaul tpagesize to old correct value
452  sqliteConnector_.query("UPDATE mapd_tables SET frag_page_size = 2097152 ");
453  // need to add new version info
454  string queryString("ALTER TABLE mapd_tables ADD version_num BIGINT DEFAULT " +
455  std::to_string(DEFAULT_INITIAL_VERSION));
456  sqliteConnector_.query(queryString);
457  }
458  } catch (std::exception& e) {
459  sqliteConnector_.query("ROLLBACK TRANSACTION");
460  throw;
461  }
462  sqliteConnector_.query("END TRANSACTION");
463 }
464 
467  sqliteConnector_.query("BEGIN TRANSACTION");
468  try {
469  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_columns)");
470  std::vector<std::string> cols;
471  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
472  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
473  }
474  if (std::find(cols.begin(), cols.end(), std::string("version_num")) == cols.end()) {
475  LOG(INFO) << "Updating mapd_columns updateDeletedColumnIndicator";
476  // need to add new version info
477  string queryString("ALTER TABLE mapd_columns ADD version_num BIGINT DEFAULT " +
478  std::to_string(DEFAULT_INITIAL_VERSION));
479  sqliteConnector_.query(queryString);
480  // need to add new column to table defintion to indicate deleted column, column used
481  // as bitmap for deleted rows.
483  "ALTER TABLE mapd_columns ADD is_deletedcol boolean default 0 ");
484  }
485  } catch (std::exception& e) {
486  sqliteConnector_.query("ROLLBACK TRANSACTION");
487  throw;
488  }
489  sqliteConnector_.query("END TRANSACTION");
490 }
491 
492 // introduce DB version into the dictionary tables
493 // if the DB does not have a version rename all dictionary tables
494 
497  if (currentDB_.dbName.length() == 0) {
498  // updateDictionaryNames dbName length is zero nothing to do here
499  return;
500  }
501  sqliteConnector_.query("BEGIN TRANSACTION");
502  try {
503  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_dictionaries)");
504  std::vector<std::string> cols;
505  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
506  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
507  }
508  if (std::find(cols.begin(), cols.end(), std::string("version_num")) == cols.end()) {
509  // No version number
510  // need to rename dictionaries
511  string dictQuery("SELECT dictid, name from mapd_dictionaries");
512  sqliteConnector_.query(dictQuery);
513  size_t numRows = sqliteConnector_.getNumRows();
514  for (size_t r = 0; r < numRows; ++r) {
515  int dictId = sqliteConnector_.getData<int>(r, 0);
516  std::string dictName = sqliteConnector_.getData<string>(r, 1);
517 
518  std::string oldName =
519  basePath_ + "/mapd_data/" + currentDB_.dbName + "_" + dictName;
520  std::string newName = basePath_ + "/mapd_data/DB_" +
521  std::to_string(currentDB_.dbId) + "_DICT_" +
522  std::to_string(dictId);
523 
524  int result = rename(oldName.c_str(), newName.c_str());
525 
526  if (result == 0) {
527  LOG(INFO) << "Dictionary upgrade: successfully renamed " << oldName << " to "
528  << newName;
529  } else {
530  LOG(ERROR) << "Failed to rename old dictionary directory " << oldName << " to "
531  << newName + " dbname '" << currentDB_.dbName << "' error code "
532  << std::to_string(result);
533  }
534  }
535  // need to add new version info
536  string queryString("ALTER TABLE mapd_dictionaries ADD version_num BIGINT DEFAULT " +
537  std::to_string(DEFAULT_INITIAL_VERSION));
538  sqliteConnector_.query(queryString);
539  }
540  } catch (std::exception& e) {
541  sqliteConnector_.query("ROLLBACK TRANSACTION");
542  throw;
543  }
544  sqliteConnector_.query("END TRANSACTION");
545 }
546 
549  sqliteConnector_.query("BEGIN TRANSACTION");
550  try {
552  "CREATE TABLE IF NOT EXISTS mapd_logical_to_physical("
553  "logical_table_id integer, physical_table_id integer)");
554  } catch (const std::exception& e) {
555  sqliteConnector_.query("ROLLBACK TRANSACTION");
556  throw;
557  }
558  sqliteConnector_.query("END TRANSACTION");
559 }
560 
561 void Catalog::updateLogicalToPhysicalTableMap(const int32_t logical_tb_id) {
562  /* this proc inserts/updates all pairs of (logical_tb_id, physical_tb_id) in
563  * sqlite mapd_logical_to_physical table for given logical_tb_id as needed
564  */
565 
567  sqliteConnector_.query("BEGIN TRANSACTION");
568  try {
569  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
570  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
571  const auto physicalTables = physicalTableIt->second;
572  CHECK(!physicalTables.empty());
573  for (size_t i = 0; i < physicalTables.size(); i++) {
574  int32_t physical_tb_id = physicalTables[i];
576  "INSERT OR REPLACE INTO mapd_logical_to_physical (logical_table_id, "
577  "physical_table_id) VALUES (?1, ?2)",
578  std::vector<std::string>{std::to_string(logical_tb_id),
579  std::to_string(physical_tb_id)});
580  }
581  }
582  } catch (std::exception& e) {
583  sqliteConnector_.query("ROLLBACK TRANSACTION");
584  throw;
585  }
586  sqliteConnector_.query("END TRANSACTION");
587 }
588 
591  sqliteConnector_.query("BEGIN TRANSACTION");
592  try {
593  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_dictionaries)");
594  std::vector<std::string> cols;
595  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
596  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
597  }
598  if (std::find(cols.begin(), cols.end(), std::string("refcount")) == cols.end()) {
599  sqliteConnector_.query("ALTER TABLE mapd_dictionaries ADD refcount DEFAULT 1");
600  }
601  } catch (std::exception& e) {
602  sqliteConnector_.query("ROLLBACK TRANSACTION");
603  throw;
604  }
605  sqliteConnector_.query("END TRANSACTION");
606 }
607 
610  sqliteConnector_.query("BEGIN TRANSACTION");
611  try {
615  } catch (std::exception& e) {
616  sqliteConnector_.query("ROLLBACK TRANSACTION");
617  throw;
618  }
619  sqliteConnector_.query("END TRANSACTION");
620 }
621 
623  std::vector<foreign_storage::ForeignTable> foreign_tables{};
624  {
626  sqliteConnector_.query("BEGIN TRANSACTION");
627  try {
629  "SELECT name FROM sqlite_master WHERE type='table' AND "
630  "name IN ('omnisci_foreign_servers', 'omnisci_foreign_tables', "
631  "'omnisci_user_mappings')");
632  if (sqliteConnector_.getNumRows() > 0) {
634  "SELECT tableid, name, isview, storage_type FROM mapd_tables "
635  "WHERE storage_type = 'FOREIGN_TABLE'");
637  for (size_t r = 0; r < num_rows; r++) {
638  foreign_storage::ForeignTable foreign_table{};
639  foreign_table.tableId = sqliteConnector_.getData<int>(r, 0);
640  foreign_table.tableName = sqliteConnector_.getData<std::string>(r, 1);
641  foreign_table.isView = sqliteConnector_.getData<bool>(r, 2);
642  foreign_table.storageType = sqliteConnector_.getData<std::string>(r, 3);
643  foreign_tables.emplace_back(foreign_table);
644  }
645  for (auto& foreign_table : foreign_tables) {
646  tableDescriptorMap_[to_upper(foreign_table.tableName)] = &foreign_table;
647  tableDescriptorMapById_[foreign_table.tableId] = &foreign_table;
648  executeDropTableSqliteQueries(&foreign_table);
649  }
650  sqliteConnector_.query("SELECT COUNT(*) FROM omnisci_foreign_tables");
651  CHECK_EQ(size_t(1), sqliteConnector_.getNumRows());
652  CHECK_EQ(0, sqliteConnector_.getData<int>(0, 0));
653 
654  sqliteConnector_.query("DROP TABLE omnisci_foreign_tables");
655  sqliteConnector_.query("DROP TABLE omnisci_foreign_servers");
656  }
657  } catch (std::exception& e) {
658  sqliteConnector_.query("ROLLBACK TRANSACTION");
659  throw;
660  }
661  sqliteConnector_.query("END TRANSACTION");
662  }
663 
664  for (auto& foreign_table : foreign_tables) {
666  DBObject(foreign_table.tableName, TableDBObjectType), this);
667  tableDescriptorMap_.erase(to_upper(foreign_table.tableName));
668  tableDescriptorMapById_.erase(foreign_table.tableId);
669  }
670 }
671 
672 const std::string Catalog::getForeignServerSchema(bool if_not_exists) {
673  return "CREATE TABLE " + (if_not_exists ? std::string{"IF NOT EXISTS "} : "") +
674  "omnisci_foreign_servers(id integer primary key, name text unique, " +
675  "data_wrapper_type text, owner_user_id integer, creation_time integer, " +
676  "options text)";
677 }
678 
679 const std::string Catalog::getForeignTableSchema(bool if_not_exists) {
680  return "CREATE TABLE " + (if_not_exists ? std::string{"IF NOT EXISTS "} : "") +
681  "omnisci_foreign_tables(table_id integer unique, server_id integer, " +
682  "options text, FOREIGN KEY(table_id) REFERENCES mapd_tables(tableid), " +
683  "FOREIGN KEY(server_id) REFERENCES omnisci_foreign_servers(id))";
684 }
685 
688  sqliteConnector_.query("BEGIN TRANSACTION");
689  std::vector<DBObject> objects;
690  try {
692  "SELECT name FROM sqlite_master WHERE type='table' AND "
693  "name='mapd_record_ownership_marker'");
694  // check if mapd catalog - marker exists
695  if (sqliteConnector_.getNumRows() != 0 && currentDB_.dbId == 1) {
696  // already done
697  sqliteConnector_.query("END TRANSACTION");
698  return;
699  }
700  // check if different catalog - marker exists
701  else if (sqliteConnector_.getNumRows() != 0 && currentDB_.dbId != 1) {
702  sqliteConnector_.query("SELECT dummy FROM mapd_record_ownership_marker");
703  // Check if migration is being performed on existing non mapd catalogs
704  // Older non mapd dbs will have table but no record in them
705  if (sqliteConnector_.getNumRows() != 0) {
706  // already done
707  sqliteConnector_.query("END TRANSACTION");
708  return;
709  }
710  }
711  // marker not exists - create one
712  else {
713  sqliteConnector_.query("CREATE TABLE mapd_record_ownership_marker (dummy integer)");
714  }
715 
716  DBMetadata db;
717  CHECK(SysCatalog::instance().getMetadataForDB(currentDB_.dbName, db));
718  // place dbId as a refernce for migration being performed
720  "INSERT INTO mapd_record_ownership_marker (dummy) VALUES (?1)",
721  std::vector<std::string>{std::to_string(db.dbOwner)});
722 
723  static const std::map<const DBObjectType, const AccessPrivileges>
724  object_level_all_privs_lookup{
730 
731  // grant owner all permissions on DB
732  DBObjectKey key;
733  key.dbId = currentDB_.dbId;
734  auto _key_place = [&key](auto type) {
735  key.permissionType = type;
736  return key;
737  };
738  for (auto& it : object_level_all_privs_lookup) {
739  objects.emplace_back(_key_place(it.first), it.second, db.dbOwner);
740  objects.back().setName(currentDB_.dbName);
741  }
742 
743  {
744  // other users tables and views
745  string tableQuery(
746  "SELECT tableid, name, userid, isview FROM mapd_tables WHERE userid > 0");
747  sqliteConnector_.query(tableQuery);
748  size_t numRows = sqliteConnector_.getNumRows();
749  for (size_t r = 0; r < numRows; ++r) {
750  int32_t tableid = sqliteConnector_.getData<int>(r, 0);
751  std::string tableName = sqliteConnector_.getData<string>(r, 1);
752  int32_t ownerid = sqliteConnector_.getData<int>(r, 2);
753  bool isview = sqliteConnector_.getData<bool>(r, 3);
754 
757  DBObjectKey key;
758  key.dbId = currentDB_.dbId;
759  key.objectId = tableid;
760  key.permissionType = type;
761 
762  DBObject obj(tableName, type);
763  obj.setObjectKey(key);
764  obj.setOwner(ownerid);
767 
768  objects.push_back(obj);
769  }
770  }
771 
772  {
773  // other users dashboards
774  string tableQuery("SELECT id, name, userid FROM mapd_dashboards WHERE userid > 0");
775  sqliteConnector_.query(tableQuery);
776  size_t numRows = sqliteConnector_.getNumRows();
777  for (size_t r = 0; r < numRows; ++r) {
778  int32_t dashId = sqliteConnector_.getData<int>(r, 0);
779  std::string dashName = sqliteConnector_.getData<string>(r, 1);
780  int32_t ownerid = sqliteConnector_.getData<int>(r, 2);
781 
783  DBObjectKey key;
784  key.dbId = currentDB_.dbId;
785  key.objectId = dashId;
786  key.permissionType = type;
787 
788  DBObject obj(dashName, type);
789  obj.setObjectKey(key);
790  obj.setOwner(ownerid);
792 
793  objects.push_back(obj);
794  }
795  }
796  } catch (const std::exception& e) {
797  sqliteConnector_.query("ROLLBACK TRANSACTION");
798  throw;
799  }
800  sqliteConnector_.query("END TRANSACTION");
801 
802  // now apply the objects to the syscat to track the permisisons
803  // moved outside transaction to avoid lock in sqlite
804  try {
806  } catch (const std::exception& e) {
807  LOG(ERROR) << " Issue during migration of DB " << name() << " issue was " << e.what();
808  throw std::runtime_error(" Issue during migration of DB " + name() + " issue was " +
809  e.what());
810  // will need to remove the mapd_record_ownership_marker table and retry
811  }
812 }
813 
818 }
819 
821  std::unordered_map<std::string, std::pair<int, std::string>> dashboards;
822  std::vector<std::string> dashboard_ids;
823  static const std::string migration_name{"dashboard_roles_migration"};
824  {
826  sqliteConnector_.query("BEGIN TRANSACTION");
827  try {
828  // migration_history should be present in all catalogs by now
829  // if not then would be created before this migration
831  "select * from mapd_version_history where migration_history = '" +
832  migration_name + "'");
833  if (sqliteConnector_.getNumRows() != 0) {
834  // no need for further execution
835  sqliteConnector_.query("END TRANSACTION");
836  return;
837  }
838  LOG(INFO) << "Performing dashboard internal roles Migration.";
839  sqliteConnector_.query("select id, userid, metadata from mapd_dashboards");
840  for (size_t i = 0; i < sqliteConnector_.getNumRows(); ++i) {
843  sqliteConnector_.getData<string>(i, 0)))) {
844  // Successfully created roles during previous migration/crash
845  // No need to include them
846  continue;
847  }
848  dashboards[sqliteConnector_.getData<string>(i, 0)] = std::make_pair(
849  sqliteConnector_.getData<int>(i, 1), sqliteConnector_.getData<string>(i, 2));
850  dashboard_ids.push_back(sqliteConnector_.getData<string>(i, 0));
851  }
852  } catch (const std::exception& e) {
853  sqliteConnector_.query("ROLLBACK TRANSACTION");
854  throw;
855  }
856  sqliteConnector_.query("END TRANSACTION");
857  }
858  // All current grantees with shared dashboards.
859  const auto active_grantees =
861 
862  try {
863  // NOTE(wamsi): Transactionally unsafe
864  for (auto dash : dashboards) {
865  createOrUpdateDashboardSystemRole(dash.second.second,
866  dash.second.first,
868  std::to_string(currentDB_.dbId), dash.first));
869  auto result = active_grantees.find(dash.first);
870  if (result != active_grantees.end()) {
873  dash.first)},
874  result->second);
875  }
876  }
878  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
879  std::vector<std::string>{std::to_string(MAPD_VERSION), migration_name});
880  } catch (const std::exception& e) {
881  LOG(ERROR) << "Failed to create dashboard system roles during migration: "
882  << e.what();
883  throw;
884  }
885  LOG(INFO) << "Successfully created dashboard system roles during migration.";
886 }
887 
898  updatePageSize();
902 
903  if (g_enable_fsi) {
905  } else {
907  }
908 }
909 
913 }
914 
915 namespace {
916 std::string getUserFromId(const int32_t id) {
917  UserMetadata user;
918  if (SysCatalog::instance().getMetadataForUserById(id, user)) {
919  return user.userName;
920  }
921  // a user could be deleted and a dashboard still exist?
922  return "Unknown";
923 }
924 } // namespace
925 
929 
930  string dictQuery(
931  "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
932  sqliteConnector_.query(dictQuery);
933  size_t numRows = sqliteConnector_.getNumRows();
934  for (size_t r = 0; r < numRows; ++r) {
935  int dictId = sqliteConnector_.getData<int>(r, 0);
936  std::string dictName = sqliteConnector_.getData<string>(r, 1);
937  int dictNBits = sqliteConnector_.getData<int>(r, 2);
938  bool is_shared = sqliteConnector_.getData<bool>(r, 3);
939  int refcount = sqliteConnector_.getData<int>(r, 4);
940  std::string fname = basePath_ + "/mapd_data/DB_" + std::to_string(currentDB_.dbId) +
941  "_DICT_" + std::to_string(dictId);
942  DictRef dict_ref(currentDB_.dbId, dictId);
943  DictDescriptor* dd = new DictDescriptor(
944  dict_ref, dictName, dictNBits, is_shared, refcount, fname, false);
945  dictDescriptorMapByRef_[dict_ref].reset(dd);
946  }
947 
948  string tableQuery(
949  "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
950  "max_chunk_size, frag_page_size, "
951  "max_rows, partitions, shard_column_id, shard, num_shards, key_metainfo, userid, "
952  "sort_column_id, storage_type "
953  "from mapd_tables");
954  sqliteConnector_.query(tableQuery);
955  numRows = sqliteConnector_.getNumRows();
956  for (size_t r = 0; r < numRows; ++r) {
957  TableDescriptor* td;
958  const auto& storage_type = sqliteConnector_.getData<string>(r, 17);
959  if (!storage_type.empty() &&
960  (!g_enable_fsi || storage_type != StorageType::FOREIGN_TABLE)) {
961  const auto table_id = sqliteConnector_.getData<int>(r, 0);
962  const auto& table_name = sqliteConnector_.getData<string>(r, 1);
963  LOG(FATAL) << "Unable to read Catalog metadata: storage type is currently not a "
964  "supported table option (table "
965  << table_name << " [" << table_id << "] in database "
966  << currentDB_.dbName << ").";
967  }
968 
969  if (storage_type == StorageType::FOREIGN_TABLE) {
971  } else {
972  td = new TableDescriptor();
973  }
974 
975  td->storageType = storage_type;
976  td->tableId = sqliteConnector_.getData<int>(r, 0);
977  td->tableName = sqliteConnector_.getData<string>(r, 1);
978  td->nColumns = sqliteConnector_.getData<int>(r, 2);
979  td->isView = sqliteConnector_.getData<bool>(r, 3);
980  td->fragments = sqliteConnector_.getData<string>(r, 4);
981  td->fragType =
983  td->maxFragRows = sqliteConnector_.getData<int>(r, 6);
984  td->maxChunkSize = sqliteConnector_.getData<int>(r, 7);
985  td->fragPageSize = sqliteConnector_.getData<int>(r, 8);
986  td->maxRows = sqliteConnector_.getData<int64_t>(r, 9);
987  td->partitions = sqliteConnector_.getData<string>(r, 10);
988  td->shardedColumnId = sqliteConnector_.getData<int>(r, 11);
989  td->shard = sqliteConnector_.getData<int>(r, 12);
990  td->nShards = sqliteConnector_.getData<int>(r, 13);
991  td->keyMetainfo = sqliteConnector_.getData<string>(r, 14);
992  td->userId = sqliteConnector_.getData<int>(r, 15);
993  td->sortedColumnId =
994  sqliteConnector_.isNull(r, 16) ? 0 : sqliteConnector_.getData<int>(r, 16);
995  if (!td->isView) {
996  td->fragmenter = nullptr;
997  }
998  td->hasDeletedCol = false;
999 
1001  tableDescriptorMapById_[td->tableId] = td;
1002  }
1003 
1004  if (g_enable_fsi) {
1007  }
1008 
1009  string columnQuery(
1010  "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
1011  "is_notnull, compression, comp_param, "
1012  "size, chunks, is_systemcol, is_virtualcol, virtual_expr, is_deletedcol from "
1013  "mapd_columns ORDER BY tableid, "
1014  "columnid");
1015  sqliteConnector_.query(columnQuery);
1016  numRows = sqliteConnector_.getNumRows();
1017  int32_t skip_physical_cols = 0;
1018  for (size_t r = 0; r < numRows; ++r) {
1019  ColumnDescriptor* cd = new ColumnDescriptor();
1020  cd->tableId = sqliteConnector_.getData<int>(r, 0);
1021  cd->columnId = sqliteConnector_.getData<int>(r, 1);
1022  cd->columnName = sqliteConnector_.getData<string>(r, 2);
1026  cd->columnType.set_scale(sqliteConnector_.getData<int>(r, 6));
1030  cd->columnType.set_size(sqliteConnector_.getData<int>(r, 10));
1031  cd->chunks = sqliteConnector_.getData<string>(r, 11);
1032  cd->isSystemCol = sqliteConnector_.getData<bool>(r, 12);
1033  cd->isVirtualCol = sqliteConnector_.getData<bool>(r, 13);
1034  cd->virtualExpr = sqliteConnector_.getData<string>(r, 14);
1035  cd->isDeletedCol = sqliteConnector_.getData<bool>(r, 15);
1036  cd->isGeoPhyCol = skip_physical_cols > 0;
1037  ColumnKey columnKey(cd->tableId, to_upper(cd->columnName));
1038  columnDescriptorMap_[columnKey] = cd;
1039  ColumnIdKey columnIdKey(cd->tableId, cd->columnId);
1040  columnDescriptorMapById_[columnIdKey] = cd;
1041 
1042  if (skip_physical_cols <= 0) {
1043  skip_physical_cols = cd->columnType.get_physical_cols();
1044  }
1045 
1046  auto td_itr = tableDescriptorMapById_.find(cd->tableId);
1047  CHECK(td_itr != tableDescriptorMapById_.end());
1048 
1049  if (cd->isDeletedCol) {
1050  td_itr->second->hasDeletedCol = true;
1051  setDeletedColumnUnlocked(td_itr->second, cd);
1052  } else if (cd->columnType.is_geometry() || skip_physical_cols-- <= 0) {
1053  tableDescriptorMapById_[cd->tableId]->columnIdBySpi_.push_back(cd->columnId);
1054  }
1055  }
1056  // sort columnIdBySpi_ based on columnId
1057  for (auto& tit : tableDescriptorMapById_) {
1058  std::sort(tit.second->columnIdBySpi_.begin(),
1059  tit.second->columnIdBySpi_.end(),
1060  [](const size_t a, const size_t b) -> bool { return a < b; });
1061  }
1062 
1063  string viewQuery("SELECT tableid, sql FROM mapd_views");
1064  sqliteConnector_.query(viewQuery);
1065  numRows = sqliteConnector_.getNumRows();
1066  for (size_t r = 0; r < numRows; ++r) {
1067  int32_t tableId = sqliteConnector_.getData<int>(r, 0);
1068  TableDescriptor* td = tableDescriptorMapById_[tableId];
1069  td->viewSQL = sqliteConnector_.getData<string>(r, 1);
1070  td->fragmenter = nullptr;
1071  }
1072 
1073  string frontendViewQuery(
1074  "SELECT id, state, name, image_hash, strftime('%Y-%m-%dT%H:%M:%SZ', update_time), "
1075  "userid, "
1076  "metadata "
1077  "FROM mapd_dashboards");
1078  sqliteConnector_.query(frontendViewQuery);
1079  numRows = sqliteConnector_.getNumRows();
1080  for (size_t r = 0; r < numRows; ++r) {
1081  std::shared_ptr<DashboardDescriptor> vd = std::make_shared<DashboardDescriptor>();
1082  vd->dashboardId = sqliteConnector_.getData<int>(r, 0);
1083  vd->dashboardState = sqliteConnector_.getData<string>(r, 1);
1084  vd->dashboardName = sqliteConnector_.getData<string>(r, 2);
1085  vd->imageHash = sqliteConnector_.getData<string>(r, 3);
1086  vd->updateTime = sqliteConnector_.getData<string>(r, 4);
1087  vd->userId = sqliteConnector_.getData<int>(r, 5);
1088  vd->dashboardMetadata = sqliteConnector_.getData<string>(r, 6);
1089  vd->user = getUserFromId(vd->userId);
1090  vd->dashboardSystemRoleName = generate_dashboard_system_rolename(
1092  dashboardDescriptorMap_[std::to_string(vd->userId) + ":" + vd->dashboardName] = vd;
1093  }
1094 
1095  string linkQuery(
1096  "SELECT linkid, userid, link, view_state, strftime('%Y-%m-%dT%H:%M:%SZ', "
1097  "update_time), view_metadata "
1098  "FROM mapd_links");
1099  sqliteConnector_.query(linkQuery);
1100  numRows = sqliteConnector_.getNumRows();
1101  for (size_t r = 0; r < numRows; ++r) {
1102  LinkDescriptor* ld = new LinkDescriptor();
1103  ld->linkId = sqliteConnector_.getData<int>(r, 0);
1104  ld->userId = sqliteConnector_.getData<int>(r, 1);
1105  ld->link = sqliteConnector_.getData<string>(r, 2);
1106  ld->viewState = sqliteConnector_.getData<string>(r, 3);
1107  ld->updateTime = sqliteConnector_.getData<string>(r, 4);
1108  ld->viewMetadata = sqliteConnector_.getData<string>(r, 5);
1110  linkDescriptorMapById_[ld->linkId] = ld;
1111  }
1112 
1113  /* rebuild map linking logical tables to corresponding physical ones */
1114  string logicalToPhysicalTableMapQuery(
1115  "SELECT logical_table_id, physical_table_id "
1116  "FROM mapd_logical_to_physical");
1117  sqliteConnector_.query(logicalToPhysicalTableMapQuery);
1118  numRows = sqliteConnector_.getNumRows();
1119  for (size_t r = 0; r < numRows; ++r) {
1120  int32_t logical_tb_id = sqliteConnector_.getData<int>(r, 0);
1121  int32_t physical_tb_id = sqliteConnector_.getData<int>(r, 1);
1122  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
1123  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
1124  /* add new entity to the map logicalToPhysicalTableMapById_ */
1125  std::vector<int32_t> physicalTables;
1126  physicalTables.push_back(physical_tb_id);
1127  const auto it_ok =
1128  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
1129  CHECK(it_ok.second);
1130  } else {
1131  /* update map logicalToPhysicalTableMapById_ */
1132  physicalTableIt->second.push_back(physical_tb_id);
1133  }
1134  }
1135 }
1136 
1138  const list<ColumnDescriptor>& columns,
1139  const list<DictDescriptor>& dicts) {
1140  cat_write_lock write_lock(this);
1141  TableDescriptor* new_td;
1142 
1143  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
1144  if (foreign_table) {
1145  auto new_foreign_table = new foreign_storage::ForeignTable();
1146  *new_foreign_table = *foreign_table;
1147  new_td = new_foreign_table;
1148  } else {
1149  new_td = new TableDescriptor();
1150  *new_td = *td;
1151  }
1152 
1153  new_td->mutex_ = std::make_shared<std::mutex>();
1154  tableDescriptorMap_[to_upper(td->tableName)] = new_td;
1155  tableDescriptorMapById_[td->tableId] = new_td;
1156  for (auto cd : columns) {
1157  ColumnDescriptor* new_cd = new ColumnDescriptor();
1158  *new_cd = cd;
1159  ColumnKey columnKey(new_cd->tableId, to_upper(new_cd->columnName));
1160  columnDescriptorMap_[columnKey] = new_cd;
1161  ColumnIdKey columnIdKey(new_cd->tableId, new_cd->columnId);
1162  columnDescriptorMapById_[columnIdKey] = new_cd;
1163 
1164  // Add deleted column to the map
1165  if (cd.isDeletedCol) {
1166  CHECK(new_td->hasDeletedCol);
1167  setDeletedColumnUnlocked(new_td, new_cd);
1168  }
1169  }
1170 
1171  std::sort(new_td->columnIdBySpi_.begin(),
1172  new_td->columnIdBySpi_.end(),
1173  [](const size_t a, const size_t b) -> bool { return a < b; });
1174 
1175  std::unique_ptr<StringDictionaryClient> client;
1176  DictRef dict_ref(currentDB_.dbId, -1);
1177  if (!string_dict_hosts_.empty()) {
1178  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1179  }
1180  for (auto dd : dicts) {
1181  if (!dd.dictRef.dictId) {
1182  // Dummy entry created for a shard of a logical table, nothing to do.
1183  continue;
1184  }
1185  dict_ref.dictId = dd.dictRef.dictId;
1186  if (client) {
1187  client->create(dict_ref, dd.dictIsTemp);
1188  }
1189  DictDescriptor* new_dd = new DictDescriptor(dd);
1190  dictDescriptorMapByRef_[dict_ref].reset(new_dd);
1191  if (!dd.dictIsTemp) {
1192  boost::filesystem::create_directory(new_dd->dictFolderPath);
1193  }
1194  }
1195 }
1196 
1197 void Catalog::removeTableFromMap(const string& tableName,
1198  const int tableId,
1199  const bool is_on_error) {
1200  cat_write_lock write_lock(this);
1201  TableDescriptorMapById::iterator tableDescIt = tableDescriptorMapById_.find(tableId);
1202  if (tableDescIt == tableDescriptorMapById_.end()) {
1203  throw runtime_error("Table " + tableName + " does not exist.");
1204  }
1205 
1206  TableDescriptor* td = tableDescIt->second;
1207 
1208  if (td->hasDeletedCol) {
1209  const auto ret = deletedColumnPerTable_.erase(td);
1210  CHECK_EQ(ret, size_t(1));
1211  }
1212 
1213  tableDescriptorMapById_.erase(tableDescIt);
1214  tableDescriptorMap_.erase(to_upper(tableName));
1215  td->fragmenter = nullptr;
1216 
1218  delete td;
1219 
1220  std::unique_ptr<StringDictionaryClient> client;
1221  if (SysCatalog::instance().isAggregator()) {
1222  CHECK(!string_dict_hosts_.empty());
1223  DictRef dict_ref(currentDB_.dbId, -1);
1224  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1225  }
1226 
1227  // delete all column descriptors for the table
1228  // no more link columnIds to sequential indexes!
1229  for (auto cit = columnDescriptorMapById_.begin();
1230  cit != columnDescriptorMapById_.end();) {
1231  if (tableId != std::get<0>(cit->first)) {
1232  ++cit;
1233  } else {
1234  int i = std::get<1>(cit++->first);
1235  ColumnIdKey cidKey(tableId, i);
1236  ColumnDescriptorMapById::iterator colDescIt = columnDescriptorMapById_.find(cidKey);
1237  ColumnDescriptor* cd = colDescIt->second;
1238  columnDescriptorMapById_.erase(colDescIt);
1239  ColumnKey cnameKey(tableId, to_upper(cd->columnName));
1240  columnDescriptorMap_.erase(cnameKey);
1241  const int dictId = cd->columnType.get_comp_param();
1242  // Dummy dictionaries created for a shard of a logical table have the id set to
1243  // zero.
1244  if (cd->columnType.get_compression() == kENCODING_DICT && dictId) {
1245  INJECT_TIMER(removingDicts);
1246  DictRef dict_ref(currentDB_.dbId, dictId);
1247  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
1248  // If we're removing this table due to an error, it is possible that the string
1249  // dictionary reference was never populated. Don't crash, just continue cleaning
1250  // up the TableDescriptor and ColumnDescriptors
1251  if (!is_on_error) {
1252  CHECK(dictIt != dictDescriptorMapByRef_.end());
1253  } else {
1254  if (dictIt == dictDescriptorMapByRef_.end()) {
1255  continue;
1256  }
1257  }
1258  const auto& dd = dictIt->second;
1259  CHECK_GE(dd->refcount, 1);
1260  --dd->refcount;
1261  if (!dd->refcount) {
1262  dd->stringDict.reset();
1263  if (!isTemp) {
1264  File_Namespace::renameForDelete(dd->dictFolderPath);
1265  }
1266  if (client) {
1267  client->drop(dict_ref);
1268  }
1269  dictDescriptorMapByRef_.erase(dictIt);
1270  }
1271  }
1272 
1273  delete cd;
1274  }
1275  }
1276 }
1277 
1279  cat_write_lock write_lock(this);
1281 }
1282 
1284  cat_write_lock write_lock(this);
1286  std::make_shared<DashboardDescriptor>(vd);
1287 }
1288 
1289 std::vector<DBObject> Catalog::parseDashboardObjects(const std::string& view_meta,
1290  const int& user_id) {
1291  std::vector<DBObject> objects;
1292  DBObjectKey key;
1293  key.dbId = currentDB_.dbId;
1294  auto _key_place = [&key](auto type, auto id) {
1295  key.permissionType = type;
1296  key.objectId = id;
1297  return key;
1298  };
1299  for (auto object_name : parse_underlying_dashboard_objects(view_meta)) {
1300  auto td = getMetadataForTable(object_name);
1301  if (!td) {
1302  // Parsed object source is not present in current database
1303  // LOG the info and ignore
1304  LOG(INFO) << "Ignoring dashboard source Table/View: " << object_name
1305  << " no longer exists in current DB.";
1306  continue;
1307  }
1308  // Dashboard source can be Table or View
1309  const auto object_type = td->isView ? ViewDBObjectType : TableDBObjectType;
1310  const auto priv = td->isView ? AccessPrivileges::SELECT_FROM_VIEW
1312  objects.emplace_back(_key_place(object_type, td->tableId), priv, user_id);
1313  objects.back().setObjectType(td->isView ? ViewDBObjectType : TableDBObjectType);
1314  objects.back().setName(td->tableName);
1315  }
1316  return objects;
1317 }
1318 
1319 void Catalog::createOrUpdateDashboardSystemRole(const std::string& view_meta,
1320  const int32_t& user_id,
1321  const std::string& dash_role_name) {
1322  auto objects = parseDashboardObjects(view_meta, user_id);
1323  Role* rl = SysCatalog::instance().getRoleGrantee(dash_role_name);
1324  if (!rl) {
1325  // Dashboard role does not exist
1326  // create role and grant privileges
1327  // NOTE(wamsi): Transactionally unsafe
1328  SysCatalog::instance().createRole(dash_role_name, false);
1329  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1330  } else {
1331  // Dashboard system role already exists
1332  // Add/remove privileges on objects
1333  auto ex_objects = rl->getDbObjects(true);
1334  for (auto key : *ex_objects | boost::adaptors::map_keys) {
1335  if (key.permissionType != TableDBObjectType &&
1336  key.permissionType != ViewDBObjectType) {
1337  continue;
1338  }
1339  bool found = false;
1340  for (auto obj : objects) {
1341  found = key == obj.getObjectKey() ? true : false;
1342  if (found) {
1343  break;
1344  }
1345  }
1346  if (!found) {
1347  // revoke privs on object since the object is no
1348  // longer used by the dashboard as source
1349  // NOTE(wamsi): Transactionally unsafe
1351  dash_role_name, *rl->findDbObject(key, true), *this);
1352  }
1353  }
1354  // Update privileges on remaining objects
1355  // NOTE(wamsi): Transactionally unsafe
1356  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1357  }
1358 }
1359 
1361  cat_write_lock write_lock(this);
1362  LinkDescriptor* new_ld = new LinkDescriptor();
1363  *new_ld = ld;
1365  linkDescriptorMapById_[ld.linkId] = new_ld;
1366 }
1367 
1369  auto time_ms = measure<>::execution([&]() {
1370  // instanciate table fragmenter upon first use
1371  // assume only insert order fragmenter is supported
1373  vector<Chunk> chunkVec;
1374  list<const ColumnDescriptor*> columnDescs;
1375  getAllColumnMetadataForTableImpl(td, columnDescs, true, false, true);
1376  Chunk::translateColumnDescriptorsToChunkVec(columnDescs, chunkVec);
1377  ChunkKey chunkKeyPrefix = {currentDB_.dbId, td->tableId};
1378  if (td->sortedColumnId > 0) {
1379  td->fragmenter = std::make_shared<SortedOrderFragmenter>(chunkKeyPrefix,
1380  chunkVec,
1381  dataMgr_.get(),
1382  const_cast<Catalog*>(this),
1383  td->tableId,
1384  td->shard,
1385  td->maxFragRows,
1386  td->maxChunkSize,
1387  td->fragPageSize,
1388  td->maxRows,
1389  td->persistenceLevel);
1390  } else {
1391  td->fragmenter = std::make_shared<InsertOrderFragmenter>(chunkKeyPrefix,
1392  chunkVec,
1393  dataMgr_.get(),
1394  const_cast<Catalog*>(this),
1395  td->tableId,
1396  td->shard,
1397  td->maxFragRows,
1398  td->maxChunkSize,
1399  td->fragPageSize,
1400  td->maxRows,
1401  td->persistenceLevel,
1402  !td->storageType.empty());
1403  }
1404  });
1405  LOG(INFO) << "Instantiating Fragmenter for table " << td->tableName << " took "
1406  << time_ms << "ms";
1407 }
1408 
1409 const TableDescriptor* Catalog::getMetadataForTable(const string& tableName,
1410  const bool populateFragmenter) const {
1411  // we give option not to populate fragmenter (default true/yes) as it can be heavy for
1412  // pure metadata calls
1413  cat_read_lock read_lock(this);
1414  auto tableDescIt = tableDescriptorMap_.find(to_upper(tableName));
1415  if (tableDescIt == tableDescriptorMap_.end()) { // check to make sure table exists
1416  return nullptr;
1417  }
1418  TableDescriptor* td = tableDescIt->second;
1419  std::unique_lock<std::mutex> td_lock(*td->mutex_.get());
1420  if (populateFragmenter && td->fragmenter == nullptr && !td->isView) {
1422  }
1423  return td; // returns pointer to table descriptor
1424 }
1425 
1427  int tableId,
1428  const bool populateFragmenter) const {
1429  auto tableDescIt = tableDescriptorMapById_.find(tableId);
1430  if (tableDescIt == tableDescriptorMapById_.end()) { // check to make sure table exists
1431  return nullptr;
1432  }
1433  TableDescriptor* td = tableDescIt->second;
1434  if (populateFragmenter) {
1435  std::unique_lock<std::mutex> td_lock(*td->mutex_.get());
1436  if (td->fragmenter == nullptr && !td->isView) {
1438  }
1439  }
1440  return td; // returns pointer to table descriptor
1441 }
1442 
1444  bool populateFragmenter) const {
1445  cat_read_lock read_lock(this);
1446  return getMetadataForTableImpl(tableId, populateFragmenter);
1447 }
1448 
1450  const bool load_dict) const {
1451  cat_read_lock read_lock(this);
1452  return getMetadataForDictUnlocked(dict_id, load_dict);
1453 }
1454 
1456  const bool loadDict) const {
1457  const DictRef dictRef(currentDB_.dbId, dictId);
1458  auto dictDescIt = dictDescriptorMapByRef_.find(dictRef);
1459  if (dictDescIt ==
1460  dictDescriptorMapByRef_.end()) { // check to make sure dictionary exists
1461  return nullptr;
1462  }
1463  auto& dd = dictDescIt->second;
1464 
1465  if (loadDict) {
1466  std::lock_guard string_dict_lock(*dd->string_dict_mutex);
1467  if (!dd->stringDict) {
1468  auto time_ms = measure<>::execution([&]() {
1469  if (string_dict_hosts_.empty()) {
1470  if (dd->dictIsTemp) {
1471  dd->stringDict = std::make_shared<StringDictionary>(
1472  dd->dictFolderPath, true, true, g_cache_string_hash);
1473  } else {
1474  dd->stringDict = std::make_shared<StringDictionary>(
1475  dd->dictFolderPath, false, true, g_cache_string_hash);
1476  }
1477  } else {
1478  dd->stringDict =
1479  std::make_shared<StringDictionary>(string_dict_hosts_.front(), dd->dictRef);
1480  }
1481  });
1482  LOG(INFO) << "Time to load Dictionary " << dd->dictRef.dbId << "_"
1483  << dd->dictRef.dictId << " was " << time_ms << "ms";
1484  }
1485  }
1486 
1487  return dd.get();
1488 }
1489 
1490 const std::vector<LeafHostInfo>& Catalog::getStringDictionaryHosts() const {
1491  return string_dict_hosts_;
1492 }
1493 
1495  const string& columnName) const {
1496  cat_read_lock read_lock(this);
1497 
1498  ColumnKey columnKey(tableId, to_upper(columnName));
1499  auto colDescIt = columnDescriptorMap_.find(columnKey);
1500  if (colDescIt ==
1501  columnDescriptorMap_.end()) { // need to check to make sure column exists for table
1502  return nullptr;
1503  }
1504  return colDescIt->second;
1505 }
1506 
1507 const ColumnDescriptor* Catalog::getMetadataForColumn(int table_id, int column_id) const {
1508  cat_read_lock read_lock(this);
1509  return getMetadataForColumnUnlocked(table_id, column_id);
1510 }
1511 
1513  int columnId) const {
1514  ColumnIdKey columnIdKey(tableId, columnId);
1515  auto colDescIt = columnDescriptorMapById_.find(columnIdKey);
1516  if (colDescIt == columnDescriptorMapById_
1517  .end()) { // need to check to make sure column exists for table
1518  return nullptr;
1519  }
1520  return colDescIt->second;
1521 }
1522 
1523 const int Catalog::getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const {
1524  const auto tabDescIt = tableDescriptorMapById_.find(table_id);
1525  CHECK(tableDescriptorMapById_.end() != tabDescIt);
1526  const auto& columnIdBySpi = tabDescIt->second->columnIdBySpi_;
1527 
1528  auto spx = spi;
1529  int phi = 0;
1530  if (spx >= SPIMAP_MAGIC1) // see Catalog.h
1531  {
1532  phi = (spx - SPIMAP_MAGIC1) % SPIMAP_MAGIC2;
1533  spx = (spx - SPIMAP_MAGIC1) / SPIMAP_MAGIC2;
1534  }
1535 
1536  CHECK(0 < spx && spx <= columnIdBySpi.size());
1537  return columnIdBySpi[spx - 1] + phi;
1538 }
1539 
1540 const int Catalog::getColumnIdBySpi(const int table_id, const size_t spi) const {
1541  cat_read_lock read_lock(this);
1542  return getColumnIdBySpiUnlocked(table_id, spi);
1543 }
1544 
1546  const size_t spi) const {
1547  cat_read_lock read_lock(this);
1548 
1549  const auto columnId = getColumnIdBySpiUnlocked(tableId, spi);
1550  ColumnIdKey columnIdKey(tableId, columnId);
1551  const auto colDescIt = columnDescriptorMapById_.find(columnIdKey);
1552  return columnDescriptorMapById_.end() == colDescIt ? nullptr : colDescIt->second;
1553 }
1554 
1555 void Catalog::deleteMetadataForDashboard(const std::string& userId,
1556  const std::string& dashName) {
1557  cat_write_lock write_lock(this);
1558 
1559  auto viewDescIt = dashboardDescriptorMap_.find(userId + ":" + dashName);
1560  if (viewDescIt == dashboardDescriptorMap_.end()) { // check to make sure view exists
1561  LOG(ERROR) << "No metadata for dashboard for user " << userId << " dashboard "
1562  << dashName << " does not exist in map";
1563  throw runtime_error("No metadata for dashboard for user " + userId + " dashboard " +
1564  dashName + " does not exist in map");
1565  }
1566  // found view in Map now remove it
1567  dashboardDescriptorMap_.erase(viewDescIt);
1568  // remove from DB
1570  sqliteConnector_.query("BEGIN TRANSACTION");
1571  try {
1573  "DELETE FROM mapd_dashboards WHERE name = ? and userid = ?",
1574  std::vector<std::string>{dashName, userId});
1575  } catch (std::exception& e) {
1576  sqliteConnector_.query("ROLLBACK TRANSACTION");
1577  throw;
1578  }
1579  sqliteConnector_.query("END TRANSACTION");
1580 }
1581 
1583  const string& userId,
1584  const string& dashName) const {
1585  cat_read_lock read_lock(this);
1586 
1587  auto viewDescIt = dashboardDescriptorMap_.find(userId + ":" + dashName);
1588  if (viewDescIt == dashboardDescriptorMap_.end()) { // check to make sure view exists
1589  return nullptr;
1590  }
1591  return viewDescIt->second.get(); // returns pointer to view descriptor
1592 }
1593 
1595  cat_read_lock read_lock(this);
1596  std::string userId;
1597  std::string name;
1598  bool found{false};
1599  {
1600  for (auto descp : dashboardDescriptorMap_) {
1601  auto dash = descp.second.get();
1602  if (dash->dashboardId == id) {
1603  userId = std::to_string(dash->userId);
1604  name = dash->dashboardName;
1605  found = true;
1606  break;
1607  }
1608  }
1609  }
1610  if (found) {
1611  return getMetadataForDashboard(userId, name);
1612  }
1613  return nullptr;
1614 }
1615 
1616 void Catalog::deleteMetadataForDashboard(const int32_t id) {
1617  std::string userId;
1618  std::string name;
1619  bool found{false};
1620  {
1621  cat_read_lock read_lock(this);
1622  for (auto descp : dashboardDescriptorMap_) {
1623  auto dash = descp.second.get();
1624  if (dash->dashboardId == id) {
1625  userId = std::to_string(dash->userId);
1626  name = dash->dashboardName;
1627  found = true;
1628  break;
1629  }
1630  }
1631  }
1632  if (found) {
1633  // TODO: transactionally unsafe
1635  DBObject(id, DashboardDBObjectType), this);
1636  deleteMetadataForDashboard(userId, name);
1637  }
1638 }
1639 
1640 const LinkDescriptor* Catalog::getMetadataForLink(const string& link) const {
1641  cat_read_lock read_lock(this);
1642  auto linkDescIt = linkDescriptorMap_.find(link);
1643  if (linkDescIt == linkDescriptorMap_.end()) { // check to make sure view exists
1644  return nullptr;
1645  }
1646  return linkDescIt->second; // returns pointer to view descriptor
1647 }
1648 
1650  cat_read_lock read_lock(this);
1651  auto linkDescIt = linkDescriptorMapById_.find(linkId);
1652  if (linkDescIt == linkDescriptorMapById_.end()) { // check to make sure view exists
1653  return nullptr;
1654  }
1655  return linkDescIt->second;
1656 }
1657 
1659  const TableDescriptor* td,
1660  list<const ColumnDescriptor*>& columnDescriptors,
1661  const bool fetchSystemColumns,
1662  const bool fetchVirtualColumns,
1663  const bool fetchPhysicalColumns) const {
1664  int32_t skip_physical_cols = 0;
1665  for (const auto& columnDescriptor : columnDescriptorMapById_) {
1666  if (!fetchPhysicalColumns && skip_physical_cols > 0) {
1667  --skip_physical_cols;
1668  continue;
1669  }
1670  auto cd = columnDescriptor.second;
1671  if (cd->tableId != td->tableId) {
1672  continue;
1673  }
1674  if (!fetchSystemColumns && cd->isSystemCol) {
1675  continue;
1676  }
1677  if (!fetchVirtualColumns && cd->isVirtualCol) {
1678  continue;
1679  }
1680  if (!fetchPhysicalColumns) {
1681  const auto& col_ti = cd->columnType;
1682  skip_physical_cols = col_ti.get_physical_cols();
1683  }
1684  columnDescriptors.push_back(cd);
1685  }
1686 }
1687 
1688 std::list<const ColumnDescriptor*> Catalog::getAllColumnMetadataForTable(
1689  const int tableId,
1690  const bool fetchSystemColumns,
1691  const bool fetchVirtualColumns,
1692  const bool fetchPhysicalColumns) const {
1693  cat_read_lock read_lock(this);
1695  tableId, fetchSystemColumns, fetchVirtualColumns, fetchPhysicalColumns);
1696 }
1697 
1698 std::list<const ColumnDescriptor*> Catalog::getAllColumnMetadataForTableUnlocked(
1699  const int tableId,
1700  const bool fetchSystemColumns,
1701  const bool fetchVirtualColumns,
1702  const bool fetchPhysicalColumns) const {
1703  std::list<const ColumnDescriptor*> columnDescriptors;
1704  const TableDescriptor* td =
1705  getMetadataForTableImpl(tableId, false); // dont instantiate fragmenter
1707  columnDescriptors,
1708  fetchSystemColumns,
1709  fetchVirtualColumns,
1710  fetchPhysicalColumns);
1711  return columnDescriptors;
1712 }
1713 
1714 list<const TableDescriptor*> Catalog::getAllTableMetadata() const {
1715  cat_read_lock read_lock(this);
1716  list<const TableDescriptor*> table_list;
1717  for (auto p : tableDescriptorMapById_) {
1718  table_list.push_back(p.second);
1719  }
1720  return table_list;
1721 }
1722 
1723 list<const DashboardDescriptor*> Catalog::getAllDashboardsMetadata() const {
1724  list<const DashboardDescriptor*> view_list;
1725  for (auto p : dashboardDescriptorMap_) {
1726  view_list.push_back(p.second.get());
1727  }
1728  return view_list;
1729 }
1730 
1732  const auto& td = *tableDescriptorMapById_[cd.tableId];
1733  list<DictDescriptor> dds;
1734  setColumnDictionary(cd, dds, td, true);
1735  auto& dd = dds.back();
1736  CHECK(dd.dictRef.dictId);
1737 
1738  std::unique_ptr<StringDictionaryClient> client;
1739  if (!string_dict_hosts_.empty()) {
1740  client.reset(new StringDictionaryClient(
1741  string_dict_hosts_.front(), DictRef(currentDB_.dbId, -1), true));
1742  }
1743  if (client) {
1744  client->create(dd.dictRef, dd.dictIsTemp);
1745  }
1746 
1747  DictDescriptor* new_dd = new DictDescriptor(dd);
1748  dictDescriptorMapByRef_[dd.dictRef].reset(new_dd);
1749  if (!dd.dictIsTemp) {
1750  boost::filesystem::create_directory(new_dd->dictFolderPath);
1751  }
1752  return dd.dictRef;
1753 }
1754 
1756  cat_write_lock write_lock(this);
1758  if (!(cd.columnType.is_string() || cd.columnType.is_string_array())) {
1759  return;
1760  }
1761  if (!(cd.columnType.get_compression() == kENCODING_DICT)) {
1762  return;
1763  }
1764  const auto dictId = cd.columnType.get_comp_param();
1765  CHECK_GT(dictId, 0);
1766  // decrement and zero check dict ref count
1767  const auto td = getMetadataForTable(cd.tableId);
1768  CHECK(td);
1770  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
1771  std::to_string(dictId));
1773  "SELECT refcount FROM mapd_dictionaries WHERE dictid = ?", std::to_string(dictId));
1774  const auto refcount = sqliteConnector_.getData<int>(0, 0);
1775  VLOG(3) << "Dictionary " << dictId << "from dropped table has reference count "
1776  << refcount;
1777  if (refcount > 0) {
1778  return;
1779  }
1780  const DictRef dictRef(currentDB_.dbId, dictId);
1781  sqliteConnector_.query_with_text_param("DELETE FROM mapd_dictionaries WHERE dictid = ?",
1782  std::to_string(dictId));
1783  File_Namespace::renameForDelete(basePath_ + "/mapd_data/DB_" +
1784  std::to_string(currentDB_.dbId) + "_DICT_" +
1785  std::to_string(dictId));
1786 
1787  std::unique_ptr<StringDictionaryClient> client;
1788  if (!string_dict_hosts_.empty()) {
1789  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dictRef, true));
1790  }
1791  if (client) {
1792  client->drop(dictRef);
1793  }
1794 
1795  dictDescriptorMapByRef_.erase(dictRef);
1796 }
1797 
1799  std::map<int, StringDictionary*>& stringDicts) {
1800  // learn 'committed' ColumnDescriptor of this column
1801  auto cit = columnDescriptorMap_.find(ColumnKey(cd.tableId, to_upper(cd.columnName)));
1802  CHECK(cit != columnDescriptorMap_.end());
1803  auto& ccd = *cit->second;
1804 
1805  if (!(ccd.columnType.is_string() || ccd.columnType.is_string_array())) {
1806  return;
1807  }
1808  if (!(ccd.columnType.get_compression() == kENCODING_DICT)) {
1809  return;
1810  }
1811  if (!(ccd.columnType.get_comp_param() > 0)) {
1812  return;
1813  }
1814 
1815  auto dictId = ccd.columnType.get_comp_param();
1816  getMetadataForDict(dictId);
1817 
1818  const DictRef dictRef(currentDB_.dbId, dictId);
1819  auto dit = dictDescriptorMapByRef_.find(dictRef);
1820  CHECK(dit != dictDescriptorMapByRef_.end());
1821  CHECK(dit->second);
1822  CHECK(dit->second.get()->stringDict);
1823  stringDicts[ccd.columnId] = dit->second.get()->stringDict.get();
1824 }
1825 
1827  cat_write_lock write_lock(this);
1828  // caller must handle sqlite/chunk transaction TOGETHER
1829  cd.tableId = td.tableId;
1830  if (td.nShards > 0 && td.shard < 0) {
1831  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
1832  auto shard_cd = cd;
1833  addColumn(*shard, shard_cd);
1834  }
1835  }
1837  addDictionary(cd);
1838  }
1839 
1841  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
1842  "colscale, is_notnull, "
1843  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
1844  "is_deletedcol) "
1845  "VALUES (?, "
1846  "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
1847  "?, ?, ?, "
1848  "?, "
1849  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1850  std::vector<std::string>{std::to_string(td.tableId),
1851  std::to_string(td.tableId),
1852  cd.columnName,
1861  "",
1864  cd.virtualExpr,
1866 
1868  "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
1869  std::vector<std::string>{std::to_string(td.tableId)});
1870 
1872  "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
1873  std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
1874  cd.columnId = sqliteConnector_.getData<int>(0, 0);
1875 
1876  ++tableDescriptorMapById_[td.tableId]->nColumns;
1877  auto ncd = new ColumnDescriptor(cd);
1880  columnDescriptorsForRoll.emplace_back(nullptr, ncd);
1881 }
1882 
1884  cat_write_lock write_lock(this);
1886  // caller must handle sqlite/chunk transaction TOGETHER
1888  "DELETE FROM mapd_columns where tableid = ? and columnid = ?",
1889  std::vector<std::string>{std::to_string(td.tableId), std::to_string(cd.columnId)});
1890 
1892  "UPDATE mapd_tables SET ncolumns = ncolumns - 1 WHERE tableid = ?",
1893  std::vector<std::string>{std::to_string(td.tableId)});
1894 
1895  ColumnDescriptorMap::iterator columnDescIt =
1897  CHECK(columnDescIt != columnDescriptorMap_.end());
1898 
1899  columnDescriptorsForRoll.emplace_back(columnDescIt->second, nullptr);
1900 
1901  columnDescriptorMap_.erase(columnDescIt);
1903  --tableDescriptorMapById_[td.tableId]->nColumns;
1904  // for each shard
1905  if (td.nShards > 0 && td.shard < 0) {
1906  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
1907  const auto shard_cd = getMetadataForColumn(shard->tableId, cd.columnId);
1908  CHECK(shard_cd);
1909  dropColumn(*shard, *shard_cd);
1910  }
1911  }
1912 }
1913 
1914 void Catalog::roll(const bool forward) {
1915  cat_write_lock write_lock(this);
1916  std::set<const TableDescriptor*> tds;
1917 
1918  for (const auto& cdr : columnDescriptorsForRoll) {
1919  auto ocd = cdr.first;
1920  auto ncd = cdr.second;
1921  CHECK(ocd || ncd);
1922  auto tabDescIt = tableDescriptorMapById_.find((ncd ? ncd : ocd)->tableId);
1923  CHECK(tableDescriptorMapById_.end() != tabDescIt);
1924  auto td = tabDescIt->second;
1925  auto& vc = td->columnIdBySpi_;
1926  if (forward) {
1927  if (ocd) {
1928  if (nullptr == ncd ||
1929  ncd->columnType.get_comp_param() != ocd->columnType.get_comp_param()) {
1930  delDictionary(*ocd);
1931  }
1932 
1933  vc.erase(std::remove(vc.begin(), vc.end(), ocd->columnId), vc.end());
1934 
1935  delete ocd;
1936  }
1937  if (ncd) {
1938  // append columnId if its new and not phy geo
1939  if (vc.end() == std::find(vc.begin(), vc.end(), ncd->columnId)) {
1940  if (!ncd->isGeoPhyCol) {
1941  vc.push_back(ncd->columnId);
1942  }
1943  }
1944  }
1945  tds.insert(td);
1946  } else {
1947  if (ocd) {
1948  columnDescriptorMap_[ColumnKey(ocd->tableId, to_upper(ocd->columnName))] = ocd;
1949  columnDescriptorMapById_[ColumnIdKey(ocd->tableId, ocd->columnId)] = ocd;
1950  }
1951  // roll back the dict of new column
1952  if (ncd) {
1953  columnDescriptorMap_.erase(ColumnKey(ncd->tableId, to_upper(ncd->columnName)));
1954  columnDescriptorMapById_.erase(ColumnIdKey(ncd->tableId, ncd->columnId));
1955  if (nullptr == ocd ||
1956  ocd->columnType.get_comp_param() != ncd->columnType.get_comp_param()) {
1957  delDictionary(*ncd);
1958  }
1959  delete ncd;
1960  }
1961  }
1962  }
1963  columnDescriptorsForRoll.clear();
1964 
1965  if (forward) {
1966  for (const auto td : tds) {
1967  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
1968  }
1969  }
1970 }
1971 
1973  list<ColumnDescriptor>& columns) {
1974  const auto& col_ti = cd.columnType;
1975  if (IS_GEO(col_ti.get_type())) {
1976  switch (col_ti.get_type()) {
1977  case kPOINT: {
1978  ColumnDescriptor physical_cd_coords(true);
1979  physical_cd_coords.columnName = cd.columnName + "_coords";
1980  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
1981  // Raw data: compressed/uncompressed coords
1982  coords_ti.set_subtype(kTINYINT);
1983  size_t unit_size;
1984  if (col_ti.get_compression() == kENCODING_GEOINT &&
1985  col_ti.get_comp_param() == 32) {
1986  unit_size = 4 * sizeof(int8_t);
1987  } else {
1988  CHECK(col_ti.get_compression() == kENCODING_NONE);
1989  unit_size = 8 * sizeof(int8_t);
1990  }
1991  coords_ti.set_size(2 * unit_size);
1992  physical_cd_coords.columnType = coords_ti;
1993  columns.push_back(physical_cd_coords);
1994 
1995  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1996 
1997  break;
1998  }
1999  case kLINESTRING: {
2000  ColumnDescriptor physical_cd_coords(true);
2001  physical_cd_coords.columnName = cd.columnName + "_coords";
2002  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2003  // Raw data: compressed/uncompressed coords
2004  coords_ti.set_subtype(kTINYINT);
2005  physical_cd_coords.columnType = coords_ti;
2006  columns.push_back(physical_cd_coords);
2007 
2008  ColumnDescriptor physical_cd_bounds(true);
2009  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2010  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2011  bounds_ti.set_subtype(kDOUBLE);
2012  bounds_ti.set_size(4 * sizeof(double));
2013  physical_cd_bounds.columnType = bounds_ti;
2014  columns.push_back(physical_cd_bounds);
2015 
2016  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2017 
2018  break;
2019  }
2020  case kPOLYGON: {
2021  ColumnDescriptor physical_cd_coords(true);
2022  physical_cd_coords.columnName = cd.columnName + "_coords";
2023  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2024  // Raw data: compressed/uncompressed coords
2025  coords_ti.set_subtype(kTINYINT);
2026  physical_cd_coords.columnType = coords_ti;
2027  columns.push_back(physical_cd_coords);
2028 
2029  ColumnDescriptor physical_cd_ring_sizes(true);
2030  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
2031  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2032  ring_sizes_ti.set_subtype(kINT);
2033  physical_cd_ring_sizes.columnType = ring_sizes_ti;
2034  columns.push_back(physical_cd_ring_sizes);
2035 
2036  ColumnDescriptor physical_cd_bounds(true);
2037  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2038  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2039  bounds_ti.set_subtype(kDOUBLE);
2040  bounds_ti.set_size(4 * sizeof(double));
2041  physical_cd_bounds.columnType = bounds_ti;
2042  columns.push_back(physical_cd_bounds);
2043 
2044  ColumnDescriptor physical_cd_render_group(true);
2045  physical_cd_render_group.columnName = cd.columnName + "_render_group";
2046  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, col_ti.get_notnull());
2047  physical_cd_render_group.columnType = render_group_ti;
2048  columns.push_back(physical_cd_render_group);
2049 
2050  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2051 
2052  break;
2053  }
2054  case kMULTIPOLYGON: {
2055  ColumnDescriptor physical_cd_coords(true);
2056  physical_cd_coords.columnName = cd.columnName + "_coords";
2057  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2058  // Raw data: compressed/uncompressed coords
2059  coords_ti.set_subtype(kTINYINT);
2060  physical_cd_coords.columnType = coords_ti;
2061  columns.push_back(physical_cd_coords);
2062 
2063  ColumnDescriptor physical_cd_ring_sizes(true);
2064  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
2065  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2066  ring_sizes_ti.set_subtype(kINT);
2067  physical_cd_ring_sizes.columnType = ring_sizes_ti;
2068  columns.push_back(physical_cd_ring_sizes);
2069 
2070  ColumnDescriptor physical_cd_poly_rings(true);
2071  physical_cd_poly_rings.columnName = cd.columnName + "_poly_rings";
2072  SQLTypeInfo poly_rings_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2073  poly_rings_ti.set_subtype(kINT);
2074  physical_cd_poly_rings.columnType = poly_rings_ti;
2075  columns.push_back(physical_cd_poly_rings);
2076 
2077  ColumnDescriptor physical_cd_bounds(true);
2078  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2079  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2080  bounds_ti.set_subtype(kDOUBLE);
2081  bounds_ti.set_size(4 * sizeof(double));
2082  physical_cd_bounds.columnType = bounds_ti;
2083  columns.push_back(physical_cd_bounds);
2084 
2085  ColumnDescriptor physical_cd_render_group(true);
2086  physical_cd_render_group.columnName = cd.columnName + "_render_group";
2087  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, col_ti.get_notnull());
2088  physical_cd_render_group.columnType = render_group_ti;
2089  columns.push_back(physical_cd_render_group);
2090 
2091  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2092 
2093  break;
2094  }
2095  default:
2096  throw runtime_error("Unrecognized geometry type.");
2097  break;
2098  }
2099  }
2100 }
2101 
2103  TableDescriptor& td,
2104  const list<ColumnDescriptor>& cols,
2105  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs,
2106  bool isLogicalTable) {
2107  cat_write_lock write_lock(this);
2108  list<ColumnDescriptor> cds = cols;
2109  list<DictDescriptor> dds;
2110  std::set<std::string> toplevel_column_names;
2111  list<ColumnDescriptor> columns;
2112 
2113  if (!td.storageType.empty() &&
2116  throw std::runtime_error("Only temporary tables can be backed by foreign storage.");
2117  }
2119  }
2120 
2121  for (auto cd : cds) {
2122  if (cd.columnName == "rowid") {
2123  throw std::runtime_error(
2124  "Cannot create column with name rowid. rowid is a system defined column.");
2125  }
2126  columns.push_back(cd);
2127  toplevel_column_names.insert(cd.columnName);
2128  if (cd.columnType.is_geometry()) {
2129  expandGeoColumn(cd, columns);
2130  }
2131  }
2132  cds.clear();
2133 
2134  ColumnDescriptor cd;
2135  // add row_id column -- Must be last column in the table
2136  cd.columnName = "rowid";
2137  cd.isSystemCol = true;
2138  cd.columnType = SQLTypeInfo(kBIGINT, true);
2139 #ifdef MATERIALIZED_ROWID
2140  cd.isVirtualCol = false;
2141 #else
2142  cd.isVirtualCol = true;
2143  cd.virtualExpr = "MAPD_FRAG_ID * MAPD_ROWS_PER_FRAG + MAPD_FRAG_ROW_ID";
2144 #endif
2145  columns.push_back(cd);
2146  toplevel_column_names.insert(cd.columnName);
2147 
2148  if (td.hasDeletedCol) {
2149  ColumnDescriptor cd_del;
2150  cd_del.columnName = "$deleted$";
2151  cd_del.isSystemCol = true;
2152  cd_del.isVirtualCol = false;
2153  cd_del.columnType = SQLTypeInfo(kBOOLEAN, true);
2154  cd_del.isDeletedCol = true;
2155 
2156  columns.push_back(cd_del);
2157  }
2158 
2159  td.nColumns = columns.size();
2161  sqliteConnector_.query("BEGIN TRANSACTION");
2163  try {
2165  R"(INSERT INTO mapd_tables (name, userid, ncolumns, isview, fragments, frag_type, max_frag_rows, max_chunk_size, frag_page_size, max_rows, partitions, shard_column_id, shard, num_shards, sort_column_id, storage_type, key_metainfo) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?))",
2166  std::vector<std::string>{td.tableName,
2167  std::to_string(td.userId),
2169  std::to_string(td.isView),
2170  "",
2175  std::to_string(td.maxRows),
2176  td.partitions,
2178  std::to_string(td.shard),
2179  std::to_string(td.nShards),
2181  td.storageType,
2182  td.keyMetainfo});
2183 
2184  // now get the auto generated tableid
2186  "SELECT tableid FROM mapd_tables WHERE name = ?", td.tableName);
2187  td.tableId = sqliteConnector_.getData<int>(0, 0);
2188  int colId = 1;
2189  for (auto cd : columns) {
2191  const bool is_foreign_col =
2192  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2193  if (!is_foreign_col) {
2194  setColumnDictionary(cd, dds, td, isLogicalTable);
2195  }
2196  }
2197 
2198  if (toplevel_column_names.count(cd.columnName)) {
2199  // make up colId gap for sanity test (begin with 1 bc much code depends on it!)
2200  if (colId > 1) {
2201  colId += g_test_against_columnId_gap;
2202  }
2203  if (!cd.isGeoPhyCol) {
2204  td.columnIdBySpi_.push_back(colId);
2205  }
2206  }
2207 
2209  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, "
2210  "coldim, colscale, is_notnull, "
2211  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
2212  "virtual_expr, is_deletedcol) "
2213  "VALUES (?, ?, ?, ?, ?, "
2214  "?, "
2215  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2216  std::vector<std::string>{std::to_string(td.tableId),
2217  std::to_string(colId),
2218  cd.columnName,
2227  "",
2230  cd.virtualExpr,
2232  cd.tableId = td.tableId;
2233  cd.columnId = colId++;
2234  cds.push_back(cd);
2235  }
2236  if (td.isView) {
2238  "INSERT INTO mapd_views (tableid, sql) VALUES (?,?)",
2239  std::vector<std::string>{std::to_string(td.tableId), td.viewSQL});
2240  }
2242  auto& foreignTable = dynamic_cast<foreign_storage::ForeignTable&>(td);
2244  "INSERT INTO omnisci_foreign_tables (table_id, server_id, options) VALUES "
2245  "(?, ?, ?)",
2246  std::vector<std::string>{std::to_string(foreignTable.tableId),
2247  std::to_string(foreignTable.foreign_server->id),
2248  foreignTable.getOptionsAsJsonString()});
2249  }
2250  } catch (std::exception& e) {
2251  sqliteConnector_.query("ROLLBACK TRANSACTION");
2252  throw;
2253  }
2254  } else { // Temporary table
2255  td.tableId = nextTempTableId_++;
2256  int colId = 1;
2257  for (auto cd : columns) {
2259  const bool is_foreign_col =
2260  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2261 
2262  if (!is_foreign_col) {
2263  // Create a new temporary dictionary
2264  std::string fileName("");
2265  std::string folderPath("");
2267  nextTempDictId_++;
2268  DictDescriptor dd(dict_ref,
2269  fileName,
2271  false,
2272  1,
2273  folderPath,
2274  true); // Is dictName (2nd argument) used?
2275  dds.push_back(dd);
2276  if (!cd.columnType.is_array()) {
2278  }
2279  cd.columnType.set_comp_param(dict_ref.dictId);
2280  }
2281  }
2282  if (toplevel_column_names.count(cd.columnName)) {
2283  // make up colId gap for sanity test (begin with 1 bc much code depends on it!)
2284  if (colId > 1) {
2285  colId += g_test_against_columnId_gap;
2286  }
2287  if (!cd.isGeoPhyCol) {
2288  td.columnIdBySpi_.push_back(colId);
2289  }
2290  }
2291  cd.tableId = td.tableId;
2292  cd.columnId = colId++;
2293  cds.push_back(cd);
2294  }
2295 
2297  serializeTableJsonUnlocked(&td, cds);
2298  }
2299  }
2300 
2301  try {
2302  addTableToMap(&td, cds, dds);
2303  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
2304  if (!td.storageType.empty() && td.storageType != StorageType::FOREIGN_TABLE) {
2306  }
2307  } catch (std::exception& e) {
2308  sqliteConnector_.query("ROLLBACK TRANSACTION");
2309  removeTableFromMap(td.tableName, td.tableId, true);
2310  throw;
2311  }
2312  sqliteConnector_.query("END TRANSACTION");
2313 }
2314 
2316  const std::list<ColumnDescriptor>& cds) const {
2317  // relies on the catalog write lock
2318  using namespace rapidjson;
2319 
2320  VLOG(1) << "Serializing temporary table " << td->tableName << " to JSON for Calcite.";
2321 
2322  const auto db_name = currentDB_.dbName;
2323  const auto file_path = table_json_filepath(basePath_, db_name);
2324 
2325  Document d;
2326  if (boost::filesystem::exists(file_path)) {
2327  // look for an existing file for this database
2328  std::ifstream reader(file_path.string());
2329  CHECK(reader.is_open());
2330  IStreamWrapper json_read_wrapper(reader);
2331  d.ParseStream(json_read_wrapper);
2332  } else {
2333  d.SetObject();
2334  }
2335  CHECK(d.IsObject());
2336  CHECK(!d.HasMember(StringRef(td->tableName.c_str())));
2337 
2338  Value table(kObjectType);
2339  table.AddMember(
2340  "name", Value().SetString(StringRef(td->tableName.c_str())), d.GetAllocator());
2341  table.AddMember("id", Value().SetInt(td->tableId), d.GetAllocator());
2342  table.AddMember("columns", Value(kArrayType), d.GetAllocator());
2343 
2344  for (const auto& cd : cds) {
2345  Value column(kObjectType);
2346  column.AddMember(
2347  "name", Value().SetString(StringRef(cd.columnName)), d.GetAllocator());
2348  column.AddMember("coltype",
2349  Value().SetInt(static_cast<int>(cd.columnType.get_type())),
2350  d.GetAllocator());
2351  column.AddMember("colsubtype",
2352  Value().SetInt(static_cast<int>(cd.columnType.get_subtype())),
2353  d.GetAllocator());
2354  column.AddMember(
2355  "coldim", Value().SetInt(cd.columnType.get_dimension()), d.GetAllocator());
2356  column.AddMember(
2357  "colscale", Value().SetInt(cd.columnType.get_scale()), d.GetAllocator());
2358  column.AddMember(
2359  "is_notnull", Value().SetBool(cd.columnType.get_notnull()), d.GetAllocator());
2360  column.AddMember("is_systemcol", Value().SetBool(cd.isSystemCol), d.GetAllocator());
2361  column.AddMember("is_virtualcol", Value().SetBool(cd.isVirtualCol), d.GetAllocator());
2362  column.AddMember("is_deletedcol", Value().SetBool(cd.isDeletedCol), d.GetAllocator());
2363  table["columns"].PushBack(column, d.GetAllocator());
2364  }
2365  d.AddMember(StringRef(td->tableName.c_str()), table, d.GetAllocator());
2366 
2367  // Overwrite the existing file
2368  std::ofstream writer(file_path.string(), writer.trunc | writer.out);
2369  CHECK(writer.is_open());
2370  OStreamWrapper json_wrapper(writer);
2371 
2372  Writer<OStreamWrapper> json_writer(json_wrapper);
2373  d.Accept(json_writer);
2374  writer.close();
2375 }
2376 
2377 void Catalog::dropTableFromJsonUnlocked(const std::string& table_name) const {
2378  // relies on the catalog write lock
2379  using namespace rapidjson;
2380 
2381  VLOG(1) << "Dropping temporary table " << table_name << " to JSON for Calcite.";
2382 
2383  const auto db_name = currentDB_.dbName;
2384  const auto file_path = table_json_filepath(basePath_, db_name);
2385 
2386  CHECK(boost::filesystem::exists(file_path));
2387  Document d;
2388 
2389  std::ifstream reader(file_path.string());
2390  CHECK(reader.is_open());
2391  IStreamWrapper json_read_wrapper(reader);
2392  d.ParseStream(json_read_wrapper);
2393 
2394  CHECK(d.IsObject());
2395  auto table_name_ref = StringRef(table_name.c_str());
2396  CHECK(d.HasMember(table_name_ref));
2397  CHECK(d.RemoveMember(table_name_ref));
2398 
2399  // Overwrite the existing file
2400  std::ofstream writer(file_path.string(), writer.trunc | writer.out);
2401  CHECK(writer.is_open());
2402  OStreamWrapper json_wrapper(writer);
2403 
2404  Writer<OStreamWrapper> json_writer(json_wrapper);
2405  d.Accept(json_writer);
2406  writer.close();
2407 }
2408 
2410  std::unique_ptr<foreign_storage::ForeignServer> foreign_server,
2411  bool if_not_exists) {
2412  cat_write_lock write_lock(this);
2414  createForeignServerNoLocks(std::move(foreign_server), if_not_exists);
2415 }
2416 
2418  std::unique_ptr<foreign_storage::ForeignServer> foreign_server,
2419  bool if_not_exists) {
2421  "SELECT name from omnisci_foreign_servers where name = ?",
2422  std::vector<std::string>{foreign_server->name});
2423 
2424  if (sqliteConnector_.getNumRows() == 0) {
2425  foreign_server->creation_time = std::time(nullptr);
2427  "INSERT INTO omnisci_foreign_servers (name, data_wrapper_type, owner_user_id, "
2428  "creation_time, "
2429  "options) "
2430  "VALUES (?, ?, ?, ?, ?)",
2431  std::vector<std::string>{foreign_server->name,
2432  foreign_server->data_wrapper_type,
2433  std::to_string(foreign_server->user_id),
2434  std::to_string(foreign_server->creation_time),
2435  foreign_server->getOptionsAsJsonString()});
2437  "SELECT id from omnisci_foreign_servers where name = ?",
2438  std::vector<std::string>{foreign_server->name});
2439  CHECK_EQ(sqliteConnector_.getNumRows(), size_t(1));
2440  foreign_server->id = sqliteConnector_.getData<int32_t>(0, 0);
2441  } else if (!if_not_exists) {
2442  throw std::runtime_error{"A foreign server with name \"" + foreign_server->name +
2443  "\" already exists."};
2444  }
2445 
2446  std::shared_ptr<foreign_storage::ForeignServer> foreign_server_shared =
2447  std::move(foreign_server);
2448  foreignServerMap_[foreign_server_shared->name] = foreign_server_shared;
2449  foreignServerMapById_[foreign_server_shared->id] = foreign_server_shared;
2450 }
2451 
2453  const std::string& server_name) const {
2454  foreign_storage::ForeignServer* foreign_server = nullptr;
2455  cat_read_lock read_lock(this);
2456  if (foreignServerMap_.find(server_name) != foreignServerMap_.end()) {
2457  foreign_server = foreignServerMap_.find(server_name)->second.get();
2458  }
2459  return foreign_server;
2460 }
2461 
2462 const std::unique_ptr<const foreign_storage::ForeignServer>
2463 Catalog::getForeignServerFromStorage(const std::string& server_name) {
2464  std::unique_ptr<foreign_storage::ForeignServer> foreign_server = nullptr;
2467  "SELECT id, name, data_wrapper_type, options, owner_user_id, creation_time "
2468  "FROM omnisci_foreign_servers WHERE name = ?",
2469  std::vector<std::string>{server_name});
2470  if (sqliteConnector_.getNumRows() > 0) {
2471  foreign_server = std::make_unique<foreign_storage::ForeignServer>(
2472  sqliteConnector_.getData<int>(0, 0),
2473  sqliteConnector_.getData<std::string>(0, 1),
2474  sqliteConnector_.getData<std::string>(0, 2),
2475  sqliteConnector_.getData<std::string>(0, 3),
2476  sqliteConnector_.getData<std::int32_t>(0, 4),
2477  sqliteConnector_.getData<std::int32_t>(0, 5));
2478  }
2479  return foreign_server;
2480 }
2481 
2482 void Catalog::changeForeignServerOwner(const std::string& server_name,
2483  const int new_owner_id) {
2484  cat_write_lock write_lock(this);
2485  foreign_storage::ForeignServer* foreign_server =
2486  foreignServerMap_.find(server_name)->second.get();
2487  CHECK(foreign_server);
2488  setForeignServerProperty(server_name, "owner_user_id", std::to_string(new_owner_id));
2489  // update in-memory server
2490  foreign_server->user_id = new_owner_id;
2491 }
2492 
2493 void Catalog::setForeignServerDataWrapper(const std::string& server_name,
2494  const std::string& data_wrapper) {
2495  cat_write_lock write_lock(this);
2496  auto data_wrapper_type = to_upper(data_wrapper);
2497  // update in-memory server
2498  foreign_storage::ForeignServer* foreign_server =
2499  foreignServerMap_.find(server_name)->second.get();
2500  CHECK(foreign_server);
2501  std::string saved_data_wrapper_type = foreign_server->data_wrapper_type;
2502  foreign_server->data_wrapper_type = data_wrapper_type;
2503  try {
2504  foreign_server->validate();
2505  } catch (const std::exception& e) {
2506  // validation did not succeed:
2507  // revert to saved data_wrapper_type & throw exception
2508  foreign_server->data_wrapper_type = saved_data_wrapper_type;
2509  throw;
2510  }
2511  setForeignServerProperty(server_name, "data_wrapper_type", data_wrapper_type);
2512 }
2513 
2514 void Catalog::setForeignServerOptions(const std::string& server_name,
2515  const std::string& options) {
2516  cat_write_lock write_lock(this);
2517  // update in-memory server
2518  foreign_storage::ForeignServer* foreign_server =
2519  foreignServerMap_.find(server_name)->second.get();
2520  CHECK(foreign_server);
2521  std::string saved_options_string = foreign_server->getOptionsAsJsonString();
2522  foreign_server->populateOptionsMap(options, true);
2523  try {
2524  foreign_server->validate();
2525  } catch (const std::exception& e) {
2526  // validation did not succeed:
2527  // revert to saved options & throw exception
2528  foreign_server->populateOptionsMap(saved_options_string, true);
2529  throw;
2530  }
2531  setForeignServerProperty(server_name, "options", options);
2532 }
2533 
2534 void Catalog::renameForeignServer(const std::string& server_name,
2535  const std::string& name) {
2536  cat_write_lock write_lock(this);
2537  auto foreign_server_it = foreignServerMap_.find(server_name);
2538  CHECK(foreign_server_it != foreignServerMap_.end());
2539  setForeignServerProperty(server_name, "name", name);
2540  auto foreign_server_shared = foreign_server_it->second;
2541  foreign_server_shared->name = name;
2542  foreignServerMap_[name] = foreign_server_shared;
2543  foreignServerMap_.erase(foreign_server_it);
2544 }
2545 
2546 void Catalog::dropForeignServer(const std::string& server_name) {
2547  cat_write_lock write_lock(this);
2549 
2551  "SELECT id from omnisci_foreign_servers where name = ?",
2552  std::vector<std::string>{server_name});
2554  if (num_rows > 0) {
2555  CHECK_EQ(size_t(1), num_rows);
2556  auto server_id = sqliteConnector_.getData<int32_t>(0, 0);
2558  "SELECT table_id from omnisci_foreign_tables where server_id = ?",
2559  std::to_string(server_id));
2560  if (sqliteConnector_.getNumRows() > 0) {
2561  throw std::runtime_error{"Foreign server \"" + server_name +
2562  "\" is referenced "
2563  "by existing foreign tables and cannot be dropped."};
2564  }
2565  sqliteConnector_.query("BEGIN TRANSACTION");
2566  try {
2568  "DELETE FROM omnisci_foreign_servers WHERE name = ?",
2569  std::vector<std::string>{server_name});
2570  } catch (const std::exception& e) {
2571  sqliteConnector_.query("ROLLBACK TRANSACTION");
2572  throw;
2573  }
2574  sqliteConnector_.query("END TRANSACTION");
2575  foreignServerMap_.erase(server_name);
2576  foreignServerMapById_.erase(server_id);
2577  }
2578 }
2579 
2581  const rapidjson::Value* filters,
2582  const UserMetadata& user,
2583  std::vector<const foreign_storage::ForeignServer*>& results) {
2584  sys_read_lock syscat_read_lock(&SysCatalog::instance());
2585  cat_read_lock read_lock(this);
2587  // Customer facing and internal SQlite names
2588  std::map<std::string, std::string> col_names{{"server_name", "name"},
2589  {"data_wrapper", "data_wrapper_type"},
2590  {"created_at", "creation_time"},
2591  {"options", "options"}};
2592 
2593  // TODO add "owner" when FSI privilege is implemented
2594  std::stringstream filter_string;
2595  std::vector<std::string> arguments;
2596 
2597  if (filters != nullptr) {
2598  // Create SQL WHERE clause for SQLite query
2599  int num_filters = 0;
2600  filter_string << " WHERE";
2601  for (auto& filter_def : filters->GetArray()) {
2602  if (num_filters > 0) {
2603  filter_string << " " << std::string(filter_def["chain"].GetString());
2604  ;
2605  }
2606 
2607  if (col_names.find(std::string(filter_def["attribute"].GetString())) ==
2608  col_names.end()) {
2609  throw std::runtime_error{"Attribute with name \"" +
2610  std::string(filter_def["attribute"].GetString()) +
2611  "\" does not exist."};
2612  }
2613 
2614  filter_string << " " << col_names[std::string(filter_def["attribute"].GetString())];
2615 
2616  bool equals_operator = false;
2617  if (std::strcmp(filter_def["operation"].GetString(), "EQUALS") == 0) {
2618  filter_string << " = ? ";
2619  equals_operator = true;
2620  } else {
2621  filter_string << " LIKE ? ";
2622  }
2623 
2624  bool timestamp_column =
2625  (std::strcmp(filter_def["attribute"].GetString(), "created_at") == 0);
2626 
2627  if (timestamp_column && !equals_operator) {
2628  throw std::runtime_error{"LIKE operator is incompatible with TIMESTAMP data"};
2629  }
2630 
2631  if (timestamp_column && equals_operator) {
2632  arguments.push_back(std::to_string(
2633  DateTimeStringValidate<kTIMESTAMP>()(filter_def["value"].GetString(), 0)));
2634  } else {
2635  arguments.push_back(filter_def["value"].GetString());
2636  }
2637 
2638  num_filters++;
2639  }
2640  }
2641  // Create select query for the omnisci_foreign_servers table
2642  std::string query = std::string("SELECT name from omnisci_foreign_servers ");
2643  query += filter_string.str();
2644 
2645  sqliteConnector_.query_with_text_params(query, arguments);
2647 
2648  if (sqliteConnector_.getNumRows() == 0)
2649  return;
2650 
2652  // Return pointers to objects
2653  results.reserve(num_rows);
2654  for (size_t row = 0; row < num_rows; ++row) {
2655  const foreign_storage::ForeignServer* foreign_server =
2656  getForeignServer(sqliteConnector_.getData<std::string>(row, 0));
2657 
2658  CHECK(foreign_server != nullptr);
2659 
2660  DBObject dbObject(foreign_server->name, ServerDBObjectType);
2661  dbObject.loadKey(*this);
2662  std::vector<DBObject> privObjects = {dbObject};
2663  if (!SysCatalog::instance().hasAnyPrivileges(user, privObjects)) {
2664  // skip server, as there are no privileges to access it
2665  continue;
2666  }
2667  results.push_back(foreign_server);
2668  }
2669 }
2670 
2671 // returns the table epoch or -1 if there is something wrong with the shared epoch
2672 int32_t Catalog::getTableEpoch(const int32_t db_id, const int32_t table_id) const {
2673  cat_read_lock read_lock(this);
2674  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(table_id);
2675  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2676  // check all shards have same checkpoint
2677  const auto physicalTables = physicalTableIt->second;
2678  CHECK(!physicalTables.empty());
2679  size_t curr_epoch = 0;
2680  for (size_t i = 0; i < physicalTables.size(); i++) {
2681  int32_t physical_tb_id = physicalTables[i];
2682  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2683  CHECK(phys_td);
2684  if (i == 0) {
2685  curr_epoch = dataMgr_->getTableEpoch(db_id, physical_tb_id);
2686  } else {
2687  if (curr_epoch != dataMgr_->getTableEpoch(db_id, physical_tb_id)) {
2688  // oh dear the leaves do not agree on the epoch for this table
2689  LOG(ERROR) << "Epochs on shards do not all agree on table id " << table_id
2690  << " db id " << db_id << " epoch " << curr_epoch << " leaf_epoch "
2691  << dataMgr_->getTableEpoch(db_id, physical_tb_id);
2692  return -1;
2693  }
2694  }
2695  }
2696  return curr_epoch;
2697  } else {
2698  return dataMgr_->getTableEpoch(db_id, table_id);
2699  }
2700 }
2701 
2702 void Catalog::setTableEpoch(const int db_id, const int table_id, int new_epoch) {
2703  cat_read_lock read_lock(this);
2704  LOG(INFO) << "Set table epoch db:" << db_id << " Table ID " << table_id
2705  << " back to new epoch " << new_epoch;
2706  removeChunks(table_id);
2707  dataMgr_->setTableEpoch(db_id, table_id, new_epoch);
2708 
2709  // check if sharded
2710  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(table_id);
2711  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2712  const auto physicalTables = physicalTableIt->second;
2713  CHECK(!physicalTables.empty());
2714  for (size_t i = 0; i < physicalTables.size(); i++) {
2715  int32_t physical_tb_id = physicalTables[i];
2716  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2717  CHECK(phys_td);
2718  LOG(INFO) << "Set sharded table epoch db:" << db_id << " Table ID "
2719  << physical_tb_id << " back to new epoch " << new_epoch;
2720  removeChunks(physical_tb_id);
2721  dataMgr_->setTableEpoch(db_id, physical_tb_id, new_epoch);
2722  }
2723  }
2724 }
2725 
2727  cat_read_lock read_lock(this);
2728  const auto it = deletedColumnPerTable_.find(td);
2729  return it != deletedColumnPerTable_.end() ? it->second : nullptr;
2730 }
2731 
2733  int delete_column_id) const {
2734  // check if there are rows deleted by examining the deletedColumn metadata
2735  CHECK(td);
2736 
2737  if (table_is_temporary(td)) {
2738  auto fragmenter = td->fragmenter;
2739  CHECK(fragmenter);
2740  return fragmenter->hasDeletedRows(delete_column_id);
2741  } else {
2742  ChunkKey chunk_key_prefix = {currentDB_.dbId, td->tableId, delete_column_id};
2743  ChunkMetadataVector chunk_metadata_vec;
2744  dataMgr_->getChunkMetadataVecForKeyPrefix(chunk_metadata_vec, chunk_key_prefix);
2745  int64_t chunk_max{0};
2746 
2747  for (auto chunk_metadata : chunk_metadata_vec) {
2748  chunk_max = chunk_metadata.second->chunkStats.max.tinyintval;
2749  // delete has occured
2750  if (chunk_max == 1) {
2751  return true;
2752  }
2753  }
2754  return false;
2755  }
2756 }
2757 
2759  const TableDescriptor* td) const {
2760  cat_read_lock read_lock(this);
2761 
2762  const auto it = deletedColumnPerTable_.find(td);
2763  // if not a table that supports delete return nullptr, nothing more to do
2764  if (it == deletedColumnPerTable_.end()) {
2765  return nullptr;
2766  }
2767  const ColumnDescriptor* cd = it->second;
2768 
2769  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
2770 
2771  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2772  // check all shards
2773  const auto physicalTables = physicalTableIt->second;
2774  CHECK(!physicalTables.empty());
2775  for (size_t i = 0; i < physicalTables.size(); i++) {
2776  int32_t physical_tb_id = physicalTables[i];
2777  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2778  CHECK(phys_td);
2779  if (checkMetadataForDeletedRecs(phys_td, cd->columnId)) {
2780  return cd;
2781  }
2782  }
2783  } else {
2784  if (checkMetadataForDeletedRecs(td, cd->columnId)) {
2785  return cd;
2786  }
2787  }
2788  // no deletes so far recorded in metadata
2789  return nullptr;
2790 }
2791 
2793  cat_write_lock write_lock(this);
2794  setDeletedColumnUnlocked(td, cd);
2795 }
2796 
2798  const ColumnDescriptor* cd) {
2799  cat_write_lock write_lock(this);
2800  const auto it_ok = deletedColumnPerTable_.emplace(td, cd);
2801  CHECK(it_ok.second);
2802 }
2803 
2804 namespace {
2805 
2807  const Catalog& cat,
2808  const Parser::SharedDictionaryDef& shared_dict_def) {
2809  const auto& table_name = shared_dict_def.get_foreign_table();
2810  const auto td = cat.getMetadataForTable(table_name);
2811  CHECK(td);
2812  const auto& foreign_col_name = shared_dict_def.get_foreign_column();
2813  return cat.getMetadataForColumn(td->tableId, foreign_col_name);
2814 }
2815 
2816 } // namespace
2817 
2819  Parser::SharedDictionaryDef shared_dict_def,
2820  const bool persist_reference) {
2821  cat_write_lock write_lock(this);
2822  const auto foreign_ref_col = get_foreign_col(*this, shared_dict_def);
2823  CHECK(foreign_ref_col);
2824  referencing_column.columnType = foreign_ref_col->columnType;
2825  const int dict_id = referencing_column.columnType.get_comp_param();
2826  const DictRef dict_ref(currentDB_.dbId, dict_id);
2827  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
2828  CHECK(dictIt != dictDescriptorMapByRef_.end());
2829  const auto& dd = dictIt->second;
2830  CHECK_GE(dd->refcount, 1);
2831  ++dd->refcount;
2832  if (persist_reference) {
2835  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
2836  {std::to_string(dict_id)});
2837  }
2838 }
2839 
2841  ColumnDescriptor& cd,
2842  std::list<ColumnDescriptor>& cdd,
2843  std::list<DictDescriptor>& dds,
2844  const TableDescriptor td,
2845  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs) {
2846  cat_write_lock write_lock(this);
2848 
2849  if (shared_dict_defs.empty()) {
2850  return false;
2851  }
2852  for (const auto& shared_dict_def : shared_dict_defs) {
2853  // check if the current column is a referencing column
2854  const auto& column = shared_dict_def.get_column();
2855  if (cd.columnName == column) {
2856  if (!shared_dict_def.get_foreign_table().compare(td.tableName)) {
2857  // Dictionaries are being shared in table to be created
2858  const auto& ref_column = shared_dict_def.get_foreign_column();
2859  auto colIt =
2860  std::find_if(cdd.begin(), cdd.end(), [ref_column](const ColumnDescriptor it) {
2861  return !ref_column.compare(it.columnName);
2862  });
2863  CHECK(colIt != cdd.end());
2864  cd.columnType = colIt->columnType;
2865 
2866  const int dict_id = colIt->columnType.get_comp_param();
2867  CHECK_GE(dict_id, 1);
2868  auto dictIt = std::find_if(
2869  dds.begin(), dds.end(), [this, dict_id](const DictDescriptor it) {
2870  return it.dictRef.dbId == this->currentDB_.dbId &&
2871  it.dictRef.dictId == dict_id;
2872  });
2873  if (dictIt != dds.end()) {
2874  // There exists dictionary definition of a dictionary column
2875  CHECK_GE(dictIt->refcount, 1);
2876  ++dictIt->refcount;
2877  if (!table_is_temporary(&td)) {
2878  // Persist reference count
2880  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
2881  {std::to_string(dict_id)});
2882  }
2883  } else {
2884  // The dictionary is referencing a column which is referencing a column in
2885  // diffrent table
2886  auto root_dict_def = compress_reference_path(shared_dict_def, shared_dict_defs);
2887  addReferenceToForeignDict(cd, root_dict_def, !table_is_temporary(&td));
2888  }
2889  } else {
2890  const auto& foreign_table_name = shared_dict_def.get_foreign_table();
2891  const auto foreign_td = getMetadataForTable(foreign_table_name, false);
2892  if (table_is_temporary(foreign_td)) {
2893  if (!table_is_temporary(&td)) {
2894  throw std::runtime_error(
2895  "Only temporary tables can share dictionaries with other temporary "
2896  "tables.");
2897  }
2898  addReferenceToForeignDict(cd, shared_dict_def, false);
2899  } else {
2900  addReferenceToForeignDict(cd, shared_dict_def, !table_is_temporary(&td));
2901  }
2902  }
2903  return true;
2904  }
2905  }
2906  return false;
2907 }
2908 
2910  std::list<DictDescriptor>& dds,
2911  const TableDescriptor& td,
2912  const bool isLogicalTable) {
2913  cat_write_lock write_lock(this);
2914 
2915  std::string dictName{"Initial_key"};
2916  int dictId{0};
2917  std::string folderPath;
2918  if (isLogicalTable) {
2920 
2922  "INSERT INTO mapd_dictionaries (name, nbits, is_shared, refcount) VALUES (?, ?, "
2923  "?, 1)",
2924  std::vector<std::string>{
2925  dictName, std::to_string(cd.columnType.get_comp_param()), "0"});
2927  "SELECT dictid FROM mapd_dictionaries WHERE name = ?", dictName);
2928  dictId = sqliteConnector_.getData<int>(0, 0);
2929  dictName = td.tableName + "_" + cd.columnName + "_dict" + std::to_string(dictId);
2931  "UPDATE mapd_dictionaries SET name = ? WHERE name = 'Initial_key'", dictName);
2932  folderPath = basePath_ + "/mapd_data/DB_" + std::to_string(currentDB_.dbId) +
2933  "_DICT_" + std::to_string(dictId);
2934  }
2936  dictId,
2937  dictName,
2939  false,
2940  1,
2941  folderPath,
2942  false);
2943  dds.push_back(dd);
2944  if (!cd.columnType.is_array()) {
2946  }
2947  cd.columnType.set_comp_param(dictId);
2948 }
2949 
2951  TableDescriptor& td,
2952  const list<ColumnDescriptor>& cols,
2953  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs) {
2954  cat_write_lock write_lock(this);
2955 
2956  /* create logical table */
2957  TableDescriptor* tdl = &td;
2958  createTable(*tdl, cols, shared_dict_defs, true); // create logical table
2959  int32_t logical_tb_id = tdl->tableId;
2960 
2961  /* create physical tables and link them to the logical table */
2962  std::vector<int32_t> physicalTables;
2963  for (int32_t i = 1; i <= td.nShards; i++) {
2964  TableDescriptor* tdp = &td;
2966  tdp->shard = i - 1;
2967  createTable(*tdp, cols, shared_dict_defs, false); // create physical table
2968  int32_t physical_tb_id = tdp->tableId;
2969 
2970  /* add physical table to the vector of physical tables */
2971  physicalTables.push_back(physical_tb_id);
2972  }
2973 
2974  if (!physicalTables.empty()) {
2975  /* add logical to physical tables correspondence to the map */
2976  const auto it_ok =
2977  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
2978  CHECK(it_ok.second);
2979  /* update sqlite mapd_logical_to_physical in sqlite database */
2980  if (!table_is_temporary(&td)) {
2981  updateLogicalToPhysicalTableMap(logical_tb_id);
2982  }
2983  }
2984 }
2985 
2987  cat_write_lock write_lock(this);
2988 
2989  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
2990  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2991  // truncate all corresponding physical tables if this is a logical table
2992  const auto physicalTables = physicalTableIt->second;
2993  CHECK(!physicalTables.empty());
2994  for (size_t i = 0; i < physicalTables.size(); i++) {
2995  int32_t physical_tb_id = physicalTables[i];
2996  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2997  CHECK(phys_td);
2998  doTruncateTable(phys_td);
2999  }
3000  }
3001  doTruncateTable(td);
3002 }
3003 
3005  cat_write_lock write_lock(this);
3006 
3007  const int tableId = td->tableId;
3008  // must destroy fragmenter before deleteChunks is called.
3009  if (td->fragmenter != nullptr) {
3010  auto tableDescIt = tableDescriptorMapById_.find(tableId);
3011  CHECK(tableDescIt != tableDescriptorMapById_.end());
3012  tableDescIt->second->fragmenter = nullptr;
3013  CHECK(td->fragmenter == nullptr);
3014  }
3015  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
3016  // assuming deleteChunksWithPrefix is atomic
3017  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
3018  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
3019 
3020  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
3021 
3022  std::unique_ptr<StringDictionaryClient> client;
3023  if (SysCatalog::instance().isAggregator()) {
3024  CHECK(!string_dict_hosts_.empty());
3025  DictRef dict_ref(currentDB_.dbId, -1);
3026  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
3027  }
3028  // clean up any dictionaries
3029  // delete all column descriptors for the table
3030  for (const auto& columnDescriptor : columnDescriptorMapById_) {
3031  auto cd = columnDescriptor.second;
3032  if (cd->tableId != td->tableId) {
3033  continue;
3034  }
3035  const int dict_id = cd->columnType.get_comp_param();
3036  // Dummy dictionaries created for a shard of a logical table have the id set to zero.
3037  if (cd->columnType.get_compression() == kENCODING_DICT && dict_id) {
3038  const DictRef dict_ref(currentDB_.dbId, dict_id);
3039  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
3040  CHECK(dictIt != dictDescriptorMapByRef_.end());
3041  const auto& dd = dictIt->second;
3042  CHECK_GE(dd->refcount, 1);
3043  // if this is the only table using this dict reset the dict
3044  if (dd->refcount == 1) {
3045  // close the dictionary
3046  dd->stringDict.reset();
3047  File_Namespace::renameForDelete(dd->dictFolderPath);
3048  if (client) {
3049  client->drop(dd->dictRef);
3050  }
3051  if (!dd->dictIsTemp) {
3052  boost::filesystem::create_directory(dd->dictFolderPath);
3053  }
3054  }
3055 
3056  DictDescriptor* new_dd = new DictDescriptor(dd->dictRef,
3057  dd->dictName,
3058  dd->dictNBits,
3059  dd->dictIsShared,
3060  dd->refcount,
3061  dd->dictFolderPath,
3062  dd->dictIsTemp);
3063  dictDescriptorMapByRef_.erase(dictIt);
3064  // now create new Dict -- need to figure out what to do here for temp tables
3065  if (client) {
3066  client->create(new_dd->dictRef, new_dd->dictIsTemp);
3067  }
3068  dictDescriptorMapByRef_[new_dd->dictRef].reset(new_dd);
3070  }
3071  }
3072 }
3073 
3074 // used by rollback_table_epoch to clean up in memory artifacts after a rollback
3075 void Catalog::removeChunks(const int table_id) {
3076  auto td = getMetadataForTable(table_id);
3077 
3078  if (td->fragmenter != nullptr) {
3080  if (td->fragmenter != nullptr) {
3081  auto tableDescIt = tableDescriptorMapById_.find(table_id);
3082  CHECK(tableDescIt != tableDescriptorMapById_.end());
3083  tableDescIt->second->fragmenter = nullptr;
3084  CHECK(td->fragmenter == nullptr);
3085  }
3086  }
3087 
3088  // remove the chunks from in memory structures
3089  ChunkKey chunkKey = {currentDB_.dbId, table_id};
3090 
3091  dataMgr_->deleteChunksWithPrefix(chunkKey, MemoryLevel::CPU_LEVEL);
3092  dataMgr_->deleteChunksWithPrefix(chunkKey, MemoryLevel::GPU_LEVEL);
3093 }
3094 
3098  cat_write_lock write_lock(this);
3100  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
3101  sqliteConnector_.query("BEGIN TRANSACTION");
3102  try {
3103  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
3104  // remove all corresponding physical tables if this is a logical table
3105  const auto physicalTables = physicalTableIt->second;
3106  CHECK(!physicalTables.empty());
3107  for (size_t i = 0; i < physicalTables.size(); i++) {
3108  int32_t physical_tb_id = physicalTables[i];
3109  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
3110  CHECK(phys_td);
3111  doDropTable(phys_td);
3112  }
3113 
3114  // remove corresponding record from the logicalToPhysicalTableMap in sqlite database
3116  "DELETE FROM mapd_logical_to_physical WHERE logical_table_id = ?",
3117  std::to_string(td->tableId));
3119  }
3120  doDropTable(td);
3121  } catch (std::exception& e) {
3122  sqliteConnector_.query("ROLLBACK TRANSACTION");
3123  throw;
3124  }
3125  sqliteConnector_.query("END TRANSACTION");
3126 }
3127 
3132  }
3134 }
3135 
3137  const int tableId = td->tableId;
3138  sqliteConnector_.query_with_text_param("DELETE FROM mapd_tables WHERE tableid = ?",
3139  std::to_string(tableId));
3141  "select comp_param from mapd_columns where compression = ? and tableid = ?",
3142  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
3143  int numRows = sqliteConnector_.getNumRows();
3144  std::vector<int> dict_id_list;
3145  for (int r = 0; r < numRows; ++r) {
3146  dict_id_list.push_back(sqliteConnector_.getData<int>(r, 0));
3147  }
3148  for (auto dict_id : dict_id_list) {
3150  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
3151  std::vector<std::string>{std::to_string(dict_id)});
3152  }
3154  "DELETE FROM mapd_dictionaries WHERE dictid in (select comp_param from "
3155  "mapd_columns where compression = ? "
3156  "and tableid = ?) and refcount = 0",
3157  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
3158  sqliteConnector_.query_with_text_param("DELETE FROM mapd_columns WHERE tableid = ?",
3159  std::to_string(tableId));
3160  if (td->isView) {
3161  sqliteConnector_.query_with_text_param("DELETE FROM mapd_views WHERE tableid = ?",
3162  std::to_string(tableId));
3163  }
3166  "DELETE FROM omnisci_foreign_tables WHERE table_id = ?", std::to_string(tableId));
3167  }
3168 }
3169 
3170 void Catalog::renamePhysicalTable(const TableDescriptor* td, const string& newTableName) {
3171  cat_write_lock write_lock(this);
3173 
3174  sqliteConnector_.query("BEGIN TRANSACTION");
3175  try {
3177  "UPDATE mapd_tables SET name = ? WHERE tableid = ?",
3178  std::vector<std::string>{newTableName, std::to_string(td->tableId)});
3179  } catch (std::exception& e) {
3180  sqliteConnector_.query("ROLLBACK TRANSACTION");
3181  throw;
3182  }
3183  sqliteConnector_.query("END TRANSACTION");
3184  TableDescriptorMap::iterator tableDescIt =
3186  CHECK(tableDescIt != tableDescriptorMap_.end());
3187  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
3188  // Get table descriptor to change it
3189  TableDescriptor* changeTd = tableDescIt->second;
3190  changeTd->tableName = newTableName;
3191  tableDescriptorMap_.erase(tableDescIt); // erase entry under old name
3192  tableDescriptorMap_[to_upper(newTableName)] = changeTd;
3193  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
3194 }
3195 
3196 void Catalog::renameTable(const TableDescriptor* td, const string& newTableName) {
3197  {
3198  cat_write_lock write_lock(this);
3200  // rename all corresponding physical tables if this is a logical table
3201  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
3202  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
3203  const auto physicalTables = physicalTableIt->second;
3204  CHECK(!physicalTables.empty());
3205  for (size_t i = 0; i < physicalTables.size(); i++) {
3206  int32_t physical_tb_id = physicalTables[i];
3207  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
3208  CHECK(phys_td);
3209  std::string newPhysTableName =
3210  generatePhysicalTableName(newTableName, static_cast<int32_t>(i + 1));
3211  renamePhysicalTable(phys_td, newPhysTableName);
3212  }
3213  }
3214  renamePhysicalTable(td, newTableName);
3215  }
3216  {
3217  DBObject object(newTableName, TableDBObjectType);
3218  // update table name in direct and effective priv map
3219  DBObjectKey key;
3220  key.dbId = currentDB_.dbId;
3221  key.objectId = td->tableId;
3222  key.permissionType = static_cast<int>(DBObjectType::TableDBObjectType);
3223  object.setObjectKey(key);
3224  auto objdescs = SysCatalog::instance().getMetadataForObject(
3225  currentDB_.dbId, static_cast<int>(DBObjectType::TableDBObjectType), td->tableId);
3226  for (auto obj : objdescs) {
3227  Grantee* grnt = SysCatalog::instance().getGrantee(obj->roleName);
3228  if (grnt) {
3229  grnt->renameDbObject(object);
3230  }
3231  }
3233  }
3234 }
3235 
3237  const ColumnDescriptor* cd,
3238  const string& newColumnName) {
3239  cat_write_lock write_lock(this);
3241  sqliteConnector_.query("BEGIN TRANSACTION");
3242  try {
3243  for (int i = 0; i <= cd->columnType.get_physical_cols(); ++i) {
3244  auto cdx = getMetadataForColumn(td->tableId, cd->columnId + i);
3245  CHECK(cdx);
3246  std::string new_column_name = cdx->columnName;
3247  new_column_name.replace(0, cd->columnName.size(), newColumnName);
3249  "UPDATE mapd_columns SET name = ? WHERE tableid = ? AND columnid = ?",
3250  std::vector<std::string>{new_column_name,
3251  std::to_string(td->tableId),
3252  std::to_string(cdx->columnId)});
3253  }
3254  } catch (std::exception& e) {
3255  sqliteConnector_.query("ROLLBACK TRANSACTION");
3256  throw;
3257  }
3258  sqliteConnector_.query("END TRANSACTION");
3259  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
3260  for (int i = 0; i <= cd->columnType.get_physical_cols(); ++i) {
3261  auto cdx = getMetadataForColumn(td->tableId, cd->columnId + i);
3262  CHECK(cdx);
3263  ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.find(
3264  std::make_tuple(td->tableId, to_upper(cdx->columnName)));
3265  CHECK(columnDescIt != columnDescriptorMap_.end());
3266  ColumnDescriptor* changeCd = columnDescIt->second;
3267  changeCd->columnName.replace(0, cd->columnName.size(), newColumnName);
3268  columnDescriptorMap_.erase(columnDescIt); // erase entry under old name
3269  columnDescriptorMap_[std::make_tuple(td->tableId, to_upper(changeCd->columnName))] =
3270  changeCd;
3271  }
3272  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
3273 }
3274 
3276  {
3277  cat_write_lock write_lock(this);
3279  sqliteConnector_.query("BEGIN TRANSACTION");
3280  try {
3281  // TODO(andrew): this should be an upsert
3283  "SELECT id FROM mapd_dashboards WHERE name = ? and userid = ?",
3284  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
3285  if (sqliteConnector_.getNumRows() > 0) {
3287  "UPDATE mapd_dashboards SET state = ?, image_hash = ?, metadata = ?, "
3288  "update_time = "
3289  "datetime('now') where name = ? "
3290  "and userid = ?",
3291  std::vector<std::string>{vd.dashboardState,
3292  vd.imageHash,
3293  vd.dashboardMetadata,
3294  vd.dashboardName,
3295  std::to_string(vd.userId)});
3296  } else {
3298  "INSERT INTO mapd_dashboards (name, state, image_hash, metadata, "
3299  "update_time, "
3300  "userid) "
3301  "VALUES "
3302  "(?,?,?,?, "
3303  "datetime('now'), ?)",
3304  std::vector<std::string>{vd.dashboardName,
3305  vd.dashboardState,
3306  vd.imageHash,
3307  vd.dashboardMetadata,
3308  std::to_string(vd.userId)});
3309  }
3310  } catch (std::exception& e) {
3311  sqliteConnector_.query("ROLLBACK TRANSACTION");
3312  throw;
3313  }
3314  sqliteConnector_.query("END TRANSACTION");
3315 
3316  // now get the auto generated dashboardId
3317  try {
3319  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_dashboards "
3320  "WHERE name = ? and userid = ?",
3321  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
3322  vd.dashboardId = sqliteConnector_.getData<int>(0, 0);
3323  vd.updateTime = sqliteConnector_.getData<std::string>(0, 1);
3324  } catch (std::exception& e) {
3325  throw;
3326  }
3330  }
3331  // NOTE(wamsi): Transactionally unsafe
3334  return vd.dashboardId;
3335 }
3336 
3338  cat_write_lock write_lock(this);
3340 
3341  sqliteConnector_.query("BEGIN TRANSACTION");
3342  try {
3344  "SELECT id FROM mapd_dashboards WHERE id = ?",
3345  std::vector<std::string>{std::to_string(vd.dashboardId)});
3346  if (sqliteConnector_.getNumRows() > 0) {
3348  "UPDATE mapd_dashboards SET name = ?, state = ?, image_hash = ?, metadata = ?, "
3349  "update_time = "
3350  "datetime('now') where id = ? ",
3351  std::vector<std::string>{vd.dashboardName,
3352  vd.dashboardState,
3353  vd.imageHash,
3354  vd.dashboardMetadata,
3356  } else {
3357  LOG(ERROR) << "Error replacing dashboard id " << vd.dashboardId
3358  << " does not exist in db";
3359  throw runtime_error("Error replacing dashboard id " +
3360  std::to_string(vd.dashboardId) + " does not exist in db");
3361  }
3362  } catch (std::exception& e) {
3363  sqliteConnector_.query("ROLLBACK TRANSACTION");
3364  throw;
3365  }
3366  sqliteConnector_.query("END TRANSACTION");
3367 
3368  bool found{false};
3369  for (auto descp : dashboardDescriptorMap_) {
3370  auto dash = descp.second.get();
3371  if (dash->dashboardId == vd.dashboardId) {
3372  found = true;
3373  auto viewDescIt = dashboardDescriptorMap_.find(std::to_string(dash->userId) + ":" +
3374  dash->dashboardName);
3375  if (viewDescIt ==
3376  dashboardDescriptorMap_.end()) { // check to make sure view exists
3377  LOG(ERROR) << "No metadata for dashboard for user " << dash->userId
3378  << " dashboard " << dash->dashboardName << " does not exist in map";
3379  throw runtime_error("No metadata for dashboard for user " +
3380  std::to_string(dash->userId) + " dashboard " +
3381  dash->dashboardName + " does not exist in map");
3382  }
3383  dashboardDescriptorMap_.erase(viewDescIt);
3384  break;
3385  }
3386  }
3387  if (!found) {
3388  LOG(ERROR) << "Error replacing dashboard id " << vd.dashboardId
3389  << " does not exist in map";
3390  throw runtime_error("Error replacing dashboard id " + std::to_string(vd.dashboardId) +
3391  " does not exist in map");
3392  }
3393 
3394  // now reload the object
3396  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM "
3397  "mapd_dashboards "
3398  "WHERE id = ?",
3399  std::vector<std::string>{std::to_string(vd.dashboardId)});
3400  vd.updateTime = sqliteConnector_.getData<string>(0, 1);
3404  // NOTE(wamsi): Transactionally unsafe
3407 }
3408 
3409 std::string Catalog::calculateSHA1(const std::string& data) {
3410  boost::uuids::detail::sha1 sha1;
3411  unsigned int digest[5];
3412  sha1.process_bytes(data.c_str(), data.length());
3413  sha1.get_digest(digest);
3414  std::stringstream ss;
3415  for (size_t i = 0; i < 5; i++) {
3416  ss << std::hex << digest[i];
3417  }
3418  return ss.str();
3419 }
3420 
3421 std::string Catalog::createLink(LinkDescriptor& ld, size_t min_length) {
3422  cat_write_lock write_lock(this);
3424  sqliteConnector_.query("BEGIN TRANSACTION");
3425  try {
3427  .substr(0, 8);
3429  "SELECT linkid FROM mapd_links WHERE link = ? and userid = ?",
3430  std::vector<std::string>{ld.link, std::to_string(ld.userId)});
3431  if (sqliteConnector_.getNumRows() > 0) {
3433  "UPDATE mapd_links SET update_time = datetime('now') WHERE userid = ? AND link "
3434  "= ?",
3435  std::vector<std::string>{std::to_string(ld.userId), ld.link});
3436  } else {
3438  "INSERT INTO mapd_links (userid, link, view_state, view_metadata, update_time) "
3439  "VALUES (?,?,?,?, "
3440  "datetime('now'))",
3441  std::vector<std::string>{
3442  std::to_string(ld.userId), ld.link, ld.viewState, ld.viewMetadata});
3443  }
3444  // now get the auto generated dashid
3446  "SELECT linkid, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_links "
3447  "WHERE link = ?",
3448  ld.link);
3449  ld.linkId = sqliteConnector_.getData<int>(0, 0);
3450  ld.updateTime = sqliteConnector_.getData<std::string>(0, 1);
3451  } catch (std::exception& e) {
3452  sqliteConnector_.query("ROLLBACK TRANSACTION");
3453  throw;
3454  }
3455  sqliteConnector_.query("END TRANSACTION");
3456  addLinkToMap(ld);
3457  return ld.link;
3458 }
3459 
3461  const TableDescriptor* td) const {
3462  cat_read_lock read_lock(this);
3463 
3464  const auto column_descriptors =
3465  getAllColumnMetadataForTable(td->tableId, false, true, true);
3466 
3467  const ColumnDescriptor* shard_cd{nullptr};
3468  int i = 1;
3469  for (auto cd_itr = column_descriptors.begin(); cd_itr != column_descriptors.end();
3470  ++cd_itr, ++i) {
3471  if (i == td->shardedColumnId) {
3472  shard_cd = *cd_itr;
3473  }
3474  }
3475  return shard_cd;
3476 }
3477 
3478 std::vector<const TableDescriptor*> Catalog::getPhysicalTablesDescriptors(
3479  const TableDescriptor* logicalTableDesc) const {
3480  cat_read_lock read_lock(this);
3481  const auto physicalTableIt =
3482  logicalToPhysicalTableMapById_.find(logicalTableDesc->tableId);
3483  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
3484  return {logicalTableDesc};
3485  }
3486 
3487  const auto physicalTablesIds = physicalTableIt->second;
3488  CHECK(!physicalTablesIds.empty());
3489  std::vector<const TableDescriptor*> physicalTables;
3490  for (size_t i = 0; i < physicalTablesIds.size(); i++) {
3491  physicalTables.push_back(getMetadataForTable(physicalTablesIds[i]));
3492  }
3493 
3494  return physicalTables;
3495 }
3496 
3497 const std::map<int, const ColumnDescriptor*> Catalog::getDictionaryToColumnMapping() {
3498  cat_read_lock read_lock(this);
3499 
3500  std::map<int, const ColumnDescriptor*> mapping;
3501 
3502  const auto tables = getAllTableMetadata();
3503  for (const auto td : tables) {
3504  if (td->shard >= 0) {
3505  // skip shards, they're not standalone tables
3506  continue;
3507  }
3508 
3509  for (auto& cd : getAllColumnMetadataForTable(td->tableId, false, false, true)) {
3510  const auto& ti = cd->columnType;
3511  if (ti.is_string()) {
3512  if (ti.get_compression() == kENCODING_DICT) {
3513  // if foreign reference, get referenced tab.col
3514  const auto dict_id = ti.get_comp_param();
3515 
3516  // ignore temp (negative) dictionaries
3517  if (dict_id > 0 && mapping.end() == mapping.find(dict_id)) {
3518  mapping[dict_id] = cd;
3519  }
3520  }
3521  }
3522  }
3523  }
3524 
3525  return mapping;
3526 }
3527 
3528 std::vector<std::string> Catalog::getTableNamesForUser(
3529  const UserMetadata& user_metadata,
3530  const GetTablesType get_tables_type) const {
3531  sys_read_lock syscat_read_lock(&SysCatalog::instance());
3532  cat_read_lock read_lock(this);
3533 
3534  std::vector<std::string> table_names;
3535  const auto tables = getAllTableMetadata();
3536  for (const auto td : tables) {
3537  if (td->shard >= 0) {
3538  // skip shards, they're not standalone tables
3539  continue;
3540  }
3541  switch (get_tables_type) {
3542  case GET_PHYSICAL_TABLES: {
3543  if (td->isView) {
3544  continue;
3545  }
3546  break;
3547  }
3548  case GET_VIEWS: {
3549  if (!td->isView) {
3550  continue;
3551  }
3552  }
3553  default:
3554  break;
3555  }
3556  DBObject dbObject(td->tableName, td->isView ? ViewDBObjectType : TableDBObjectType);
3557  dbObject.loadKey(*this);
3558  std::vector<DBObject> privObjects = {dbObject};
3559  if (!SysCatalog::instance().hasAnyPrivileges(user_metadata, privObjects)) {
3560  // skip table, as there are no privileges to access it
3561  continue;
3562  }
3563  table_names.push_back(td->tableName);
3564  }
3565  return table_names;
3566 }
3567 
3568 int Catalog::getLogicalTableId(const int physicalTableId) const {
3569  cat_read_lock read_lock(this);
3570  for (const auto& l : logicalToPhysicalTableMapById_) {
3571  if (l.second.end() != std::find_if(l.second.begin(),
3572  l.second.end(),
3573  [&](decltype(*l.second.begin()) tid) -> bool {
3574  return physicalTableId == tid;
3575  })) {
3576  return l.first;
3577  }
3578  }
3579  return physicalTableId;
3580 }
3581 
3582 void Catalog::checkpoint(const int logicalTableId) const {
3583  const auto td = getMetadataForTable(logicalTableId);
3584  const auto shards = getPhysicalTablesDescriptors(td);
3585  for (const auto shard : shards) {
3586  getDataMgr().checkpoint(getCurrentDB().dbId, shard->tableId);
3587  }
3588 }
3589 
3591  cat_write_lock write_lock(this);
3592  // Physically erase all tables and dictionaries from disc and memory
3593  const auto tables = getAllTableMetadata();
3594  for (const auto table : tables) {
3595  eraseTablePhysicalData(table);
3596  }
3597  // Physically erase database metadata
3598  boost::filesystem::remove(basePath_ + "/mapd_catalogs/" + currentDB_.dbName);
3599  calciteMgr_->updateMetadata(currentDB_.dbName, "");
3600 }
3601 
3603  const int tableId = td->tableId;
3604  // must destroy fragmenter before deleteChunks is called.
3605  if (td->fragmenter != nullptr) {
3606  auto tableDescIt = tableDescriptorMapById_.find(tableId);
3607  CHECK(tableDescIt != tableDescriptorMapById_.end());
3608  {
3609  INJECT_TIMER(deleting_fragmenter);
3610  tableDescIt->second->fragmenter = nullptr;
3611  }
3612  }
3613  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
3614  {
3615  INJECT_TIMER(deleteChunksWithPrefix);
3616  // assuming deleteChunksWithPrefix is atomic
3617  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
3618  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
3619  }
3620  if (!td->isView) {
3621  INJECT_TIMER(Remove_Table);
3622  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
3623  }
3624  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
3625  {
3626  INJECT_TIMER(removeTableFromMap_);
3627  removeTableFromMap(td->tableName, tableId);
3628  }
3629 }
3630 
3631 std::string Catalog::generatePhysicalTableName(const std::string& logicalTableName,
3632  const int32_t& shardNumber) {
3633  std::string physicalTableName =
3634  logicalTableName + physicalTableNameTag_ + std::to_string(shardNumber);
3635  return (physicalTableName);
3636 }
3637 
3638 void Catalog::set(const std::string& dbName, std::shared_ptr<Catalog> cat) {
3639  mapd_cat_map_[dbName] = cat;
3640 }
3641 
3642 std::shared_ptr<Catalog> Catalog::get(const std::string& dbName) {
3643  auto cat_it = mapd_cat_map_.find(dbName);
3644  if (cat_it != mapd_cat_map_.end()) {
3645  return cat_it->second;
3646  }
3647  return nullptr;
3648 }
3649 
3650 std::shared_ptr<Catalog> Catalog::get(const int32_t db_id) {
3651  for (const auto& entry : mapd_cat_map_) {
3652  if (entry.second->currentDB_.dbId == db_id) {
3653  return entry.second;
3654  }
3655  }
3656  return nullptr;
3657 }
3658 
3659 std::shared_ptr<Catalog> Catalog::get(const string& basePath,
3660  const DBMetadata& curDB,
3661  std::shared_ptr<Data_Namespace::DataMgr> dataMgr,
3662  const std::vector<LeafHostInfo>& string_dict_hosts,
3663  std::shared_ptr<Calcite> calcite,
3664  bool is_new_db) {
3665  auto cat = Catalog::get(curDB.dbName);
3666 
3667  if (cat) {
3668  return cat;
3669  } else {
3670  cat = std::make_shared<Catalog>(
3671  basePath, curDB, dataMgr, string_dict_hosts, calcite, is_new_db);
3672  Catalog::set(curDB.dbName, cat);
3673  return cat;
3674  }
3675 }
3676 
3677 void Catalog::remove(const std::string& dbName) {
3678  mapd_cat_map_.erase(dbName);
3679 }
3680 
3681 void Catalog::vacuumDeletedRows(const int logicalTableId) const {
3682  // shard here to serve request from TableOptimizer and elsewhere
3683  const auto td = getMetadataForTable(logicalTableId);
3684  const auto shards = getPhysicalTablesDescriptors(td);
3685  for (const auto shard : shards) {
3686  vacuumDeletedRows(shard);
3687  }
3688 }
3689 
3691  // "if not a table that supports delete return nullptr, nothing more to do"
3692  const ColumnDescriptor* cd = getDeletedColumn(td);
3693  if (nullptr == cd) {
3694  return;
3695  }
3696  // vacuum chunks which show sign of deleted rows in metadata
3697  ChunkKey chunkKeyPrefix = {currentDB_.dbId, td->tableId, cd->columnId};
3698  ChunkMetadataVector chunkMetadataVec;
3699  dataMgr_->getChunkMetadataVecForKeyPrefix(chunkMetadataVec, chunkKeyPrefix);
3700  for (auto cm : chunkMetadataVec) {
3701  // "delete has occured"
3702  if (cm.second->chunkStats.max.tinyintval == 1) {
3703  UpdelRoll updel_roll;
3704  updel_roll.catalog = this;
3705  updel_roll.logicalTableId = getLogicalTableId(td->tableId);
3707  const auto cd = getMetadataForColumn(td->tableId, cm.first[2]);
3708  const auto chunk = Chunk_NS::Chunk::getChunk(cd,
3709  &getDataMgr(),
3710  cm.first,
3711  updel_roll.memoryLevel,
3712  0,
3713  cm.second->numBytes,
3714  cm.second->numElements);
3715  td->fragmenter->compactRows(this,
3716  td,
3717  cm.first[3],
3718  td->fragmenter->getVacuumOffsets(chunk),
3719  updel_roll.memoryLevel,
3720  updel_roll);
3721  updel_roll.commitUpdate();
3722  }
3723  }
3724 }
3725 
3728  "SELECT id, name, data_wrapper_type, options, owner_user_id, creation_time FROM "
3729  "omnisci_foreign_servers");
3731  for (size_t row = 0; row < num_rows; row++) {
3732  auto foreign_server = std::make_shared<foreign_storage::ForeignServer>(
3733  sqliteConnector_.getData<int>(row, 0),
3734  sqliteConnector_.getData<std::string>(row, 1),
3735  sqliteConnector_.getData<std::string>(row, 2),
3736  sqliteConnector_.getData<std::string>(row, 3),
3737  sqliteConnector_.getData<std::int32_t>(row, 4),
3738  sqliteConnector_.getData<std::int32_t>(row, 5));
3739  foreignServerMap_[foreign_server->name] = foreign_server;
3740  foreignServerMapById_[foreign_server->id] = foreign_server;
3741  }
3742 }
3743 
3746  "SELECT table_id, server_id, options from omnisci_foreign_tables");
3748  for (size_t r = 0; r < num_rows; r++) {
3749  const auto table_id = sqliteConnector_.getData<int32_t>(r, 0);
3750  const auto server_id = sqliteConnector_.getData<int32_t>(r, 1);
3751  const auto& options = sqliteConnector_.getData<std::string>(r, 2);
3752 
3753  CHECK(tableDescriptorMapById_.find(table_id) != tableDescriptorMapById_.end());
3754  auto foreign_table =
3755  dynamic_cast<foreign_storage::ForeignTable*>(tableDescriptorMapById_[table_id]);
3756  CHECK(foreign_table);
3757  foreign_table->foreign_server = foreignServerMapById_[server_id].get();
3758  CHECK(foreign_table->foreign_server);
3759  foreign_table->populateOptionsMap(options);
3760  }
3761 }
3762 
3763 void Catalog::setForeignServerProperty(const std::string& server_name,
3764  const std::string& property,
3765  const std::string& value) {
3768  "SELECT id from omnisci_foreign_servers where name = ?",
3769  std::vector<std::string>{server_name});
3771  if (num_rows > 0) {
3772  CHECK_EQ(size_t(1), num_rows);
3773  auto server_id = sqliteConnector_.getData<int32_t>(0, 0);
3775  "UPDATE omnisci_foreign_servers SET " + property + " = ? WHERE id = ?",
3776  std::vector<std::string>{value, std::to_string(server_id)});
3777  } else {
3778  throw std::runtime_error{"Can not change property \"" + property +
3779  "\" for foreign server." + " Foreign server \"" +
3780  server_name + "\" is not found."};
3781  }
3782 }
3783 
3786  std::map<std::string, std::string, std::less<>> options;
3787  options[std::string{ForeignServer::STORAGE_TYPE_KEY}] =
3788  ForeignServer::LOCAL_FILE_STORAGE_TYPE;
3789  options[std::string{ForeignServer::BASE_PATH_KEY}] =
3790  boost::filesystem::path::preferred_separator;
3791 
3792  auto local_csv_server =
3793  std::make_unique<ForeignServer>("omnisci_local_csv",
3795  options,
3797  local_csv_server->validate();
3798  createForeignServerNoLocks(std::move(local_csv_server), true);
3799 
3800  auto local_parquet_server =
3801  std::make_unique<ForeignServer>("omnisci_local_parquet",
3803  options,
3805  local_parquet_server->validate();
3806  createForeignServerNoLocks(std::move(local_parquet_server), true);
3807 }
3808 
3809 // prepare a fresh file reload on next table access
3810 void Catalog::setForReload(const int32_t tableId) {
3811  cat_read_lock read_lock(this);
3812  const auto td = getMetadataForTable(tableId);
3813  for (const auto shard : getPhysicalTablesDescriptors(td)) {
3814  const auto tableEpoch = getTableEpoch(currentDB_.dbId, shard->tableId);
3815  setTableEpoch(currentDB_.dbId, shard->tableId, tableEpoch);
3816  }
3817 }
3818 
3819 // get a table's data dirs
3820 std::vector<std::string> Catalog::getTableDataDirectories(
3821  const TableDescriptor* td) const {
3822  const auto global_file_mgr = getDataMgr().getGlobalFileMgr();
3823  std::vector<std::string> file_paths;
3824  for (auto shard : getPhysicalTablesDescriptors(td)) {
3825  const auto file_mgr = dynamic_cast<File_Namespace::FileMgr*>(
3826  global_file_mgr->getFileMgr(currentDB_.dbId, shard->tableId));
3827  boost::filesystem::path file_path(file_mgr->getFileMgrBasePath());
3828  file_paths.push_back(file_path.filename().string());
3829  }
3830  return file_paths;
3831 }
3832 
3833 // get a column's dict dir basename
3835  if ((cd->columnType.is_string() || cd->columnType.is_string_array()) &&
3837  cd->columnType.get_comp_param() > 0) {
3838  const auto dictId = cd->columnType.get_comp_param();
3839  const DictRef dictRef(currentDB_.dbId, dictId);
3840  const auto dit = dictDescriptorMapByRef_.find(dictRef);
3841  CHECK(dit != dictDescriptorMapByRef_.end());
3842  CHECK(dit->second);
3843  boost::filesystem::path file_path(dit->second->dictFolderPath);
3844  return file_path.filename().string();
3845  }
3846  return std::string();
3847 }
3848 
3849 // get a table's dict dirs
3850 std::vector<std::string> Catalog::getTableDictDirectories(
3851  const TableDescriptor* td) const {
3852  std::vector<std::string> file_paths;
3853  for (auto cd : getAllColumnMetadataForTable(td->tableId, false, false, true)) {
3854  auto file_base = getColumnDictDirectory(cd);
3855  if (!file_base.empty() &&
3856  file_paths.end() == std::find(file_paths.begin(), file_paths.end(), file_base)) {
3857  file_paths.push_back(file_base);
3858  }
3859  }
3860  return file_paths;
3861 }
3862 
3863 // returns table schema in a string
3864 // NOTE(sy): Might be able to replace dumpSchema() later with
3865 // dumpCreateTable() after a deeper review of the TableArchiver code.
3866 std::string Catalog::dumpSchema(const TableDescriptor* td) const {
3867  cat_read_lock read_lock(this);
3868 
3869  std::ostringstream os;
3870  os << "CREATE TABLE @T (";
3871  // gather column defines
3872  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
3873  std::string comma;
3874  std::vector<std::string> shared_dicts;
3875  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
3876  for (const auto cd : cds) {
3877  if (!(cd->isSystemCol || cd->isVirtualCol)) {
3878  const auto& ti = cd->columnType;
3879  os << comma << cd->columnName;
3880  // CHAR is perculiar... better dump it as TEXT(32) like \d does
3881  if (ti.get_type() == SQLTypes::kCHAR) {
3882  os << " "
3883  << "TEXT";
3884  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
3885  os << " "
3886  << "TEXT[]";
3887  } else {
3888  os << " " << ti.get_type_name();
3889  }
3890  os << (ti.get_notnull() ? " NOT NULL" : "");
3891  if (ti.is_string()) {
3892  if (ti.get_compression() == kENCODING_DICT) {
3893  // if foreign reference, get referenced tab.col
3894  const auto dict_id = ti.get_comp_param();
3895  const DictRef dict_ref(currentDB_.dbId, dict_id);
3896  const auto dict_it = dictDescriptorMapByRef_.find(dict_ref);
3897  CHECK(dict_it != dictDescriptorMapByRef_.end());
3898  const auto dict_name = dict_it->second->dictName;
3899  // when migrating a table, any foreign dict ref will be dropped
3900  // and the first cd of a dict will become root of the dict
3901  if (dict_root_cds.end() == dict_root_cds.find(dict_name)) {
3902  dict_root_cds[dict_name] = cd;
3903  os << " ENCODING " << ti.get_compression_name() << "(" << (ti.get_size() * 8)
3904  << ")";
3905  } else {
3906  const auto dict_root_cd = dict_root_cds[dict_name];
3907  shared_dicts.push_back("SHARED DICTIONARY (" + cd->columnName +
3908  ") REFERENCES @T(" + dict_root_cd->columnName + ")");
3909  // "... shouldn't specify an encoding, it borrows from the referenced column"
3910  }
3911  } else {
3912  os << " ENCODING NONE";
3913  }
3914  } else if (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size()) {
3915  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
3916  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
3917  }
3918  comma = ", ";
3919  }
3920  }
3921  // gather SHARED DICTIONARYs
3922  if (shared_dicts.size()) {
3923  os << ", " << boost::algorithm::join(shared_dicts, ", ");
3924  }
3925  // gather WITH options ...
3926  std::vector<std::string> with_options;
3927  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
3928  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
3929  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
3930  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
3931  with_options.emplace_back(td->hasDeletedCol ? "VACUUM='DELAYED'"
3932  : "VACUUM='IMMEDIATE'");
3933  if (!td->partitions.empty()) {
3934  with_options.push_back("PARTITIONS='" + td->partitions + "'");
3935  }
3936  if (td->nShards > 0) {
3937  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
3938  CHECK(shard_cd);
3939  os << ", SHARD KEY(" << shard_cd->columnName << ")";
3940  with_options.push_back("SHARD_COUNT=" + std::to_string(td->nShards));
3941  }
3942  if (td->sortedColumnId > 0) {
3943  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
3944  CHECK(sort_cd);
3945  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
3946  }
3947  os << ") WITH (" + boost::algorithm::join(with_options, ", ") + ");";
3948  return os.str();
3949 }
3950 
3951 namespace {
3952 
3953 void unserialize_key_metainfo(std::vector<std::string>& shared_dicts,
3954  std::set<std::string>& shared_dict_column_names,
3955  const std::string keyMetainfo) {
3956  rapidjson::Document document;
3957  document.Parse(keyMetainfo.c_str());
3958  CHECK(!document.HasParseError());
3959  CHECK(document.IsArray());
3960  for (auto it = document.Begin(); it != document.End(); ++it) {
3961  const auto& key_with_spec_json = *it;
3962  CHECK(key_with_spec_json.IsObject());
3963  const std::string type = key_with_spec_json["type"].GetString();
3964  const std::string name = key_with_spec_json["name"].GetString();
3965  auto key_with_spec = type + " (" + name + ")";
3966  if (type == "SHARED DICTIONARY") {
3967  shared_dict_column_names.insert(name);
3968  key_with_spec += " REFERENCES ";
3969  const std::string foreign_table = key_with_spec_json["foreign_table"].GetString();
3970  const std::string foreign_column = key_with_spec_json["foreign_column"].GetString();
3971  key_with_spec += foreign_table + "(" + foreign_column + ")";
3972  } else {
3973  CHECK(type == "SHARD KEY");
3974  }
3975  shared_dicts.push_back(key_with_spec);
3976  }
3977 }
3978 
3979 } // namespace
3980 
3981 // returns a "CREATE TABLE" statement in a string
3983  bool multiline_formatting,
3984  bool dump_defaults) const {
3985  cat_read_lock read_lock(this);
3986 
3987  std::ostringstream os;
3988 
3989  if (!td->isView) {
3990  os << "CREATE ";
3992  os << "TEMPORARY ";
3993  }
3994  os << "TABLE " + td->tableName + " (";
3995  } else {
3996  os << "CREATE VIEW " + td->tableName + " AS " << td->viewSQL;
3997  return os.str();
3998  }
3999  // scan column defines
4000  std::vector<std::string> shared_dicts;
4001  std::set<std::string> shared_dict_column_names;
4002  unserialize_key_metainfo(shared_dicts, shared_dict_column_names, td->keyMetainfo);
4003  // gather column defines
4004  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
4005  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
4006  bool first = true;
4007  for (const auto cd : cds) {
4008  if (!(cd->isSystemCol || cd->isVirtualCol)) {
4009  const auto& ti = cd->columnType;
4010  if (!first) {
4011  os << ",";
4012  if (!multiline_formatting) {
4013  os << " ";
4014  }
4015  } else {
4016  first = false;
4017  }
4018  if (multiline_formatting) {
4019  os << "\n ";
4020  }
4021  os << cd->columnName;
4022  // CHAR is perculiar... better dump it as TEXT(32) like \d does
4023  if (ti.get_type() == SQLTypes::kCHAR) {
4024  os << " "
4025  << "TEXT";
4026  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
4027  os << " "
4028  << "TEXT[]";
4029  } else {
4030  os << " " << ti.get_type_name();
4031  }
4032  os << (ti.get_notnull() ? " NOT NULL" : "");
4033  if (shared_dict_column_names.find(cd->columnName) ==
4034  shared_dict_column_names.end()) {
4035  // avoids "Exception: Column ... shouldn't specify an encoding, it borrows it from
4036  // the referenced column"
4037  if (ti.is_string()) {
4038  if (ti.get_compression() == kENCODING_DICT) {
4039  os << " ENCODING " << ti.get_compression_name() << "(" << (ti.get_size() * 8)
4040  << ")";
4041  } else {
4042  os << " ENCODING NONE";
4043  }
4044  } else if (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size()) {
4045  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
4046  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
4047  }
4048  }
4049  }
4050  }
4051  // gather SHARED DICTIONARYs
4052  if (shared_dicts.size()) {
4053  std::string comma;
4054  if (!multiline_formatting) {
4055  comma = ", ";
4056  } else {
4057  comma = ",\n ";
4058  }
4059  os << comma;
4060  os << boost::algorithm::join(shared_dicts, comma);
4061  }
4062  // gather WITH options ...
4063  std::vector<std::string> with_options;
4064  if (dump_defaults || td->maxFragRows != DEFAULT_FRAGMENT_ROWS) {
4065  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
4066  }
4067  if (dump_defaults || td->maxChunkSize != DEFAULT_MAX_CHUNK_SIZE) {
4068  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
4069  }
4070  if (dump_defaults || td->fragPageSize != DEFAULT_PAGE_SIZE) {
4071  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
4072  }
4073  if (dump_defaults || td->maxRows != DEFAULT_MAX_ROWS) {
4074  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
4075  }
4076  if (dump_defaults || !td->hasDeletedCol) {
4077  with_options.push_back(td->hasDeletedCol ? "VACUUM='DELAYED'" : "VACUUM='IMMEDIATE'");
4078  }
4079  if (!td->partitions.empty()) {
4080  with_options.push_back("PARTITIONS='" + td->partitions + "'");
4081  }
4082  if (td->nShards > 0) {
4083  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
4084  CHECK(shard_cd);
4085  with_options.push_back("SHARD_COUNT=" + std::to_string(td->nShards));
4086  }
4087  if (td->sortedColumnId > 0) {
4088  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
4089  CHECK(sort_cd);
4090  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
4091  }
4092  os << ")";
4093  if (!with_options.empty()) {
4094  if (!multiline_formatting) {
4095  os << " ";
4096  } else {
4097  os << "\n";
4098  }
4099  os << "WITH (" + boost::algorithm::join(with_options, ", ") + ")";
4100  }
4101  os << ";";
4102  return os.str();
4103 }
4104 
4106  const bool if_not_exists) {
4107  if (getMetadataForTable(name, false)) {
4108  if (if_not_exists) {
4109  return false;
4110  }
4111  throw std::runtime_error("Table or View with name \"" + name + "\" already exists.");
4112  }
4113  return true;
4114 }
4115 
4116 } // namespace Catalog_Namespace
void serializeTableJsonUnlocked(const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
Definition: Catalog.cpp:2315
const Parser::SharedDictionaryDef compress_reference_path(Parser::SharedDictionaryDef cur_node, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Data_Namespace::MemoryLevel memoryLevel
Definition: UpdelRoll.h:65
std::string virtualExpr
void set_compression(EncodingType c)
Definition: sqltypes.h:357
void set_size(int s)
Definition: sqltypes.h:355
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:1714
#define CHECK_EQ(x, y)
Definition: Logger.h:205
void executeDropTableSqliteQueries(const TableDescriptor *td)
Definition: Catalog.cpp:3136
const int MAPD_TEMP_TABLE_START_ID
Definition: Catalog.cpp:100
std::string partitions
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:3602
std::string dictFolderPath
std::tuple< int, std::string > ColumnKey
Definition: Types.h:36
void doDropTable(const TableDescriptor *td)
Definition: Catalog.cpp:3128
~Catalog()
Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the ...
Definition: Catalog.cpp:190
bool is_array() const
Definition: sqltypes.h:416
T getData(const int row, const int col)
bool is_string() const
Definition: sqltypes.h:408
mapd_shared_mutex sharedMutex_
Definition: Catalog.h:518
void commitUpdate()
const int8_t const int64_t * num_rows
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:86
SQLTypes
Definition: sqltypes.h:39
const std::vector< LeafHostInfo > & getStringDictionaryHosts() const
Definition: Catalog.cpp:1490
#define SPIMAP_MAGIC1
Definition: Catalog.h:73
std::string tableName
void addReferenceToForeignDict(ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def, const bool persist_reference)
Definition: Catalog.cpp:2818
void setDeletedColumn(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:2792
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:462
static const AccessPrivileges ALL_DATABASE
Definition: DBObject.h:152
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:2102
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:407
EncodingType
Definition: encodetypes.h:22
virtual void query_with_text_params(std::string const &query_only)
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
DBObjectType
Definition: DBObject.h:42
void createForeignServer(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2409
std::string getOptionsAsJsonString() const
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:195
void addColumn(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:1826
void checkpoint(const int db_id, const int tb_id)
Definition: DataMgr.cpp:477
void roll(const bool forward)
Definition: Catalog.cpp:1914
bool validateNonExistentTableOrView(const std::string &name, const bool if_not_exists)
Definition: Catalog.cpp:4105
#define LOG(tag)
Definition: Logger.h:188
SqliteConnector sqliteConnector_
Definition: Catalog.h:471
static std::shared_ptr< Catalog > get(const std::string &dbName)
Definition: Catalog.cpp:3642
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:475
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:464
std::string storageType
void revokeDBObjectPrivilegesFromAll(DBObject object, Catalog *catalog)
const ColumnDescriptor * getDeletedColumnIfRowsDeleted(const TableDescriptor *td) const
Definition: Catalog.cpp:2758
void revokeDBObjectPrivileges(const std::string &grantee, const DBObject &object, const Catalog_Namespace::Catalog &catalog)
std::string join(T const &container, std::string const &delim)
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:266
Catalog(const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
Constructor - takes basePath to already extant data directory for writing.
Definition: Catalog.cpp:160
#define DEFAULT_MAX_CHUNK_SIZE
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:473
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:348
DBObject * findDbObject(const DBObjectKey &objectKey, bool only_direct) const
Definition: Grantee.cpp:72
void populateOptionsMap(const rapidjson::Value &ddl_options)
virtual void query(const std::string &queryString)
#define CHECK_GE(x, y)
Definition: Logger.h:210
void setObjectKey(const DBObjectKey &objectKey)
Definition: DBObject.h:215
int32_t getTableEpoch(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:2672
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:152
HOST DEVICE int get_size() const
Definition: sqltypes.h:267
std::string fragments
void setForReload(const int32_t tableId)
Definition: Catalog.cpp:3810
int32_t objectId
Definition: DBObject.h:57
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
void replaceDashboard(DashboardDescriptor &vd)
Definition: Catalog.cpp:3337
void unserialize_key_metainfo(std::vector< std::string > &shared_dicts, std::set< std::string > &shared_dict_column_names, const std::string keyMetainfo)
Definition: Catalog.cpp:3953
const int MAPD_TEMP_DICT_START_ID
Definition: Catalog.cpp:102
void getForeignServersForUser(const rapidjson::Value *filters, const UserMetadata &user, std::vector< const foreign_storage::ForeignServer *> &results)
Definition: Catalog.cpp:2580
Definition: Grantee.h:76
std::string generatePhysicalTableName(const std::string &logicalTableName, const int32_t &shardNumber)
Definition: Catalog.cpp:3631
The InsertOrderFragmenter is a child class of AbstractFragmenter, and fragments data in insert order...
void dropColumn(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:1883
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logicalTableDesc) const
Definition: Catalog.cpp:3478
void checkDateInDaysColumnMigration()
Definition: Catalog.cpp:814
void setPrivileges(const AccessPrivileges &privs)
Definition: DBObject.h:217
void addFrontendViewToMap(DashboardDescriptor &vd)
Definition: Catalog.cpp:1278
void deleteMetadataForDashboard(const std::string &userId, const std::string &dashName)
Definition: Catalog.cpp:1555
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:265
void vacuumDeletedRows(const TableDescriptor *td) const
Definition: Catalog.cpp:3690
static const AccessPrivileges SELECT_FROM_TABLE
Definition: DBObject.h:161
std::vector< int > columnIdBySpi_
std::string dumpCreateTable(const TableDescriptor *td, bool multiline_formatting=true, bool dump_defaults=false) const
Definition: Catalog.cpp:3982
#define CHECK_GT(x, y)
Definition: Logger.h:209
Grantee * getGrantee(const std::string &name) const
void changeForeignServerOwner(const std::string &server_name, const int new_owner_id)
Definition: Catalog.cpp:2482
std::atomic< std::thread::id > thread_holding_sqlite_lock
Definition: Catalog.h:519
HOST DEVICE int get_scale() const
Definition: sqltypes.h:262
void renameColumn(const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
Definition: Catalog.cpp:3236
DeletedColumnPerTableMap deletedColumnPerTable_
Definition: Catalog.h:494
const ColumnDescriptor * get_foreign_col(const Catalog &cat, const Parser::SharedDictionaryDef &shared_dict_def)
Definition: Catalog.cpp:2806
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:463
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:465
void setForeignServerOptions(const std::string &server_name, const std::string &options)
Definition: Catalog.cpp:2514
std::string to_string(char const *&&v)
std::shared_ptr< std::mutex > mutex_
void addTableToMap(const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
Definition: Catalog.cpp:1137
std::string dumpSchema(const TableDescriptor *td) const
Definition: Catalog.cpp:3866
static const AccessPrivileges ALL_VIEW
Definition: DBObject.h:178
std::string chunks
void grantRoleBatch(const std::vector< std::string > &roles, const std::vector< std::string > &grantees)
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:686
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:119
std::list< const ColumnDescriptor * > getAllColumnMetadataForTableUnlocked(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Definition: Catalog.cpp:1698
#define SPIMAP_MAGIC2
Definition: Catalog.h:74
const ColumnDescriptor * getShardColumnMetadataForTable(const TableDescriptor *td) const
Definition: Catalog.cpp:3460
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:264
void createRole(const std::string &roleName, const bool &userPrivateRole=false)
std::string dashboardSystemRoleName
This file contains the class specification and related data structures for Catalog.
int getLogicalTableId(const int physicalTableId) const
Definition: Catalog.cpp:3568
std::vector< std::string > getTableDictDirectories(const TableDescriptor *td) const
Definition: Catalog.cpp:3850
#define DEFAULT_MAX_ROWS
void checkpoint(const int logicalTableId) const
Definition: Catalog.cpp:3582
static SysCatalog & instance()
Definition: SysCatalog.h:284
This file contains the class specification and related data structures for SysCatalog.
std::string cat(Ts &&... args)
Classes representing a parse tree.
const Catalog_Namespace::Catalog * catalog
Definition: UpdelRoll.h:63
std::string calculateSHA1(const std::string &data)
Definition: Catalog.cpp:3409
std::string getUserFromId(const int32_t id)
Definition: Catalog.cpp:916
void addLinkToMap(LinkDescriptor &ld)
Definition: Catalog.cpp:1360
static const std::string getForeignTableSchema(bool if_not_exists=false)
Definition: Catalog.cpp:679
#define INJECT_TIMER(DESC)
Definition: measure.h:91
std::string getColumnDictDirectory(const ColumnDescriptor *cd) const
Definition: Catalog.cpp:3834
void delDictionary(const ColumnDescriptor &cd)
Definition: Catalog.cpp:1755
const std::string & get_foreign_column() const
Definition: ParserNode.h:901
GetTablesType
Definition: Catalog.h:61
const TableDescriptor * getMetadataForTableImpl(int tableId, const bool populateFragmenter) const
Definition: Catalog.cpp:1426
void renameObjectsInDescriptorMap(DBObject &object, const Catalog_Namespace::Catalog &cat)
void removeChunks(const int table_id)
Definition: Catalog.cpp:3075
void set_scale(int s)
Definition: sqltypes.h:352
void dropForeignServer(const std::string &server_name)
Definition: Catalog.cpp:2546
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:258
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:2797
void dropTable(const TableDescriptor *td)
Definition: Catalog.cpp:3095
std::vector< std::string > parse_underlying_dashboard_objects(const std::string &meta)
static void migrateDateInDaysMetadata(const Catalog_Namespace::TableDescriptorMapById &table_descriptors_by_id, const int database_id, const Catalog_Namespace::Catalog *cat, SqliteConnector &sqlite)
static const int32_t MAPD_VERSION
Definition: release.h:33
static const std::string getForeignServerSchema(bool if_not_exists=false)
Definition: Catalog.cpp:672
static const AccessPrivileges ALL_SERVER
Definition: DBObject.h:188
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:461
void setOwner(int32_t userId)
Definition: DBObject.h:225
static std::shared_ptr< Chunk > getChunk(const ColumnDescriptor *cd, DataMgr *data_mgr, const ChunkKey &key, const MemoryLevel mem_level, const int deviceId, const size_t num_bytes, const size_t num_elems)
Definition: Chunk.cpp:28
int get_physical_cols() const
Definition: sqltypes.h:278
void grantDBObjectPrivilegesBatch(const std::vector< std::string > &grantees, const std::vector< DBObject > &objects, const Catalog_Namespace::Catalog &catalog)
specifies the content in-memory of a row in the column metadata table
int32_t dictId
Definition: DictRef.h:10
static void registerTable(Catalog_Namespace::Catalog *catalog, const TableDescriptor &td, const std::list< ColumnDescriptor > &cols)
ids are created
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:194
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:1972
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1319
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:469
std::string keyMetainfo
int g_test_against_columnId_gap
Definition: Catalog.cpp:89
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1449
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
bool g_serialize_temp_tables
Definition: Catalog.cpp:95
struct dict_ref_t DictRef
Definition: DictRef.h:37
void createShardedTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:2950
ForeignServerMap foreignServerMap_
Definition: Catalog.h:468
std::string to_upper(const std::string &str)
#define DEFAULT_PAGE_SIZE
void set_comp_param(int p)
Definition: sqltypes.h:358
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2417
void loadKey()
Definition: DBObject.cpp:179
const int DEFAULT_INITIAL_VERSION
Definition: Catalog.cpp:99
const bool checkMetadataForDeletedRecs(const TableDescriptor *td, int column_id) const
Definition: Catalog.cpp:2732
bool is_geometry() const
Definition: sqltypes.h:420
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:476
static const std::string physicalTableNameTag_
Definition: Catalog.h:480
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1283
static void remove(const std::string &dbName)
Definition: Catalog.cpp:3677
const ColumnDescriptor * getMetadataForColumnUnlocked(int tableId, int columnId) const
Definition: Catalog.cpp:1512
static const AccessPrivileges SELECT_FROM_VIEW
Definition: DBObject.h:181
bool table_is_temporary(const TableDescriptor *const td)
bool is_string_array() const
Definition: sqltypes.h:409
std::vector< std::string > getTableNamesForUser(const UserMetadata &user, const GetTablesType get_tables_type) const
Definition: Catalog.cpp:3528
void setTableEpoch(const int db_id, const int table_id, const int new_epoch)
Definition: Catalog.cpp:2702
static void set(const std::string &dbName, std::shared_ptr< Catalog > cat)
Definition: Catalog.cpp:3638
std::vector< ObjectRoleDescriptor * > getMetadataForObject(int32_t dbId, int32_t dbType, int32_t objectId) const
const DBObjectMap * getDbObjects(bool only_direct) const
Definition: Grantee.h:56
void set_dimension(int d)
Definition: sqltypes.h:349
#define DEFAULT_FRAGMENT_ROWS
const int OMNISCI_ROOT_USER_ID
Definition: SysCatalog.h:60
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:259
Fragmenter_Namespace::FragmenterType fragType
int32_t dbId
Definition: DBObject.h:56
Data_Namespace::MemoryLevel persistenceLevel
void dropTableFromJsonUnlocked(const std::string &table_name) const
Definition: Catalog.cpp:2377
std::string name() const
Definition: Catalog.h:250
const std::string & get_foreign_table() const
Definition: ParserNode.h:899
int32_t createDashboard(DashboardDescriptor &vd)
Definition: Catalog.cpp:3275
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1197
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
Definition: DataMgr.cpp:522
static const AccessPrivileges ALL_DASHBOARD
Definition: DBObject.h:170
Definition: sqltypes.h:42
static const AccessPrivileges ALL_TABLE
Definition: DBObject.h:158
int logicalTableId
Definition: UpdelRoll.h:64
bool g_cache_string_hash
Definition: Execute.cpp:93
bool isNull(const int row, const int col) const
const ColumnDescriptor * getDeletedColumn(const TableDescriptor *td) const
Definition: Catalog.cpp:2726
const ColumnDescriptor * getMetadataForColumnBySpi(const int tableId, const size_t spi) const
Definition: Catalog.cpp:1545
const int getColumnIdBySpi(const int tableId, const size_t spi) const
Definition: Catalog.cpp:1540
bool setColumnSharedDictionary(ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:2840
mapd_shared_lock< mapd_shared_mutex > read_lock
const foreign_storage::ForeignServer * getForeignServer(const std::string &server_name) const
Definition: Catalog.cpp:2452
void set_notnull(bool n)
Definition: sqltypes.h:354
#define CHECK(condition)
Definition: Logger.h:197
static constexpr char const * CSV
Definition: ForeignServer.h:35
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:257
std::vector< int > ChunkKey
Definition: types.h:35
void getDictionary(const ColumnDescriptor &cd, std::map< int, StringDictionary *> &stringDicts)
Definition: Catalog.cpp:1798
std::string createLink(LinkDescriptor &ld, size_t min_length)
Definition: Catalog.cpp:3421
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const
void createFsiSchemasAndDefaultServers()
Definition: Catalog.cpp:608
Descriptor for a dictionary for a string columne.
void updateLogicalToPhysicalTableLinkSchema()
Definition: Catalog.cpp:547
std::vector< std::string > getTableDataDirectories(const TableDescriptor *td) const
Definition: Catalog.cpp:3820
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1688
FileBuffer Chunk
A Chunk is the fundamental unit of execution in Map-D.
Definition: FileMgr.h:67
const int getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const
Definition: Catalog.cpp:1523
std::unordered_map< std::string, std::vector< std::string > > getGranteesOfSharedDashboards(const std::vector< std::string > &dashboard_ids)
std::atomic< std::thread::id > thread_holding_write_lock
Definition: Catalog.h:520
static TimeT::rep execution(F func, Args &&... args)
Definition: sample.cpp:29
int32_t permissionType
Definition: DBObject.h:55
virtual size_t getNumRows() const
void populateRoleDbObjects(const std::vector< DBObject > &objects)
void setForeignServerDataWrapper(const std::string &server_name, const std::string &data_wrapper)
Definition: Catalog.cpp:2493
std::string viewSQL
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata > >> ChunkMetadataVector
mapd_unique_lock< mapd_shared_mutex > write_lock
Definition: sqltypes.h:46
SQLTypeInfo columnT