OmniSciDB  0b528656ed
Catalog.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
29 #pragma once
30 
31 #include <atomic>
32 #include <cstdint>
33 #include <ctime>
34 #include <limits>
35 #include <list>
36 #include <map>
37 #include <mutex>
38 #include <string>
39 #include <utility>
40 #include <vector>
41 
42 #include "Calcite/Calcite.h"
45 #include "Catalog/DictDescriptor.h"
46 #include "Catalog/ForeignServer.h"
47 #include "Catalog/ForeignTable.h"
48 #include "Catalog/LinkDescriptor.h"
49 #include "Catalog/SessionInfo.h"
50 #include "Catalog/SysCatalog.h"
52 #include "Catalog/Types.h"
53 #include "DataMgr/DataMgr.h"
54 #include "LockMgr/LockMgrImpl.h"
58 
59 #include "LeafHostInfo.h"
60 
62 
63 namespace Parser {
64 
66 
67 } // namespace Parser
68 
69 class TableArchiver;
70 
71 // SPI means Sequential Positional Index which is equivalent to the input index in a
72 // RexInput node
73 #define SPIMAP_MAGIC1 (std::numeric_limits<unsigned>::max() / 4)
74 #define SPIMAP_MAGIC2 8
75 #define SPIMAP_GEO_PHYSICAL_INPUT(c, i) \
76  (SPIMAP_MAGIC1 + (unsigned)(SPIMAP_MAGIC2 * ((c) + 1) + (i)))
77 
78 namespace Catalog_Namespace {
79 
86 class Catalog final {
87  public:
96  Catalog(const std::string& basePath,
97  const DBMetadata& curDB,
98  std::shared_ptr<Data_Namespace::DataMgr> dataMgr,
99  const std::vector<LeafHostInfo>& string_dict_hosts,
100  std::shared_ptr<Calcite> calcite,
101  bool is_new_db);
102 
109  ~Catalog();
110 
111  static void expandGeoColumn(const ColumnDescriptor& cd,
112  std::list<ColumnDescriptor>& columns);
113  void createTable(TableDescriptor& td,
114  const std::list<ColumnDescriptor>& columns,
115  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs,
116  bool isLogicalTable);
117  void createShardedTable(
118  TableDescriptor& td,
119  const std::list<ColumnDescriptor>& columns,
120  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs);
121  int32_t createDashboard(DashboardDescriptor& vd);
122  void replaceDashboard(DashboardDescriptor& vd);
123  std::string createLink(LinkDescriptor& ld, size_t min_length);
124  void dropTable(const TableDescriptor* td);
125  void truncateTable(const TableDescriptor* td);
126  void renameTable(const TableDescriptor* td, const std::string& newTableName);
127  void renameColumn(const TableDescriptor* td,
128  const ColumnDescriptor* cd,
129  const std::string& newColumnName);
130  void addColumn(const TableDescriptor& td, ColumnDescriptor& cd);
131  void dropColumn(const TableDescriptor& td, const ColumnDescriptor& cd);
132  void removeChunks(const int table_id);
133 
134  const std::map<int, const ColumnDescriptor*> getDictionaryToColumnMapping();
135 
144  const TableDescriptor* getMetadataForTable(const std::string& tableName,
145  const bool populateFragmenter = true) const;
146  const TableDescriptor* getMetadataForTableImpl(int tableId,
147  const bool populateFragmenter) const;
148  const TableDescriptor* getMetadataForTable(int tableId,
149  bool populateFragmenter = true) const;
150 
151  const ColumnDescriptor* getMetadataForColumn(int tableId,
152  const std::string& colName) const;
153  const ColumnDescriptor* getMetadataForColumn(int tableId, int columnId) const;
154  const ColumnDescriptor* getMetadataForColumnUnlocked(int tableId, int columnId) const;
155 
156  const int getColumnIdBySpi(const int tableId, const size_t spi) const;
157  const ColumnDescriptor* getMetadataForColumnBySpi(const int tableId,
158  const size_t spi) const;
159 
160  const DashboardDescriptor* getMetadataForDashboard(const std::string& userId,
161  const std::string& dashName) const;
162  void deleteMetadataForDashboard(const std::string& userId, const std::string& dashName);
163 
164  const DashboardDescriptor* getMetadataForDashboard(const int32_t dashboard_id) const;
165  void deleteMetadataForDashboard(const int32_t dashboard_id);
166 
167  const LinkDescriptor* getMetadataForLink(const std::string& link) const;
168  const LinkDescriptor* getMetadataForLink(int linkId) const;
169 
178  std::list<const ColumnDescriptor*> getAllColumnMetadataForTable(
179  const int tableId,
180  const bool fetchSystemColumns,
181  const bool fetchVirtualColumns,
182  const bool fetchPhysicalColumns) const;
186  std::list<const ColumnDescriptor*> getAllColumnMetadataForTableUnlocked(
187  const int tableId,
188  const bool fetchSystemColumns,
189  const bool fetchVirtualColumns,
190  const bool fetchPhysicalColumns) const;
191 
192  std::list<const TableDescriptor*> getAllTableMetadata() const;
193  std::list<const DashboardDescriptor*> getAllDashboardsMetadata() const;
194  const DBMetadata& getCurrentDB() const { return currentDB_; }
195  Data_Namespace::DataMgr& getDataMgr() const { return *dataMgr_; }
196  std::shared_ptr<Calcite> getCalciteMgr() const { return calciteMgr_; }
197  const std::string& getBasePath() const { return basePath_; }
198 
199  const DictDescriptor* getMetadataForDict(int dict_ref, bool loadDict = true) const;
200  const DictDescriptor* getMetadataForDictUnlocked(int dict_ref, bool loadDict) const;
201 
202  const std::vector<LeafHostInfo>& getStringDictionaryHosts() const;
203 
204  const ColumnDescriptor* getShardColumnMetadataForTable(const TableDescriptor* td) const;
205 
206  std::vector<const TableDescriptor*> getPhysicalTablesDescriptors(
207  const TableDescriptor* logicalTableDesc) const;
208 
217  std::vector<std::string> getTableNamesForUser(
218  const UserMetadata& user,
219  const GetTablesType get_tables_type) const;
220 
221  int32_t getTableEpoch(const int32_t db_id, const int32_t table_id) const;
222  void setTableEpoch(const int db_id, const int table_id, const int new_epoch);
223  int getDatabaseId() const { return currentDB_.dbId; }
224  SqliteConnector& getSqliteConnector() { return sqliteConnector_; }
225  void roll(const bool forward);
226  DictRef addDictionary(ColumnDescriptor& cd);
227  void delDictionary(const ColumnDescriptor& cd);
228  void getDictionary(const ColumnDescriptor& cd,
229  std::map<int, StringDictionary*>& stringDicts);
230 
231  static void set(const std::string& dbName, std::shared_ptr<Catalog> cat);
232  static std::shared_ptr<Catalog> get(const std::string& dbName);
233  static std::shared_ptr<Catalog> get(const int32_t db_id);
234  static std::shared_ptr<Catalog> get(const std::string& basePath,
235  const DBMetadata& curDB,
236  std::shared_ptr<Data_Namespace::DataMgr> dataMgr,
237  const std::vector<LeafHostInfo>& string_dict_hosts,
238  std::shared_ptr<Calcite> calcite,
239  bool is_new_db);
240  static void remove(const std::string& dbName);
241 
242  const bool checkMetadataForDeletedRecs(const TableDescriptor* td, int column_id) const;
243  const ColumnDescriptor* getDeletedColumn(const TableDescriptor* td) const;
244  const ColumnDescriptor* getDeletedColumnIfRowsDeleted(const TableDescriptor* td) const;
245 
246  void setDeletedColumn(const TableDescriptor* td, const ColumnDescriptor* cd);
247  void setDeletedColumnUnlocked(const TableDescriptor* td, const ColumnDescriptor* cd);
248  int getLogicalTableId(const int physicalTableId) const;
249  void checkpoint(const int logicalTableId) const;
250  std::string name() const { return getCurrentDB().dbName; }
251  void eraseDBData();
252  void eraseTablePhysicalData(const TableDescriptor* td);
253  void vacuumDeletedRows(const TableDescriptor* td) const;
254  void vacuumDeletedRows(const int logicalTableId) const;
255  void setForReload(const int32_t tableId);
256 
257  std::vector<std::string> getTableDataDirectories(const TableDescriptor* td) const;
258  std::vector<std::string> getTableDictDirectories(const TableDescriptor* td) const;
259  std::string getColumnDictDirectory(const ColumnDescriptor* cd) const;
260  std::string dumpSchema(const TableDescriptor* td) const;
261  std::string dumpCreateTable(const TableDescriptor* td,
262  bool multiline_formatting = true,
263  bool dump_defaults = false) const;
264 
272  static const std::string getForeignTableSchema(bool if_not_exists = false);
273 
281  static const std::string getForeignServerSchema(bool if_not_exists = false);
282 
292  void createForeignServer(std::unique_ptr<foreign_storage::ForeignServer> foreign_server,
293  bool if_not_exists);
294 
302  const foreign_storage::ForeignServer* getForeignServer(
303  const std::string& server_name) const;
304 
314  const std::unique_ptr<const foreign_storage::ForeignServer> getForeignServerFromStorage(
315  const std::string& server_name);
316 
323  void changeForeignServerOwner(const std::string& server_name, const int new_owner_id);
324 
331  void setForeignServerDataWrapper(const std::string& server_name,
332  const std::string& data_wrapper);
339  void setForeignServerOptions(const std::string& server_name,
340  const std::string& options);
347  void renameForeignServer(const std::string& server_name, const std::string& name);
348 
354  void dropForeignServer(const std::string& server_name);
355 
368  void getForeignServersForUser(
369  const rapidjson::Value* filters,
370  const UserMetadata& user,
371  std::vector<const foreign_storage::ForeignServer*>& results);
372 
376  void createDefaultServersIfNotExists();
377 
388  bool validateNonExistentTableOrView(const std::string& name, const bool if_not_exists);
389 
390  protected:
391  void CheckAndExecuteMigrations();
392  void CheckAndExecuteMigrationsPostBuildMaps();
393  void updateDictionaryNames();
394  void updateTableDescriptorSchema();
395  void updateFixlenArrayColumns();
396  void updateGeoColumns();
397  void updateFrontendViewSchema();
398  void updateLinkSchema();
399  void updateFrontendViewAndLinkUsers();
400  void updateLogicalToPhysicalTableLinkSchema();
401  void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id);
402  void updateDictionarySchema();
403  void updatePageSize();
404  void updateDeletedColumnIndicator();
405  void updateFrontendViewsToDashboards();
406  void createFsiSchemasAndDefaultServers();
407  void dropFsiSchemasAndTables();
408  void recordOwnershipOfObjectsInObjectPermissions();
409  void checkDateInDaysColumnMigration();
410  void createDashboardSystemRoles();
411  void buildMaps();
412  void addTableToMap(const TableDescriptor* td,
413  const std::list<ColumnDescriptor>& columns,
414  const std::list<DictDescriptor>& dicts);
415  void addReferenceToForeignDict(ColumnDescriptor& referencing_column,
416  Parser::SharedDictionaryDef shared_dict_def,
417  const bool persist_reference);
418  bool setColumnSharedDictionary(
419  ColumnDescriptor& cd,
420  std::list<ColumnDescriptor>& cdd,
421  std::list<DictDescriptor>& dds,
422  const TableDescriptor td,
423  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs);
424  void setColumnDictionary(ColumnDescriptor& cd,
425  std::list<DictDescriptor>& dds,
426  const TableDescriptor& td,
427  const bool isLogicalTable);
428  void addFrontendViewToMap(DashboardDescriptor& vd);
429  void addFrontendViewToMapNoLock(DashboardDescriptor& vd);
430  void addLinkToMap(LinkDescriptor& ld);
431  void removeTableFromMap(const std::string& tableName,
432  const int tableId,
433  const bool is_on_error = false);
434  void doDropTable(const TableDescriptor* td);
435  void executeDropTableSqliteQueries(const TableDescriptor* td);
436  void doTruncateTable(const TableDescriptor* td);
437  void renamePhysicalTable(const TableDescriptor* td, const std::string& newTableName);
438  void instantiateFragmenter(TableDescriptor* td) const;
439  void getAllColumnMetadataForTableImpl(const TableDescriptor* td,
440  std::list<const ColumnDescriptor*>& colDescs,
441  const bool fetchSystemColumns,
442  const bool fetchVirtualColumns,
443  const bool fetchPhysicalColumns) const;
444  std::string calculateSHA1(const std::string& data);
445  std::string generatePhysicalTableName(const std::string& logicalTableName,
446  const int32_t& shardNumber);
447  std::vector<DBObject> parseDashboardObjects(const std::string& view_meta,
448  const int& user_id);
449  void createOrUpdateDashboardSystemRole(const std::string& view_meta,
450  const int32_t& user_id,
451  const std::string& dash_role_name);
452 
453  const int getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const;
454 
455  void serializeTableJsonUnlocked(const TableDescriptor* td,
456  const std::list<ColumnDescriptor>& cds) const;
457  void dropTableFromJsonUnlocked(const std::string& table_name) const;
458 
459  std::string basePath_;
470 
473  std::shared_ptr<Data_Namespace::DataMgr> dataMgr_;
474 
475  const std::vector<LeafHostInfo> string_dict_hosts_;
476  std::shared_ptr<Calcite> calciteMgr_;
477 
479  static const std::string
480  physicalTableNameTag_; // extra component added to the name of each physical table
483 
484  // this tuple is for rolling forw/back once after ALTER ADD/DEL/MODIFY columns
485  // succeeds/fails
486  // get(0) = old ColumnDescriptor*
487  // get(1) = new ColumnDescriptor*
489  std::vector<std::pair<ColumnDescriptor*, ColumnDescriptor*>>;
491 
492  private:
493  static std::map<std::string, std::shared_ptr<Catalog>> mapd_cat_map_;
495  void adjustAlteredTableFiles(
496  const std::string& temp_data_dir,
497  const std::unordered_map<int, int>& all_column_ids_map) const;
498  void renameTableDirectories(const std::string& temp_data_dir,
499  const std::vector<std::string>& target_paths,
500  const std::string& name_prefix) const;
501  void buildForeignServerMap();
502  void addForeignTableDetails();
503 
504  void setForeignServerProperty(const std::string& server_name,
505  const std::string& property,
506  const std::string& value);
507 
512  void createForeignServerNoLocks(
513  std::unique_ptr<foreign_storage::ForeignServer> foreign_server,
514  bool if_not_exists);
515 
516  public:
517  mutable std::mutex sqliteMutex_;
519  mutable std::atomic<std::thread::id> thread_holding_sqlite_lock;
520  mutable std::atomic<std::thread::id> thread_holding_write_lock;
521  // assuming that you never call into a catalog from another catalog via the same thread
522  static thread_local bool thread_holds_read_lock;
523 };
524 
525 } // namespace Catalog_Namespace
std::unordered_map< const TableDescriptor *, const ColumnDescriptor * > DeletedColumnPerTableMap
Definition: Types.h:46
mapd_shared_mutex sharedMutex_
Definition: Catalog.h:518
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:86
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:462
std::map< ColumnKey, ColumnDescriptor * > ColumnDescriptorMap
Definition: Types.h:37
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:195
SqliteConnector sqliteConnector_
Definition: Catalog.h:471
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:475
std::shared_ptr< Calcite > getCalciteMgr() const
Definition: Catalog.h:196
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:464
std::map< std::string, std::shared_ptr< foreign_storage::ForeignServer > > ForeignServerMap
Definition: Types.h:48
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:473
std::atomic< std::thread::id > thread_holding_sqlite_lock
Definition: Catalog.h:519
DeletedColumnPerTableMap deletedColumnPerTable_
Definition: Catalog.h:494
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:463
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:465
std::map< int32_t, std::vector< int32_t > > LogicalToPhysicalTableMapById
Definition: Types.h:35
int getDatabaseId() const
Definition: Catalog.h:223
This file contains the class specification and related data structures for SysCatalog.
std::string cat(Ts &&... args)
std::map< ColumnIdKey, ColumnDescriptor * > ColumnDescriptorMapById
Definition: Types.h:39
std::map< int, TableDescriptor * > TableDescriptorMapById
Definition: Types.h:34
std::map< std::string, LinkDescriptor * > LinkDescriptorMap
Definition: Types.h:43
GetTablesType
Definition: Catalog.h:61
std::shared_timed_mutex mapd_shared_mutex
std::vector< std::pair< ColumnDescriptor *, ColumnDescriptor * > > ColumnDescriptorsForRoll
Definition: Catalog.h:489
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:461
specifies the content in-memory of a row in the column metadata table
std::map< int, std::shared_ptr< foreign_storage::ForeignServer > > ForeignServerMapById
Definition: Types.h:50
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:194
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:469
ForeignServerMap foreignServerMap_
Definition: Catalog.h:468
std::map< std::string, std::shared_ptr< DashboardDescriptor > > DashboardDescriptorMap
Definition: Types.h:42
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:476
static const std::string physicalTableNameTag_
Definition: Catalog.h:480
const std::string & getBasePath() const
Definition: Catalog.h:197
std::string name() const
Definition: Catalog.h:250
Definition: Catalog.h:63
std::map< int, LinkDescriptor * > LinkDescriptorMapById
Definition: Types.h:44
Descriptor for a dictionary for a string columne.
std::atomic< std::thread::id > thread_holding_write_lock
Definition: Catalog.h:520
SqliteConnector & getSqliteConnector()
Definition: Catalog.h:224
specifies the content in-memory of a row in the table metadata table
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:467
static std::map< std::string, std::shared_ptr< Catalog > > mapd_cat_map_
Definition: Catalog.h:493
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:490
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:478
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:460
static thread_local bool thread_holds_read_lock
Definition: Catalog.h:522
std::map< DictRef, std::unique_ptr< DictDescriptor > > DictDescriptorMapById
Definition: Types.h:40
std::map< std::string, TableDescriptor * > TableDescriptorMap
Definition: Types.h:33
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:466