OmniSciDB  91042dcc5b
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Catalog.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
29 #pragma once
30 
31 #include <array>
32 #include <atomic>
33 #include <cstdint>
34 #include <ctime>
35 #include <limits>
36 #include <list>
37 #include <map>
38 #include <mutex>
39 #include <string>
40 #include <utility>
41 #include <vector>
42 
43 #include "Calcite/Calcite.h"
47 #include "Catalog/DictDescriptor.h"
48 #include "Catalog/ForeignServer.h"
49 #include "Catalog/ForeignTable.h"
50 #include "Catalog/LinkDescriptor.h"
51 #include "Catalog/SessionInfo.h"
52 #include "Catalog/SysCatalog.h"
54 #include "Catalog/TableMetadata.h"
55 #include "Catalog/Types.h"
56 #include "DataMgr/DataMgr.h"
60 
61 #include "LeafHostInfo.h"
62 
64 
65 namespace Parser {
66 
68 
69 } // namespace Parser
70 
71 class TableArchiver;
72 
73 // SPI means Sequential Positional Index which is equivalent to the input index in a
74 // RexInput node
75 #define SPIMAP_MAGIC1 (std::numeric_limits<unsigned>::max() / 4)
76 #define SPIMAP_MAGIC2 8
77 #define SPIMAP_GEO_PHYSICAL_INPUT(c, i) \
78  (SPIMAP_MAGIC1 + (unsigned)(SPIMAP_MAGIC2 * ((c) + 1) + (i)))
79 
80 namespace File_Namespace {
81 struct FileMgrParams;
82 }
83 namespace Catalog_Namespace {
86 
87  TableEpochInfo(const int32_t table_id_param, const int32_t table_epoch_param)
88  : table_id(table_id_param), table_epoch(table_epoch_param) {}
89  TableEpochInfo(const int32_t table_id_param,
90  const int32_t table_epoch_param,
91  const size_t leaf_index_param)
92  : table_id(table_id_param)
93  , table_epoch(table_epoch_param)
94  , leaf_index(leaf_index_param) {}
95 };
96 
97 static constexpr const char* USERS_SYS_TABLE_NAME{"users"};
98 static constexpr const char* TABLES_SYS_TABLE_NAME{"tables"};
99 static constexpr const char* DASHBOARDS_SYS_TABLE_NAME{"dashboards"};
100 static constexpr const char* DATABASES_SYS_TABLE_NAME{"databases"};
101 static constexpr const char* PERMISSIONS_SYS_TABLE_NAME{"permissions"};
102 static constexpr const char* ROLES_SYS_TABLE_NAME{"roles"};
103 static constexpr const char* ROLE_ASSIGNMENTS_SYS_TABLE_NAME{"role_assignments"};
104 static constexpr const char* MEMORY_SUMMARY_SYS_TABLE_NAME{"memory_summary"};
105 static constexpr const char* MEMORY_DETAILS_SYS_TABLE_NAME{"memory_details"};
106 static constexpr const char* STORAGE_DETAILS_SYS_TABLE_NAME{"storage_details"};
107 
114 class Catalog final {
115  public:
124  Catalog(const std::string& basePath,
125  const DBMetadata& curDB,
126  std::shared_ptr<Data_Namespace::DataMgr> dataMgr,
127  const std::vector<LeafHostInfo>& string_dict_hosts,
128  std::shared_ptr<Calcite> calcite,
129  bool is_new_db);
134  Catalog();
135 
142  ~Catalog();
143 
144  static void expandGeoColumn(const ColumnDescriptor& cd,
145  std::list<ColumnDescriptor>& columns);
146  void createTable(TableDescriptor& td,
147  const std::list<ColumnDescriptor>& columns,
148  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs,
149  bool isLogicalTable);
150  void createShardedTable(
151  TableDescriptor& td,
152  const std::list<ColumnDescriptor>& columns,
153  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs);
155  bool skip_system_role_creation = false);
157  std::string createLink(LinkDescriptor& ld, size_t min_length);
158  void dropTable(const TableDescriptor* td);
159  void truncateTable(const TableDescriptor* td);
160  void renameTable(const TableDescriptor* td, const std::string& newTableName);
161  void renameTable(std::vector<std::pair<std::string, std::string>>& names);
162  void renameColumn(const TableDescriptor* td,
163  const ColumnDescriptor* cd,
164  const std::string& newColumnName);
165  void addColumn(const TableDescriptor& td, ColumnDescriptor& cd);
166  void dropColumn(const TableDescriptor& td, const ColumnDescriptor& cd);
167  void removeFragmenterForTable(const int table_id) const;
168 
169  const std::map<int, const ColumnDescriptor*> getDictionaryToColumnMapping();
170 
179  const TableDescriptor* getMetadataForTable(const std::string& tableName,
180  const bool populateFragmenter = true) const;
181  const TableDescriptor* getMetadataForTable(int tableId,
182  bool populateFragmenter = true) const;
183 
184  std::optional<std::string> getTableName(int32_t table_id) const;
185  std::optional<int32_t> getTableId(const std::string& table_name) const;
186 
187  const ColumnDescriptor* getMetadataForColumn(int tableId,
188  const std::string& colName) const;
189  const ColumnDescriptor* getMetadataForColumn(int tableId, int columnId) const;
190 
191  const int getColumnIdBySpi(const int tableId, const size_t spi) const;
192  const ColumnDescriptor* getMetadataForColumnBySpi(const int tableId,
193  const size_t spi) const;
194 
195  const DashboardDescriptor* getMetadataForDashboard(const std::string& userId,
196  const std::string& dashName) const;
197 
198  const DashboardDescriptor* getMetadataForDashboard(const int32_t dashboard_id) const;
199  void deleteMetadataForDashboards(const std::vector<int32_t> ids,
200  const UserMetadata& user);
201 
202  const LinkDescriptor* getMetadataForLink(const std::string& link) const;
203  const LinkDescriptor* getMetadataForLink(int linkId) const;
204 
206  const std::string& tableName) const;
207 
208  const foreign_storage::ForeignTable* getForeignTable(int table_id) const;
209 
218  std::list<const ColumnDescriptor*> getAllColumnMetadataForTable(
219  const int tableId,
220  const bool fetchSystemColumns,
221  const bool fetchVirtualColumns,
222  const bool fetchPhysicalColumns) const;
223 
224  std::list<const TableDescriptor*> getAllTableMetadata() const;
225  std::vector<TableDescriptor> getAllTableMetadataCopy() const;
226  std::list<const DashboardDescriptor*> getAllDashboardsMetadata() const;
227  std::vector<DashboardDescriptor> getAllDashboardsMetadataCopy() const;
228  const DBMetadata& getCurrentDB() const { return currentDB_; }
230  std::shared_ptr<Calcite> getCalciteMgr() const { return calciteMgr_; }
231  const std::string& getCatalogBasePath() const { return basePath_; }
232 
233  const DictDescriptor* getMetadataForDict(int dict_ref, bool loadDict = true) const;
234 
235  const std::vector<LeafHostInfo>& getStringDictionaryHosts() const;
236 
238 
239  std::vector<const TableDescriptor*> getPhysicalTablesDescriptors(
240  const TableDescriptor* logical_table_desc,
241  bool populate_fragmenter = true) const;
242 
243  std::vector<std::pair<int32_t, int32_t>> getAllPersistedTableAndShardIds() const;
244 
253  std::vector<std::string> getTableNamesForUser(
254  const UserMetadata& user,
255  const GetTablesType get_tables_type) const;
256 
266  std::vector<TableMetadata> getTablesMetadataForUser(
267  const UserMetadata& user_metadata,
268  const GetTablesType get_tables_type,
269  const std::string& filter_table_name) const;
270 
271  int32_t getTableEpoch(const int32_t db_id, const int32_t table_id) const;
272  void setTableEpoch(const int db_id, const int table_id, const int new_epoch);
273  void setMaxRollbackEpochs(const int32_t table_id, const int32_t max_rollback_epochs);
274  void setMaxRows(const int32_t table_id, const int64_t max_rows);
275 
276  std::vector<TableEpochInfo> getTableEpochs(const int32_t db_id,
277  const int32_t table_id) const;
278  void setTableEpochs(const int32_t db_id,
279  const std::vector<TableEpochInfo>& table_epochs) const;
280 
281  void setTableEpochsLogExceptions(const int32_t db_id,
282  const std::vector<TableEpochInfo>& table_epochs) const;
283 
284  int getDatabaseId() const { return currentDB_.dbId; }
286  void roll(const bool forward);
288  void delDictionary(const ColumnDescriptor& cd);
289  void getDictionary(const ColumnDescriptor& cd,
290  std::map<int, StringDictionary*>& stringDicts);
291 
292  const bool checkMetadataForDeletedRecs(const TableDescriptor* td, int column_id) const;
293  const ColumnDescriptor* getDeletedColumn(const TableDescriptor* td) const;
295 
296  void setDeletedColumn(const TableDescriptor* td, const ColumnDescriptor* cd);
297  int getLogicalTableId(const int physicalTableId) const;
298  void checkpoint(const int logicalTableId) const;
299  void checkpointWithAutoRollback(const int logical_table_id) const;
300  void resetTableEpochFloor(const int logicalTableId) const;
301  std::string name() const { return getCurrentDB().dbName; }
302  void eraseDbMetadata();
303  void eraseDbPhysicalData();
304  void eraseTablePhysicalData(const TableDescriptor* td);
305  void setForReload(const int32_t tableId);
306 
307  std::vector<std::string> getTableDataDirectories(const TableDescriptor* td) const;
308  std::vector<std::string> getTableDictDirectories(const TableDescriptor* td) const;
309  std::set<std::string> getTableDictDirectoryPaths(int32_t table_id) const;
310  std::string getColumnDictDirectory(const ColumnDescriptor* cd,
311  bool file_name_only = true) const;
312  std::string dumpSchema(const TableDescriptor* td) const;
313  std::string dumpCreateTable(const TableDescriptor* td,
314  bool multiline_formatting = true,
315  bool dump_defaults = false) const;
316 
324  static const std::string getForeignTableSchema(bool if_not_exists = false);
325 
333  static const std::string getForeignServerSchema(bool if_not_exists = false);
334 
344  void createForeignServer(std::unique_ptr<foreign_storage::ForeignServer> foreign_server,
345  bool if_not_exists);
346 
355  const std::string& server_name) const;
356 
366  const std::unique_ptr<const foreign_storage::ForeignServer> getForeignServerFromStorage(
367  const std::string& server_name);
368 
378  const std::unique_ptr<const foreign_storage::ForeignTable> getForeignTableFromStorage(
379  int table_id);
380 
387  void changeForeignServerOwner(const std::string& server_name, const int new_owner_id);
388 
395  void setForeignServerDataWrapper(const std::string& server_name,
396  const std::string& data_wrapper);
403  void setForeignServerOptions(const std::string& server_name,
404  const std::string& options);
411  void renameForeignServer(const std::string& server_name, const std::string& name);
412 
418  void dropForeignServer(const std::string& server_name);
419 
433  const rapidjson::Value* filters,
434  const UserMetadata& user,
435  std::vector<const foreign_storage::ForeignServer*>& results);
436 
441 
452  bool validateNonExistentTableOrView(const std::string& name, const bool if_not_exists);
453 
461  std::vector<const TableDescriptor*> getAllForeignTablesForRefresh() const;
462 
469  void updateForeignTableRefreshTimes(const int32_t table_id);
470 
477  void setForeignTableOptions(const std::string& table_name,
478  foreign_storage::OptionsMap& options_map,
479  bool clear_existing_options = true);
480 
481  void updateLeaf(const LeafHostInfo& string_dict_host);
482 
483  // For testing purposes only
484  void setUncappedTableEpoch(const std::string& table_name);
485 
486  // Methods for accessing and updating custom expressions
487 
495  static const std::string getCustomExpressionsSchema(bool if_not_exists = false);
496 
504  int32_t createCustomExpression(std::unique_ptr<CustomExpression> custom_expression);
505 
513  const CustomExpression* getCustomExpression(int32_t custom_expression_id) const;
514 
524  const std::unique_ptr<const CustomExpression> getCustomExpressionFromStorage(
525  int32_t custom_expression_id);
526 
535  std::vector<const CustomExpression*> getCustomExpressionsForUser(
536  const UserMetadata& user) const;
537 
544  void updateCustomExpression(int32_t custom_expression_id,
545  const std::string& expression_json);
546 
553  void deleteCustomExpressions(const std::vector<int32_t>& custom_expression_ids,
554  bool do_soft_delete);
555 
564  void reassignOwners(const std::set<std::string>& old_owners,
565  const std::string& new_owner);
566 
567  protected:
570  void updateDictionaryNames();
573  void updateGeoColumns();
575  void updateLinkSchema();
578  void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id);
579  void updateDictionarySchema();
580  void updatePageSize();
585  void updateFsiSchemas();
589  void buildMaps();
590  void addTableToMap(const TableDescriptor* td,
591  const std::list<ColumnDescriptor>& columns,
592  const std::list<DictDescriptor>& dicts);
593  void addReferenceToForeignDict(ColumnDescriptor& referencing_column,
594  Parser::SharedDictionaryDef shared_dict_def,
595  const bool persist_reference);
597  ColumnDescriptor& cd,
598  std::list<ColumnDescriptor>& cdd,
599  std::list<DictDescriptor>& dds,
600  const TableDescriptor td,
601  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs);
603  std::list<DictDescriptor>& dds,
604  const TableDescriptor& td,
605  const bool isLogicalTable);
608  void addLinkToMap(LinkDescriptor& ld);
609  void removeTableFromMap(const std::string& tableName,
610  const int tableId,
611  const bool is_on_error = false);
612  void eraseTableMetadata(const TableDescriptor* td);
614  void doTruncateTable(const TableDescriptor* td);
615  void renamePhysicalTable(const TableDescriptor* td, const std::string& newTableName);
616  void renamePhysicalTable(std::vector<std::pair<std::string, std::string>>& names,
617  std::vector<int>& tableIds);
618  void instantiateFragmenter(TableDescriptor* td) const;
620  std::list<const ColumnDescriptor*>& colDescs,
621  const bool fetchSystemColumns,
622  const bool fetchVirtualColumns,
623  const bool fetchPhysicalColumns) const;
624  std::string calculateSHA1(const std::string& data);
625  std::string generatePhysicalTableName(const std::string& logicalTableName,
626  const int32_t& shardNumber);
627  std::vector<DBObject> parseDashboardObjects(const std::string& view_meta,
628  const int& user_id);
629  void createOrUpdateDashboardSystemRole(const std::string& view_meta,
630  const int32_t& user_id,
631  const std::string& dash_role_name);
632 
633  const int getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const;
634 
636  const std::list<ColumnDescriptor>& cds) const;
637  void dropTableFromJsonUnlocked(const std::string& table_name) const;
638 
639  std::string basePath_;
651 
654  std::shared_ptr<Data_Namespace::DataMgr> dataMgr_;
655 
656  const std::vector<LeafHostInfo> string_dict_hosts_;
657  std::shared_ptr<Calcite> calciteMgr_;
658 
660  static const std::string
661  physicalTableNameTag_; // extra component added to the name of each physical table
664 
665  // this tuple is for rolling forw/back once after ALTER ADD/DEL/MODIFY columns
666  // succeeds/fails
667  // get(0) = old ColumnDescriptor*
668  // get(1) = new ColumnDescriptor*
670  std::vector<std::pair<ColumnDescriptor*, ColumnDescriptor*>>;
672 
673  private:
674  void gatherAdditionalInfo(std::vector<std::string>& additional_info,
675  std::set<std::string>& shared_dict_column_names,
676  const TableDescriptor* td) const;
677  std::string quoteIfRequired(const std::string& column_name) const;
680  const std::string& temp_data_dir,
681  const std::unordered_map<int, int>& all_column_ids_map) const;
682  void renameTableDirectories(const std::string& temp_data_dir,
683  const std::vector<std::string>& target_paths,
684  const std::string& name_prefix) const;
685  void buildForeignServerMap();
686  void addForeignTableDetails();
687 
688  void setForeignServerProperty(const std::string& server_name,
689  const std::string& property,
690  const std::string& value);
691 
693  const std::string& property,
694  const std::string& value);
695 
697  const TableDescriptorUpdateParams& table_update_params);
698  void alterTableMetadata(const TableDescriptor* td,
699  const TableDescriptorUpdateParams& table_update_params);
700  void setTableFileMgrParams(const int table_id,
701  const File_Namespace::FileMgrParams& file_mgr_params);
703  const UserMetadata& user_metadata,
704  const GetTablesType get_tables_type) const;
705 
708  const std::string& table_name) const;
709 
715  std::unique_ptr<foreign_storage::ForeignServer> foreign_server,
716  bool if_not_exists);
717 
719  const std::string& tableName) const;
720 
721  const Catalog* getObjForLock();
722  void removeChunks(const int table_id) const;
723 
725  std::unique_ptr<CustomExpression> getCustomExpressionFromConnector(size_t row);
726 
727  void restoreOldOwners(
728  const std::map<int32_t, std::vector<DBObject>>& old_owner_db_objects,
729  int32_t new_owner_id);
731  const std::map<int32_t, std::vector<DBObject>>& old_owner_db_objects,
732  int32_t new_owner_id);
733 
736  void initializeSystemTables();
737  void createSystemTableServer(const std::string& server_name,
738  const std::string& data_wrapper_type);
739  void createSystemTable(
740  const std::string& table_name,
741  const std::string& server_name,
742  const std::vector<std::pair<std::string, SQLTypeInfo>>& column_type_by_name);
743 
744  void setDeletedColumnUnlocked(const TableDescriptor* td, const ColumnDescriptor* cd);
745 
746  static constexpr const char* CATALOG_SERVER_NAME{"omnisci_catalog_server"};
747  static constexpr const char* MEMORY_STATS_SERVER_NAME{"omnisci_memory_stats_server"};
748  static constexpr const char* STORAGE_STATS_SERVER_NAME{"omnisci_storage_stats_server"};
749  static constexpr std::array<const char*, 3> INTERNAL_SERVERS{CATALOG_SERVER_NAME,
752 
753  public:
754  mutable std::mutex sqliteMutex_;
756  mutable std::atomic<std::thread::id> thread_holding_sqlite_lock;
757  mutable std::atomic<std::thread::id> thread_holding_write_lock;
758  // assuming that you never call into a catalog from another catalog via the same thread
759  static thread_local bool thread_holds_read_lock;
760  bool initialized_ = false;
761 };
762 
763 } // namespace Catalog_Namespace
std::string dumpCreateTable(const TableDescriptor *td, bool multiline_formatting=true, bool dump_defaults=false) const
Definition: Catalog.cpp:4667
static constexpr const char * MEMORY_DETAILS_SYS_TABLE_NAME
Definition: Catalog.h:105
void serializeTableJsonUnlocked(const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
Definition: Catalog.cpp:2526
std::unordered_map< const TableDescriptor *, const ColumnDescriptor * > DeletedColumnPerTableMap
Definition: Types.h:47
void instantiateFragmenter(TableDescriptor *td) const
Definition: Catalog.cpp:1453
const foreign_storage::ForeignTable * getForeignTable(const std::string &tableName) const
Definition: Catalog.cpp:1502
void deleteMetadataForDashboards(const std::vector< int32_t > ids, const UserMetadata &user)
Definition: Catalog.cpp:1675
static constexpr std::array< const char *, 3 > INTERNAL_SERVERS
Definition: Catalog.h:749
void executeDropTableSqliteQueries(const TableDescriptor *td)
Definition: Catalog.cpp:3599
void renameTableDirectories(const std::string &temp_data_dir, const std::vector< std::string > &target_paths, const std::string &name_prefix) const
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:4347
void conditionallyInitializeSystemObjects()
Definition: Catalog.cpp:5430
~Catalog()
Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the ...
Definition: Catalog.cpp:200
std::string getColumnDictDirectory(const ColumnDescriptor *cd, bool file_name_only=true) const
Definition: Catalog.cpp:4491
void getDictionary(const ColumnDescriptor &cd, std::map< int, StringDictionary * > &stringDicts)
Definition: Catalog.cpp:1958
mapd_shared_mutex sharedMutex_
Definition: Catalog.h:755
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:114
void restoreOldOwners(const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects, int32_t new_owner_id)
Definition: Catalog.cpp:5349
std::vector< TableDescriptor > getAllTableMetadataCopy() const
Definition: Catalog.cpp:1861
void deleteCustomExpressions(const std::vector< int32_t > &custom_expression_ids, bool do_soft_delete)
Definition: Catalog.cpp:5127
void addReferenceToForeignDict(ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def, const bool persist_reference)
Definition: Catalog.cpp:3277
const ColumnDescriptor * getDeletedColumn(const TableDescriptor *td) const
Definition: Catalog.cpp:3208
void setDeletedColumn(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3251
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:642
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:2286
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:436
void createForeignServer(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2620
TableEpochInfo(const int32_t table_id_param, const int32_t table_epoch_param, const size_t leaf_index_param)
Definition: Catalog.h:89
std::map< ColumnKey, ColumnDescriptor * > ColumnDescriptorMap
Definition: Types.h:38
std::map< int32_t, std::vector< int32_t >> LogicalToPhysicalTableMapById
Definition: Types.h:36
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:229
void addColumn(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:1986
int32_t createCustomExpression(std::unique_ptr< CustomExpression > custom_expression)
Definition: Catalog.cpp:5002
void roll(const bool forward)
Definition: Catalog.cpp:2084
bool validateNonExistentTableOrView(const std::string &name, const bool if_not_exists)
Definition: Catalog.cpp:4830
void setTableFileMgrParams(const int table_id, const File_Namespace::FileMgrParams &file_mgr_params)
Definition: Catalog.cpp:3105
std::vector< const TableDescriptor * > getAllForeignTablesForRefresh() const
Definition: Catalog.cpp:4841
std::map< int, std::unique_ptr< CustomExpression >> CustomExpressionMapById
Definition: Types.h:52
const foreign_storage::ForeignServer * getForeignServer(const std::string &server_name) const
Definition: Catalog.cpp:2665
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:1852
static constexpr const char * DASHBOARDS_SYS_TABLE_NAME
Definition: Catalog.h:99
SqliteConnector sqliteConnector_
Definition: Catalog.h:652
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:656
const DBMetadata currentDB_
Definition: Catalog.h:653
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:644
void createSystemTable(const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name)
Definition: Catalog.cpp:5581
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:654
const LinkDescriptor * getMetadataForLink(const std::string &link) const
std::vector< DashboardDescriptor > getAllDashboardsMetadataCopy() const
Definition: Catalog.cpp:1880
void dropTableFromJsonUnlocked(const std::string &table_name) const
Definition: Catalog.cpp:2588
void setForReload(const int32_t tableId)
Definition: Catalog.cpp:4468
int32_t getTableEpoch(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:2910
void replaceDashboard(DashboardDescriptor &vd)
Definition: Catalog.cpp:4007
void createSystemTableServer(const std::string &server_name, const std::string &data_wrapper_type)
Definition: Catalog.cpp:5570
std::vector< std::string > getTableDataDirectories(const TableDescriptor *td) const
Definition: Catalog.cpp:4477
void updateCustomExpression(int32_t custom_expression_id, const std::string &expression_json)
Definition: Catalog.cpp:5099
std::string generatePhysicalTableName(const std::string &logicalTableName, const int32_t &shardNumber)
Definition: Catalog.cpp:4365
void dropColumn(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:2049
void checkDateInDaysColumnMigration()
Definition: Catalog.cpp:833
void addFrontendViewToMap(DashboardDescriptor &vd)
Definition: Catalog.cpp:1358
std::vector< TableMetadata > getTablesMetadataForUser(const UserMetadata &user_metadata, const GetTablesType get_tables_type, const std::string &filter_table_name) const
Definition: Catalog.cpp:4265
std::unique_ptr< CustomExpression > getCustomExpressionFromConnector(size_t row)
Definition: Catalog.cpp:1201
std::vector< std::pair< int32_t, int32_t > > getAllPersistedTableAndShardIds() const
Definition: Catalog.cpp:4170
TableDescriptor * getMutableMetadataForTableUnlocked(int table_id) const
Definition: Catalog.cpp:1571
const ColumnDescriptor * getShardColumnMetadataForTable(const TableDescriptor *td) const
Definition: Catalog.cpp:4132
void changeForeignServerOwner(const std::string &server_name, const int new_owner_id)
Definition: Catalog.cpp:2717
std::optional< int32_t > getTableId(const std::string &table_name) const
Definition: Catalog.cpp:1553
std::atomic< std::thread::id > thread_holding_sqlite_lock
Definition: Catalog.h:756
void renameColumn(const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
Definition: Catalog.cpp:3903
DeletedColumnPerTableMap deletedColumnPerTable_
Definition: Catalog.h:678
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:643
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:645
void setForeignServerOptions(const std::string &server_name, const std::string &options)
Definition: Catalog.cpp:2749
std::vector< std::string > getTableNamesForUser(const UserMetadata &user, const GetTablesType get_tables_type) const
Definition: Catalog.cpp:4250
void addTableToMap(const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
Definition: Catalog.cpp:1217
void getAllColumnMetadataForTableImpl(const TableDescriptor *td, std::list< const ColumnDescriptor * > &colDescs, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Definition: Catalog.cpp:1806
void resetTableEpochFloor(const int logicalTableId) const
Definition: Catalog.cpp:4321
TableEpochInfo(const int32_t table_id_param, const int32_t table_epoch_param)
Definition: Catalog.h:87
void setForeignTableOptions(const std::string &table_name, foreign_storage::OptionsMap &options_map, bool clear_existing_options=true)
Definition: Catalog.cpp:4885
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:705
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:124
bool filterTableByTypeAndUser(const TableDescriptor *td, const UserMetadata &user_metadata, const GetTablesType get_tables_type) const
Definition: Catalog.cpp:4217
const bool checkMetadataForDeletedRecs(const TableDescriptor *td, int column_id) const
Definition: Catalog.cpp:3214
std::string quoteIfRequired(const std::string &column_name) const
Definition: Catalog.cpp:4925
std::string name() const
Definition: Catalog.h:301
std::list< const DashboardDescriptor * > getAllDashboardsMetadata() const
Definition: Catalog.cpp:1871
void updateLeaf(const LeafHostInfo &string_dict_host)
void removeChunks(const int table_id) const
Definition: Catalog.cpp:3530
static constexpr const char * ROLES_SYS_TABLE_NAME
Definition: Catalog.h:102
This file contains the class specification and related data structures for SysCatalog.
std::map< ColumnIdKey, ColumnDescriptor * > ColumnDescriptorMapById
Definition: Types.h:40
void restoreOldOwnersInMemory(const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects, int32_t new_owner_id)
Definition: Catalog.cpp:5396
std::string calculateSHA1(const std::string &data)
Definition: Catalog.cpp:4082
void gatherAdditionalInfo(std::vector< std::string > &additional_info, std::set< std::string > &shared_dict_column_names, const TableDescriptor *td) const
Definition: Catalog.cpp:4936
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:228
void addLinkToMap(LinkDescriptor &ld)
Definition: Catalog.cpp:1445
std::shared_ptr< Calcite > getCalciteMgr() const
Definition: Catalog.h:230
static const std::string getForeignTableSchema(bool if_not_exists=false)
Definition: Catalog.cpp:690
std::map< int, TableDescriptor * > TableDescriptorMapById
Definition: Types.h:35
const ColumnDescriptor * getDeletedColumnIfRowsDeleted(const TableDescriptor *td) const
Definition: Catalog.cpp:3226
void adjustAlteredTableFiles(const std::string &temp_data_dir, const std::unordered_map< int, int > &all_column_ids_map) const
std::map< std::string, LinkDescriptor * > LinkDescriptorMap
Definition: Types.h:44
void delDictionary(const ColumnDescriptor &cd)
Definition: Catalog.cpp:1915
GetTablesType
Definition: Catalog.h:63
std::shared_timed_mutex mapd_shared_mutex
void dropForeignServer(const std::string &server_name)
Definition: Catalog.cpp:2781
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3256
void dropTable(const TableDescriptor *td)
Definition: Catalog.cpp:3540
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
static const std::string getForeignServerSchema(bool if_not_exists=false)
Definition: Catalog.cpp:683
int getDatabaseId() const
Definition: Catalog.h:284
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const
static constexpr const char * MEMORY_SUMMARY_SYS_TABLE_NAME
Definition: Catalog.h:104
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:641
int getLogicalTableId(const int physicalTableId) const
Definition: Catalog.cpp:4289
const CustomExpression * getCustomExpression(int32_t custom_expression_id) const
Definition: Catalog.cpp:5051
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1579
specifies the content in-memory of a row in the column metadata table
static constexpr const char * ROLE_ASSIGNMENTS_SYS_TABLE_NAME
Definition: Catalog.h:103
void setForeignTableProperty(const foreign_storage::ForeignTable *table, const std::string &property, const std::string &value)
Definition: Catalog.cpp:4905
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:2142
const std::vector< LeafHostInfo > & getStringDictionaryHosts() const
Definition: Catalog.cpp:1615
void checkpointWithAutoRollback(const int logical_table_id) const
Definition: Catalog.cpp:4311
static constexpr const char * USERS_SYS_TABLE_NAME
Definition: Catalog.h:97
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4150
void setMaxRows(const int32_t table_id, const int64_t max_rows)
Definition: Catalog.cpp:3071
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1399
const ColumnDescriptor * getMetadataForColumnBySpi(const int tableId, const size_t spi) const
Definition: Catalog.cpp:1665
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:649
void alterPhysicalTableMetadata(const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
Definition: Catalog.cpp:2994
static constexpr const char * DATABASES_SYS_TABLE_NAME
Definition: Catalog.h:100
ForeignServerMap foreignServerMap_
Definition: Catalog.h:648
void createShardedTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:3409
std::map< std::string, std::shared_ptr< foreign_storage::ForeignServer >> ForeignServerMap
Definition: Types.h:49
void reassignOwners(const std::set< std::string > &old_owners, const std::string &new_owner)
Definition: Catalog.cpp:5219
std::vector< const CustomExpression * > getCustomExpressionsForUser(const UserMetadata &user) const
Definition: Catalog.cpp:5074
void checkpoint(const int logicalTableId) const
Definition: Catalog.cpp:4303
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2628
static constexpr const char * TABLES_SYS_TABLE_NAME
Definition: Catalog.h:98
std::vector< std::pair< ColumnDescriptor *, ColumnDescriptor * >> ColumnDescriptorsForRoll
Definition: Catalog.h:670
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:657
static const std::string physicalTableNameTag_
Definition: Catalog.h:661
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1363
void setTableEpoch(const int db_id, const int table_id, const int new_epoch)
Definition: Catalog.cpp:2961
void alterTableMetadata(const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
Definition: Catalog.cpp:3021
std::vector< std::string > getTableDictDirectories(const TableDescriptor *td) const
Definition: Catalog.cpp:4512
void getForeignServersForUser(const rapidjson::Value *filters, const UserMetadata &user, std::vector< const foreign_storage::ForeignServer * > &results)
Definition: Catalog.cpp:2815
std::map< DictRef, std::unique_ptr< DictDescriptor >> DictDescriptorMapById
Definition: Types.h:41
std::map< std::string, std::shared_ptr< DashboardDescriptor >> DashboardDescriptorMap
Definition: Types.h:43
static const std::string getCustomExpressionsSchema(bool if_not_exists=false)
Definition: Catalog.cpp:698
static constexpr const char * PERMISSIONS_SYS_TABLE_NAME
Definition: Catalog.h:101
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1836
const Catalog * getObjForLock()
Definition: Catalog.cpp:225
static constexpr const char * STORAGE_DETAILS_SYS_TABLE_NAME
Definition: Catalog.h:106
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1277
void removeFragmenterForTable(const int table_id) const
Definition: Catalog.cpp:3518
std::map< int, LinkDescriptor * > LinkDescriptorMapById
Definition: Types.h:45
std::map< int, std::shared_ptr< foreign_storage::ForeignServer >> ForeignServerMapById
Definition: Types.h:51
const std::unique_ptr< const CustomExpression > getCustomExpressionFromStorage(int32_t custom_expression_id)
Definition: Catalog.cpp:5060
static constexpr const char * CATALOG_SERVER_NAME
Definition: Catalog.h:746
const int getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const
Definition: Catalog.cpp:1643
void eraseTableMetadata(const TableDescriptor *td)
Definition: Catalog.cpp:3587
bool setColumnSharedDictionary(ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:3299
int32_t createDashboard(DashboardDescriptor &vd, bool skip_system_role_creation=false)
Definition: Catalog.cpp:3942
void updateCustomExpressionsSchema()
Definition: Catalog.cpp:671
std::set< std::string > getTableDictDirectoryPaths(int32_t table_id) const
Definition: Catalog.cpp:4525
void setTableEpochs(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3160
std::map< std::string, std::string, std::less<>> OptionsMap
void setMaxRollbackEpochs(const int32_t table_id, const int32_t max_rollback_epochs)
Definition: Catalog.cpp:3046
std::optional< std::string > getTableName(int32_t table_id) const
Definition: Catalog.cpp:1544
std::string createLink(LinkDescriptor &ld, size_t min_length)
Definition: Catalog.cpp:4094
Descriptor for a dictionary for a string columne.
void updateLogicalToPhysicalTableLinkSchema()
Definition: Catalog.cpp:597
void setUncappedTableEpoch(const std::string &table_name)
Definition: Catalog.cpp:3090
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:650
std::atomic< std::thread::id > thread_holding_write_lock
Definition: Catalog.h:757
foreign_storage::ForeignTable * getForeignTableUnlocked(const std::string &tableName) const
Definition: Catalog.cpp:1493
void setForeignServerDataWrapper(const std::string &server_name, const std::string &data_wrapper)
Definition: Catalog.cpp:2728
void setTableEpochsLogExceptions(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3196
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
SqliteConnector & getSqliteConnector()
Definition: Catalog.h:285
const std::string & getCatalogBasePath() const
Definition: Catalog.h:231
DictRef addDictionary(ColumnDescriptor &cd)
Definition: Catalog.cpp:1890
const std::unique_ptr< const foreign_storage::ForeignTable > getForeignTableFromStorage(int table_id)
Definition: Catalog.cpp:2697
void renameTable(const TableDescriptor *td, const std::string &newTableName)
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:647
std::string dumpSchema(const TableDescriptor *td) const
Definition: Catalog.cpp:4540
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:3368
void updateForeignTableRefreshTimes(const int32_t table_id)
Definition: Catalog.cpp:4863
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:671
static constexpr const char * MEMORY_STATS_SERVER_NAME
Definition: Catalog.h:747
void doTruncateTable(const TableDescriptor *td)
Definition: Catalog.cpp:3453
const std::map< int, const ColumnDescriptor * > getDictionaryToColumnMapping()
Definition: Catalog.cpp:4186
void renamePhysicalTable(const TableDescriptor *td, const std::string &newTableName)
void renameForeignServer(const std::string &server_name, const std::string &name)
Definition: Catalog.cpp:2769
std::vector< DBObject > parseDashboardObjects(const std::string &view_meta, const int &user_id)
Definition: Catalog.cpp:1369
const std::unique_ptr< const foreign_storage::ForeignServer > getForeignServerFromStorage(const std::string &server_name)
Definition: Catalog.cpp:2677
static constexpr const char * STORAGE_STATS_SERVER_NAME
Definition: Catalog.h:748
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:659
Catalog()
Constructor builds a hollow catalog used during constructor of other catalogs.
Definition: Catalog.cpp:198
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:640
void setForeignServerProperty(const std::string &server_name, const std::string &property, const std::string &value)
Definition: Catalog.cpp:4414
static thread_local bool thread_holds_read_lock
Definition: Catalog.h:759
std::map< std::string, TableDescriptor * > TableDescriptorMap
Definition: Types.h:34
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:646
void CheckAndExecuteMigrationsPostBuildMaps()
Definition: Catalog.cpp:936
std::vector< TableEpochInfo > getTableEpochs(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:3132
const int getColumnIdBySpi(const int tableId, const size_t spi) const
Definition: Catalog.cpp:1660
void createDefaultServersIfNotExists()
Definition: Catalog.cpp:4435
void truncateTable(const TableDescriptor *td)
Definition: Catalog.cpp:3445
void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id)
Definition: Catalog.cpp:611