OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Catalog.h
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
28 #pragma once
29 
30 #include <array>
31 #include <atomic>
32 #include <cstdint>
33 #include <ctime>
34 #include <limits>
35 #include <list>
36 #include <map>
37 #include <mutex>
38 #include <string>
39 #include <utility>
40 #include <vector>
41 
42 #include "Calcite/Calcite.h"
46 #include "Catalog/DictDescriptor.h"
47 #include "Catalog/ForeignServer.h"
48 #include "Catalog/ForeignTable.h"
49 #include "Catalog/LinkDescriptor.h"
50 #include "Catalog/SessionInfo.h"
51 #include "Catalog/SysCatalog.h"
53 #include "Catalog/TableMetadata.h"
54 #include "Catalog/Types.h"
55 #include "DataMgr/DataMgr.h"
60 
61 #include "LeafHostInfo.h"
62 
64 
65 namespace lockmgr {
66 
67 template <class T>
69 
70 } // namespace lockmgr
71 
72 namespace Parser {
73 
75 
76 } // namespace Parser
77 
78 class TableArchiver;
79 
80 // SPI means Sequential Positional Index which is equivalent to the input index in a
81 // RexInput node
82 #define SPIMAP_MAGIC1 (std::numeric_limits<unsigned>::max() / 4)
83 #define SPIMAP_MAGIC2 8
84 #define SPIMAP_GEO_PHYSICAL_INPUT(c, i) \
85  (SPIMAP_MAGIC1 + (unsigned)(SPIMAP_MAGIC2 * ((c) + 1) + (i)))
86 
87 namespace File_Namespace {
88 struct FileMgrParams;
89 }
90 namespace Catalog_Namespace {
93 
94  TableEpochInfo(const int32_t table_id_param, const int32_t table_epoch_param)
95  : table_id(table_id_param), table_epoch(table_epoch_param) {}
96  TableEpochInfo(const int32_t table_id_param,
97  const int32_t table_epoch_param,
98  const size_t leaf_index_param)
99  : table_id(table_id_param)
100  , table_epoch(table_epoch_param)
101  , leaf_index(leaf_index_param) {}
102 };
103 
104 struct TableNotFoundException : public std::runtime_error {
105  TableNotFoundException(const std::string& table_name,
106  const std::string& db_name,
107  const std::string& context_msg = {})
108  : std::runtime_error("Table/View " + table_name + " for catalog " + db_name +
109  " does not exist." + context_msg) {}
110 };
111 
112 static constexpr const char* USERS_SYS_TABLE_NAME{"users"};
113 static constexpr const char* TABLES_SYS_TABLE_NAME{"tables"};
114 static constexpr const char* DASHBOARDS_SYS_TABLE_NAME{"dashboards"};
115 static constexpr const char* DATABASES_SYS_TABLE_NAME{"databases"};
116 static constexpr const char* PERMISSIONS_SYS_TABLE_NAME{"permissions"};
117 static constexpr const char* ROLES_SYS_TABLE_NAME{"roles"};
118 static constexpr const char* ROLE_ASSIGNMENTS_SYS_TABLE_NAME{"role_assignments"};
119 static constexpr const char* MEMORY_SUMMARY_SYS_TABLE_NAME{"memory_summary"};
120 static constexpr const char* MEMORY_DETAILS_SYS_TABLE_NAME{"memory_details"};
121 static constexpr const char* STORAGE_DETAILS_SYS_TABLE_NAME{"storage_details"};
123  "executor_pool_summary"};
124 static constexpr const char* ML_MODEL_METADATA_SYS_TABLE_NAME{"ml_models"};
125 static constexpr const char* SERVER_LOGS_SYS_TABLE_NAME{"server_logs"};
126 static constexpr const char* REQUEST_LOGS_SYS_TABLE_NAME{"request_logs"};
127 static constexpr const char* WS_SERVER_LOGS_SYS_TABLE_NAME{"web_server_logs"};
128 static constexpr const char* WS_SERVER_ACCESS_LOGS_SYS_TABLE_NAME{
129  "web_server_access_logs"};
130 
131 static const std::array<std::string, 4> kAggregatorOnlySystemTables{
136 
143 class Catalog final {
144  public:
153  Catalog(const std::string& basePath,
154  const DBMetadata& curDB,
155  std::shared_ptr<Data_Namespace::DataMgr> dataMgr,
156  const std::vector<LeafHostInfo>& string_dict_hosts,
157  std::shared_ptr<Calcite> calcite,
158  bool is_new_db);
159 
164  Catalog();
165 
172  ~Catalog();
173 
174  static void expandGeoColumn(const ColumnDescriptor& cd,
175  std::list<ColumnDescriptor>& columns);
176  void createTable(TableDescriptor& td,
177  const std::list<ColumnDescriptor>& columns,
178  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs,
179  bool isLogicalTable);
180  void createShardedTable(
181  TableDescriptor& td,
182  const std::list<ColumnDescriptor>& columns,
183  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs);
186  std::string createLink(LinkDescriptor& ld, size_t min_length);
187  void dropTable(const TableDescriptor* td);
188  void truncateTable(const TableDescriptor* td);
189  void renameTable(const TableDescriptor* td, const std::string& newTableName);
190  void renameTables(const std::vector<std::pair<std::string, std::string>>& names);
191  void renameColumn(const TableDescriptor* td,
192  const ColumnDescriptor* cd,
193  const std::string& newColumnName);
194  int getNextAddedColumnId(const TableDescriptor& td);
195  void addColumn(const TableDescriptor& td, ColumnDescriptor& cd);
198  void dropColumnTransactional(const TableDescriptor& td, const ColumnDescriptor& cd);
200  void dropColumn(const TableDescriptor& td, const ColumnDescriptor& cd);
201  void dropColumnPolicies(const TableDescriptor& td, const ColumnDescriptor& cd);
202  void invalidateCachesForTable(const int table_id);
203  void removeFragmenterForTable(const int table_id) const;
204 
205  const std::map<int, const ColumnDescriptor*> getDictionaryToColumnMapping();
206 
215  const TableDescriptor* getMetadataForTable(const std::string& tableName,
216  const bool populateFragmenter = true) const;
217  const TableDescriptor* getMetadataForTable(int tableId,
218  bool populateFragmenter = true) const;
219 
220  std::optional<std::string> getTableName(int32_t table_id) const;
221  std::optional<int32_t> getTableId(const std::string& table_name) const;
222 
223  const ColumnDescriptor* getMetadataForColumn(int tableId,
224  const std::string& colName) const;
225  const ColumnDescriptor* getMetadataForColumn(int tableId, int columnId) const;
226  const std::optional<std::string> getColumnName(int table_id, int column_id) const;
227 
228  const int getColumnIdBySpi(const int tableId, const size_t spi) const;
229  const ColumnDescriptor* getMetadataForColumnBySpi(const int tableId,
230  const size_t spi) const;
231 
232  const DashboardDescriptor* getMetadataForDashboard(const std::string& userId,
233  const std::string& dashName) const;
234 
235  const DashboardDescriptor* getMetadataForDashboard(const int32_t dashboard_id) const;
236  void deleteMetadataForDashboards(const std::vector<int32_t> ids,
237  const UserMetadata& user);
238 
239  const LinkDescriptor* getMetadataForLink(const std::string& link) const;
240  const LinkDescriptor* getMetadataForLink(int linkId) const;
241 
243  const std::string& tableName) const;
244 
245  const foreign_storage::ForeignTable* getForeignTable(int table_id) const;
246 
255  std::list<const ColumnDescriptor*> getAllColumnMetadataForTable(
256  const int tableId,
257  const bool fetchSystemColumns,
258  const bool fetchVirtualColumns,
259  const bool fetchPhysicalColumns) const;
260 
261  std::list<const TableDescriptor*> getAllTableMetadata() const;
262  std::vector<TableDescriptor> getAllTableMetadataCopy() const;
263  std::list<const DashboardDescriptor*> getAllDashboardsMetadata() const;
264  std::vector<DashboardDescriptor> getAllDashboardsMetadataCopy() const;
265  const DBMetadata& getCurrentDB() const { return currentDB_; }
267  std::shared_ptr<Calcite> getCalciteMgr() const { return calciteMgr_; }
268  void setCalciteMgr(const std::shared_ptr<Calcite>& new_calcite_mgr) {
269  // Used for testing.
270  calciteMgr_ = new_calcite_mgr;
271  }
272 
273  const std::string& getCatalogBasePath() const { return basePath_; }
274 
275  const DictDescriptor* getMetadataForDict(int dict_ref, bool loadDict = true) const;
276 
277  const std::vector<LeafHostInfo>& getStringDictionaryHosts() const;
278 
280 
281  std::vector<const TableDescriptor*> getPhysicalTablesDescriptors(
282  const TableDescriptor* logical_table_desc,
283  bool populate_fragmenter = true) const;
284 
285  std::vector<std::pair<int32_t, int32_t>> getAllPersistedTableAndShardIds() const;
286 
295  std::vector<std::string> getTableNamesForUser(
296  const UserMetadata& user,
297  const GetTablesType get_tables_type) const;
298 
308  std::vector<TableMetadata> getTablesMetadataForUser(
309  const UserMetadata& user_metadata,
310  const GetTablesType get_tables_type,
311  const std::string& filter_table_name) const;
312 
313  int32_t getTableEpoch(const int32_t db_id, const int32_t table_id) const;
314  void setTableEpoch(const int db_id, const int table_id, const int new_epoch);
315  void setMaxRollbackEpochs(const int32_t table_id, const int32_t max_rollback_epochs);
316  void setMaxRows(const int32_t table_id, const int64_t max_rows);
317 
318  std::vector<TableEpochInfo> getTableEpochs(const int32_t db_id,
319  const int32_t table_id) const;
320  void setTableEpochs(const int32_t db_id,
321  const std::vector<TableEpochInfo>& table_epochs) const;
322 
323  void setTableEpochsLogExceptions(const int32_t db_id,
324  const std::vector<TableEpochInfo>& table_epochs) const;
325 
326  int getDatabaseId() const { return currentDB_.dbId; }
328  void rollLegacy(const bool forward);
329  void getDictionary(const ColumnDescriptor& cd,
330  std::map<int, StringDictionary*>& stringDicts);
332 
335 
336  std::list<const DictDescriptor*> getAllDictionariesWithColumnInName(
337  const ColumnDescriptor* cd);
338 
339  const bool checkMetadataForDeletedRecs(const TableDescriptor* td, int column_id) const;
340  const ColumnDescriptor* getDeletedColumn(const TableDescriptor* td) const;
342 
343  void setDeletedColumn(const TableDescriptor* td, const ColumnDescriptor* cd);
344  int getLogicalTableId(const int physicalTableId) const;
345  void checkpoint(const int logicalTableId) const;
346  void checkpointWithAutoRollback(const int logical_table_id) const;
347  void resetTableEpochFloor(const int logicalTableId) const;
348  std::string name() const { return getCurrentDB().dbName; }
349  void eraseDbMetadata();
350  void eraseDbPhysicalData();
351  void eraseTablePhysicalData(const TableDescriptor* td);
352  void setForReload(const int32_t tableId);
353 
354  std::vector<std::string> getTableDataDirectories(const TableDescriptor* td) const;
355  std::vector<std::string> getTableDictDirectories(const TableDescriptor* td) const;
356  std::set<std::string> getTableDictDirectoryPaths(int32_t table_id) const;
357  std::string getColumnDictDirectory(const ColumnDescriptor* cd,
358  bool file_name_only = true) const;
359  std::string dumpSchema(const TableDescriptor* td) const;
360  std::string dumpCreateTable(const TableDescriptor* td,
361  bool multiline_formatting = true,
362  bool dump_defaults = false) const;
363  std::optional<std::string> dumpCreateTable(int32_t table_id,
364  bool multiline_formatting = true,
365  bool dump_defaults = false) const;
366  std::string dumpCreateServer(const std::string& name,
367  bool multiline_formatting = true) const;
368 
376  static const std::string getForeignTableSchema(bool if_not_exists = false);
377 
385  static const std::string getForeignServerSchema(bool if_not_exists = false);
386 
396  void createForeignServer(std::unique_ptr<foreign_storage::ForeignServer> foreign_server,
397  bool if_not_exists);
398 
407  const std::string& server_name) const;
408 
418  const std::unique_ptr<const foreign_storage::ForeignServer> getForeignServerFromStorage(
419  const std::string& server_name);
420 
430  const std::unique_ptr<const foreign_storage::ForeignTable> getForeignTableFromStorage(
431  int table_id);
432 
439  void changeForeignServerOwner(const std::string& server_name, const int new_owner_id);
440 
447  void setForeignServerDataWrapper(const std::string& server_name,
448  const std::string& data_wrapper);
455  void setForeignServerOptions(const std::string& server_name,
456  const std::string& options);
463  void renameForeignServer(const std::string& server_name, const std::string& name);
464 
470  void dropForeignServer(const std::string& server_name);
471 
479  std::vector<const foreign_storage::ForeignTable*> getAllForeignTablesForForeignServer(
480  const int32_t foreign_server_id);
481 
495  const rapidjson::Value* filters,
496  const UserMetadata& user,
497  std::vector<const foreign_storage::ForeignServer*>& results);
498 
503 
514  bool validateNonExistentTableOrView(const std::string& name, const bool if_not_exists);
515 
523  std::vector<const TableDescriptor*> getAllForeignTablesForRefresh() const;
524 
531  void updateForeignTableRefreshTimes(const int32_t table_id);
532 
539  void setForeignTableOptions(const std::string& table_name,
540  foreign_storage::OptionsMap& options_map,
541  bool clear_existing_options = true);
542 
543  void updateLeaf(const LeafHostInfo& string_dict_host);
544 
545  // For testing purposes only
546  void setUncappedTableEpoch(const std::string& table_name);
547 
548  // Methods for accessing and updating custom expressions
549 
557  static const std::string getCustomExpressionsSchema(bool if_not_exists = false);
558 
566  int32_t createCustomExpression(std::unique_ptr<CustomExpression> custom_expression);
567 
575  const CustomExpression* getCustomExpression(int32_t custom_expression_id) const;
576 
586  const std::unique_ptr<const CustomExpression> getCustomExpressionFromStorage(
587  int32_t custom_expression_id);
588 
597  std::vector<const CustomExpression*> getCustomExpressionsForUser(
598  const UserMetadata& user) const;
599 
606  void updateCustomExpression(int32_t custom_expression_id,
607  const std::string& expression_json);
608 
615  void deleteCustomExpressions(const std::vector<int32_t>& custom_expression_ids,
616  bool do_soft_delete);
617 
626  void reassignOwners(const std::set<std::string>& old_owners,
627  const std::string& new_owner);
628 
629  bool isInfoSchemaDb() const;
631 
632  protected:
635  void updateDictionaryNames();
638  void updateGeoColumns();
640  void updateLinkSchema();
643  void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id);
644  void updateDictionarySchema();
645  void updatePageSize();
650  void updateFsiSchemas();
655  void buildMaps();
656  void addTableToMap(const TableDescriptor* td,
657  const std::list<ColumnDescriptor>& columns,
658  const std::list<DictDescriptor>& dicts);
659  void addReferenceToForeignDict(ColumnDescriptor& referencing_column,
660  Parser::SharedDictionaryDef shared_dict_def,
661  const bool persist_reference);
663  ColumnDescriptor& cd,
664  std::list<ColumnDescriptor>& cdd,
665  std::list<DictDescriptor>& dds,
666  const TableDescriptor td,
667  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs);
669  std::list<DictDescriptor>& dds,
670  const TableDescriptor& td,
671  bool is_logical_table,
672  bool use_temp_dictionary = false);
675  void addLinkToMap(LinkDescriptor& ld);
676  void removeTableFromMap(const std::string& tableName,
677  const int tableId,
678  const bool is_on_error = false);
679  void eraseTableMetadata(const TableDescriptor* td);
681  void doTruncateTable(const TableDescriptor* td);
682  void renamePhysicalTable(const TableDescriptor* td, const std::string& newTableName);
683  void renamePhysicalTables(std::vector<std::pair<std::string, std::string>>& names,
684  std::vector<int>& tableIds);
685  void instantiateFragmenter(TableDescriptor* td) const;
687  std::list<const ColumnDescriptor*>& colDescs,
688  const bool fetchSystemColumns,
689  const bool fetchVirtualColumns,
690  const bool fetchPhysicalColumns) const;
691  std::string calculateSHA1(const std::string& data);
692  std::string generatePhysicalTableName(const std::string& logicalTableName,
693  const size_t shardNumber);
694  std::vector<DBObject> parseDashboardObjects(const std::string& view_meta,
695  const int& user_id);
696  void createOrUpdateDashboardSystemRole(const std::string& view_meta,
697  const int32_t& user_id,
698  const std::string& dash_role_name);
699 
700  const int getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const;
701 
703  const std::list<ColumnDescriptor>& cds) const;
704  void dropTableFromJsonUnlocked(const std::string& table_name) const;
705 
706  std::string basePath_;
719 
722  std::shared_ptr<Data_Namespace::DataMgr> dataMgr_;
723 
724  const std::vector<LeafHostInfo> string_dict_hosts_;
725  std::shared_ptr<Calcite> calciteMgr_;
726 
728  static const std::string
729  physicalTableNameTag_; // extra component added to the name of each physical table
732 
733  // this tuple is for rolling forw/back once after ALTER ADD/DEL/MODIFY columns
734  // succeeds/fails
735  // get(0) = old ColumnDescriptor*
736  // get(1) = new ColumnDescriptor*
738  std::vector<std::pair<ColumnDescriptor*, ColumnDescriptor*>>;
740 
741  private:
742  void addColumnDescriptor(const ColumnDescriptor* cd);
748  void reloadTableMetadata(int table_id);
749  template <class T>
750  friend class lockmgr::TableLockMgrImpl; // for reloadTableMetadataUnlocked()
751  void reloadTableMetadataUnlocked(int table_id);
752  TableDescriptor* createTableFromDiskUnlocked(int32_t table_id);
754  std::list<ColumnDescriptor*> sqliteGetColumnsForTableUnlocked(int32_t table_id);
756  void reloadCatalogMetadata(const std::map<int32_t, std::string>& user_name_by_user_id);
758  const std::map<int32_t, std::string>& user_name_by_user_id);
760  void buildTablesMapUnlocked();
765  const std::map<int32_t, std::string>& user_name_by_user_id);
766  void buildLinksMapUnlocked();
769 
770  void gatherAdditionalInfo(std::vector<std::string>& additional_info,
771  std::set<std::string>& shared_dict_column_names,
772  const TableDescriptor* td) const;
773  std::string quoteIfRequired(const std::string& column_name) const;
776  const std::string& temp_data_dir,
777  const std::unordered_map<int, int>& all_column_ids_map) const;
778  void renameTableDirectories(const std::string& temp_data_dir,
779  const std::vector<std::string>& target_paths,
780  const std::string& name_prefix) const;
782 
783  void setForeignServerProperty(const std::string& server_name,
784  const std::string& property,
785  const std::string& value);
786 
788  const std::string& property,
789  const std::string& value);
790 
792  const TableDescriptorUpdateParams& table_update_params);
793  void alterTableMetadata(const TableDescriptor* td,
794  const TableDescriptorUpdateParams& table_update_params);
795  void setTableFileMgrParams(const int table_id,
796  const File_Namespace::FileMgrParams& file_mgr_params);
798  const UserMetadata& user_metadata,
799  const GetTablesType get_tables_type) const;
800 
803  const std::string& table_name) const;
804 
810  std::unique_ptr<foreign_storage::ForeignServer> foreign_server,
811  bool if_not_exists);
812 
814  const std::string& tableName) const;
815 
816  const Catalog* getObjForLock();
817  void removeChunks(const int table_id) const;
818 
820  std::unique_ptr<CustomExpression> getCustomExpressionFromConnector(size_t row);
821 
822  void restoreOldOwners(
823  const std::map<int32_t, std::string>& old_owners_user_name_by_id,
824  const std::map<int32_t, std::vector<DBObject>>& old_owner_db_objects,
825  int32_t new_owner_id);
827  const std::map<int32_t, std::string>& old_owners_user_name_by_id,
828  const std::map<int32_t, std::vector<DBObject>>& old_owner_db_objects,
829  int32_t new_owner_id);
830 
833  void initializeSystemTables();
850 
851  void createSystemTableServer(const std::string& server_name,
852  const std::string& data_wrapper_type,
853  const foreign_storage::OptionsMap& options = {});
854  std::pair<foreign_storage::ForeignTable, std::list<ColumnDescriptor>>
856  const std::string& table_name,
857  const std::string& server_name,
858  const std::vector<std::pair<std::string, SQLTypeInfo>>& column_type_by_name,
859  bool is_in_memory_system_table);
860 
862  const std::list<ColumnDescriptor>& columns);
863 
864  void setDeletedColumnUnlocked(const TableDescriptor* td, const ColumnDescriptor* cd);
865 
869 
871  const TableDescriptor* logical_table,
872  const std::vector<const TableDescriptor*>& physical_tables);
873 
874  std::string dumpCreateTableUnlocked(const TableDescriptor* td,
875  bool multiline_formatting,
876  bool dump_defaults) const;
877 
878  void removeDiskCachedDataForMutableTable(int32_t table_id) const;
879 
880  struct NoTableFoundException : std::runtime_error {
881  NoTableFoundException(int32_t table_id)
882  : std::runtime_error{"No entry found for table: " + std::to_string(table_id)} {}
883  };
884 
885  template <typename F, typename... Args>
886  void execInTransaction(F&& f, Args&&... args);
887 
889  const std::map<std::string, int>& cached_table_map,
890  const std::string& cur_table_name);
891 
892  static constexpr const char* CATALOG_SERVER_NAME{"system_catalog_server"};
893  static constexpr const char* MEMORY_STATS_SERVER_NAME{"system_memory_stats_server"};
894  static constexpr const char* STORAGE_STATS_SERVER_NAME{"system_storage_stats_server"};
895  static constexpr const char* EXECUTOR_STATS_SERVER_NAME{"system_executor_stats_server"};
896  static constexpr const char* ML_METADATA_SERVER_NAME{"ml_metadata_server"};
897  static constexpr const char* LOGS_SERVER_NAME{"system_logs_server"};
898  static constexpr std::array<const char*, 6> INTERNAL_SERVERS{CATALOG_SERVER_NAME,
904 
905  public:
906  mutable std::unique_ptr<heavyai::DistributedSharedMutex> dcatalogMutex_;
907  mutable std::unique_ptr<heavyai::DistributedSharedMutex> dsqliteMutex_;
908  mutable std::mutex sqliteMutex_;
910  mutable std::atomic<std::thread::id> thread_holding_sqlite_lock;
911  mutable std::atomic<std::thread::id> thread_holding_write_lock;
912  // assuming that you never call into a catalog from another catalog via the same thread
913  static thread_local bool thread_holds_read_lock;
914  bool initialized_ = false;
915 };
916 
917 } // namespace Catalog_Namespace
std::string dumpCreateTable(const TableDescriptor *td, bool multiline_formatting=true, bool dump_defaults=false) const
Definition: Catalog.cpp:5567
static constexpr const char * MEMORY_DETAILS_SYS_TABLE_NAME
Definition: Catalog.h:120
void serializeTableJsonUnlocked(const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
Definition: Catalog.cpp:3180
std::unordered_map< const TableDescriptor *, const ColumnDescriptor * > DeletedColumnPerTableMap
Definition: Types.h:48
void updateViewUnlocked(TableDescriptor &)
Definition: Catalog.cpp:1395
void instantiateFragmenter(TableDescriptor *td) const
Definition: Catalog.cpp:1778
void removeFromColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6992
const foreign_storage::ForeignTable * getForeignTable(const std::string &tableName) const
Definition: Catalog.cpp:1827
static constexpr const char * WS_SERVER_ACCESS_LOGS_SYS_TABLE_NAME
Definition: Catalog.h:128
void initializePermissionsSystemTable()
Definition: Catalog.cpp:6531
void deleteMetadataForDashboards(const std::vector< int32_t > ids, const UserMetadata &user)
Definition: Catalog.cpp:2011
TableDictColumnsMap dict_columns_by_table_id_
Definition: Catalog.h:718
void executeDropTableSqliteQueries(const TableDescriptor *td)
Definition: Catalog.cpp:4344
DictRef addDictionaryNontransactional(ColumnDescriptor &cd)
Definition: Catalog.cpp:2242
void renameTableDirectories(const std::string &temp_data_dir, const std::vector< std::string > &target_paths, const std::string &name_prefix) const
std::pair< foreign_storage::ForeignTable, std::list< ColumnDescriptor > > getSystemTableSchema(const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name, bool is_in_memory_system_table)
Definition: Catalog.cpp:6860
static constexpr const char * SERVER_LOGS_SYS_TABLE_NAME
Definition: Catalog.h:125
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:5067
static constexpr const char * EXECUTOR_RESOURCE_POOL_SUMMARY_SYS_TABLE_NAME
Definition: Catalog.h:122
void conditionallyInitializeSystemObjects()
Definition: Catalog.cpp:6382
~Catalog()
Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the ...
Definition: Catalog.cpp:231
std::string getColumnDictDirectory(const ColumnDescriptor *cd, bool file_name_only=true) const
Definition: Catalog.cpp:5388
void getDictionary(const ColumnDescriptor &cd, std::map< int, StringDictionary * > &stringDicts)
Definition: Catalog.cpp:2342
heavyai::shared_mutex sharedMutex_
Definition: Catalog.h:909
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:143
std::vector< TableDescriptor > getAllTableMetadataCopy() const
Definition: Catalog.cpp:2197
void deleteCustomExpressions(const std::vector< int32_t > &custom_expression_ids, bool do_soft_delete)
Definition: Catalog.cpp:6069
void addReferenceToForeignDict(ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def, const bool persist_reference)
Definition: Catalog.cpp:3965
const ColumnDescriptor * getDeletedColumn(const TableDescriptor *td) const
Definition: Catalog.cpp:3897
void setDeletedColumn(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3940
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:709
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:2935
void renameTables(const std::vector< std::pair< std::string, std::string >> &names)
Definition: Catalog.cpp:4503
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:470
void createForeignServer(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:3283
TableEpochInfo(const int32_t table_id_param, const int32_t table_epoch_param, const size_t leaf_index_param)
Definition: Catalog.h:96
std::map< ColumnKey, ColumnDescriptor * > ColumnDescriptorMap
Definition: Types.h:38
std::string dumpCreateServer(const std::string &name, bool multiline_formatting=true) const
Definition: Catalog.cpp:5746
void addColumnNontransactional(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:2462
std::map< int32_t, std::vector< int32_t >> LogicalToPhysicalTableMapById
Definition: Types.h:36
TableNotFoundException(const std::string &table_name, const std::string &db_name, const std::string &context_msg={})
Definition: Catalog.h:105
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:266
void reloadCatalogMetadata(const std::map< int32_t, std::string > &user_name_by_user_id)
Definition: Catalog.cpp:1190
void delDictionaryTransactional(const ColumnDescriptor &cd)
Definition: Catalog.cpp:2267
void addColumn(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:2528
int32_t createCustomExpression(std::unique_ptr< CustomExpression > custom_expression)
Definition: Catalog.cpp:5944
bool validateNonExistentTableOrView(const std::string &name, const bool if_not_exists)
Definition: Catalog.cpp:5772
void setTableFileMgrParams(const int table_id, const File_Namespace::FileMgrParams &file_mgr_params)
Definition: Catalog.cpp:3794
std::vector< const TableDescriptor * > getAllForeignTablesForRefresh() const
Definition: Catalog.cpp:5783
std::map< int, std::unique_ptr< CustomExpression >> CustomExpressionMapById
Definition: Types.h:53
const foreign_storage::ForeignServer * getForeignServer(const std::string &server_name) const
Definition: Catalog.cpp:3334
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:2188
static constexpr const char * DASHBOARDS_SYS_TABLE_NAME
Definition: Catalog.h:114
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:724
const DBMetadata currentDB_
Definition: Catalog.h:721
const TableDescriptor * getCachedTableDescriptor(const std::map< std::string, int > &cached_table_map, const std::string &cur_table_name)
Definition: Catalog.cpp:4480
static constexpr const char * ML_METADATA_SERVER_NAME
Definition: Catalog.h:896
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:711
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, bool is_logical_table, bool use_temp_dictionary=false)
Definition: Catalog.cpp:4056
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:722
const LinkDescriptor * getMetadataForLink(const std::string &link) const
const std::optional< std::string > getColumnName(int table_id, int column_id) const
Definition: Catalog.cpp:1968
TableDescriptor * createTableFromDiskUnlocked(int32_t table_id)
Definition: Catalog.cpp:5243
std::vector< DashboardDescriptor > getAllDashboardsMetadataCopy() const
Definition: Catalog.cpp:2217
void dropTableFromJsonUnlocked(const std::string &table_name) const
Definition: Catalog.cpp:3251
void setForReload(const int32_t tableId)
Definition: Catalog.cpp:5365
int32_t getTableEpoch(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:3579
void deleteTableCatalogMetadata(const TableDescriptor *logical_table, const std::vector< const TableDescriptor * > &physical_tables)
Definition: Catalog.cpp:4310
void replaceDashboard(DashboardDescriptor &vd)
Definition: Catalog.cpp:4724
void initializeServerLogsSystemTables()
Definition: Catalog.cpp:6721
static constexpr std::array< const char *, 6 > INTERNAL_SERVERS
Definition: Catalog.h:898
std::string dumpCreateTableUnlocked(const TableDescriptor *td, bool multiline_formatting, bool dump_defaults) const
Definition: Catalog.cpp:5585
std::vector< std::string > getTableDataDirectories(const TableDescriptor *td) const
Definition: Catalog.cpp:5374
void updateCustomExpression(int32_t custom_expression_id, const std::string &expression_json)
Definition: Catalog.cpp:6041
void dropColumn(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:2653
void checkDateInDaysColumnMigration()
Definition: Catalog.cpp:925
static constexpr const char * LOGS_SERVER_NAME
Definition: Catalog.h:897
void addFrontendViewToMap(DashboardDescriptor &vd)
Definition: Catalog.cpp:1683
std::vector< const foreign_storage::ForeignTable * > getAllForeignTablesForForeignServer(const int32_t foreign_server_id)
Get all of the foreign tables for associated with a foreign server id.
Definition: Catalog.cpp:3631
std::vector< TableMetadata > getTablesMetadataForUser(const UserMetadata &user_metadata, const GetTablesType get_tables_type, const std::string &filter_table_name) const
Definition: Catalog.cpp:4984
void reloadCatalogMetadataUnlocked(const std::map< int32_t, std::string > &user_name_by_user_id)
Definition: Catalog.cpp:1198
std::unique_ptr< CustomExpression > getCustomExpressionFromConnector(size_t row)
Definition: Catalog.cpp:1526
std::vector< std::pair< int32_t, int32_t > > getAllPersistedTableAndShardIds() const
Definition: Catalog.cpp:4889
TableDescriptor * getMutableMetadataForTableUnlocked(int table_id) const
Definition: Catalog.cpp:1896
const ColumnDescriptor * getShardColumnMetadataForTable(const TableDescriptor *td) const
Definition: Catalog.cpp:4851
void changeForeignServerOwner(const std::string &server_name, const int new_owner_id)
Definition: Catalog.cpp:3386
std::optional< int32_t > getTableId(const std::string &table_name) const
Definition: Catalog.cpp:1878
std::atomic< std::thread::id > thread_holding_sqlite_lock
Definition: Catalog.h:910
void renameColumn(const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
Definition: Catalog.cpp:4621
DeletedColumnPerTableMap deletedColumnPerTable_
Definition: Catalog.h:774
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:710
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:712
void setForeignServerOptions(const std::string &server_name, const std::string &options)
Definition: Catalog.cpp:3418
std::string to_string(char const *&&v)
std::vector< std::string > getTableNamesForUser(const UserMetadata &user, const GetTablesType get_tables_type) const
Definition: Catalog.cpp:4969
void addTableToMap(const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
Definition: Catalog.cpp:1542
void getAllColumnMetadataForTableImpl(const TableDescriptor *td, std::list< const ColumnDescriptor * > &colDescs, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Definition: Catalog.cpp:2142
void initializeMLModelMetadataSystemTable()
Definition: Catalog.cpp:6702
void resetTableEpochFloor(const int logicalTableId) const
Definition: Catalog.cpp:5040
TableEpochInfo(const int32_t table_id_param, const int32_t table_epoch_param)
Definition: Catalog.h:94
void setForeignTableOptions(const std::string &table_name, foreign_storage::OptionsMap &options_map, bool clear_existing_options=true)
Definition: Catalog.cpp:5827
static constexpr const char * REQUEST_LOGS_SYS_TABLE_NAME
Definition: Catalog.h:126
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:797
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:129
bool filterTableByTypeAndUser(const TableDescriptor *td, const UserMetadata &user_metadata, const GetTablesType get_tables_type) const
Definition: Catalog.cpp:4936
const bool checkMetadataForDeletedRecs(const TableDescriptor *td, int column_id) const
Definition: Catalog.cpp:3903
void restoreOldOwnersInMemory(const std::map< int32_t, std::string > &old_owners_user_name_by_id, const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects, int32_t new_owner_id)
Definition: Catalog.cpp:6344
std::string quoteIfRequired(const std::string &column_name) const
Definition: Catalog.cpp:5867
void initializeMemorySummarySystemTable()
Definition: Catalog.cpp:6607
std::string name() const
Definition: Catalog.h:348
void addColumnTransactional(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:2449
std::list< const DashboardDescriptor * > getAllDashboardsMetadata() const
Definition: Catalog.cpp:2208
void updateLeaf(const LeafHostInfo &string_dict_host)
void initializeWebServerLogsSystemTables()
Definition: Catalog.cpp:6783
void removeChunks(const int table_id) const
Definition: Catalog.cpp:4272
static constexpr const char * ROLES_SYS_TABLE_NAME
Definition: Catalog.h:117
void reloadForeignTableUnlocked(foreign_storage::ForeignTable &foreign_table)
Definition: Catalog.cpp:5142
This file contains the class specification and related data structures for SysCatalog.
DictRef addDictionaryTransactional(ColumnDescriptor &cd)
Definition: Catalog.cpp:2227
std::map< ColumnIdKey, ColumnDescriptor * > ColumnDescriptorMapById
Definition: Types.h:40
void setCalciteMgr(const std::shared_ptr< Calcite > &new_calcite_mgr)
Definition: Catalog.h:268
std::string calculateSHA1(const std::string &data)
Definition: Catalog.cpp:4801
void gatherAdditionalInfo(std::vector< std::string > &additional_info, std::set< std::string > &shared_dict_column_names, const TableDescriptor *td) const
Definition: Catalog.cpp:5878
void buildLogicalToPhysicalMapUnlocked()
Definition: Catalog.cpp:1462
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:265
void addLinkToMap(LinkDescriptor &ld)
Definition: Catalog.cpp:1770
std::shared_ptr< Calcite > getCalciteMgr() const
Definition: Catalog.h:267
static const std::string getForeignTableSchema(bool if_not_exists=false)
Definition: Catalog.cpp:782
void initializeStorageDetailsSystemTable()
Definition: Catalog.cpp:6647
std::list< const DictDescriptor * > getAllDictionariesWithColumnInName(const ColumnDescriptor *cd)
Definition: Catalog.cpp:2323
std::map< int, TableDescriptor * > TableDescriptorMapById
Definition: Types.h:35
const ColumnDescriptor * getDeletedColumnIfRowsDeleted(const TableDescriptor *td) const
Definition: Catalog.cpp:3915
void adjustAlteredTableFiles(const std::string &temp_data_dir, const std::unordered_map< int, int > &all_column_ids_map) const
std::map< std::string, LinkDescriptor * > LinkDescriptorMap
Definition: Types.h:45
GetTablesType
Definition: Catalog.h:63
void restoreOldOwners(const std::map< int32_t, std::string > &old_owners_user_name_by_id, const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects, int32_t new_owner_id)
Definition: Catalog.cpp:6295
void removeColumnDescriptor(const ColumnDescriptor *cd)
Definition: Catalog.cpp:2686
void dropForeignServer(const std::string &server_name)
Definition: Catalog.cpp:3450
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3945
void dropTable(const TableDescriptor *td)
Definition: Catalog.cpp:4282
void buildDashboardsMapUnlocked(const std::map< int32_t, std::string > &user_name_by_user_id)
Definition: Catalog.cpp:1417
void initializeRoleAssignmentsSystemTable()
Definition: Catalog.cpp:6598
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
static const std::string getForeignServerSchema(bool if_not_exists=false)
Definition: Catalog.cpp:775
int getDatabaseId() const
Definition: Catalog.h:326
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const
static constexpr const char * MEMORY_SUMMARY_SYS_TABLE_NAME
Definition: Catalog.h:119
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
static constexpr const char * EXECUTOR_STATS_SERVER_NAME
Definition: Catalog.h:895
int getLogicalTableId(const int physicalTableId) const
Definition: Catalog.cpp:5008
void invalidateCachesForTable(const int table_id)
Definition: Catalog.cpp:4229
const CustomExpression * getCustomExpression(int32_t custom_expression_id) const
Definition: Catalog.cpp:5993
size_t getTotalMemorySizeForDictionariesForDatabase() const
Definition: Catalog.cpp:2370
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1904
specifies the content in-memory of a row in the column metadata table
static constexpr const char * ROLE_ASSIGNMENTS_SYS_TABLE_NAME
Definition: Catalog.h:118
void updateInColumnMap(ColumnDescriptor *cd, ColumnDescriptor *old_cd)
Definition: Catalog.cpp:6960
void execInTransaction(F &&f, Args &&...args)
Definition: Catalog.cpp:7002
void removeDiskCachedDataForMutableTable(int32_t table_id) const
void setForeignTableProperty(const foreign_storage::ForeignTable *table, const std::string &property, const std::string &value)
Definition: Catalog.cpp:5847
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:2774
void buildCustomExpressionsMapUnlocked()
Definition: Catalog.cpp:1514
const std::vector< LeafHostInfo > & getStringDictionaryHosts() const
Definition: Catalog.cpp:1940
void checkpointWithAutoRollback(const int logical_table_id) const
Definition: Catalog.cpp:5030
static constexpr const char * USERS_SYS_TABLE_NAME
Definition: Catalog.h:112
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4869
void setMaxRows(const int32_t table_id, const int64_t max_rows)
Definition: Catalog.cpp:3757
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1724
const ColumnDescriptor * getMetadataForColumnBySpi(const int tableId, const size_t spi) const
Definition: Catalog.cpp:2001
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:716
void alterPhysicalTableMetadata(const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
Definition: Catalog.cpp:3680
static constexpr const char * DATABASES_SYS_TABLE_NAME
Definition: Catalog.h:115
void rollLegacy(const bool forward)
Definition: Catalog.cpp:2718
ForeignServerMap foreignServerMap_
Definition: Catalog.h:715
void createShardedTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:4099
std::map< std::string, std::shared_ptr< foreign_storage::ForeignServer >> ForeignServerMap
Definition: Types.h:50
void reassignOwners(const std::set< std::string > &old_owners, const std::string &new_owner)
Definition: Catalog.cpp:6161
std::vector< const CustomExpression * > getCustomExpressionsForUser(const UserMetadata &user) const
Definition: Catalog.cpp:6016
void checkpoint(const int logicalTableId) const
Definition: Catalog.cpp:5022
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:3291
void initializeMemoryDetailsSystemTable()
Definition: Catalog.cpp:6623
static constexpr const char * TABLES_SYS_TABLE_NAME
Definition: Catalog.h:113
std::vector< std::pair< ColumnDescriptor *, ColumnDescriptor * >> ColumnDescriptorsForRoll
Definition: Catalog.h:738
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:725
static const std::string physicalTableNameTag_
Definition: Catalog.h:729
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1688
void reloadTableMetadata(int table_id)
Definition: Catalog.cpp:1093
void dropColumnPolicies(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:2640
void setTableEpoch(const int db_id, const int table_id, const int new_epoch)
Definition: Catalog.cpp:3647
void alterTableMetadata(const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
Definition: Catalog.cpp:3707
static constexpr const char * ML_MODEL_METADATA_SYS_TABLE_NAME
Definition: Catalog.h:124
std::vector< std::string > getTableDictDirectories(const TableDescriptor *td) const
Definition: Catalog.cpp:5409
void getForeignServersForUser(const rapidjson::Value *filters, const UserMetadata &user, std::vector< const foreign_storage::ForeignServer * > &results)
Definition: Catalog.cpp:3484
int getNextAddedColumnId(const TableDescriptor &td)
Definition: Catalog.cpp:2441
std::map< DictRef, std::unique_ptr< DictDescriptor >> DictDescriptorMapById
Definition: Types.h:42
std::map< std::string, std::shared_ptr< DashboardDescriptor >> DashboardDescriptorMap
Definition: Types.h:44
static const std::string getCustomExpressionsSchema(bool if_not_exists=false)
Definition: Catalog.cpp:790
static constexpr const char * PERMISSIONS_SYS_TABLE_NAME
Definition: Catalog.h:116
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:2172
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
void dropColumnTransactional(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:2591
static constexpr const char * STORAGE_DETAILS_SYS_TABLE_NAME
Definition: Catalog.h:121
int32_t createDashboard(DashboardDescriptor &vd)
Definition: Catalog.cpp:4660
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1601
std::map< int32_t, std::set< const ColumnDescriptor * >> TableDictColumnsMap
Definition: Types.h:41
void removeFragmenterForTable(const int table_id) const
Definition: Catalog.cpp:4260
torch::Tensor f(torch::Tensor x, torch::Tensor W_target, torch::Tensor b_target)
bool checkDropRenderGroupColumnsMigration()
Definition: Catalog.cpp:931
std::map< int, LinkDescriptor * > LinkDescriptorMapById
Definition: Types.h:46
std::map< int, std::shared_ptr< foreign_storage::ForeignServer >> ForeignServerMapById
Definition: Types.h:52
const std::unique_ptr< const CustomExpression > getCustomExpressionFromStorage(int32_t custom_expression_id)
Definition: Catalog.cpp:6002
void initializeRequestLogsSystemTables()
Definition: Catalog.cpp:6746
static constexpr const char * CATALOG_SERVER_NAME
Definition: Catalog.h:892
std::list< ColumnDescriptor * > sqliteGetColumnsForTableUnlocked(int32_t table_id)
Definition: Catalog.cpp:5194
const int getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const
Definition: Catalog.cpp:1978
void eraseTableMetadata(const TableDescriptor *td)
Definition: Catalog.cpp:4332
bool setColumnSharedDictionary(ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:3987
void updateCustomExpressionsSchema()
Definition: Catalog.cpp:763
void delDictionaryNontransactional(const ColumnDescriptor &cd)
Definition: Catalog.cpp:2280
std::set< std::string > getTableDictDirectoryPaths(int32_t table_id) const
Definition: Catalog.cpp:5422
void initializeExecutorResourcePoolSummarySystemTable()
Definition: Catalog.cpp:6673
void renamePhysicalTables(std::vector< std::pair< std::string, std::string >> &names, std::vector< int > &tableIds)
Definition: Catalog.cpp:4444
std::unique_ptr< heavyai::DistributedSharedMutex > dsqliteMutex_
Definition: Catalog.h:907
bool isInfoSchemaDb() const
Definition: Catalog.cpp:6389
void setTableEpochs(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3849
std::map< std::string, std::string, std::less<>> OptionsMap
void setMaxRollbackEpochs(const int32_t table_id, const int32_t max_rollback_epochs)
Definition: Catalog.cpp:3732
std::string generatePhysicalTableName(const std::string &logicalTableName, const size_t shardNumber)
Definition: Catalog.cpp:5085
std::optional< std::string > getTableName(int32_t table_id) const
Definition: Catalog.cpp:1869
void reloadTableMetadataUnlocked(int table_id)
Definition: Catalog.cpp:1100
void updateForeignTablesInMapUnlocked()
Definition: Catalog.cpp:5112
std::string createLink(LinkDescriptor &ld, size_t min_length)
Definition: Catalog.cpp:4813
Descriptor for a dictionary for a string columne.
void updateLogicalToPhysicalTableLinkSchema()
Definition: Catalog.cpp:631
void setUncappedTableEpoch(const std::string &table_name)
Definition: Catalog.cpp:3776
void addColumnDescriptor(const ColumnDescriptor *cd)
Definition: Catalog.cpp:2701
void alterColumnTypeTransactional(const ColumnDescriptor &cd)
Definition: Catalog.cpp:2383
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:717
std::atomic< std::thread::id > thread_holding_write_lock
Definition: Catalog.h:911
foreign_storage::ForeignTable * getForeignTableUnlocked(const std::string &tableName) const
Definition: Catalog.cpp:1818
void setForeignServerDataWrapper(const std::string &server_name, const std::string &data_wrapper)
Definition: Catalog.cpp:3397
std::unique_ptr< heavyai::DistributedSharedMutex > dcatalogMutex_
Definition: Catalog.h:906
void setTableEpochsLogExceptions(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3885
static const std::array< std::string, 4 > kAggregatorOnlySystemTables
Definition: Catalog.h:131
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
SqliteConnector & getSqliteConnector()
Definition: Catalog.h:327
const std::string & getCatalogBasePath() const
Definition: Catalog.h:273
std::shared_timed_mutex shared_mutex
const std::unique_ptr< const foreign_storage::ForeignTable > getForeignTableFromStorage(int table_id)
Definition: Catalog.cpp:3366
void refreshDictionaryCachesForTableUnlocked(const TableDescriptor &td)
Definition: Catalog.cpp:4209
void renameTable(const TableDescriptor *td, const std::string &newTableName)
Definition: Catalog.cpp:4404
void reloadDictionariesFromDiskUnlocked()
Definition: Catalog.cpp:5170
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:714
std::string dumpSchema(const TableDescriptor *td) const
Definition: Catalog.cpp:5440
void updateForeignTableRefreshTimes(const int32_t table_id)
Definition: Catalog.cpp:5805
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:739
static constexpr const char * MEMORY_STATS_SERVER_NAME
Definition: Catalog.h:893
bool recreateSystemTableIfUpdated(foreign_storage::ForeignTable &foreign_table, const std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:6898
void doTruncateTable(const TableDescriptor *td)
Definition: Catalog.cpp:4143
void createSystemTableServer(const std::string &server_name, const std::string &data_wrapper_type, const foreign_storage::OptionsMap &options={})
Definition: Catalog.cpp:6833
const std::map< int, const ColumnDescriptor * > getDictionaryToColumnMapping()
Definition: Catalog.cpp:4905
void renamePhysicalTable(const TableDescriptor *td, const std::string &newTableName)
Definition: Catalog.cpp:4378
void renameForeignServer(const std::string &server_name, const std::string &name)
Definition: Catalog.cpp:3438
std::vector< DBObject > parseDashboardObjects(const std::string &view_meta, const int &user_id)
Definition: Catalog.cpp:1694
static constexpr const char * WS_SERVER_LOGS_SYS_TABLE_NAME
Definition: Catalog.h:127
void dropColumnNontransactional(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:2607
void addToColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6984
const std::unique_ptr< const foreign_storage::ForeignServer > getForeignServerFromStorage(const std::string &server_name)
Definition: Catalog.cpp:3346
static constexpr const char * STORAGE_STATS_SERVER_NAME
Definition: Catalog.h:894
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:727
Catalog()
Constructor builds a hollow catalog used during constructor of other catalogs.
Definition: Catalog.cpp:171
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:707
void setForeignServerProperty(const std::string &server_name, const std::string &property, const std::string &value)
Definition: Catalog.cpp:5310
static thread_local bool thread_holds_read_lock
Definition: Catalog.h:913
std::map< std::string, TableDescriptor * > TableDescriptorMap
Definition: Types.h:34
void initializeDashboardsSystemTable()
Definition: Catalog.cpp:6582
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:713
void CheckAndExecuteMigrationsPostBuildMaps()
Definition: Catalog.cpp:1036
std::vector< TableEpochInfo > getTableEpochs(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:3821
const int getColumnIdBySpi(const int tableId, const size_t spi) const
Definition: Catalog.cpp:1996
void createDefaultServersIfNotExists()
Definition: Catalog.cpp:5331
void truncateTable(const TableDescriptor *td)
Definition: Catalog.cpp:4135
void initializeWebServerAccessLogsSystemTables()
Definition: Catalog.cpp:6806
void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id)
Definition: Catalog.cpp:645