OmniSciDB  eb3a3d0a03
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Catalog.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
29 #pragma once
30 
31 #include <atomic>
32 #include <cstdint>
33 #include <ctime>
34 #include <limits>
35 #include <list>
36 #include <map>
37 #include <mutex>
38 #include <string>
39 #include <utility>
40 #include <vector>
41 
42 #include "Calcite/Calcite.h"
46 #include "Catalog/DictDescriptor.h"
47 #include "Catalog/ForeignServer.h"
48 #include "Catalog/ForeignTable.h"
49 #include "Catalog/LinkDescriptor.h"
50 #include "Catalog/SessionInfo.h"
51 #include "Catalog/SysCatalog.h"
53 #include "Catalog/TableMetadata.h"
54 #include "Catalog/Types.h"
55 #include "DataMgr/DataMgr.h"
59 
60 #include "LeafHostInfo.h"
61 
63 
64 namespace Parser {
65 
67 
68 } // namespace Parser
69 
70 class TableArchiver;
71 
72 // SPI means Sequential Positional Index which is equivalent to the input index in a
73 // RexInput node
74 #define SPIMAP_MAGIC1 (std::numeric_limits<unsigned>::max() / 4)
75 #define SPIMAP_MAGIC2 8
76 #define SPIMAP_GEO_PHYSICAL_INPUT(c, i) \
77  (SPIMAP_MAGIC1 + (unsigned)(SPIMAP_MAGIC2 * ((c) + 1) + (i)))
78 
79 namespace File_Namespace {
80 struct FileMgrParams;
81 }
82 namespace Catalog_Namespace {
85 
86  TableEpochInfo(const int32_t table_id_param, const int32_t table_epoch_param)
87  : table_id(table_id_param), table_epoch(table_epoch_param) {}
88  TableEpochInfo(const int32_t table_id_param,
89  const int32_t table_epoch_param,
90  const size_t leaf_index_param)
91  : table_id(table_id_param)
92  , table_epoch(table_epoch_param)
93  , leaf_index(leaf_index_param) {}
94 };
95 
102 class Catalog final {
103  public:
112  Catalog(const std::string& basePath,
113  const DBMetadata& curDB,
114  std::shared_ptr<Data_Namespace::DataMgr> dataMgr,
115  const std::vector<LeafHostInfo>& string_dict_hosts,
116  std::shared_ptr<Calcite> calcite,
117  bool is_new_db);
122  Catalog();
123 
130  ~Catalog();
131 
132  static void expandGeoColumn(const ColumnDescriptor& cd,
133  std::list<ColumnDescriptor>& columns);
134  void createTable(TableDescriptor& td,
135  const std::list<ColumnDescriptor>& columns,
136  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs,
137  bool isLogicalTable);
138  void createShardedTable(
139  TableDescriptor& td,
140  const std::list<ColumnDescriptor>& columns,
141  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs);
144  std::string createLink(LinkDescriptor& ld, size_t min_length);
145  void dropTable(const TableDescriptor* td);
146  void truncateTable(const TableDescriptor* td);
147  void renameTable(const TableDescriptor* td, const std::string& newTableName);
148  void renameTable(std::vector<std::pair<std::string, std::string>>& names);
149  void renameColumn(const TableDescriptor* td,
150  const ColumnDescriptor* cd,
151  const std::string& newColumnName);
152  void addColumn(const TableDescriptor& td, ColumnDescriptor& cd);
153  void dropColumn(const TableDescriptor& td, const ColumnDescriptor& cd);
154  void removeFragmenterForTable(const int table_id) const;
155 
156  const std::map<int, const ColumnDescriptor*> getDictionaryToColumnMapping();
157 
166  const TableDescriptor* getMetadataForTable(const std::string& tableName,
167  const bool populateFragmenter = true) const;
168  const TableDescriptor* getMetadataForTableImpl(int tableId,
169  const bool populateFragmenter) const;
170  const TableDescriptor* getMetadataForTable(int tableId,
171  bool populateFragmenter = true) const;
172 
173  const ColumnDescriptor* getMetadataForColumn(int tableId,
174  const std::string& colName) const;
175  const ColumnDescriptor* getMetadataForColumn(int tableId, int columnId) const;
176  const ColumnDescriptor* getMetadataForColumnUnlocked(int tableId, int columnId) const;
177 
178  const int getColumnIdBySpi(const int tableId, const size_t spi) const;
179  const ColumnDescriptor* getMetadataForColumnBySpi(const int tableId,
180  const size_t spi) const;
181 
182  const DashboardDescriptor* getMetadataForDashboard(const std::string& userId,
183  const std::string& dashName) const;
184 
185  const DashboardDescriptor* getMetadataForDashboard(const int32_t dashboard_id) const;
186  void deleteMetadataForDashboards(const std::vector<int32_t> ids,
187  const UserMetadata& user);
188 
189  const LinkDescriptor* getMetadataForLink(const std::string& link) const;
190  const LinkDescriptor* getMetadataForLink(int linkId) const;
191 
192  const foreign_storage::ForeignTable* getForeignTableUnlocked(int tableId) const;
194  const std::string& tableName) const;
195 
196  const foreign_storage::ForeignTable* getForeignTable(int table_id) const;
197 
206  std::list<const ColumnDescriptor*> getAllColumnMetadataForTable(
207  const int tableId,
208  const bool fetchSystemColumns,
209  const bool fetchVirtualColumns,
210  const bool fetchPhysicalColumns) const;
214  std::list<const ColumnDescriptor*> getAllColumnMetadataForTableUnlocked(
215  const int tableId,
216  const bool fetchSystemColumns,
217  const bool fetchVirtualColumns,
218  const bool fetchPhysicalColumns) const;
219 
220  std::list<const TableDescriptor*> getAllTableMetadata() const;
221  std::list<const DashboardDescriptor*> getAllDashboardsMetadata() const;
222  const DBMetadata& getCurrentDB() const { return currentDB_; }
224  std::shared_ptr<Calcite> getCalciteMgr() const { return calciteMgr_; }
225  const std::string& getCatalogBasePath() const { return basePath_; }
226 
227  const DictDescriptor* getMetadataForDict(int dict_ref, bool loadDict = true) const;
228  const DictDescriptor* getMetadataForDictUnlocked(int dict_ref, bool loadDict) const;
229 
230  const std::vector<LeafHostInfo>& getStringDictionaryHosts() const;
231 
233 
234  std::vector<const TableDescriptor*> getPhysicalTablesDescriptors(
235  const TableDescriptor* logical_table_desc,
236  bool populate_fragmenter = true) const;
237 
246  std::vector<std::string> getTableNamesForUser(
247  const UserMetadata& user,
248  const GetTablesType get_tables_type) const;
249 
259  std::vector<TableMetadata> getTablesMetadataForUser(
260  const UserMetadata& user_metadata,
261  const GetTablesType get_tables_type,
262  const std::string& filter_table_name) const;
263 
264  int32_t getTableEpoch(const int32_t db_id, const int32_t table_id) const;
265  void setTableEpoch(const int db_id, const int table_id, const int new_epoch);
266  void setMaxRollbackEpochs(const int32_t table_id, const int32_t max_rollback_epochs);
267  void setMaxRows(const int32_t table_id, const int64_t max_rows);
268 
269  std::vector<TableEpochInfo> getTableEpochs(const int32_t db_id,
270  const int32_t table_id) const;
271  void setTableEpochs(const int32_t db_id,
272  const std::vector<TableEpochInfo>& table_epochs) const;
273 
274  void setTableEpochsLogExceptions(const int32_t db_id,
275  const std::vector<TableEpochInfo>& table_epochs) const;
276 
277  int getDatabaseId() const { return currentDB_.dbId; }
279  void roll(const bool forward);
281  void delDictionary(const ColumnDescriptor& cd);
282  void getDictionary(const ColumnDescriptor& cd,
283  std::map<int, StringDictionary*>& stringDicts);
284 
285  const bool checkMetadataForDeletedRecs(const TableDescriptor* td, int column_id) const;
286  const ColumnDescriptor* getDeletedColumn(const TableDescriptor* td) const;
288 
289  void setDeletedColumn(const TableDescriptor* td, const ColumnDescriptor* cd);
290  void setDeletedColumnUnlocked(const TableDescriptor* td, const ColumnDescriptor* cd);
291  int getLogicalTableId(const int physicalTableId) const;
292  void checkpoint(const int logicalTableId) const;
293  void checkpointWithAutoRollback(const int logical_table_id) const;
294  std::string name() const { return getCurrentDB().dbName; }
295  void eraseDBData();
296  void eraseTablePhysicalData(const TableDescriptor* td);
297  void setForReload(const int32_t tableId);
298 
299  std::vector<std::string> getTableDataDirectories(const TableDescriptor* td) const;
300  std::vector<std::string> getTableDictDirectories(const TableDescriptor* td) const;
301  std::string getColumnDictDirectory(const ColumnDescriptor* cd) const;
302  std::string dumpSchema(const TableDescriptor* td) const;
303  std::string dumpCreateTable(const TableDescriptor* td,
304  bool multiline_formatting = true,
305  bool dump_defaults = false) const;
306 
314  static const std::string getForeignTableSchema(bool if_not_exists = false);
315 
323  static const std::string getForeignServerSchema(bool if_not_exists = false);
324 
334  void createForeignServer(std::unique_ptr<foreign_storage::ForeignServer> foreign_server,
335  bool if_not_exists);
336 
345  const std::string& server_name) const;
346 
356  const std::unique_ptr<const foreign_storage::ForeignServer> getForeignServerFromStorage(
357  const std::string& server_name);
358 
368  const std::unique_ptr<const foreign_storage::ForeignTable> getForeignTableFromStorage(
369  int table_id);
370 
377  void changeForeignServerOwner(const std::string& server_name, const int new_owner_id);
378 
385  void setForeignServerDataWrapper(const std::string& server_name,
386  const std::string& data_wrapper);
393  void setForeignServerOptions(const std::string& server_name,
394  const std::string& options);
401  void renameForeignServer(const std::string& server_name, const std::string& name);
402 
408  void dropForeignServer(const std::string& server_name);
409 
423  const rapidjson::Value* filters,
424  const UserMetadata& user,
425  std::vector<const foreign_storage::ForeignServer*>& results);
426 
431 
442  bool validateNonExistentTableOrView(const std::string& name, const bool if_not_exists);
443 
451  std::vector<const TableDescriptor*> getAllForeignTablesForRefresh() const;
452 
459  void updateForeignTableRefreshTimes(const int32_t table_id);
460 
467  void setForeignTableOptions(const std::string& table_name,
468  foreign_storage::OptionsMap& options_map,
469  bool clear_existing_options = true);
470 
471  void updateLeaf(const LeafHostInfo& string_dict_host);
472 
473  // For testing purposes only
474  void setUncappedTableEpoch(const std::string& table_name);
475 
476  // Methods for accessing and updating custom expressions
477 
485  static const std::string getCustomExpressionsSchema(bool if_not_exists = false);
486 
494  int32_t createCustomExpression(std::unique_ptr<CustomExpression> custom_expression);
495 
503  const CustomExpression* getCustomExpression(int32_t custom_expression_id) const;
504 
514  const std::unique_ptr<const CustomExpression> getCustomExpressionFromStorage(
515  int32_t custom_expression_id);
516 
525  std::vector<const CustomExpression*> getCustomExpressionsForUser(
526  const UserMetadata& user) const;
527 
534  void updateCustomExpression(int32_t custom_expression_id,
535  const std::string& expression_json);
536 
543  void deleteCustomExpressions(const std::vector<int32_t>& custom_expression_ids,
544  bool do_soft_delete);
545 
554  void reassignOwners(const std::set<std::string>& old_owners,
555  const std::string& new_owner);
556 
557  protected:
560  void updateDictionaryNames();
563  void updateGeoColumns();
565  void updateLinkSchema();
568  void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id);
569  void updateDictionarySchema();
570  void updatePageSize();
575  void updateFsiSchemas();
579  void buildMaps();
580  void addTableToMap(const TableDescriptor* td,
581  const std::list<ColumnDescriptor>& columns,
582  const std::list<DictDescriptor>& dicts);
583  void addReferenceToForeignDict(ColumnDescriptor& referencing_column,
584  Parser::SharedDictionaryDef shared_dict_def,
585  const bool persist_reference);
587  ColumnDescriptor& cd,
588  std::list<ColumnDescriptor>& cdd,
589  std::list<DictDescriptor>& dds,
590  const TableDescriptor td,
591  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs);
593  std::list<DictDescriptor>& dds,
594  const TableDescriptor& td,
595  const bool isLogicalTable);
598  void addLinkToMap(LinkDescriptor& ld);
599  void removeTableFromMap(const std::string& tableName,
600  const int tableId,
601  const bool is_on_error = false);
602  void doDropTable(const TableDescriptor* td);
604  void doTruncateTable(const TableDescriptor* td);
605  void renamePhysicalTable(const TableDescriptor* td, const std::string& newTableName);
606  void renamePhysicalTable(std::vector<std::pair<std::string, std::string>>& names,
607  std::vector<int>& tableIds);
608  void instantiateFragmenter(TableDescriptor* td) const;
610  std::list<const ColumnDescriptor*>& colDescs,
611  const bool fetchSystemColumns,
612  const bool fetchVirtualColumns,
613  const bool fetchPhysicalColumns) const;
614  std::string calculateSHA1(const std::string& data);
615  std::string generatePhysicalTableName(const std::string& logicalTableName,
616  const int32_t& shardNumber);
617  std::vector<DBObject> parseDashboardObjects(const std::string& view_meta,
618  const int& user_id);
619  void createOrUpdateDashboardSystemRole(const std::string& view_meta,
620  const int32_t& user_id,
621  const std::string& dash_role_name);
622 
623  const int getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const;
624 
626  const std::list<ColumnDescriptor>& cds) const;
627  void dropTableFromJsonUnlocked(const std::string& table_name) const;
628 
629  std::string basePath_;
641 
644  std::shared_ptr<Data_Namespace::DataMgr> dataMgr_;
645 
646  const std::vector<LeafHostInfo> string_dict_hosts_;
647  std::shared_ptr<Calcite> calciteMgr_;
648 
650  static const std::string
651  physicalTableNameTag_; // extra component added to the name of each physical table
654 
655  // this tuple is for rolling forw/back once after ALTER ADD/DEL/MODIFY columns
656  // succeeds/fails
657  // get(0) = old ColumnDescriptor*
658  // get(1) = new ColumnDescriptor*
660  std::vector<std::pair<ColumnDescriptor*, ColumnDescriptor*>>;
662 
663  private:
664  void gatherAdditionalInfo(std::vector<std::string>& additional_info,
665  std::set<std::string>& shared_dict_column_names,
666  const TableDescriptor* td) const;
667  std::string quoteIfRequired(const std::string& column_name) const;
670  const std::string& temp_data_dir,
671  const std::unordered_map<int, int>& all_column_ids_map) const;
672  void renameTableDirectories(const std::string& temp_data_dir,
673  const std::vector<std::string>& target_paths,
674  const std::string& name_prefix) const;
675  void buildForeignServerMap();
676  void addForeignTableDetails();
677 
678  void setForeignServerProperty(const std::string& server_name,
679  const std::string& property,
680  const std::string& value);
681 
683  const std::string& property,
684  const std::string& value);
685 
687  const TableDescriptorUpdateParams& table_update_params);
688  void alterTableMetadata(const TableDescriptor* td,
689  const TableDescriptorUpdateParams& table_update_params);
690  void setTableFileMgrParams(const int table_id,
691  const File_Namespace::FileMgrParams& file_mgr_params);
693  const UserMetadata& user_metadata,
694  const GetTablesType get_tables_type) const;
695 
697 
703  std::unique_ptr<foreign_storage::ForeignServer> foreign_server,
704  bool if_not_exists);
705 
707  const std::string& tableName) const;
708 
709  const Catalog* getObjForLock();
710  void removeChunksUnlocked(const int table_id) const;
711 
713  std::unique_ptr<CustomExpression> getCustomExpressionFromConnector(size_t row);
714 
715  void restoreOldOwners(
716  const std::map<int32_t, std::vector<DBObject>>& old_owner_db_objects,
717  int32_t new_owner_id);
719  const std::map<int32_t, std::vector<DBObject>>& old_owner_db_objects,
720  int32_t new_owner_id);
721 
722  public:
723  mutable std::mutex sqliteMutex_;
725  mutable std::atomic<std::thread::id> thread_holding_sqlite_lock;
726  mutable std::atomic<std::thread::id> thread_holding_write_lock;
727  // assuming that you never call into a catalog from another catalog via the same thread
728  static thread_local bool thread_holds_read_lock;
729  bool initialized_ = false;
730 };
731 
732 } // namespace Catalog_Namespace
std::string dumpCreateTable(const TableDescriptor *td, bool multiline_formatting=true, bool dump_defaults=false) const
Definition: Catalog.cpp:4594
std::list< const ColumnDescriptor * > getAllColumnMetadataForTableUnlocked(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Definition: Catalog.cpp:1811
void serializeTableJsonUnlocked(const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
Definition: Catalog.cpp:2473
std::unordered_map< const TableDescriptor *, const ColumnDescriptor * > DeletedColumnPerTableMap
Definition: Types.h:47
void instantiateFragmenter(TableDescriptor *td) const
Definition: Catalog.cpp:1428
const foreign_storage::ForeignTable * getForeignTable(const std::string &tableName) const
Definition: Catalog.cpp:1478
void deleteMetadataForDashboards(const std::vector< int32_t > ids, const UserMetadata &user)
Definition: Catalog.cpp:1638
void executeDropTableSqliteQueries(const TableDescriptor *td)
Definition: Catalog.cpp:3573
void renameTableDirectories(const std::string &temp_data_dir, const std::vector< std::string > &target_paths, const std::string &name_prefix) const
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:4290
void doDropTable(const TableDescriptor *td)
Definition: Catalog.cpp:3565
~Catalog()
Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the ...
Definition: Catalog.cpp:197
void getDictionary(const ColumnDescriptor &cd, std::map< int, StringDictionary * > &stringDicts)
Definition: Catalog.cpp:1911
mapd_shared_mutex sharedMutex_
Definition: Catalog.h:724
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:102
void restoreOldOwners(const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects, int32_t new_owner_id)
Definition: Catalog.cpp:5280
void deleteCustomExpressions(const std::vector< int32_t > &custom_expression_ids, bool do_soft_delete)
Definition: Catalog.cpp:5058
void addReferenceToForeignDict(ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def, const bool persist_reference)
Definition: Catalog.cpp:3245
const ColumnDescriptor * getDeletedColumn(const TableDescriptor *td) const
Definition: Catalog.cpp:3176
void setDeletedColumn(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3219
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:632
const foreign_storage::ForeignTable * getForeignTableUnlocked(int tableId) const
Definition: Catalog.cpp:1757
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:2235
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:428
void createForeignServer(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2567
TableEpochInfo(const int32_t table_id_param, const int32_t table_epoch_param, const size_t leaf_index_param)
Definition: Catalog.h:88
std::map< ColumnKey, ColumnDescriptor * > ColumnDescriptorMap
Definition: Types.h:38
std::map< int32_t, std::vector< int32_t >> LogicalToPhysicalTableMapById
Definition: Types.h:36
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:223
void addColumn(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:1939
int32_t createCustomExpression(std::unique_ptr< CustomExpression > custom_expression)
Definition: Catalog.cpp:4933
void roll(const bool forward)
Definition: Catalog.cpp:2034
bool validateNonExistentTableOrView(const std::string &name, const bool if_not_exists)
Definition: Catalog.cpp:4756
void setTableFileMgrParams(const int table_id, const File_Namespace::FileMgrParams &file_mgr_params)
Definition: Catalog.cpp:3064
std::vector< const TableDescriptor * > getAllForeignTablesForRefresh() const
Definition: Catalog.cpp:4767
std::map< int, std::unique_ptr< CustomExpression >> CustomExpressionMapById
Definition: Types.h:52
const foreign_storage::ForeignServer * getForeignServer(const std::string &server_name) const
Definition: Catalog.cpp:2612
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:1827
SqliteConnector sqliteConnector_
Definition: Catalog.h:642
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:646
const DBMetadata currentDB_
Definition: Catalog.h:643
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:634
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:644
const LinkDescriptor * getMetadataForLink(const std::string &link) const
void dropTableFromJsonUnlocked(const std::string &table_name) const
Definition: Catalog.cpp:2535
void setForReload(const int32_t tableId)
Definition: Catalog.cpp:4412
int32_t getTableEpoch(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:2854
void replaceDashboard(DashboardDescriptor &vd)
Definition: Catalog.cpp:3978
std::vector< std::string > getTableDataDirectories(const TableDescriptor *td) const
Definition: Catalog.cpp:4422
void updateCustomExpression(int32_t custom_expression_id, const std::string &expression_json)
Definition: Catalog.cpp:5030
std::string generatePhysicalTableName(const std::string &logicalTableName, const int32_t &shardNumber)
Definition: Catalog.cpp:4319
void dropColumn(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:2003
void checkDateInDaysColumnMigration()
Definition: Catalog.cpp:825
void addFrontendViewToMap(DashboardDescriptor &vd)
Definition: Catalog.cpp:1333
std::vector< TableMetadata > getTablesMetadataForUser(const UserMetadata &user_metadata, const GetTablesType get_tables_type, const std::string &filter_table_name) const
Definition: Catalog.cpp:4222
std::unique_ptr< CustomExpression > getCustomExpressionFromConnector(size_t row)
Definition: Catalog.cpp:1176
const ColumnDescriptor * getShardColumnMetadataForTable(const TableDescriptor *td) const
Definition: Catalog.cpp:4103
void changeForeignServerOwner(const std::string &server_name, const int new_owner_id)
Definition: Catalog.cpp:2664
std::atomic< std::thread::id > thread_holding_sqlite_lock
Definition: Catalog.h:725
void renameColumn(const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
Definition: Catalog.cpp:3877
DeletedColumnPerTableMap deletedColumnPerTable_
Definition: Catalog.h:668
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:633
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:635
void setForeignServerOptions(const std::string &server_name, const std::string &options)
Definition: Catalog.cpp:2696
std::vector< std::string > getTableNamesForUser(const UserMetadata &user, const GetTablesType get_tables_type) const
Definition: Catalog.cpp:4206
void addTableToMap(const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
Definition: Catalog.cpp:1192
void getAllColumnMetadataForTableImpl(const TableDescriptor *td, std::list< const ColumnDescriptor * > &colDescs, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Definition: Catalog.cpp:1771
TableEpochInfo(const int32_t table_id_param, const int32_t table_epoch_param)
Definition: Catalog.h:86
void setForeignTableOptions(const std::string &table_name, foreign_storage::OptionsMap &options_map, bool clear_existing_options=true)
Definition: Catalog.cpp:4816
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:697
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:122
bool filterTableByTypeAndUser(const TableDescriptor *td, const UserMetadata &user_metadata, const GetTablesType get_tables_type) const
Definition: Catalog.cpp:4173
const bool checkMetadataForDeletedRecs(const TableDescriptor *td, int column_id) const
Definition: Catalog.cpp:3182
std::string quoteIfRequired(const std::string &column_name) const
Definition: Catalog.cpp:4856
std::string getColumnDictDirectory(const ColumnDescriptor *cd) const
Definition: Catalog.cpp:4436
std::string name() const
Definition: Catalog.h:294
std::list< const DashboardDescriptor * > getAllDashboardsMetadata() const
Definition: Catalog.cpp:1836
void updateLeaf(const LeafHostInfo &string_dict_host)
const ColumnDescriptor * getMetadataForColumnUnlocked(int tableId, int columnId) const
Definition: Catalog.cpp:1595
This file contains the class specification and related data structures for SysCatalog.
std::map< ColumnIdKey, ColumnDescriptor * > ColumnDescriptorMapById
Definition: Types.h:40
void restoreOldOwnersInMemory(const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects, int32_t new_owner_id)
Definition: Catalog.cpp:5327
void removeChunksUnlocked(const int table_id) const
Definition: Catalog.cpp:3514
std::string calculateSHA1(const std::string &data)
Definition: Catalog.cpp:4053
void gatherAdditionalInfo(std::vector< std::string > &additional_info, std::set< std::string > &shared_dict_column_names, const TableDescriptor *td) const
Definition: Catalog.cpp:4867
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:222
void addLinkToMap(LinkDescriptor &ld)
Definition: Catalog.cpp:1420
std::shared_ptr< Calcite > getCalciteMgr() const
Definition: Catalog.h:224
static const std::string getForeignTableSchema(bool if_not_exists=false)
Definition: Catalog.cpp:682
std::map< int, TableDescriptor * > TableDescriptorMapById
Definition: Types.h:35
const ColumnDescriptor * getDeletedColumnIfRowsDeleted(const TableDescriptor *td) const
Definition: Catalog.cpp:3194
void adjustAlteredTableFiles(const std::string &temp_data_dir, const std::unordered_map< int, int > &all_column_ids_map) const
std::map< std::string, LinkDescriptor * > LinkDescriptorMap
Definition: Types.h:44
void delDictionary(const ColumnDescriptor &cd)
Definition: Catalog.cpp:1868
GetTablesType
Definition: Catalog.h:62
std::shared_timed_mutex mapd_shared_mutex
void dropForeignServer(const std::string &server_name)
Definition: Catalog.cpp:2728
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3224
void dropTable(const TableDescriptor *td)
Definition: Catalog.cpp:3532
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
static const std::string getForeignServerSchema(bool if_not_exists=false)
Definition: Catalog.cpp:675
int getDatabaseId() const
Definition: Catalog.h:277
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:631
int getLogicalTableId(const int physicalTableId) const
Definition: Catalog.cpp:4246
const CustomExpression * getCustomExpression(int32_t custom_expression_id) const
Definition: Catalog.cpp:4982
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1524
specifies the content in-memory of a row in the column metadata table
void setForeignTableProperty(const foreign_storage::ForeignTable *table, const std::string &property, const std::string &value)
Definition: Catalog.cpp:4836
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:2092
const std::vector< LeafHostInfo > & getStringDictionaryHosts() const
Definition: Catalog.cpp:1573
void checkpointWithAutoRollback(const int logical_table_id) const
Definition: Catalog.cpp:4268
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4121
void setMaxRows(const int32_t table_id, const int64_t max_rows)
Definition: Catalog.cpp:3031
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1374
const ColumnDescriptor * getMetadataForColumnBySpi(const int tableId, const size_t spi) const
Definition: Catalog.cpp:1628
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:639
void alterPhysicalTableMetadata(const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
Definition: Catalog.cpp:2949
ForeignServerMap foreignServerMap_
Definition: Catalog.h:638
void createShardedTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:3377
std::map< std::string, std::shared_ptr< foreign_storage::ForeignServer >> ForeignServerMap
Definition: Types.h:49
void reassignOwners(const std::set< std::string > &old_owners, const std::string &new_owner)
Definition: Catalog.cpp:5150
std::vector< const CustomExpression * > getCustomExpressionsForUser(const UserMetadata &user) const
Definition: Catalog.cpp:5005
void checkpoint(const int logicalTableId) const
Definition: Catalog.cpp:4260
const DictDescriptor * getMetadataForDictUnlocked(int dict_ref, bool loadDict) const
Definition: Catalog.cpp:1538
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2575
std::vector< std::pair< ColumnDescriptor *, ColumnDescriptor * >> ColumnDescriptorsForRoll
Definition: Catalog.h:660
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:647
static const std::string physicalTableNameTag_
Definition: Catalog.h:651
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1338
void setTableEpoch(const int db_id, const int table_id, const int new_epoch)
Definition: Catalog.cpp:2905
void alterTableMetadata(const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
Definition: Catalog.cpp:2976
TableDescriptor * getMutableMetadataForTableUnlocked(int tableId)
Definition: Catalog.cpp:1530
std::vector< std::string > getTableDictDirectories(const TableDescriptor *td) const
Definition: Catalog.cpp:4452
void getForeignServersForUser(const rapidjson::Value *filters, const UserMetadata &user, std::vector< const foreign_storage::ForeignServer * > &results)
Definition: Catalog.cpp:2762
std::map< DictRef, std::unique_ptr< DictDescriptor >> DictDescriptorMapById
Definition: Types.h:41
std::map< std::string, std::shared_ptr< DashboardDescriptor >> DashboardDescriptorMap
Definition: Types.h:43
static const std::string getCustomExpressionsSchema(bool if_not_exists=false)
Definition: Catalog.cpp:690
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1801
const Catalog * getObjForLock()
Definition: Catalog.cpp:222
int32_t createDashboard(DashboardDescriptor &vd)
Definition: Catalog.cpp:3916
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1252
void removeFragmenterForTable(const int table_id) const
Definition: Catalog.cpp:3502
std::map< int, LinkDescriptor * > LinkDescriptorMapById
Definition: Types.h:45
std::map< int, std::shared_ptr< foreign_storage::ForeignServer >> ForeignServerMapById
Definition: Types.h:51
const std::unique_ptr< const CustomExpression > getCustomExpressionFromStorage(int32_t custom_expression_id)
Definition: Catalog.cpp:4991
const int getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const
Definition: Catalog.cpp:1606
bool setColumnSharedDictionary(ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:3267
void updateCustomExpressionsSchema()
Definition: Catalog.cpp:663
void setTableEpochs(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3127
std::map< std::string, std::string, std::less<>> OptionsMap
void setMaxRollbackEpochs(const int32_t table_id, const int32_t max_rollback_epochs)
Definition: Catalog.cpp:3001
const TableDescriptor * getMetadataForTableImpl(int tableId, const bool populateFragmenter) const
Definition: Catalog.cpp:1501
std::string createLink(LinkDescriptor &ld, size_t min_length)
Definition: Catalog.cpp:4065
Descriptor for a dictionary for a string columne.
void updateLogicalToPhysicalTableLinkSchema()
Definition: Catalog.cpp:589
void setUncappedTableEpoch(const std::string &table_name)
Definition: Catalog.cpp:3050
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:640
std::atomic< std::thread::id > thread_holding_write_lock
Definition: Catalog.h:726
void setForeignServerDataWrapper(const std::string &server_name, const std::string &data_wrapper)
Definition: Catalog.cpp:2675
void setTableEpochsLogExceptions(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3164
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
SqliteConnector & getSqliteConnector()
Definition: Catalog.h:278
const std::string & getCatalogBasePath() const
Definition: Catalog.h:225
DictRef addDictionary(ColumnDescriptor &cd)
Definition: Catalog.cpp:1844
const std::unique_ptr< const foreign_storage::ForeignTable > getForeignTableFromStorage(int table_id)
Definition: Catalog.cpp:2644
void renameTable(const TableDescriptor *td, const std::string &newTableName)
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:637
std::string dumpSchema(const TableDescriptor *td) const
Definition: Catalog.cpp:4468
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:3336
void updateForeignTableRefreshTimes(const int32_t table_id)
Definition: Catalog.cpp:4791
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:661
void doTruncateTable(const TableDescriptor *td)
Definition: Catalog.cpp:3432
const std::map< int, const ColumnDescriptor * > getDictionaryToColumnMapping()
Definition: Catalog.cpp:4142
void renamePhysicalTable(const TableDescriptor *td, const std::string &newTableName)
void renameForeignServer(const std::string &server_name, const std::string &name)
Definition: Catalog.cpp:2716
std::vector< DBObject > parseDashboardObjects(const std::string &view_meta, const int &user_id)
Definition: Catalog.cpp:1344
const std::unique_ptr< const foreign_storage::ForeignServer > getForeignServerFromStorage(const std::string &server_name)
Definition: Catalog.cpp:2624
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:649
Catalog()
Constructor builds a hollow catalog used during constructor of other catalogs.
Definition: Catalog.cpp:195
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:630
void setForeignServerProperty(const std::string &server_name, const std::string &property, const std::string &value)
Definition: Catalog.cpp:4368
static thread_local bool thread_holds_read_lock
Definition: Catalog.h:728
std::map< std::string, TableDescriptor * > TableDescriptorMap
Definition: Types.h:34
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:636
void CheckAndExecuteMigrationsPostBuildMaps()
Definition: Catalog.cpp:928
std::vector< TableEpochInfo > getTableEpochs(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:3099
const int getColumnIdBySpi(const int tableId, const size_t spi) const
Definition: Catalog.cpp:1623
void createDefaultServersIfNotExists()
Definition: Catalog.cpp:4389
void truncateTable(const TableDescriptor *td)
Definition: Catalog.cpp:3414
void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id)
Definition: Catalog.cpp:603