OmniSciDB  8fa3bf436f
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Catalog.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
29 #pragma once
30 
31 #include <atomic>
32 #include <cstdint>
33 #include <ctime>
34 #include <limits>
35 #include <list>
36 #include <map>
37 #include <mutex>
38 #include <string>
39 #include <utility>
40 #include <vector>
41 
42 #include "Calcite/Calcite.h"
46 #include "Catalog/DictDescriptor.h"
47 #include "Catalog/ForeignServer.h"
48 #include "Catalog/ForeignTable.h"
49 #include "Catalog/LinkDescriptor.h"
50 #include "Catalog/SessionInfo.h"
51 #include "Catalog/SysCatalog.h"
53 #include "Catalog/TableMetadata.h"
54 #include "Catalog/Types.h"
55 #include "DataMgr/DataMgr.h"
59 
60 #include "LeafHostInfo.h"
61 
63 
64 namespace Parser {
65 
67 
68 } // namespace Parser
69 
70 class TableArchiver;
71 
72 // SPI means Sequential Positional Index which is equivalent to the input index in a
73 // RexInput node
74 #define SPIMAP_MAGIC1 (std::numeric_limits<unsigned>::max() / 4)
75 #define SPIMAP_MAGIC2 8
76 #define SPIMAP_GEO_PHYSICAL_INPUT(c, i) \
77  (SPIMAP_MAGIC1 + (unsigned)(SPIMAP_MAGIC2 * ((c) + 1) + (i)))
78 
79 namespace File_Namespace {
80 struct FileMgrParams;
81 }
82 namespace Catalog_Namespace {
85 
86  TableEpochInfo(const int32_t table_id_param, const int32_t table_epoch_param)
87  : table_id(table_id_param), table_epoch(table_epoch_param) {}
88  TableEpochInfo(const int32_t table_id_param,
89  const int32_t table_epoch_param,
90  const size_t leaf_index_param)
91  : table_id(table_id_param)
92  , table_epoch(table_epoch_param)
93  , leaf_index(leaf_index_param) {}
94 };
95 
102 class Catalog final {
103  public:
112  Catalog(const std::string& basePath,
113  const DBMetadata& curDB,
114  std::shared_ptr<Data_Namespace::DataMgr> dataMgr,
115  const std::vector<LeafHostInfo>& string_dict_hosts,
116  std::shared_ptr<Calcite> calcite,
117  bool is_new_db);
122  Catalog();
123 
130  ~Catalog();
131 
132  static void expandGeoColumn(const ColumnDescriptor& cd,
133  std::list<ColumnDescriptor>& columns);
134  void createTable(TableDescriptor& td,
135  const std::list<ColumnDescriptor>& columns,
136  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs,
137  bool isLogicalTable);
138  void createShardedTable(
139  TableDescriptor& td,
140  const std::list<ColumnDescriptor>& columns,
141  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs);
144  std::string createLink(LinkDescriptor& ld, size_t min_length);
145  void dropTable(const TableDescriptor* td);
146  void truncateTable(const TableDescriptor* td);
147  void renameTable(const TableDescriptor* td, const std::string& newTableName);
148  void renameTable(std::vector<std::pair<std::string, std::string>>& names);
149  void renameColumn(const TableDescriptor* td,
150  const ColumnDescriptor* cd,
151  const std::string& newColumnName);
152  void addColumn(const TableDescriptor& td, ColumnDescriptor& cd);
153  void dropColumn(const TableDescriptor& td, const ColumnDescriptor& cd);
154  void removeFragmenterForTable(const int table_id) const;
155 
156  const std::map<int, const ColumnDescriptor*> getDictionaryToColumnMapping();
157 
166  const TableDescriptor* getMetadataForTable(const std::string& tableName,
167  const bool populateFragmenter = true) const;
168  const TableDescriptor* getMetadataForTableImpl(int tableId,
169  const bool populateFragmenter) const;
170  const TableDescriptor* getMetadataForTable(int tableId,
171  bool populateFragmenter = true) const;
172 
173  const ColumnDescriptor* getMetadataForColumn(int tableId,
174  const std::string& colName) const;
175  const ColumnDescriptor* getMetadataForColumn(int tableId, int columnId) const;
176  const ColumnDescriptor* getMetadataForColumnUnlocked(int tableId, int columnId) const;
177 
178  const int getColumnIdBySpi(const int tableId, const size_t spi) const;
179  const ColumnDescriptor* getMetadataForColumnBySpi(const int tableId,
180  const size_t spi) const;
181 
182  const DashboardDescriptor* getMetadataForDashboard(const std::string& userId,
183  const std::string& dashName) const;
184 
185  const DashboardDescriptor* getMetadataForDashboard(const int32_t dashboard_id) const;
186  void deleteMetadataForDashboards(const std::vector<int32_t> ids,
187  const UserMetadata& user);
188 
189  const LinkDescriptor* getMetadataForLink(const std::string& link) const;
190  const LinkDescriptor* getMetadataForLink(int linkId) const;
191 
192  const foreign_storage::ForeignTable* getForeignTableUnlocked(int tableId) const;
194  const std::string& tableName) const;
195 
196  const foreign_storage::ForeignTable* getForeignTable(int table_id) const;
197 
206  std::list<const ColumnDescriptor*> getAllColumnMetadataForTable(
207  const int tableId,
208  const bool fetchSystemColumns,
209  const bool fetchVirtualColumns,
210  const bool fetchPhysicalColumns) const;
214  std::list<const ColumnDescriptor*> getAllColumnMetadataForTableUnlocked(
215  const int tableId,
216  const bool fetchSystemColumns,
217  const bool fetchVirtualColumns,
218  const bool fetchPhysicalColumns) const;
219 
220  std::list<const TableDescriptor*> getAllTableMetadata() const;
221  std::list<const DashboardDescriptor*> getAllDashboardsMetadata() const;
222  const DBMetadata& getCurrentDB() const { return currentDB_; }
224  std::shared_ptr<Calcite> getCalciteMgr() const { return calciteMgr_; }
225  const std::string& getCatalogBasePath() const { return basePath_; }
226 
227  const DictDescriptor* getMetadataForDict(int dict_ref, bool loadDict = true) const;
228  const DictDescriptor* getMetadataForDictUnlocked(int dict_ref, bool loadDict) const;
229 
230  const std::vector<LeafHostInfo>& getStringDictionaryHosts() const;
231 
233 
234  std::vector<const TableDescriptor*> getPhysicalTablesDescriptors(
235  const TableDescriptor* logical_table_desc,
236  bool populate_fragmenter = true) const;
237 
246  std::vector<std::string> getTableNamesForUser(
247  const UserMetadata& user,
248  const GetTablesType get_tables_type) const;
249 
259  std::vector<TableMetadata> getTablesMetadataForUser(
260  const UserMetadata& user_metadata,
261  const GetTablesType get_tables_type,
262  const std::string& filter_table_name) const;
263 
264  int32_t getTableEpoch(const int32_t db_id, const int32_t table_id) const;
265  void setTableEpoch(const int db_id, const int table_id, const int new_epoch);
266  void setMaxRollbackEpochs(const int32_t table_id, const int32_t max_rollback_epochs);
267  void setMaxRows(const int32_t table_id, const int64_t max_rows);
268 
269  std::vector<TableEpochInfo> getTableEpochs(const int32_t db_id,
270  const int32_t table_id) const;
271  void setTableEpochs(const int32_t db_id,
272  const std::vector<TableEpochInfo>& table_epochs) const;
273 
274  void setTableEpochsLogExceptions(const int32_t db_id,
275  const std::vector<TableEpochInfo>& table_epochs) const;
276 
277  int getDatabaseId() const { return currentDB_.dbId; }
279  void roll(const bool forward);
281  void delDictionary(const ColumnDescriptor& cd);
282  void getDictionary(const ColumnDescriptor& cd,
283  std::map<int, StringDictionary*>& stringDicts);
284 
285  const bool checkMetadataForDeletedRecs(const TableDescriptor* td, int column_id) const;
286  const ColumnDescriptor* getDeletedColumn(const TableDescriptor* td) const;
288 
289  void setDeletedColumn(const TableDescriptor* td, const ColumnDescriptor* cd);
290  void setDeletedColumnUnlocked(const TableDescriptor* td, const ColumnDescriptor* cd);
291  int getLogicalTableId(const int physicalTableId) const;
292  void checkpoint(const int logicalTableId) const;
293  void checkpointWithAutoRollback(const int logical_table_id) const;
294  std::string name() const { return getCurrentDB().dbName; }
295  void eraseDBData();
296  void eraseTablePhysicalData(const TableDescriptor* td);
297  void setForReload(const int32_t tableId);
298 
299  std::vector<std::string> getTableDataDirectories(const TableDescriptor* td) const;
300  std::vector<std::string> getTableDictDirectories(const TableDescriptor* td) const;
301  std::string getColumnDictDirectory(const ColumnDescriptor* cd) const;
302  std::string dumpSchema(const TableDescriptor* td) const;
303  std::string dumpCreateTable(const TableDescriptor* td,
304  bool multiline_formatting = true,
305  bool dump_defaults = false) const;
306 
314  static const std::string getForeignTableSchema(bool if_not_exists = false);
315 
323  static const std::string getForeignServerSchema(bool if_not_exists = false);
324 
334  void createForeignServer(std::unique_ptr<foreign_storage::ForeignServer> foreign_server,
335  bool if_not_exists);
336 
345  const std::string& server_name) const;
346 
356  const std::unique_ptr<const foreign_storage::ForeignServer> getForeignServerFromStorage(
357  const std::string& server_name);
358 
368  const std::unique_ptr<const foreign_storage::ForeignTable> getForeignTableFromStorage(
369  int table_id);
370 
377  void changeForeignServerOwner(const std::string& server_name, const int new_owner_id);
378 
385  void setForeignServerDataWrapper(const std::string& server_name,
386  const std::string& data_wrapper);
393  void setForeignServerOptions(const std::string& server_name,
394  const std::string& options);
401  void renameForeignServer(const std::string& server_name, const std::string& name);
402 
408  void dropForeignServer(const std::string& server_name);
409 
423  const rapidjson::Value* filters,
424  const UserMetadata& user,
425  std::vector<const foreign_storage::ForeignServer*>& results);
426 
431 
442  bool validateNonExistentTableOrView(const std::string& name, const bool if_not_exists);
443 
451  std::vector<const TableDescriptor*> getAllForeignTablesForRefresh() const;
452 
459  void updateForeignTableRefreshTimes(const int32_t table_id);
460 
467  void setForeignTableOptions(const std::string& table_name,
468  foreign_storage::OptionsMap& options_map,
469  bool clear_existing_options = true);
470 
471  void updateLeaf(const LeafHostInfo& string_dict_host);
472 
473  // For testing purposes only
474  void setUncappedTableEpoch(const std::string& table_name);
475 
476  // Methods for accessing and updating custom expressions
477 
485  static const std::string getCustomExpressionsSchema(bool if_not_exists = false);
486 
494  int32_t createCustomExpression(std::unique_ptr<CustomExpression> custom_expression);
495 
503  const CustomExpression* getCustomExpression(int32_t custom_expression_id) const;
504 
514  const std::unique_ptr<const CustomExpression> getCustomExpressionFromStorage(
515  int32_t custom_expression_id);
516 
525  std::vector<const CustomExpression*> getCustomExpressionsForUser(
526  const UserMetadata& user) const;
527 
534  void updateCustomExpression(int32_t custom_expression_id,
535  const std::string& expression_json);
536 
543  void deleteCustomExpressions(const std::vector<int32_t>& custom_expression_ids,
544  bool do_soft_delete);
545 
546  protected:
549  void updateDictionaryNames();
552  void updateGeoColumns();
554  void updateLinkSchema();
557  void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id);
558  void updateDictionarySchema();
559  void updatePageSize();
563  void updateFsiSchemas();
567  void buildMaps();
568  void addTableToMap(const TableDescriptor* td,
569  const std::list<ColumnDescriptor>& columns,
570  const std::list<DictDescriptor>& dicts);
571  void addReferenceToForeignDict(ColumnDescriptor& referencing_column,
572  Parser::SharedDictionaryDef shared_dict_def,
573  const bool persist_reference);
575  ColumnDescriptor& cd,
576  std::list<ColumnDescriptor>& cdd,
577  std::list<DictDescriptor>& dds,
578  const TableDescriptor td,
579  const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs);
581  std::list<DictDescriptor>& dds,
582  const TableDescriptor& td,
583  const bool isLogicalTable);
586  void addLinkToMap(LinkDescriptor& ld);
587  void removeTableFromMap(const std::string& tableName,
588  const int tableId,
589  const bool is_on_error = false);
590  void doDropTable(const TableDescriptor* td);
592  void doTruncateTable(const TableDescriptor* td);
593  void renamePhysicalTable(const TableDescriptor* td, const std::string& newTableName);
594  void renamePhysicalTable(std::vector<std::pair<std::string, std::string>>& names,
595  std::vector<int>& tableIds);
596  void instantiateFragmenter(TableDescriptor* td) const;
598  std::list<const ColumnDescriptor*>& colDescs,
599  const bool fetchSystemColumns,
600  const bool fetchVirtualColumns,
601  const bool fetchPhysicalColumns) const;
602  std::string calculateSHA1(const std::string& data);
603  std::string generatePhysicalTableName(const std::string& logicalTableName,
604  const int32_t& shardNumber);
605  std::vector<DBObject> parseDashboardObjects(const std::string& view_meta,
606  const int& user_id);
607  void createOrUpdateDashboardSystemRole(const std::string& view_meta,
608  const int32_t& user_id,
609  const std::string& dash_role_name);
610 
611  const int getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const;
612 
614  const std::list<ColumnDescriptor>& cds) const;
615  void dropTableFromJsonUnlocked(const std::string& table_name) const;
616 
617  std::string basePath_;
629 
632  std::shared_ptr<Data_Namespace::DataMgr> dataMgr_;
633 
634  const std::vector<LeafHostInfo> string_dict_hosts_;
635  std::shared_ptr<Calcite> calciteMgr_;
636 
638  static const std::string
639  physicalTableNameTag_; // extra component added to the name of each physical table
642 
643  // this tuple is for rolling forw/back once after ALTER ADD/DEL/MODIFY columns
644  // succeeds/fails
645  // get(0) = old ColumnDescriptor*
646  // get(1) = new ColumnDescriptor*
648  std::vector<std::pair<ColumnDescriptor*, ColumnDescriptor*>>;
650 
651  private:
652  void gatherAdditionalInfo(std::vector<std::string>& additional_info,
653  std::set<std::string>& shared_dict_column_names,
654  const TableDescriptor* td) const;
655  std::string quoteIfRequired(const std::string& column_name) const;
658  const std::string& temp_data_dir,
659  const std::unordered_map<int, int>& all_column_ids_map) const;
660  void renameTableDirectories(const std::string& temp_data_dir,
661  const std::vector<std::string>& target_paths,
662  const std::string& name_prefix) const;
663  void buildForeignServerMap();
664  void addForeignTableDetails();
665 
666  void setForeignServerProperty(const std::string& server_name,
667  const std::string& property,
668  const std::string& value);
669 
671  const std::string& property,
672  const std::string& value);
673 
675  const TableDescriptorUpdateParams& table_update_params);
676  void alterTableMetadata(const TableDescriptor* td,
677  const TableDescriptorUpdateParams& table_update_params);
678  void setTableFileMgrParams(const int table_id,
679  const File_Namespace::FileMgrParams& file_mgr_params);
681  const UserMetadata& user_metadata,
682  const GetTablesType get_tables_type) const;
683 
685 
691  std::unique_ptr<foreign_storage::ForeignServer> foreign_server,
692  bool if_not_exists);
693 
695  const std::string& tableName) const;
696 
697  const Catalog* getObjForLock();
698  void removeChunksUnlocked(const int table_id) const;
699 
701  std::unique_ptr<CustomExpression> getCustomExpressionFromConnector(size_t row);
702 
703  public:
704  mutable std::mutex sqliteMutex_;
706  mutable std::atomic<std::thread::id> thread_holding_sqlite_lock;
707  mutable std::atomic<std::thread::id> thread_holding_write_lock;
708  // assuming that you never call into a catalog from another catalog via the same thread
709  static thread_local bool thread_holds_read_lock;
710  bool initialized_ = false;
711 };
712 
713 } // namespace Catalog_Namespace
std::string dumpCreateTable(const TableDescriptor *td, bool multiline_formatting=true, bool dump_defaults=false) const
Definition: Catalog.cpp:4533
std::list< const ColumnDescriptor * > getAllColumnMetadataForTableUnlocked(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Definition: Catalog.cpp:1781
void serializeTableJsonUnlocked(const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
Definition: Catalog.cpp:2415
std::unordered_map< const TableDescriptor *, const ColumnDescriptor * > DeletedColumnPerTableMap
Definition: Types.h:47
void instantiateFragmenter(TableDescriptor *td) const
Definition: Catalog.cpp:1398
const foreign_storage::ForeignTable * getForeignTable(const std::string &tableName) const
Definition: Catalog.cpp:1448
void deleteMetadataForDashboards(const std::vector< int32_t > ids, const UserMetadata &user)
Definition: Catalog.cpp:1608
void executeDropTableSqliteQueries(const TableDescriptor *td)
Definition: Catalog.cpp:3515
void renameTableDirectories(const std::string &temp_data_dir, const std::vector< std::string > &target_paths, const std::string &name_prefix) const
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:4232
void doDropTable(const TableDescriptor *td)
Definition: Catalog.cpp:3507
~Catalog()
Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the ...
Definition: Catalog.cpp:196
void getDictionary(const ColumnDescriptor &cd, std::map< int, StringDictionary * > &stringDicts)
Definition: Catalog.cpp:1881
mapd_shared_mutex sharedMutex_
Definition: Catalog.h:705
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:102
void deleteCustomExpressions(const std::vector< int32_t > &custom_expression_ids, bool do_soft_delete)
Definition: Catalog.cpp:4994
void addReferenceToForeignDict(ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def, const bool persist_reference)
Definition: Catalog.cpp:3187
const ColumnDescriptor * getDeletedColumn(const TableDescriptor *td) const
Definition: Catalog.cpp:3118
void setDeletedColumn(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3161
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:620
const foreign_storage::ForeignTable * getForeignTableUnlocked(int tableId) const
Definition: Catalog.cpp:1727
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:2198
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:427
void createForeignServer(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2509
TableEpochInfo(const int32_t table_id_param, const int32_t table_epoch_param, const size_t leaf_index_param)
Definition: Catalog.h:88
std::map< ColumnKey, ColumnDescriptor * > ColumnDescriptorMap
Definition: Types.h:38
std::map< int32_t, std::vector< int32_t >> LogicalToPhysicalTableMapById
Definition: Types.h:36
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:223
void addColumn(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:1909
int32_t createCustomExpression(std::unique_ptr< CustomExpression > custom_expression)
Definition: Catalog.cpp:4869
void roll(const bool forward)
Definition: Catalog.cpp:1997
bool validateNonExistentTableOrView(const std::string &name, const bool if_not_exists)
Definition: Catalog.cpp:4692
void setTableFileMgrParams(const int table_id, const File_Namespace::FileMgrParams &file_mgr_params)
Definition: Catalog.cpp:3006
std::vector< const TableDescriptor * > getAllForeignTablesForRefresh() const
Definition: Catalog.cpp:4703
std::map< int, std::unique_ptr< CustomExpression >> CustomExpressionMapById
Definition: Types.h:52
const foreign_storage::ForeignServer * getForeignServer(const std::string &server_name) const
Definition: Catalog.cpp:2554
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:1797
SqliteConnector sqliteConnector_
Definition: Catalog.h:630
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:634
const DBMetadata currentDB_
Definition: Catalog.h:631
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:622
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:632
const LinkDescriptor * getMetadataForLink(const std::string &link) const
void dropTableFromJsonUnlocked(const std::string &table_name) const
Definition: Catalog.cpp:2477
void setForReload(const int32_t tableId)
Definition: Catalog.cpp:4354
int32_t getTableEpoch(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:2796
void replaceDashboard(DashboardDescriptor &vd)
Definition: Catalog.cpp:3920
std::vector< std::string > getTableDataDirectories(const TableDescriptor *td) const
Definition: Catalog.cpp:4364
void updateCustomExpression(int32_t custom_expression_id, const std::string &expression_json)
Definition: Catalog.cpp:4966
std::string generatePhysicalTableName(const std::string &logicalTableName, const int32_t &shardNumber)
Definition: Catalog.cpp:4261
void dropColumn(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:1966
void checkDateInDaysColumnMigration()
Definition: Catalog.cpp:803
void addFrontendViewToMap(DashboardDescriptor &vd)
Definition: Catalog.cpp:1304
std::vector< TableMetadata > getTablesMetadataForUser(const UserMetadata &user_metadata, const GetTablesType get_tables_type, const std::string &filter_table_name) const
Definition: Catalog.cpp:4164
std::unique_ptr< CustomExpression > getCustomExpressionFromConnector(size_t row)
Definition: Catalog.cpp:1147
const ColumnDescriptor * getShardColumnMetadataForTable(const TableDescriptor *td) const
Definition: Catalog.cpp:4045
void changeForeignServerOwner(const std::string &server_name, const int new_owner_id)
Definition: Catalog.cpp:2606
std::atomic< std::thread::id > thread_holding_sqlite_lock
Definition: Catalog.h:706
void renameColumn(const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
Definition: Catalog.cpp:3819
DeletedColumnPerTableMap deletedColumnPerTable_
Definition: Catalog.h:656
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:621
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:623
void setForeignServerOptions(const std::string &server_name, const std::string &options)
Definition: Catalog.cpp:2638
std::vector< std::string > getTableNamesForUser(const UserMetadata &user, const GetTablesType get_tables_type) const
Definition: Catalog.cpp:4148
void addTableToMap(const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
Definition: Catalog.cpp:1163
void getAllColumnMetadataForTableImpl(const TableDescriptor *td, std::list< const ColumnDescriptor * > &colDescs, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Definition: Catalog.cpp:1741
TableEpochInfo(const int32_t table_id_param, const int32_t table_epoch_param)
Definition: Catalog.h:86
void setForeignTableOptions(const std::string &table_name, foreign_storage::OptionsMap &options_map, bool clear_existing_options=true)
Definition: Catalog.cpp:4752
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:675
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:121
bool filterTableByTypeAndUser(const TableDescriptor *td, const UserMetadata &user_metadata, const GetTablesType get_tables_type) const
Definition: Catalog.cpp:4115
const bool checkMetadataForDeletedRecs(const TableDescriptor *td, int column_id) const
Definition: Catalog.cpp:3124
std::string quoteIfRequired(const std::string &column_name) const
Definition: Catalog.cpp:4792
std::string getColumnDictDirectory(const ColumnDescriptor *cd) const
Definition: Catalog.cpp:4378
std::string name() const
Definition: Catalog.h:294
std::list< const DashboardDescriptor * > getAllDashboardsMetadata() const
Definition: Catalog.cpp:1806
void updateLeaf(const LeafHostInfo &string_dict_host)
const ColumnDescriptor * getMetadataForColumnUnlocked(int tableId, int columnId) const
Definition: Catalog.cpp:1565
This file contains the class specification and related data structures for SysCatalog.
std::map< ColumnIdKey, ColumnDescriptor * > ColumnDescriptorMapById
Definition: Types.h:40
void removeChunksUnlocked(const int table_id) const
Definition: Catalog.cpp:3456
std::string calculateSHA1(const std::string &data)
Definition: Catalog.cpp:3995
void gatherAdditionalInfo(std::vector< std::string > &additional_info, std::set< std::string > &shared_dict_column_names, const TableDescriptor *td) const
Definition: Catalog.cpp:4803
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:222
void addLinkToMap(LinkDescriptor &ld)
Definition: Catalog.cpp:1390
std::shared_ptr< Calcite > getCalciteMgr() const
Definition: Catalog.h:224
static const std::string getForeignTableSchema(bool if_not_exists=false)
Definition: Catalog.cpp:660
std::map< int, TableDescriptor * > TableDescriptorMapById
Definition: Types.h:35
const ColumnDescriptor * getDeletedColumnIfRowsDeleted(const TableDescriptor *td) const
Definition: Catalog.cpp:3136
void adjustAlteredTableFiles(const std::string &temp_data_dir, const std::unordered_map< int, int > &all_column_ids_map) const
std::map< std::string, LinkDescriptor * > LinkDescriptorMap
Definition: Types.h:44
void delDictionary(const ColumnDescriptor &cd)
Definition: Catalog.cpp:1838
GetTablesType
Definition: Catalog.h:62
std::shared_timed_mutex mapd_shared_mutex
void dropForeignServer(const std::string &server_name)
Definition: Catalog.cpp:2670
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3166
void dropTable(const TableDescriptor *td)
Definition: Catalog.cpp:3474
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
static const std::string getForeignServerSchema(bool if_not_exists=false)
Definition: Catalog.cpp:653
int getDatabaseId() const
Definition: Catalog.h:277
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:619
int getLogicalTableId(const int physicalTableId) const
Definition: Catalog.cpp:4188
const CustomExpression * getCustomExpression(int32_t custom_expression_id) const
Definition: Catalog.cpp:4918
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1494
specifies the content in-memory of a row in the column metadata table
void setForeignTableProperty(const foreign_storage::ForeignTable *table, const std::string &property, const std::string &value)
Definition: Catalog.cpp:4772
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:2055
const std::vector< LeafHostInfo > & getStringDictionaryHosts() const
Definition: Catalog.cpp:1543
void checkpointWithAutoRollback(const int logical_table_id) const
Definition: Catalog.cpp:4210
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4063
void setMaxRows(const int32_t table_id, const int64_t max_rows)
Definition: Catalog.cpp:2973
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1345
const ColumnDescriptor * getMetadataForColumnBySpi(const int tableId, const size_t spi) const
Definition: Catalog.cpp:1598
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:627
void alterPhysicalTableMetadata(const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
Definition: Catalog.cpp:2891
ForeignServerMap foreignServerMap_
Definition: Catalog.h:626
void createShardedTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:3319
std::map< std::string, std::shared_ptr< foreign_storage::ForeignServer >> ForeignServerMap
Definition: Types.h:49
std::vector< const CustomExpression * > getCustomExpressionsForUser(const UserMetadata &user) const
Definition: Catalog.cpp:4941
void checkpoint(const int logicalTableId) const
Definition: Catalog.cpp:4202
const DictDescriptor * getMetadataForDictUnlocked(int dict_ref, bool loadDict) const
Definition: Catalog.cpp:1508
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2517
std::vector< std::pair< ColumnDescriptor *, ColumnDescriptor * >> ColumnDescriptorsForRoll
Definition: Catalog.h:648
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:635
static const std::string physicalTableNameTag_
Definition: Catalog.h:639
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1309
void setTableEpoch(const int db_id, const int table_id, const int new_epoch)
Definition: Catalog.cpp:2847
void alterTableMetadata(const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
Definition: Catalog.cpp:2918
TableDescriptor * getMutableMetadataForTableUnlocked(int tableId)
Definition: Catalog.cpp:1500
std::vector< std::string > getTableDictDirectories(const TableDescriptor *td) const
Definition: Catalog.cpp:4394
void getForeignServersForUser(const rapidjson::Value *filters, const UserMetadata &user, std::vector< const foreign_storage::ForeignServer * > &results)
Definition: Catalog.cpp:2704
std::map< DictRef, std::unique_ptr< DictDescriptor >> DictDescriptorMapById
Definition: Types.h:41
std::map< std::string, std::shared_ptr< DashboardDescriptor >> DashboardDescriptorMap
Definition: Types.h:43
static const std::string getCustomExpressionsSchema(bool if_not_exists=false)
Definition: Catalog.cpp:668
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1771
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
int32_t createDashboard(DashboardDescriptor &vd)
Definition: Catalog.cpp:3858
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1223
void removeFragmenterForTable(const int table_id) const
Definition: Catalog.cpp:3444
std::map< int, LinkDescriptor * > LinkDescriptorMapById
Definition: Types.h:45
std::map< int, std::shared_ptr< foreign_storage::ForeignServer >> ForeignServerMapById
Definition: Types.h:51
const std::unique_ptr< const CustomExpression > getCustomExpressionFromStorage(int32_t custom_expression_id)
Definition: Catalog.cpp:4927
const int getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const
Definition: Catalog.cpp:1576
bool setColumnSharedDictionary(ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:3209
void updateCustomExpressionsSchema()
Definition: Catalog.cpp:641
void setTableEpochs(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3069
std::map< std::string, std::string, std::less<>> OptionsMap
void setMaxRollbackEpochs(const int32_t table_id, const int32_t max_rollback_epochs)
Definition: Catalog.cpp:2943
const TableDescriptor * getMetadataForTableImpl(int tableId, const bool populateFragmenter) const
Definition: Catalog.cpp:1471
std::string createLink(LinkDescriptor &ld, size_t min_length)
Definition: Catalog.cpp:4007
Descriptor for a dictionary for a string columne.
void updateLogicalToPhysicalTableLinkSchema()
Definition: Catalog.cpp:567
void setUncappedTableEpoch(const std::string &table_name)
Definition: Catalog.cpp:2992
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:628
std::atomic< std::thread::id > thread_holding_write_lock
Definition: Catalog.h:707
void setForeignServerDataWrapper(const std::string &server_name, const std::string &data_wrapper)
Definition: Catalog.cpp:2617
void setTableEpochsLogExceptions(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3106
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
SqliteConnector & getSqliteConnector()
Definition: Catalog.h:278
const std::string & getCatalogBasePath() const
Definition: Catalog.h:225
DictRef addDictionary(ColumnDescriptor &cd)
Definition: Catalog.cpp:1814
const std::unique_ptr< const foreign_storage::ForeignTable > getForeignTableFromStorage(int table_id)
Definition: Catalog.cpp:2586
void renameTable(const TableDescriptor *td, const std::string &newTableName)
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:625
std::string dumpSchema(const TableDescriptor *td) const
Definition: Catalog.cpp:4410
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:3278
void updateForeignTableRefreshTimes(const int32_t table_id)
Definition: Catalog.cpp:4727
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:649
void doTruncateTable(const TableDescriptor *td)
Definition: Catalog.cpp:3374
const std::map< int, const ColumnDescriptor * > getDictionaryToColumnMapping()
Definition: Catalog.cpp:4084
void renamePhysicalTable(const TableDescriptor *td, const std::string &newTableName)
void renameForeignServer(const std::string &server_name, const std::string &name)
Definition: Catalog.cpp:2658
std::vector< DBObject > parseDashboardObjects(const std::string &view_meta, const int &user_id)
Definition: Catalog.cpp:1315
const std::unique_ptr< const foreign_storage::ForeignServer > getForeignServerFromStorage(const std::string &server_name)
Definition: Catalog.cpp:2566
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:637
Catalog()
Constructor builds a hollow catalog used during constructor of other catalogs.
Definition: Catalog.cpp:194
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:618
void setForeignServerProperty(const std::string &server_name, const std::string &property, const std::string &value)
Definition: Catalog.cpp:4310
static thread_local bool thread_holds_read_lock
Definition: Catalog.h:709
std::map< std::string, TableDescriptor * > TableDescriptorMap
Definition: Types.h:34
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:624
void CheckAndExecuteMigrationsPostBuildMaps()
Definition: Catalog.cpp:905
std::vector< TableEpochInfo > getTableEpochs(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:3041
const int getColumnIdBySpi(const int tableId, const size_t spi) const
Definition: Catalog.cpp:1593
void createDefaultServersIfNotExists()
Definition: Catalog.cpp:4331
void truncateTable(const TableDescriptor *td)
Definition: Catalog.cpp:3356
void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id)
Definition: Catalog.cpp:581