49 #include "../DataMgr/DataMgr.h"
50 #include "../QueryEngine/CompilationOptions.h"
51 #include "../SqliteConnector/SqliteConnector.h"
54 #include "../Calcite/Calcite.h"
55 #include "../Shared/mapd_shared_mutex.h"
68 #define SPIMAP_MAGIC1 (std::numeric_limits<unsigned>::max() / 4)
69 #define SPIMAP_MAGIC2 8
70 #define SPIMAP_GEO_PHYSICAL_INPUT(c, i) \
71 (SPIMAP_MAGIC1 + (unsigned)(SPIMAP_MAGIC2 * ((c) + 1) + (i)))
73 namespace Catalog_Namespace {
91 Catalog(
const std::string& basePath,
93 std::shared_ptr<Data_Namespace::DataMgr> dataMgr,
94 const std::vector<LeafHostInfo>& string_dict_hosts,
95 std::shared_ptr<Calcite> calcite,
107 std::list<ColumnDescriptor>& columns);
109 const std::list<ColumnDescriptor>& columns,
110 const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs,
111 bool isLogicalTable);
114 const std::list<ColumnDescriptor>& columns,
115 const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs);
124 const std::string& newColumnName);
137 const bool populateFragmenter =
true)
const;
139 const bool populateFragmenter)
const;
143 const std::string& colName)
const;
148 const size_t spi)
const;
151 const std::string& dashName)
const;
170 const bool fetchSystemColumns,
171 const bool fetchVirtualColumns,
172 const bool fetchPhysicalColumns)
const;
190 int32_t
getTableEpoch(
const int32_t db_id,
const int32_t table_id)
const;
191 void setTableEpoch(
const int db_id,
const int table_id,
const int new_epoch);
195 void roll(
const bool forward);
199 std::map<int, StringDictionary*>& stringDicts);
201 static void set(
const std::string& dbName, std::shared_ptr<Catalog> cat);
202 static std::shared_ptr<Catalog>
get(
const std::string& dbName);
203 static std::shared_ptr<Catalog>
get(
const std::string& basePath,
205 std::shared_ptr<Data_Namespace::DataMgr> dataMgr,
206 const std::vector<LeafHostInfo>& string_dict_hosts,
207 std::shared_ptr<Calcite> calcite,
209 static void remove(
const std::string& dbName);
218 void checkpoint(
const int logicalTableId)
const;
227 const std::string& path,
228 const std::string& compression)
const;
231 const std::string& file_path,
232 const std::string& compression);
234 const std::string& table_name,
235 const std::string& file_path,
236 const std::string& compression);
251 typedef std::map<std::string, std::shared_ptr<DashboardDescriptor>>
255 typedef std::unordered_map<const TableDescriptor*, const ColumnDescriptor*>
277 const std::list<ColumnDescriptor>& columns,
278 const std::list<DictDescriptor>& dicts);
283 std::list<ColumnDescriptor>& cdd,
284 std::list<DictDescriptor>& dds,
286 const std::vector<Parser::SharedDictionaryDef>& shared_dict_defs);
288 std::list<DictDescriptor>& dds,
290 const bool isLogicalTable);
296 const bool is_on_error =
false);
302 std::list<const ColumnDescriptor*>& colDescs,
303 const bool fetchSystemColumns,
304 const bool fetchVirtualColumns,
305 const bool fetchPhysicalColumns)
const;
308 const int32_t& shardNumber);
312 const int32_t& user_id,
313 const std::string& dash_role_name);
334 static const std::string
344 std::vector<std::pair<ColumnDescriptor*, ColumnDescriptor*>>;
351 const std::string& temp_data_dir,
352 const std::unordered_map<int, int>& all_column_ids_map)
const;
354 const std::vector<std::string>& target_paths,
355 const std::string& name_prefix)
const;
void instantiateFragmenter(TableDescriptor *td) const
void renameTableDirectories(const std::string &temp_data_dir, const std::vector< std::string > &target_paths, const std::string &name_prefix) const
specifies the content in-memory of a row in the link metadata view
void eraseTablePhysicalData(const TableDescriptor *td)
void CheckAndExecuteMigrations()
void doDropTable(const TableDescriptor *td)
~Catalog()
Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the ...
void getDictionary(const ColumnDescriptor &cd, std::map< int, StringDictionary * > &stringDicts)
mapd_shared_mutex sharedMutex_
class for a per-database catalog. also includes metadata for the current database and the current use...
const ColumnDescriptor * getDeletedColumn(const TableDescriptor *td) const
void setDeletedColumn(const TableDescriptor *td, const ColumnDescriptor *cd)
ColumnDescriptorMap columnDescriptorMap_
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
void updateFrontendViewAndLinkUsers()
Data_Namespace::DataMgr & getDataMgr() const
void addColumn(const TableDescriptor &td, ColumnDescriptor &cd)
void roll(const bool forward)
std::map< ColumnKey, ColumnDescriptor * > ColumnDescriptorMap
std::map< std::string, TableDescriptor * > TableDescriptorMap
std::list< const TableDescriptor * > getAllTableMetadata() const
std::unordered_map< const TableDescriptor *, const ColumnDescriptor * > DeletedColumnPerTableMap
std::map< int32_t, std::vector< int32_t > > LogicalToPhysicalTableMapById
SqliteConnector sqliteConnector_
const std::vector< LeafHostInfo > string_dict_hosts_
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logicalTableDesc) const
DictDescriptorMapById dictDescriptorMapByRef_
Catalog(const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
Constructor - takes basePath to already extant data directory for writing.
void updateDictionarySchema()
void updateDictionaryNames()
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
const LinkDescriptor * getMetadataForLink(const std::string &link) const
int32_t getTableEpoch(const int32_t db_id, const int32_t table_id) const
void replaceDashboard(DashboardDescriptor &vd)
std::vector< std::string > getTableDataDirectories(const TableDescriptor *td) const
std::string generatePhysicalTableName(const std::string &logicalTableName, const int32_t &shardNumber)
void addTableToMap(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
void checkDateInDaysColumnMigration()
void addFrontendViewToMap(DashboardDescriptor &vd)
void deleteMetadataForDashboard(const std::string &userId, const std::string &dashName)
const ColumnDescriptor * getShardColumnMetadataForTable(const TableDescriptor *td) const
std::atomic< std::thread::id > thread_holding_sqlite_lock
void updateFrontendViewSchema()
void renameColumn(const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
DeletedColumnPerTableMap deletedColumnPerTable_
ColumnDescriptorMapById columnDescriptorMapById_
DashboardDescriptorMap dashboardDescriptorMap_
void recordOwnershipOfObjectsInObjectPermissions()
void updateFrontendViewsToDashboards()
std::map< std::string, LinkDescriptor * > LinkDescriptorMap
std::string getColumnDictDirectory(const ColumnDescriptor *cd) const
std::list< const DashboardDescriptor * > getAllDashboardsMetadata() const
void dumpTable(const TableDescriptor *td, const std::string &path, const std::string &compression) const
std::map< int, LinkDescriptor * > LinkDescriptorMapById
This file contains the class specification and related data structures for SysCatalog.
std::string calculateSHA1(const std::string &data)
const DBMetadata & getCurrentDB() const
void addLinkToMap(LinkDescriptor &ld)
std::shared_ptr< Calcite > getCalciteMgr() const
std::map< DictRef, std::unique_ptr< DictDescriptor > > DictDescriptorMapById
const ColumnDescriptor * getDeletedColumnIfRowsDeleted(const TableDescriptor *td) const
void adjustAlteredTableFiles(const std::string &temp_data_dir, const std::unordered_map< int, int > &all_column_ids_map) const
void delDictionary(const ColumnDescriptor &cd)
std::shared_timed_mutex mapd_shared_mutex
void removeChunks(const int table_id)
std::map< int, TableDescriptor * > TableDescriptorMapById
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
void dropTable(const TableDescriptor *td)
std::tuple< int, int > ColumnIdKey
const bool checkMetadataForDeletedRecs(int dbId, int tableId, int columnId) const
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
const std::string & getBasePath() const
std::tuple< int, std::string > ColumnKey
int getDatabaseId() const
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const
TableDescriptorMapById tableDescriptorMapById_
int getLogicalTableId(const int physicalTableId) const
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
specifies the content in-memory of a row in the column metadata table
void updateDeletedColumnIndicator()
void restoreTable(const SessionInfo &session, const TableDescriptor *td, const std::string &file_path, const std::string &compression)
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
const std::vector< LeafHostInfo > & getStringDictionaryHosts() const
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
const ColumnDescriptor * getMetadataForColumnBySpi(const int tableId, const size_t spi) const
void createShardedTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
std::map< ColumnIdKey, ColumnDescriptor * > ColumnDescriptorMapById
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
void checkpoint(const int logicalTableId) const
void updateFixlenArrayColumns()
std::vector< std::pair< ColumnDescriptor *, ColumnDescriptor * >> ColumnDescriptorsForRoll
std::shared_ptr< Calcite > calciteMgr_
static const std::string physicalTableNameTag_
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
void setTableEpoch(const int db_id, const int table_id, const int new_epoch)
static void set(const std::string &dbName, std::shared_ptr< Catalog > cat)
std::vector< std::string > getTableDictDirectories(const TableDescriptor *td) const
void vacuumDeletedRows(const TableDescriptor *td) const
int32_t createDashboard(DashboardDescriptor &vd)
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
void addReferenceToForeignDict(ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def)
const int getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const
bool setColumnSharedDictionary(ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
const TableDescriptor * getMetadataForTableImpl(int tableId, const bool populateFragmenter) const
std::string createLink(LinkDescriptor &ld, size_t min_length)
Descriptor for a dictionary for a string columne.
void updateLogicalToPhysicalTableLinkSchema()
std::atomic< std::thread::id > thread_holding_write_lock
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
SqliteConnector & getSqliteConnector()
DictRef addDictionary(ColumnDescriptor &cd)
specifies the content in-memory of a row in the table metadata table
void renameTable(const TableDescriptor *td, const std::string &newTableName)
void updateTableDescriptorSchema()
LinkDescriptorMapById linkDescriptorMapById_
static std::map< std::string, std::shared_ptr< Catalog > > mapd_cat_map_
std::string dumpSchema(const TableDescriptor *td) const
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
ColumnDescriptorsForRoll columnDescriptorsForRoll
void doTruncateTable(const TableDescriptor *td)
std::map< std::string, std::shared_ptr< DashboardDescriptor > > DashboardDescriptorMap
void renamePhysicalTable(const TableDescriptor *td, const std::string &newTableName)
std::vector< DBObject > parseDashboardObjects(const std::string &view_meta, const int &user_id)
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
TableDescriptorMap tableDescriptorMap_
static thread_local bool thread_holds_read_lock
LinkDescriptorMap linkDescriptorMap_
void CheckAndExecuteMigrationsPostBuildMaps()
const int getColumnIdBySpi(const int tableId, const size_t spi) const
void truncateTable(const TableDescriptor *td)
void createDashboardSystemRoles()
void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id)