OmniSciDB  cde582ebc3
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Catalog_Namespace::Catalog Class Referencefinal

class for a per-database catalog. also includes metadata for the current database and the current user. More...

#include <Catalog.h>

+ Collaboration diagram for Catalog_Namespace::Catalog:

Public Member Functions

 Catalog (const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
 Constructor - takes basePath to already extant data directory for writing. More...
 
 ~Catalog ()
 Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite. More...
 
void createTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
 
void createShardedTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
int32_t createDashboard (DashboardDescriptor &vd)
 
void replaceDashboard (DashboardDescriptor &vd)
 
std::string createLink (LinkDescriptor &ld, size_t min_length)
 
void dropTable (const TableDescriptor *td)
 
void truncateTable (const TableDescriptor *td)
 
void renameTable (const TableDescriptor *td, const std::string &newTableName)
 
void renameTable (std::vector< std::pair< std::string, std::string >> &names)
 
void renameColumn (const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
 
void addColumn (const TableDescriptor &td, ColumnDescriptor &cd)
 
void dropColumn (const TableDescriptor &td, const ColumnDescriptor &cd)
 
void invalidateCachesForTable (const int table_id)
 
void removeFragmenterForTable (const int table_id) const
 
const std::map< int, const
ColumnDescriptor * > 
getDictionaryToColumnMapping ()
 
const TableDescriptorgetMetadataForTable (const std::string &tableName, const bool populateFragmenter=true) const
 Returns a pointer to a const TableDescriptor struct matching the provided tableName. More...
 
const TableDescriptorgetMetadataForTable (int tableId, bool populateFragmenter=true) const
 
std::optional< std::string > getTableName (int32_t table_id) const
 
std::optional< int32_t > getTableId (const std::string &table_name) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, const std::string &colName) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, int columnId) const
 
const std::optional< std::string > getColumnName (int table_id, int column_id) const
 
const int getColumnIdBySpi (const int tableId, const size_t spi) const
 
const ColumnDescriptorgetMetadataForColumnBySpi (const int tableId, const size_t spi) const
 
const DashboardDescriptorgetMetadataForDashboard (const std::string &userId, const std::string &dashName) const
 
const DashboardDescriptorgetMetadataForDashboard (const int32_t dashboard_id) const
 
void deleteMetadataForDashboards (const std::vector< int32_t > ids, const UserMetadata &user)
 
const LinkDescriptorgetMetadataForLink (const std::string &link) const
 
const LinkDescriptorgetMetadataForLink (int linkId) const
 
const
foreign_storage::ForeignTable
getForeignTable (const std::string &tableName) const
 
const
foreign_storage::ForeignTable
getForeignTable (int table_id) const
 
std::list< const
ColumnDescriptor * > 
getAllColumnMetadataForTable (const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular table specified by table id. More...
 
std::list< const
TableDescriptor * > 
getAllTableMetadata () const
 
std::vector< TableDescriptorgetAllTableMetadataCopy () const
 
std::list< const
DashboardDescriptor * > 
getAllDashboardsMetadata () const
 
std::vector< DashboardDescriptorgetAllDashboardsMetadataCopy () const
 
const DBMetadatagetCurrentDB () const
 
Data_Namespace::DataMgrgetDataMgr () const
 
std::shared_ptr< CalcitegetCalciteMgr () const
 
const std::string & getCatalogBasePath () const
 
const DictDescriptorgetMetadataForDict (int dict_ref, bool loadDict=true) const
 
const std::vector< LeafHostInfo > & getStringDictionaryHosts () const
 
const ColumnDescriptorgetShardColumnMetadataForTable (const TableDescriptor *td) const
 
std::vector< const
TableDescriptor * > 
getPhysicalTablesDescriptors (const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
 
std::vector< std::pair
< int32_t, int32_t > > 
getAllPersistedTableAndShardIds () const
 
std::vector< std::string > getTableNamesForUser (const UserMetadata &user, const GetTablesType get_tables_type) const
 
std::vector< TableMetadatagetTablesMetadataForUser (const UserMetadata &user_metadata, const GetTablesType get_tables_type, const std::string &filter_table_name) const
 
int32_t getTableEpoch (const int32_t db_id, const int32_t table_id) const
 
void setTableEpoch (const int db_id, const int table_id, const int new_epoch)
 
void setMaxRollbackEpochs (const int32_t table_id, const int32_t max_rollback_epochs)
 
void setMaxRows (const int32_t table_id, const int64_t max_rows)
 
std::vector< TableEpochInfogetTableEpochs (const int32_t db_id, const int32_t table_id) const
 
void setTableEpochs (const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
 
void setTableEpochsLogExceptions (const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
 
int getDatabaseId () const
 
SqliteConnectorgetSqliteConnector ()
 
void roll (const bool forward)
 
DictRef addDictionary (ColumnDescriptor &cd)
 
void delDictionary (const ColumnDescriptor &cd)
 
void getDictionary (const ColumnDescriptor &cd, std::map< int, StringDictionary * > &stringDicts)
 
const bool checkMetadataForDeletedRecs (const TableDescriptor *td, int column_id) const
 
const ColumnDescriptorgetDeletedColumn (const TableDescriptor *td) const
 
const ColumnDescriptorgetDeletedColumnIfRowsDeleted (const TableDescriptor *td) const
 
void setDeletedColumn (const TableDescriptor *td, const ColumnDescriptor *cd)
 
int getLogicalTableId (const int physicalTableId) const
 
void checkpoint (const int logicalTableId) const
 
void checkpointWithAutoRollback (const int logical_table_id) const
 
void resetTableEpochFloor (const int logicalTableId) const
 
std::string name () const
 
void eraseDbMetadata ()
 
void eraseDbPhysicalData ()
 
void eraseTablePhysicalData (const TableDescriptor *td)
 
void setForReload (const int32_t tableId)
 
std::vector< std::string > getTableDataDirectories (const TableDescriptor *td) const
 
std::vector< std::string > getTableDictDirectories (const TableDescriptor *td) const
 
std::set< std::string > getTableDictDirectoryPaths (int32_t table_id) const
 
std::string getColumnDictDirectory (const ColumnDescriptor *cd, bool file_name_only=true) const
 
std::string dumpSchema (const TableDescriptor *td) const
 
std::string dumpCreateTable (const TableDescriptor *td, bool multiline_formatting=true, bool dump_defaults=false) const
 
std::optional< std::string > dumpCreateTable (int32_t table_id, bool multiline_formatting=true, bool dump_defaults=false) const
 
std::string dumpCreateServer (const std::string &name, bool multiline_formatting=true) const
 
void createForeignServer (std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
 
const
foreign_storage::ForeignServer
getForeignServer (const std::string &server_name) const
 
const std::unique_ptr< const
foreign_storage::ForeignServer
getForeignServerFromStorage (const std::string &server_name)
 
const std::unique_ptr< const
foreign_storage::ForeignTable
getForeignTableFromStorage (int table_id)
 
void changeForeignServerOwner (const std::string &server_name, const int new_owner_id)
 
void setForeignServerDataWrapper (const std::string &server_name, const std::string &data_wrapper)
 
void setForeignServerOptions (const std::string &server_name, const std::string &options)
 
void renameForeignServer (const std::string &server_name, const std::string &name)
 
void dropForeignServer (const std::string &server_name)
 
std::vector< const
foreign_storage::ForeignTable * > 
getAllForeignTablesForForeignServer (const int32_t foreign_server_id)
 Get all of the foreign tables for associated with a foreign server id. More...
 
void getForeignServersForUser (const rapidjson::Value *filters, const UserMetadata &user, std::vector< const foreign_storage::ForeignServer * > &results)
 
void createDefaultServersIfNotExists ()
 
bool validateNonExistentTableOrView (const std::string &name, const bool if_not_exists)
 
std::vector< const
TableDescriptor * > 
getAllForeignTablesForRefresh () const
 
void updateForeignTableRefreshTimes (const int32_t table_id)
 
void setForeignTableOptions (const std::string &table_name, foreign_storage::OptionsMap &options_map, bool clear_existing_options=true)
 
void updateLeaf (const LeafHostInfo &string_dict_host)
 
void setUncappedTableEpoch (const std::string &table_name)
 
int32_t createCustomExpression (std::unique_ptr< CustomExpression > custom_expression)
 
const CustomExpressiongetCustomExpression (int32_t custom_expression_id) const
 
const std::unique_ptr< const
CustomExpression
getCustomExpressionFromStorage (int32_t custom_expression_id)
 
std::vector< const
CustomExpression * > 
getCustomExpressionsForUser (const UserMetadata &user) const
 
void updateCustomExpression (int32_t custom_expression_id, const std::string &expression_json)
 
void deleteCustomExpressions (const std::vector< int32_t > &custom_expression_ids, bool do_soft_delete)
 
void reassignOwners (const std::set< std::string > &old_owners, const std::string &new_owner)
 
bool isInfoSchemaDb () const
 

Static Public Member Functions

static void expandGeoColumn (const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
 
static const std::string getForeignTableSchema (bool if_not_exists=false)
 
static const std::string getForeignServerSchema (bool if_not_exists=false)
 
static const std::string getCustomExpressionsSchema (bool if_not_exists=false)
 

Public Attributes

std::unique_ptr
< heavyai::DistributedSharedMutex
dcatalogMutex_
 
std::unique_ptr
< heavyai::DistributedSharedMutex
dsqliteMutex_
 
std::mutex sqliteMutex_
 
heavyai::shared_mutex sharedMutex_
 
std::atomic< std::thread::id > thread_holding_sqlite_lock
 
std::atomic< std::thread::id > thread_holding_write_lock
 
bool initialized_ = false
 

Static Public Attributes

static thread_local bool thread_holds_read_lock = false
 

Protected Types

using ColumnDescriptorsForRoll = std::vector< std::pair< ColumnDescriptor *, ColumnDescriptor * >>
 

Protected Member Functions

void CheckAndExecuteMigrations ()
 
void CheckAndExecuteMigrationsPostBuildMaps ()
 
void updateDictionaryNames ()
 
void updateTableDescriptorSchema ()
 
void updateFixlenArrayColumns ()
 
void updateGeoColumns ()
 
void updateFrontendViewSchema ()
 
void updateLinkSchema ()
 
void updateFrontendViewAndLinkUsers ()
 
void updateLogicalToPhysicalTableLinkSchema ()
 
void updateLogicalToPhysicalTableMap (const int32_t logical_tb_id)
 
void updateDictionarySchema ()
 
void updatePageSize ()
 
void updateDeletedColumnIndicator ()
 
void updateDefaultColumnValues ()
 
void updateFrontendViewsToDashboards ()
 
void updateCustomExpressionsSchema ()
 
void updateFsiSchemas ()
 
void renameLegacyDataWrappers ()
 
void recordOwnershipOfObjectsInObjectPermissions ()
 
void checkDateInDaysColumnMigration ()
 
void createDashboardSystemRoles ()
 
void buildMaps ()
 
void addTableToMap (const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
 
void addReferenceToForeignDict (ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def, const bool persist_reference)
 
bool setColumnSharedDictionary (ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
void setColumnDictionary (ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, bool is_logical_table, bool use_temp_dictionary=false)
 
void addFrontendViewToMap (DashboardDescriptor &vd)
 
void addFrontendViewToMapNoLock (DashboardDescriptor &vd)
 
void addLinkToMap (LinkDescriptor &ld)
 
void removeTableFromMap (const std::string &tableName, const int tableId, const bool is_on_error=false)
 
void eraseTableMetadata (const TableDescriptor *td)
 
void executeDropTableSqliteQueries (const TableDescriptor *td)
 
void doTruncateTable (const TableDescriptor *td)
 
void renamePhysicalTable (const TableDescriptor *td, const std::string &newTableName)
 
void renamePhysicalTable (std::vector< std::pair< std::string, std::string >> &names, std::vector< int > &tableIds)
 
void instantiateFragmenter (TableDescriptor *td) const
 
void getAllColumnMetadataForTableImpl (const TableDescriptor *td, std::list< const ColumnDescriptor * > &colDescs, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 
std::string calculateSHA1 (const std::string &data)
 
std::string generatePhysicalTableName (const std::string &logicalTableName, const int32_t &shardNumber)
 
std::vector< DBObjectparseDashboardObjects (const std::string &view_meta, const int &user_id)
 
void createOrUpdateDashboardSystemRole (const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
 
const int getColumnIdBySpiUnlocked (const int table_id, const size_t spi) const
 
void serializeTableJsonUnlocked (const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
 
void dropTableFromJsonUnlocked (const std::string &table_name) const
 

Protected Attributes

std::string basePath_
 
TableDescriptorMap tableDescriptorMap_
 
TableDescriptorMapById tableDescriptorMapById_
 
ColumnDescriptorMap columnDescriptorMap_
 
ColumnDescriptorMapById columnDescriptorMapById_
 
DictDescriptorMapById dictDescriptorMapByRef_
 
DashboardDescriptorMap dashboardDescriptorMap_
 
LinkDescriptorMap linkDescriptorMap_
 
LinkDescriptorMapById linkDescriptorMapById_
 
ForeignServerMap foreignServerMap_
 
ForeignServerMapById foreignServerMapById_
 
CustomExpressionMapById custom_expr_map_by_id_
 
TableDictColumnsMap dict_columns_by_table_id_
 
SqliteConnector sqliteConnector_
 
const DBMetadata currentDB_
 
std::shared_ptr
< Data_Namespace::DataMgr
dataMgr_
 
const std::vector< LeafHostInfostring_dict_hosts_
 
std::shared_ptr< CalcitecalciteMgr_
 
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
 
int nextTempTableId_
 
int nextTempDictId_
 
ColumnDescriptorsForRoll columnDescriptorsForRoll
 

Static Protected Attributes

static const std::string physicalTableNameTag_
 

Private Member Functions

void buildDictionaryMapUnlocked ()
 
void reloadTableMetadata (int table_id)
 
void reloadTableMetadataUnlocked (int table_id)
 
void reloadCatalogMetadata (const std::map< int32_t, std::string > &user_name_by_user_id)
 
void reloadCatalogMetadataUnlocked (const std::map< int32_t, std::string > &user_name_by_user_id)
 
void buildTablesMapUnlocked ()
 
void buildColumnsMapUnlocked ()
 
void updateViewsInMapUnlocked ()
 
void buildDashboardsMapUnlocked (const std::map< int32_t, std::string > &user_name_by_user_id)
 
void buildLinksMapUnlocked ()
 
void buildLogicalToPhysicalMapUnlocked ()
 
void updateForeignTablesInMapUnlocked ()
 
void gatherAdditionalInfo (std::vector< std::string > &additional_info, std::set< std::string > &shared_dict_column_names, const TableDescriptor *td) const
 
std::string quoteIfRequired (const std::string &column_name) const
 
void adjustAlteredTableFiles (const std::string &temp_data_dir, const std::unordered_map< int, int > &all_column_ids_map) const
 
void renameTableDirectories (const std::string &temp_data_dir, const std::vector< std::string > &target_paths, const std::string &name_prefix) const
 
void buildForeignServerMapUnlocked ()
 
void setForeignServerProperty (const std::string &server_name, const std::string &property, const std::string &value)
 
void setForeignTableProperty (const foreign_storage::ForeignTable *table, const std::string &property, const std::string &value)
 
void alterPhysicalTableMetadata (const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
 
void alterTableMetadata (const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
 
void setTableFileMgrParams (const int table_id, const File_Namespace::FileMgrParams &file_mgr_params)
 
bool filterTableByTypeAndUser (const TableDescriptor *td, const UserMetadata &user_metadata, const GetTablesType get_tables_type) const
 
TableDescriptorgetMutableMetadataForTableUnlocked (int table_id) const
 
TableDescriptorgetMutableMetadataForTableUnlocked (const std::string &table_name) const
 
void createForeignServerNoLocks (std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
 
foreign_storage::ForeignTablegetForeignTableUnlocked (const std::string &tableName) const
 
void removeChunks (const int table_id) const
 
void buildCustomExpressionsMapUnlocked ()
 
std::unique_ptr< CustomExpressiongetCustomExpressionFromConnector (size_t row)
 
void restoreOldOwners (const std::map< int32_t, std::string > &old_owners_user_name_by_id, const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects, int32_t new_owner_id)
 
void restoreOldOwnersInMemory (const std::map< int32_t, std::string > &old_owners_user_name_by_id, const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects, int32_t new_owner_id)
 
void conditionallyInitializeSystemObjects ()
 
void initializeSystemServers ()
 
void initializeSystemTables ()
 
void initializeUsersSystemTable ()
 
void initializeDatabasesSystemTable ()
 
void initializePermissionsSystemTable ()
 
void initializeRolesSystemTable ()
 
void initializeTablesSystemTable ()
 
void initializeDashboardsSystemTable ()
 
void initializeRoleAssignmentsSystemTable ()
 
void initializeMemorySummarySystemTable ()
 
void initializeMemoryDetailsSystemTable ()
 
void initializeStorageDetailsSystemTable ()
 
void initializeServerLogsSystemTables ()
 
void initializeRequestLogsSystemTables ()
 
void initializeWebServerLogsSystemTables ()
 
void initializeWebServerAccessLogsSystemTables ()
 
void createSystemTableServer (const std::string &server_name, const std::string &data_wrapper_type, const foreign_storage::OptionsMap &options={})
 
std::pair
< foreign_storage::ForeignTable,
std::list< ColumnDescriptor > > 
getSystemTableSchema (const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name, bool is_in_memory_system_table)
 
bool recreateSystemTableIfUpdated (foreign_storage::ForeignTable &foreign_table, const std::list< ColumnDescriptor > &columns)
 
void setDeletedColumnUnlocked (const TableDescriptor *td, const ColumnDescriptor *cd)
 
void addToColumnMap (ColumnDescriptor *cd)
 
void removeFromColumnMap (ColumnDescriptor *cd)
 
void deleteTableCatalogMetadata (const TableDescriptor *logical_table, const std::vector< const TableDescriptor * > &physical_tables)
 
std::string dumpCreateTableUnlocked (const TableDescriptor *td, bool multiline_formatting, bool dump_defaults) const
 

Private Attributes

DeletedColumnPerTableMap deletedColumnPerTable_
 

Static Private Attributes

static constexpr const char * CATALOG_SERVER_NAME {"system_catalog_server"}
 
static constexpr const char * MEMORY_STATS_SERVER_NAME {"system_memory_stats_server"}
 
static constexpr const char * STORAGE_STATS_SERVER_NAME {"system_storage_stats_server"}
 
static constexpr const char * LOGS_SERVER_NAME {"system_logs_server"}
 
static constexpr std::array
< const char *, 4 > 
INTERNAL_SERVERS
 

Friends

template<class T >
class lockmgr::TableLockMgrImpl
 

Detailed Description

class for a per-database catalog. also includes metadata for the current database and the current user.

Catalog

Definition at line 132 of file Catalog.h.

Member Typedef Documentation

Definition at line 704 of file Catalog.h.

Constructor & Destructor Documentation

Catalog_Namespace::Catalog::Catalog ( const std::string &  basePath,
const DBMetadata curDB,
std::shared_ptr< Data_Namespace::DataMgr dataMgr,
const std::vector< LeafHostInfo > &  string_dict_hosts,
std::shared_ptr< Calcite calcite,
bool  is_new_db 
)

Constructor - takes basePath to already extant data directory for writing.

Parameters
basePathdirectory path for writing catalog
dbNamename of the database
fragmenterFragmenter object metadata - expects for this directory to already exist

Definition at line 173 of file Catalog.cpp.

References dsqliteMutex_, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_user_id_to_user_name_map(), initialized_, and reloadCatalogMetadataUnlocked().

179  : basePath_(basePath)
180  , sqliteConnector_(curDB.dbName, basePath + "/" + shared::kCatalogDirectoryName + "/")
181  , currentDB_(curDB)
182  , dataMgr_(dataMgr)
183  , string_dict_hosts_(string_dict_hosts)
184  , calciteMgr_(calcite)
187  , dcatalogMutex_(std::make_unique<heavyai::DistributedSharedMutex>(
188  std::filesystem::path(basePath_) / shared::kLockfilesDirectoryName /
190  [this](size_t) {
191  if (!initialized_) {
192  return;
193  }
194  const auto user_name_by_user_id = get_user_id_to_user_name_map();
196  *dsqliteMutex_);
197  reloadCatalogMetadataUnlocked(user_name_by_user_id);
198  }))
const int MAPD_TEMP_TABLE_START_ID
Definition: Catalog.cpp:113
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:690
const DBMetadata currentDB_
Definition: Catalog.h:687
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:688
const int MAPD_TEMP_DICT_START_ID
Definition: Catalog.cpp:115
void reloadCatalogMetadataUnlocked(const std::map< int32_t, std::string > &user_name_by_user_id)
Definition: Catalog.cpp:1269
std::unique_lock< T > unique_lock
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:691
const std::string kCatalogDirectoryName
std::unique_ptr< heavyai::DistributedSharedMutex > dsqliteMutex_
Definition: Catalog.h:840
const std::string kLockfilesDirectoryName
std::unique_ptr< heavyai::DistributedSharedMutex > dcatalogMutex_
Definition: Catalog.h:839
std::map< int32_t, std::string > get_user_id_to_user_name_map()
Definition: Catalog.cpp:1027

+ Here is the call graph for this function:

Catalog_Namespace::Catalog::~Catalog ( )

Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite.

Definition at line 231 of file Catalog.cpp.

References basePath_, columnDescriptorMap_, currentDB_, Catalog_Namespace::DBMetadata::dbName, g_serialize_temp_tables, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and tableDescriptorMap_.

231  {
232  // cat_write_lock write_lock(this);
233 
234  // must clean up heap-allocated TableDescriptor and ColumnDescriptor structs
235  for (TableDescriptorMap::iterator tableDescIt = tableDescriptorMap_.begin();
236  tableDescIt != tableDescriptorMap_.end();
237  ++tableDescIt) {
238  tableDescIt->second->fragmenter = nullptr;
239  delete tableDescIt->second;
240  }
241 
242  // TableDescriptorMapById points to the same descriptors. No need to delete
243 
244  for (ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.begin();
245  columnDescIt != columnDescriptorMap_.end();
246  ++columnDescIt) {
247  delete columnDescIt->second;
248  }
249 
250  // ColumnDescriptorMapById points to the same descriptors. No need to delete
251 
253  boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
254  }
255 }
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:675
const DBMetadata currentDB_
Definition: Catalog.h:687
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:164
bool g_serialize_temp_tables
Definition: Catalog.cpp:108
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:673

+ Here is the call graph for this function:

Member Function Documentation

void Catalog_Namespace::Catalog::addColumn ( const TableDescriptor td,
ColumnDescriptor cd 
)

Definition at line 2379 of file Catalog.cpp.

References ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::default_value, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, kENCODING_DICT, TableDescriptor::nShards, TableDescriptor::shard, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), and ColumnDescriptor::virtualExpr.

2379  {
2380  // caller must handle sqlite/chunk transaction TOGETHER
2381  cd.tableId = td.tableId;
2382  if (td.nShards > 0 && td.shard < 0) {
2383  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
2384  auto shard_cd = cd;
2385  addColumn(*shard, shard_cd);
2386  }
2387  }
2389  addDictionary(cd);
2390  }
2391 
2392  using BindType = SqliteConnector::BindType;
2393  std::vector<BindType> types(17, BindType::TEXT);
2394  if (!cd.default_value.has_value()) {
2395  types[16] = BindType::NULL_TYPE;
2396  }
2398  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
2399  "colscale, is_notnull, "
2400  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
2401  "is_deletedcol, default_value) "
2402  "VALUES (?, "
2403  "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
2404  "?, ?, ?, "
2405  "?, "
2406  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2407  std::vector<std::string>{std::to_string(td.tableId),
2408  std::to_string(td.tableId),
2409  cd.columnName,
2418  "",
2421  cd.virtualExpr,
2423  cd.default_value.value_or("NULL")},
2424  types);
2425 
2427  "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
2428  std::vector<std::string>{std::to_string(td.tableId)});
2429 
2431  "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
2432  std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
2433  cd.columnId = sqliteConnector_.getData<int>(0, 0);
2434 
2435  ++tableDescriptorMapById_[td.tableId]->nColumns;
2436  auto ncd = new ColumnDescriptor(cd);
2437  addToColumnMap(ncd);
2438  columnDescriptorsForRoll.emplace_back(nullptr, ncd);
2439 }
std::string virtualExpr
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:330
HOST DEVICE int get_size() const
Definition: sqltypes.h:339
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
void addColumn(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:2379
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
HOST DEVICE int get_scale() const
Definition: sqltypes.h:334
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:329
std::string to_string(char const *&&v)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:674
specifies the content in-memory of a row in the column metadata table
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4601
std::optional< std::string > default_value
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:337
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:331
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:338
SQLTypeInfo columnType
DictRef addDictionary(ColumnDescriptor &cd)
Definition: Catalog.cpp:2283
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:336
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:705
std::string columnName
void addToColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6468

+ Here is the call graph for this function:

DictRef Catalog_Namespace::Catalog::addDictionary ( ColumnDescriptor cd)

Definition at line 2283 of file Catalog.cpp.

References CHECK, DictDescriptor::dictFolderPath, and ColumnDescriptor::tableId.

2283  {
2284  cat_write_lock write_lock(this);
2285  const auto& td = *tableDescriptorMapById_[cd.tableId];
2286  list<DictDescriptor> dds;
2287  setColumnDictionary(cd, dds, td, true);
2288  auto& dd = dds.back();
2289  CHECK(dd.dictRef.dictId);
2290 
2291  std::unique_ptr<StringDictionaryClient> client;
2292  if (!string_dict_hosts_.empty()) {
2293  client.reset(new StringDictionaryClient(
2294  string_dict_hosts_.front(), DictRef(currentDB_.dbId, -1), true));
2295  }
2296  if (client) {
2297  client->create(dd.dictRef, dd.dictIsTemp);
2298  }
2299 
2300  DictDescriptor* new_dd = new DictDescriptor(dd);
2301  dictDescriptorMapByRef_[dd.dictRef].reset(new_dd);
2302  if (!dd.dictIsTemp) {
2303  boost::filesystem::create_directory(new_dd->dictFolderPath);
2304  }
2305  return dd.dictRef;
2306 }
std::string dictFolderPath
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:690
const DBMetadata currentDB_
Definition: Catalog.h:687
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:677
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, bool is_logical_table, bool use_temp_dictionary=false)
Definition: Catalog.cpp:3791
heavyai::unique_lock< heavyai::shared_mutex > write_lock
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:674
struct dict_ref_t DictRef
Definition: DictRef.h:49
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:124
#define CHECK(condition)
Definition: Logger.h:222
Descriptor for a dictionary for a string columne.
void Catalog_Namespace::Catalog::addFrontendViewToMap ( DashboardDescriptor vd)
protected

Definition at line 1739 of file Catalog.cpp.

1739  {
1740  cat_write_lock write_lock(this);
1742 }
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1744
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:124
void Catalog_Namespace::Catalog::addFrontendViewToMapNoLock ( DashboardDescriptor vd)
protected

Definition at line 1744 of file Catalog.cpp.

References DashboardDescriptor::dashboardName, to_string(), and DashboardDescriptor::userId.

1744  {
1745  cat_write_lock write_lock(this);
1747  std::make_shared<DashboardDescriptor>(vd);
1748 }
heavyai::unique_lock< heavyai::shared_mutex > write_lock
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:678
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:124

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addLinkToMap ( LinkDescriptor ld)
protected

Definition at line 1826 of file Catalog.cpp.

References LinkDescriptor::link, LinkDescriptor::linkId, and to_string().

1826  {
1827  cat_write_lock write_lock(this);
1828  LinkDescriptor* new_ld = new LinkDescriptor();
1829  *new_ld = ld;
1831  linkDescriptorMapById_[ld.linkId] = new_ld;
1832 }
const DBMetadata currentDB_
Definition: Catalog.h:687
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:124
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:680
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:679

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addReferenceToForeignDict ( ColumnDescriptor referencing_column,
Parser::SharedDictionaryDef  shared_dict_def,
const bool  persist_reference 
)
protected

Definition at line 3700 of file Catalog.cpp.

References CHECK, CHECK_GE, ColumnDescriptor::columnType, SQLTypeInfo::get_comp_param(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_foreign_col(), and to_string().

3702  {
3703  cat_write_lock write_lock(this);
3704  const auto foreign_ref_col = get_foreign_col(*this, shared_dict_def);
3705  CHECK(foreign_ref_col);
3706  referencing_column.columnType = foreign_ref_col->columnType;
3707  const int dict_id = referencing_column.columnType.get_comp_param();
3708  const DictRef dict_ref(currentDB_.dbId, dict_id);
3709  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
3710  CHECK(dictIt != dictDescriptorMapByRef_.end());
3711  const auto& dd = dictIt->second;
3712  CHECK_GE(dd->refcount, 1);
3713  ++dd->refcount;
3714  if (persist_reference) {
3715  cat_sqlite_lock sqlite_lock(this);
3717  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
3718  {std::to_string(dict_id)});
3719  }
3720 }
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
const DBMetadata currentDB_
Definition: Catalog.h:687
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:677
#define CHECK_GE(x, y)
Definition: Logger.h:235
heavyai::unique_lock< heavyai::shared_mutex > write_lock
const ColumnDescriptor * get_foreign_col(const Catalog &cat, const Parser::SharedDictionaryDef &shared_dict_def)
Definition: Catalog.cpp:3688
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:124
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:338
#define CHECK(condition)
Definition: Logger.h:222
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:125

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addTableToMap ( const TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::list< DictDescriptor > &  dicts 
)
protected

Definition at line 1598 of file Catalog.cpp.

References anonymous_namespace{Utm.h}::a, CHECK, TableDescriptor::columnIdBySpi_, DictDescriptor::dictFolderPath, dict_ref_t::dictId, TableDescriptor::hasDeletedCol, ColumnDescriptor::isDeletedCol, TableDescriptor::mutex_, gpu_enabled::sort(), TableDescriptor::tableId, TableDescriptor::tableName, and to_upper().

1600  {
1601  cat_write_lock write_lock(this);
1602  TableDescriptor* new_td;
1603 
1604  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
1605  if (foreign_table) {
1606  auto new_foreign_table = new foreign_storage::ForeignTable();
1607  *new_foreign_table = *foreign_table;
1608  new_td = new_foreign_table;
1609  } else {
1610  new_td = new TableDescriptor();
1611  *new_td = *td;
1612  }
1613 
1614  new_td->mutex_ = std::make_shared<std::mutex>();
1615  tableDescriptorMap_[to_upper(td->tableName)] = new_td;
1616  tableDescriptorMapById_[td->tableId] = new_td;
1617  for (auto cd : columns) {
1618  ColumnDescriptor* new_cd = new ColumnDescriptor();
1619  *new_cd = cd;
1620  addToColumnMap(new_cd);
1621 
1622  // Add deleted column to the map
1623  if (cd.isDeletedCol) {
1624  CHECK(new_td->hasDeletedCol);
1625  setDeletedColumnUnlocked(new_td, new_cd);
1626  }
1627  }
1628 
1629  std::sort(new_td->columnIdBySpi_.begin(),
1630  new_td->columnIdBySpi_.end(),
1631  [](const size_t a, const size_t b) -> bool { return a < b; });
1632  // TODO(sy): Why does addTableToMap() sort columnIdBySpi_ but not insert into it while
1633  // buildColumnsMapUnlocked() does both?
1634 
1635  std::unique_ptr<StringDictionaryClient> client;
1636  DictRef dict_ref(currentDB_.dbId, -1);
1637  if (!string_dict_hosts_.empty()) {
1638  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1639  }
1640  for (auto dd : dicts) {
1641  if (!dd.dictRef.dictId) {
1642  // Dummy entry created for a shard of a logical table, nothing to do.
1643  continue;
1644  }
1645  dict_ref.dictId = dd.dictRef.dictId;
1646  if (client) {
1647  client->create(dict_ref, dd.dictIsTemp);
1648  }
1649  DictDescriptor* new_dd = new DictDescriptor(dd);
1650  dictDescriptorMapByRef_[dict_ref].reset(new_dd);
1651  if (!dd.dictIsTemp) {
1652  boost::filesystem::create_directory(new_dd->dictFolderPath);
1653  }
1654  }
1655 }
std::string dictFolderPath
std::string tableName
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:690
const DBMetadata currentDB_
Definition: Catalog.h:687
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:677
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::vector< int > columnIdBySpi_
std::shared_ptr< std::mutex > mutex_
constexpr double a
Definition: Utm.h:32
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3680
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:674
specifies the content in-memory of a row in the column metadata table
std::string to_upper(const std::string &str)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:124
#define CHECK(condition)
Definition: Logger.h:222
Descriptor for a dictionary for a string columne.
void addToColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6468
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:673

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addToColumnMap ( ColumnDescriptor cd)
private

Definition at line 6468 of file Catalog.cpp.

References ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::is_dict_encoded_type(), ColumnDescriptor::tableId, and to_upper().

6468  {
6471  if (cd->columnType.is_dict_encoded_type()) {
6472  dict_columns_by_table_id_[cd->tableId].emplace(cd);
6473  }
6474 }
TableDictColumnsMap dict_columns_by_table_id_
Definition: Catalog.h:684
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:675
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:676
bool is_dict_encoded_type() const
Definition: sqltypes.h:560
std::string to_upper(const std::string &str)
SQLTypeInfo columnType
std::string columnName
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::adjustAlteredTableFiles ( const std::string &  temp_data_dir,
const std::unordered_map< int, int > &  all_column_ids_map 
) const
private
void Catalog_Namespace::Catalog::alterPhysicalTableMetadata ( const TableDescriptor td,
const TableDescriptorUpdateParams table_update_params 
)
private

Definition at line 3414 of file Catalog.cpp.

References CHECK, TableDescriptorUpdateParams::max_rollback_epochs, TableDescriptorUpdateParams::max_rows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::tableId, and to_string().

3416  {
3417  // Only called from parent alterTableParamMetadata, expect already to have catalog and
3418  // sqlite write locks
3419 
3420  // Sqlite transaction should have already been begun in parent alterTableCatalogMetadata
3421 
3423  CHECK(mutable_td);
3424  if (td->maxRollbackEpochs != table_update_params.max_rollback_epochs) {
3426  "UPDATE mapd_tables SET max_rollback_epochs = ? WHERE tableid = ?",
3427  std::vector<std::string>{std::to_string(table_update_params.max_rollback_epochs),
3428  std::to_string(td->tableId)});
3429  mutable_td->maxRollbackEpochs = table_update_params.max_rollback_epochs;
3430  }
3431 
3432  if (td->maxRows != table_update_params.max_rows) {
3434  "UPDATE mapd_tables SET max_rows = ? WHERE tableid = ?",
3435  std::vector<std::string>{std::to_string(table_update_params.max_rows),
3436  std::to_string(td->tableId)});
3437  mutable_td->maxRows = table_update_params.max_rows;
3438  }
3439 }
int32_t maxRollbackEpochs
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
TableDescriptor * getMutableMetadataForTableUnlocked(int table_id) const
Definition: Catalog.cpp:1952
std::string to_string(char const *&&v)
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::alterTableMetadata ( const TableDescriptor td,
const TableDescriptorUpdateParams table_update_params 
)
private

Definition at line 3441 of file Catalog.cpp.

References CHECK, logger::FATAL, LOG, TableDescriptor::tableId, and TableDescriptor::tableName.

3442  {
3443  cat_write_lock write_lock(this);
3444  cat_sqlite_lock sqlite_lock(this);
3445  sqliteConnector_.query("BEGIN TRANSACTION");
3446  try {
3447  const auto physical_table_it = logicalToPhysicalTableMapById_.find(td->tableId);
3448  if (physical_table_it != logicalToPhysicalTableMapById_.end()) {
3449  const auto physical_tables = physical_table_it->second;
3450  CHECK(!physical_tables.empty());
3451  for (size_t i = 0; i < physical_tables.size(); i++) {
3452  int32_t physical_tb_id = physical_tables[i];
3453  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id, false);
3454  CHECK(phys_td);
3455  alterPhysicalTableMetadata(phys_td, table_update_params);
3456  }
3457  }
3458  alterPhysicalTableMetadata(td, table_update_params);
3459  } catch (std::exception& e) {
3460  sqliteConnector_.query("ROLLBACK TRANSACTION");
3461  LOG(FATAL) << "Table '" << td->tableName << "' catalog update failed";
3462  }
3463  sqliteConnector_.query("END TRANSACTION");
3464 }
std::string tableName
#define LOG(tag)
Definition: Logger.h:216
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void alterPhysicalTableMetadata(const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
Definition: Catalog.cpp:3414
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:124
#define CHECK(condition)
Definition: Logger.h:222
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:125
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:693
void Catalog_Namespace::Catalog::buildColumnsMapUnlocked ( )
private

Definition at line 1401 of file Catalog.cpp.

References anonymous_namespace{Utm.h}::a, CHECK, ColumnDescriptor::chunks, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::default_value, SQLTypeInfo::get_physical_cols(), SQLTypeInfo::is_geometry(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_compression(), SQLTypeInfo::set_dimension(), SQLTypeInfo::set_notnull(), SQLTypeInfo::set_scale(), SQLTypeInfo::set_size(), SQLTypeInfo::set_subtype(), SQLTypeInfo::set_type(), gpu_enabled::sort(), ColumnDescriptor::tableId, and ColumnDescriptor::virtualExpr.

1401  {
1402  std::string columnQuery(
1403  "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
1404  "is_notnull, compression, comp_param, "
1405  "size, chunks, is_systemcol, is_virtualcol, virtual_expr, is_deletedcol, "
1406  "default_value from "
1407  "mapd_columns ORDER BY tableid, "
1408  "columnid");
1409  sqliteConnector_.query(columnQuery);
1410  auto numRows = sqliteConnector_.getNumRows();
1411  int32_t skip_physical_cols = 0;
1412  for (size_t r = 0; r < numRows; ++r) {
1413  ColumnDescriptor* cd = new ColumnDescriptor();
1414  cd->tableId = sqliteConnector_.getData<int>(r, 0);
1415  cd->columnId = sqliteConnector_.getData<int>(r, 1);
1416  cd->columnName = sqliteConnector_.getData<string>(r, 2);
1420  cd->columnType.set_scale(sqliteConnector_.getData<int>(r, 6));
1424  cd->columnType.set_size(sqliteConnector_.getData<int>(r, 10));
1425  cd->chunks = sqliteConnector_.getData<string>(r, 11);
1426  cd->isSystemCol = sqliteConnector_.getData<bool>(r, 12);
1427  cd->isVirtualCol = sqliteConnector_.getData<bool>(r, 13);
1428  cd->virtualExpr = sqliteConnector_.getData<string>(r, 14);
1429  cd->isDeletedCol = sqliteConnector_.getData<bool>(r, 15);
1430  if (sqliteConnector_.isNull(r, 16)) {
1431  cd->default_value = std::nullopt;
1432  } else {
1433  cd->default_value = std::make_optional(sqliteConnector_.getData<string>(r, 16));
1434  }
1435  cd->isGeoPhyCol = skip_physical_cols > 0;
1436  addToColumnMap(cd);
1437 
1438  if (skip_physical_cols <= 0) {
1439  skip_physical_cols = cd->columnType.get_physical_cols();
1440  }
1441 
1442  auto td_itr = tableDescriptorMapById_.find(cd->tableId);
1443  CHECK(td_itr != tableDescriptorMapById_.end());
1444 
1445  if (cd->isDeletedCol) {
1446  td_itr->second->hasDeletedCol = true;
1447  setDeletedColumnUnlocked(td_itr->second, cd);
1448  } else if (cd->columnType.is_geometry() || skip_physical_cols-- <= 0) {
1449  tableDescriptorMapById_[cd->tableId]->columnIdBySpi_.push_back(cd->columnId);
1450  }
1451  }
1452 
1453  // sort columnIdBySpi_ based on columnId
1454  for (auto& tit : tableDescriptorMapById_) {
1455  std::sort(tit.second->columnIdBySpi_.begin(),
1456  tit.second->columnIdBySpi_.end(),
1457  [](const size_t a, const size_t b) -> bool { return a < b; });
1458  }
1459 } // namespace Catalog_Namespace
std::string virtualExpr
void set_compression(EncodingType c)
Definition: sqltypes.h:440
void set_size(int s)
Definition: sqltypes.h:437
T getData(const int row, const int col)
SQLTypes
Definition: sqltypes.h:38
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:430
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
virtual void query(const std::string &queryString)
std::string chunks
constexpr double a
Definition: Utm.h:32
EncodingType
Definition: sqltypes.h:233
int get_physical_cols() const
Definition: sqltypes.h:360
void set_scale(int s)
Definition: sqltypes.h:434
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3680
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:674
specifies the content in-memory of a row in the column metadata table
void set_comp_param(int p)
Definition: sqltypes.h:441
std::optional< std::string > default_value
bool isNull(const int row, const int col) const
void set_dimension(int d)
Definition: sqltypes.h:431
void set_notnull(bool n)
Definition: sqltypes.h:436
#define CHECK(condition)
Definition: Logger.h:222
bool is_geometry() const
Definition: sqltypes.h:522
SQLTypeInfo columnType
std::string columnName
void addToColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6468
virtual size_t getNumRows() const
HOST DEVICE void set_type(SQLTypes t)
Definition: sqltypes.h:429

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildCustomExpressionsMapUnlocked ( )
private

Definition at line 1570 of file Catalog.cpp.

1570  {
1572  "SELECT id, name, expression_json, data_source_type, data_source_id, "
1573  "is_deleted "
1574  "FROM omnisci_custom_expressions");
1575  auto num_rows = sqliteConnector_.getNumRows();
1576  for (size_t row = 0; row < num_rows; row++) {
1577  auto custom_expr = getCustomExpressionFromConnector(row);
1578  custom_expr_map_by_id_[custom_expr->id] = std::move(custom_expr);
1579  }
1580 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
virtual void query(const std::string &queryString)
std::unique_ptr< CustomExpression > getCustomExpressionFromConnector(size_t row)
Definition: Catalog.cpp:1582
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:683
virtual size_t getNumRows() const
void Catalog_Namespace::Catalog::buildDashboardsMapUnlocked ( const std::map< int32_t, std::string > &  user_name_by_user_id)
private

Definition at line 1473 of file Catalog.cpp.

References generate_dashboard_system_rolename(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_user_name_from_id(), and to_string().

1474  {
1475  std::string frontendViewQuery(
1476  "SELECT id, state, name, image_hash, strftime('%Y-%m-%dT%H:%M:%SZ', update_time), "
1477  "userid, "
1478  "metadata "
1479  "FROM mapd_dashboards");
1480  sqliteConnector_.query(frontendViewQuery);
1481  auto numRows = sqliteConnector_.getNumRows();
1482  for (size_t r = 0; r < numRows; ++r) {
1483  auto vd = std::make_shared<DashboardDescriptor>();
1484  vd->dashboardId = sqliteConnector_.getData<int>(r, 0);
1485  vd->dashboardState = sqliteConnector_.getData<string>(r, 1);
1486  vd->dashboardName = sqliteConnector_.getData<string>(r, 2);
1487  vd->imageHash = sqliteConnector_.getData<string>(r, 3);
1488  vd->updateTime = sqliteConnector_.getData<string>(r, 4);
1489  vd->userId = sqliteConnector_.getData<int>(r, 5);
1490  vd->dashboardMetadata = sqliteConnector_.getData<string>(r, 6);
1491  vd->user = get_user_name_from_id(vd->userId, user_name_by_user_id);
1492  vd->dashboardSystemRoleName = generate_dashboard_system_rolename(
1494  dashboardDescriptorMap_[std::to_string(vd->userId) + ":" + vd->dashboardName] = vd;
1495  }
1496 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
const DBMetadata currentDB_
Definition: Catalog.h:687
virtual void query(const std::string &queryString)
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:678
std::string to_string(char const *&&v)
std::string get_user_name_from_id(int32_t id, const std::map< int32_t, std::string > &user_name_by_user_id)
Definition: Catalog.cpp:1036
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildDictionaryMapUnlocked ( )
private

Definition at line 1048 of file Catalog.cpp.

References currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, g_base_path, SqliteConnector::getData(), SqliteConnector::getNumRows(), shared::kDataDirectoryName, SqliteConnector::query(), sqliteConnector_, and to_string().

1048  {
1049  std::string dictQuery(
1050  "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
1051  sqliteConnector_.query(dictQuery);
1052  auto numRows = sqliteConnector_.getNumRows();
1053  for (size_t r = 0; r < numRows; ++r) {
1054  auto dictId = sqliteConnector_.getData<int>(r, 0);
1055  auto dictName = sqliteConnector_.getData<string>(r, 1);
1056  auto dictNBits = sqliteConnector_.getData<int>(r, 2);
1057  auto is_shared = sqliteConnector_.getData<bool>(r, 3);
1058  auto refcount = sqliteConnector_.getData<int>(r, 4);
1059  auto fname = g_base_path + "/" + shared::kDataDirectoryName + "/DB_" +
1060  std::to_string(currentDB_.dbId) + "_DICT_" + std::to_string(dictId);
1061  DictRef dict_ref(currentDB_.dbId, dictId);
1062  auto dd = new DictDescriptor(
1063  dict_ref, dictName, dictNBits, is_shared, refcount, fname, false);
1064  dictDescriptorMapByRef_[dict_ref].reset(dd);
1065  }
1066 }
const std::string kDataDirectoryName
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
const DBMetadata currentDB_
Definition: Catalog.h:687
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:677
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
std::string g_base_path
Definition: SysCatalog.cpp:62
Descriptor for a dictionary for a string columne.
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildForeignServerMapUnlocked ( )
private

Definition at line 4824 of file Catalog.cpp.

References CHECK, and g_enable_fsi.

4824  {
4827  "SELECT id, name, data_wrapper_type, options, owner_user_id, creation_time FROM "
4828  "omnisci_foreign_servers");
4829  auto num_rows = sqliteConnector_.getNumRows();
4830 
4831  for (size_t row = 0; row < num_rows; row++) {
4832  auto foreign_server = std::make_shared<foreign_storage::ForeignServer>(
4833  sqliteConnector_.getData<int>(row, 0),
4834  sqliteConnector_.getData<std::string>(row, 1),
4835  sqliteConnector_.getData<std::string>(row, 2),
4836  sqliteConnector_.getData<std::string>(row, 3),
4837  sqliteConnector_.getData<std::int32_t>(row, 4),
4838  sqliteConnector_.getData<std::int32_t>(row, 5));
4839  foreignServerMap_[foreign_server->name] = foreign_server;
4840  foreignServerMapById_[foreign_server->id] = foreign_server;
4841  }
4842 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
virtual void query(const std::string &queryString)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:682
ForeignServerMap foreignServerMap_
Definition: Catalog.h:681
#define CHECK(condition)
Definition: Logger.h:222
bool g_enable_fsi
Definition: Catalog.cpp:96
virtual size_t getNumRows() const
void Catalog_Namespace::Catalog::buildLinksMapUnlocked ( )
private

Definition at line 1498 of file Catalog.cpp.

References to_string().

1498  {
1499  std::string linkQuery(
1500  "SELECT linkid, userid, link, view_state, strftime('%Y-%m-%dT%H:%M:%SZ', "
1501  "update_time), view_metadata "
1502  "FROM mapd_links");
1503  sqliteConnector_.query(linkQuery);
1504  auto numRows = sqliteConnector_.getNumRows();
1505  for (size_t r = 0; r < numRows; ++r) {
1506  auto ld = new LinkDescriptor();
1507  ld->linkId = sqliteConnector_.getData<int>(r, 0);
1508  ld->userId = sqliteConnector_.getData<int>(r, 1);
1509  ld->link = sqliteConnector_.getData<string>(r, 2);
1510  ld->viewState = sqliteConnector_.getData<string>(r, 3);
1511  ld->updateTime = sqliteConnector_.getData<string>(r, 4);
1512  ld->viewMetadata = sqliteConnector_.getData<string>(r, 5);
1514  linkDescriptorMapById_[ld->linkId] = ld;
1515  }
1516 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
const DBMetadata currentDB_
Definition: Catalog.h:687
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:680
virtual size_t getNumRows() const
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:679

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildLogicalToPhysicalMapUnlocked ( )
private

Definition at line 1518 of file Catalog.cpp.

References CHECK.

1518  {
1519  /* rebuild map linking logical tables to corresponding physical ones */
1520  std::string logicalToPhysicalTableMapQuery(
1521  "SELECT logical_table_id, physical_table_id "
1522  "FROM mapd_logical_to_physical");
1523  sqliteConnector_.query(logicalToPhysicalTableMapQuery);
1524  auto numRows = sqliteConnector_.getNumRows();
1525  for (size_t r = 0; r < numRows; ++r) {
1526  auto logical_tb_id = sqliteConnector_.getData<int>(r, 0);
1527  auto physical_tb_id = sqliteConnector_.getData<int>(r, 1);
1528  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
1529  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
1530  /* add new entity to the map logicalToPhysicalTableMapById_ */
1531  std::vector<int32_t> physicalTables{physical_tb_id};
1532  const auto it_ok =
1533  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
1534  CHECK(it_ok.second);
1535  } else {
1536  /* update map logicalToPhysicalTableMapById_ */
1537  physicalTableIt->second.push_back(physical_tb_id);
1538  }
1539  }
1540 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
virtual void query(const std::string &queryString)
#define CHECK(condition)
Definition: Logger.h:222
virtual size_t getNumRows() const
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:693
void Catalog_Namespace::Catalog::buildMaps ( )
protected

Definition at line 1545 of file Catalog.cpp.

References g_enable_fsi, and Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_user_id_to_user_name_map().

1545  {
1546  // Get all user id to username mapping here in order to avoid making a call to
1547  // SysCatalog (and attempting to acquire SysCatalog locks) while holding locks for this
1548  // catalog.
1549  const auto user_name_by_user_id = get_user_id_to_user_name_map();
1550 
1551  cat_write_lock write_lock(this);
1552  cat_sqlite_lock sqlite_lock(this);
1553 
1556 
1557  if (g_enable_fsi) {
1560  }
1561 
1564  buildDashboardsMapUnlocked(user_name_by_user_id);
1568 }
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void buildLogicalToPhysicalMapUnlocked()
Definition: Catalog.cpp:1518
void buildDashboardsMapUnlocked(const std::map< int32_t, std::string > &user_name_by_user_id)
Definition: Catalog.cpp:1473
void buildCustomExpressionsMapUnlocked()
Definition: Catalog.cpp:1570
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:124
void updateForeignTablesInMapUnlocked()
Definition: Catalog.cpp:4844
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:125
bool g_enable_fsi
Definition: Catalog.cpp:96
std::map< int32_t, std::string > get_user_id_to_user_name_map()
Definition: Catalog.cpp:1027

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildTablesMapUnlocked ( )
private

Definition at line 1342 of file Catalog.cpp.

References logger::FATAL, StorageType::FOREIGN_TABLE, TableDescriptor::fragmenter, TableDescriptor::fragments, TableDescriptor::fragPageSize, TableDescriptor::fragType, TableDescriptor::hasDeletedCol, TableDescriptor::is_system_table, TableDescriptor::isView, TableDescriptor::keyMetainfo, LOG, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nColumns, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::storageType, TableDescriptor::tableId, TableDescriptor::tableName, to_upper(), and TableDescriptor::userId.

1342  {
1343  std::string tableQuery(
1344  "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
1345  "max_chunk_size, frag_page_size, "
1346  "max_rows, partitions, shard_column_id, shard, num_shards, key_metainfo, userid, "
1347  "sort_column_id, storage_type, max_rollback_epochs, is_system_table "
1348  "from mapd_tables");
1349  sqliteConnector_.query(tableQuery);
1350  auto numRows = sqliteConnector_.getNumRows();
1351  for (size_t r = 0; r < numRows; ++r) {
1352  TableDescriptor* td;
1353  const auto& storage_type = sqliteConnector_.getData<string>(r, 17);
1354  if (!storage_type.empty() && storage_type != StorageType::FOREIGN_TABLE) {
1355  const auto table_id = sqliteConnector_.getData<int>(r, 0);
1356  const auto& table_name = sqliteConnector_.getData<string>(r, 1);
1357  LOG(FATAL) << "Unable to read Catalog metadata: storage type is currently not a "
1358  "supported table option (table "
1359  << table_name << " [" << table_id << "] in database "
1360  << currentDB_.dbName << ").";
1361  }
1362 
1363  if (storage_type == StorageType::FOREIGN_TABLE) {
1364  td = new foreign_storage::ForeignTable();
1365  } else {
1366  td = new TableDescriptor();
1367  }
1368 
1369  td->storageType = storage_type;
1370  td->tableId = sqliteConnector_.getData<int>(r, 0);
1371  td->tableName = sqliteConnector_.getData<string>(r, 1);
1372  td->nColumns = sqliteConnector_.getData<int>(r, 2);
1373  td->isView = sqliteConnector_.getData<bool>(r, 3);
1374  td->fragments = sqliteConnector_.getData<string>(r, 4);
1375  td->fragType =
1377  td->maxFragRows = sqliteConnector_.getData<int>(r, 6);
1378  td->maxChunkSize = sqliteConnector_.getData<int64_t>(r, 7);
1379  td->fragPageSize = sqliteConnector_.getData<int>(r, 8);
1380  td->maxRows = sqliteConnector_.getData<int64_t>(r, 9);
1381  td->partitions = sqliteConnector_.getData<string>(r, 10);
1382  td->shardedColumnId = sqliteConnector_.getData<int>(r, 11);
1383  td->shard = sqliteConnector_.getData<int>(r, 12);
1384  td->nShards = sqliteConnector_.getData<int>(r, 13);
1385  td->keyMetainfo = sqliteConnector_.getData<string>(r, 14);
1386  td->userId = sqliteConnector_.getData<int>(r, 15);
1387  td->sortedColumnId =
1388  sqliteConnector_.isNull(r, 16) ? 0 : sqliteConnector_.getData<int>(r, 16);
1389  if (!td->isView) {
1390  td->fragmenter = nullptr;
1391  }
1392  td->maxRollbackEpochs = sqliteConnector_.getData<int>(r, 18);
1393  td->is_system_table = sqliteConnector_.getData<bool>(r, 19);
1394  td->hasDeletedCol = false;
1395 
1397  tableDescriptorMapById_[td->tableId] = td;
1398  }
1399 }
int32_t maxRollbackEpochs
std::string partitions
T getData(const int row, const int col)
std::string tableName
#define LOG(tag)
Definition: Logger.h:216
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
const DBMetadata currentDB_
Definition: Catalog.h:687
std::string storageType
virtual void query(const std::string &queryString)
std::string fragments
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:674
std::string keyMetainfo
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
std::string to_upper(const std::string &str)
bool isNull(const int row, const int col) const
Fragmenter_Namespace::FragmenterType fragType
static constexpr char const * FOREIGN_TABLE
virtual size_t getNumRows() const
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:673

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::calculateSHA1 ( const std::string &  data)
protected

Definition at line 4533 of file Catalog.cpp.

4533  {
4534  boost::uuids::detail::sha1 sha1;
4535  unsigned int digest[5];
4536  sha1.process_bytes(data.c_str(), data.length());
4537  sha1.get_digest(digest);
4538  std::stringstream ss;
4539  for (size_t i = 0; i < 5; i++) {
4540  ss << std::hex << digest[i];
4541  }
4542  return ss.str();
4543 }
void Catalog_Namespace::Catalog::changeForeignServerOwner ( const std::string &  server_name,
const int  new_owner_id 
)

Change the owner of a Foreign Server to a new owner.

Parameters
server_name- Name of the foreign server whose owner to change
new_owner_id- New owner's user id

Definition at line 3120 of file Catalog.cpp.

References CHECK, to_string(), and foreign_storage::ForeignServer::user_id.

3121  {
3122  cat_write_lock write_lock(this);
3123  foreign_storage::ForeignServer* foreign_server =
3124  foreignServerMap_.find(server_name)->second.get();
3125  CHECK(foreign_server);
3126  setForeignServerProperty(server_name, "owner_user_id", std::to_string(new_owner_id));
3127  // update in-memory server
3128  foreign_server->user_id = new_owner_id;
3129 }
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::string to_string(char const *&&v)
ForeignServerMap foreignServerMap_
Definition: Catalog.h:681
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:124
#define CHECK(condition)
Definition: Logger.h:222
void setForeignServerProperty(const std::string &server_name, const std::string &property, const std::string &value)
Definition: Catalog.cpp:4874

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::CheckAndExecuteMigrations ( )
protected

Definition at line 997 of file Catalog.cpp.

References g_enable_fsi, recordOwnershipOfObjectsInObjectPermissions(), renameLegacyDataWrappers(), updateCustomExpressionsSchema(), updateDefaultColumnValues(), updateDeletedColumnIndicator(), updateDictionaryNames(), updateDictionarySchema(), updateFixlenArrayColumns(), updateFrontendViewAndLinkUsers(), updateFrontendViewSchema(), updateFrontendViewsToDashboards(), updateFsiSchemas(), updateGeoColumns(), updateLinkSchema(), updateLogicalToPhysicalTableLinkSchema(), updatePageSize(), and updateTableDescriptorSchema().

997  {
999  cat_sqlite_lock sqlite_lock(this);
1002  updateGeoColumns();
1005  updateLinkSchema();
1009  updatePageSize();
1013  if (g_enable_fsi) {
1014  updateFsiSchemas();
1016  }
1019 }
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:460
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:787
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:131
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:124
void updateCustomExpressionsSchema()
Definition: Catalog.cpp:753
void updateLogicalToPhysicalTableLinkSchema()
Definition: Catalog.cpp:621
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:125
bool g_enable_fsi
Definition: Catalog.cpp:96

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::CheckAndExecuteMigrationsPostBuildMaps ( )
protected

Definition at line 1021 of file Catalog.cpp.

References checkDateInDaysColumnMigration(), and createDashboardSystemRoles().

1021  {
1024 }
void checkDateInDaysColumnMigration()
Definition: Catalog.cpp:915

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::checkDateInDaysColumnMigration ( )
protected

Definition at line 915 of file Catalog.cpp.

References getCurrentDB(), migrations::MigrationMgr::migrateDateInDaysMetadata(), sqliteConnector_, and tableDescriptorMapById_.

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

915  {
916  cat_sqlite_lock sqlite_lock(this);
919 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:242
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:674
static void migrateDateInDaysMetadata(const Catalog_Namespace::TableDescriptorMapById &table_descriptors_by_id, const int database_id, Catalog_Namespace::Catalog *cat, SqliteConnector &sqlite)
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:125

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const bool Catalog_Namespace::Catalog::checkMetadataForDeletedRecs ( const TableDescriptor td,
int  column_id 
) const

Definition at line 3638 of file Catalog.cpp.

References CHECK, and TableDescriptor::fragmenter.

3639  {
3640  // check if there are rows deleted by examining the deletedColumn metadata
3641  CHECK(td);
3642  auto fragmenter = td->fragmenter;
3643  if (fragmenter) {
3644  return fragmenter->hasDeletedRows(delete_column_id);
3645  } else {
3646  return false;
3647  }
3648 }
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
#define CHECK(condition)
Definition: Logger.h:222
void Catalog_Namespace::Catalog::checkpoint ( const int  logicalTableId) const

Definition at line 4754 of file Catalog.cpp.

Referenced by UpdelRoll::commitUpdate(), TableOptimizer::vacuumDeletedRows(), and TableOptimizer::vacuumFragmentsAboveMinSelectivity().

4754  {
4755  const auto td = getMetadataForTable(logicalTableId);
4756  const auto shards = getPhysicalTablesDescriptors(td);
4757  for (const auto shard : shards) {
4758  getDataMgr().checkpoint(getCurrentDB().dbId, shard->tableId);
4759  }
4760 }
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:243
void checkpoint(const int db_id, const int tb_id)
Definition: DataMgr.cpp:548
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:242
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4601
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::checkpointWithAutoRollback ( const int  logical_table_id) const

Definition at line 4762 of file Catalog.cpp.

Referenced by Fragmenter_Namespace::LocalInsertConnector::checkpoint(), import_export::Loader::checkpoint(), StorageIOFacility::TransactionParameters::finalizeTransaction(), and TableOptimizer::vacuumFragmentsAboveMinSelectivity().

4762  {
4763  auto table_epochs = getTableEpochs(getDatabaseId(), logical_table_id);
4764  try {
4765  checkpoint(logical_table_id);
4766  } catch (...) {
4767  setTableEpochsLogExceptions(getDatabaseId(), table_epochs);
4768  throw;
4769  }
4770 }
int getDatabaseId() const
Definition: Catalog.h:298
void checkpoint(const int logicalTableId) const
Definition: Catalog.cpp:4754
void setTableEpochsLogExceptions(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3620
std::vector< TableEpochInfo > getTableEpochs(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:3556

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::conditionallyInitializeSystemObjects ( )
private

Definition at line 5946 of file Catalog.cpp.

References g_enable_system_tables.

5946  {
5950  }
5951 }
bool g_enable_system_tables
Definition: SysCatalog.cpp:64
bool isInfoSchemaDb() const
Definition: Catalog.cpp:5953
int32_t Catalog_Namespace::Catalog::createCustomExpression ( std::unique_ptr< CustomExpression custom_expression)

Creates a new custom expression.

Parameters
custom_expression- unique pointer to struct containing custom expression details.
Returns
id of created custom expression

Definition at line 5508 of file Catalog.cpp.

References CHECK, CHECK_EQ, CHECK_GT, and to_string().

5509  {
5510  cat_write_lock write_lock(this);
5511  cat_sqlite_lock sqlite_lock(this);
5512  sqliteConnector_.query("BEGIN TRANSACTION");
5513  int32_t custom_expression_id{-1};
5514  try {
5515  auto data_source_type_str =
5516  CustomExpression::dataSourceTypeToString(custom_expression->data_source_type);
5517  auto data_source_id_str = std::to_string(custom_expression->data_source_id);
5518  std::string custom_expr_select_query{
5519  "SELECT id FROM omnisci_custom_expressions WHERE name = ? and data_source_type = "
5520  "? and data_source_id = ? and is_deleted = ?"};
5521  std::vector<std::string> custom_expr_select_params{custom_expression->name,
5522  data_source_type_str,
5523  data_source_id_str,
5524  std::to_string(false)};
5525  sqliteConnector_.query_with_text_params(custom_expr_select_query,
5526  custom_expr_select_params);
5527  if (sqliteConnector_.getNumRows() > 0) {
5528  throw std::runtime_error{
5529  "A custom expression with the given "
5530  "name and data source already exists."};
5531  }
5533  "INSERT INTO omnisci_custom_expressions(name, expression_json, "
5534  "data_source_type, data_source_id, is_deleted) VALUES (?,?,?,?,?)",
5535  std::vector<std::string>{custom_expression->name,
5536  custom_expression->expression_json,
5537  data_source_type_str,
5538  data_source_id_str,
5539  std::to_string(false)});
5540  sqliteConnector_.query_with_text_params(custom_expr_select_query,
5541  custom_expr_select_params);
5542  CHECK_EQ(sqliteConnector_.getNumRows(), static_cast<size_t>(1));
5543  custom_expression->id = sqliteConnector_.getData<int32_t>(0, 0);
5544  custom_expression_id = custom_expression->id;
5545  CHECK(custom_expr_map_by_id_.find(custom_expression->id) ==
5546  custom_expr_map_by_id_.end());
5547  custom_expr_map_by_id_[custom_expression->id] = std::move(custom_expression);
5548  } catch (std::exception& e) {
5549  sqliteConnector_.query("ROLLBACK TRANSACTION");
5550  throw;
5551  }
5552  sqliteConnector_.query("END TRANSACTION");
5553  CHECK_GT(custom_expression_id, 0);
5554  return custom_expression_id;
5555 }
#define CHECK_EQ(x, y)
Definition: Logger.h:230
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
#define CHECK_GT(x, y)
Definition: Logger.h:234
std::string to_string(char const *&&v)
static std::string dataSourceTypeToString(DataSourceType type_enum)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:124
#define CHECK(condition)
Definition: Logger.h:222
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:683
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:125
virtual size_t getNumRows() const

+ Here is the call graph for this function:

int32_t Catalog_Namespace::Catalog::createDashboard ( DashboardDescriptor vd)

Definition at line 4392 of file Catalog.cpp.

References DashboardDescriptor::dashboardId, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, DashboardDescriptor::dashboardSystemRoleName, generate_dashboard_system_rolename(), DashboardDescriptor::imageHash, to_string(), Catalog_Namespace::sqlite_lock< T >::unlock(), Catalog_Namespace::write_lock< T >::unlock(), DashboardDescriptor::updateTime, and DashboardDescriptor::userId.

4392  {
4393  cat_write_lock write_lock(this);
4394  cat_sqlite_lock sqlite_lock(this);
4395  sqliteConnector_.query("BEGIN TRANSACTION");
4396  try {
4397  // TODO(andrew): this should be an upsert
4399  "SELECT id FROM mapd_dashboards WHERE name = ? and userid = ?",
4400  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
4401  if (sqliteConnector_.getNumRows() > 0) {
4403  "UPDATE mapd_dashboards SET state = ?, image_hash = ?, metadata = ?, "
4404  "update_time = "
4405  "datetime('now') where name = ? "
4406  "and userid = ?",
4407  std::vector<std::string>{vd.dashboardState,
4408  vd.imageHash,
4409  vd.dashboardMetadata,
4410  vd.dashboardName,
4411  std::to_string(vd.userId)});
4412  } else {
4414  "INSERT INTO mapd_dashboards (name, state, image_hash, metadata, "
4415  "update_time, "
4416  "userid) "
4417  "VALUES "
4418  "(?,?,?,?, "
4419  "datetime('now'), ?)",
4420  std::vector<std::string>{vd.dashboardName,
4421  vd.dashboardState,
4422  vd.imageHash,
4423  vd.dashboardMetadata,
4424  std::to_string(vd.userId)});
4425  }
4426  } catch (std::exception& e) {
4427  sqliteConnector_.query("ROLLBACK TRANSACTION");
4428  throw;
4429  }
4430  sqliteConnector_.query("END TRANSACTION");
4431 
4432  // now get the auto generated dashboardId
4433  try {
4435  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_dashboards "
4436  "WHERE name = ? and userid = ?",
4437  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
4438  vd.dashboardId = sqliteConnector_.getData<int>(0, 0);
4439  vd.updateTime = sqliteConnector_.getData<std::string>(0, 1);
4440  } catch (std::exception& e) {
4441  throw;
4442  }
4446  sqlite_lock.unlock();
4447  write_lock.unlock();
4448  if (!isInfoSchemaDb()) {
4449  // NOTE(wamsi): Transactionally unsafe
4452  }
4453  return vd.dashboardId;
4454 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
const DBMetadata currentDB_
Definition: Catalog.h:687
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void addFrontendViewToMap(DashboardDescriptor &vd)
Definition: Catalog.cpp:1739
std::string to_string(char const *&&v)
std::string dashboardSystemRoleName
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1780
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:124
bool isInfoSchemaDb() const
Definition: Catalog.cpp:5953
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:125
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createDashboardSystemRoles ( )
protected

Definition at line 921 of file Catalog.cpp.

References createOrUpdateDashboardSystemRole(), currentDB_, Catalog_Namespace::DBMetadata::dbId, logger::ERROR, generate_dashboard_system_rolename(), SqliteConnector::getData(), Catalog_Namespace::SysCatalog::getGranteesOfSharedDashboards(), SqliteConnector::getNumRows(), Catalog_Namespace::SysCatalog::getRoleGrantee(), Catalog_Namespace::SysCatalog::grantRoleBatch(), logger::INFO, Catalog_Namespace::SysCatalog::instance(), LOG, MAPD_VERSION, SqliteConnector::query(), SqliteConnector::query_with_text_params(), run_benchmark_import::result, sqliteConnector_, and to_string().

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

921  {
922  std::unordered_map<std::string, std::pair<int, std::string>> dashboards;
923  std::vector<std::string> dashboard_ids;
924  static const std::string migration_name{"dashboard_roles_migration"};
925  {
926  cat_sqlite_lock sqlite_lock(this);
927  sqliteConnector_.query("BEGIN TRANSACTION");
928  try {
929  // migration_history should be present in all catalogs by now
930  // if not then would be created before this migration
932  "select * from mapd_version_history where migration_history = '" +
933  migration_name + "'");
934  if (sqliteConnector_.getNumRows() != 0) {
935  // no need for further execution
936  sqliteConnector_.query("END TRANSACTION");
937  return;
938  }
939  LOG(INFO) << "Performing dashboard internal roles Migration.";
940  sqliteConnector_.query("select id, userid, metadata from mapd_dashboards");
941  for (size_t i = 0; i < sqliteConnector_.getNumRows(); ++i) {
944  sqliteConnector_.getData<string>(i, 0)))) {
945  // Successfully created roles during previous migration/crash
946  // No need to include them
947  continue;
948  }
949  dashboards[sqliteConnector_.getData<string>(i, 0)] = std::make_pair(
950  sqliteConnector_.getData<int>(i, 1), sqliteConnector_.getData<string>(i, 2));
951  dashboard_ids.push_back(sqliteConnector_.getData<string>(i, 0));
952  }
953  } catch (const std::exception& e) {
954  sqliteConnector_.query("ROLLBACK TRANSACTION");
955  throw;
956  }
957  sqliteConnector_.query("END TRANSACTION");
958  }
959  // All current grantees with shared dashboards.
960  const auto active_grantees =
962 
963  try {
964  // NOTE(wamsi): Transactionally unsafe
965  for (auto dash : dashboards) {
966  createOrUpdateDashboardSystemRole(dash.second.second,
967  dash.second.first,
969  std::to_string(currentDB_.dbId), dash.first));
970  auto result = active_grantees.find(dash.first);
971  if (result != active_grantees.end()) {
974  dash.first)},
975  result->second);
976  }
977  }
978  cat_sqlite_lock sqlite_lock(this);
979  // check if this has already been completed
981  "select * from mapd_version_history where migration_history = '" +
982  migration_name + "'");
983  if (sqliteConnector_.getNumRows() != 0) {
984  return;
985  }
987  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
988  std::vector<std::string>{std::to_string(MAPD_VERSION), migration_name});
989  } catch (const std::exception& e) {
990  LOG(ERROR) << "Failed to create dashboard system roles during migration: "
991  << e.what();
992  throw;
993  }
994  LOG(INFO) << "Successfully created dashboard system roles during migration.";
995 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
#define LOG(tag)
Definition: Logger.h:216
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
const DBMetadata currentDB_
Definition: Catalog.h:687
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
void grantRoleBatch(const std::vector< std::string > &roles, const std::vector< std::string > &grantees)
static SysCatalog & instance()
Definition: SysCatalog.h:337
static const int32_t MAPD_VERSION
Definition: release.h:32
Role * getRoleGrantee(const std::string &name) const
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1780
std::unordered_map< std::string, std::vector< std::string > > getGranteesOfSharedDashboards(const std::vector< std::string > &dashboard_ids)
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:125
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createDefaultServersIfNotExists ( )

Creates default local file servers (if they don't already exist).

Definition at line 4895 of file Catalog.cpp.

References CHECK, foreign_storage::DataWrapperType::CSV, g_enable_fsi, shared::kRootUserId, foreign_storage::AbstractFileStorageDataWrapper::LOCAL_FILE_STORAGE_TYPE, foreign_storage::DataWrapperType::PARQUET, foreign_storage::DataWrapperType::REGEX_PARSER, and foreign_storage::AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY.

4895  {
4900 
4901  auto local_csv_server = std::make_unique<foreign_storage::ForeignServer>(
4902  "default_local_delimited",
4904  options,
4906  local_csv_server->validate();
4907  createForeignServerNoLocks(std::move(local_csv_server), true);
4908 
4909 #ifdef ENABLE_IMPORT_PARQUET
4910  auto local_parquet_server = std::make_unique<foreign_storage::ForeignServer>(
4911  "default_local_parquet",
4913  options,
4915  local_parquet_server->validate();
4916  createForeignServerNoLocks(std::move(local_parquet_server), true);
4917 #endif
4918 
4919  auto local_regex_parser_server = std::make_unique<foreign_storage::ForeignServer>(
4920  "default_local_regex_parsed",
4922  options,
4924  local_regex_parser_server->validate();
4925  createForeignServerNoLocks(std::move(local_regex_parser_server), true);
4926 }
const int kRootUserId
static constexpr char const * REGEX_PARSER
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:3025
#define CHECK(condition)
Definition: Logger.h:222
static constexpr char const * CSV
std::map< std::string, std::string, std::less<>> OptionsMap
bool g_enable_fsi
Definition: Catalog.cpp:96
static constexpr char const * PARQUET
void Catalog_Namespace::Catalog::createForeignServer ( std::unique_ptr< foreign_storage::ForeignServer foreign_server,
bool  if_not_exists 
)

Creates a new foreign server DB object.

Parameters
foreign_server- unique pointer to struct containing foreign server details
if_not_exists- flag indicating whether or not an attempt to create a new foreign server should occur if a server with the same name already exists. An exception is thrown if this flag is set to "false" and an attempt is made to create a pre-existing foreign server

Definition at line 3017 of file Catalog.cpp.

3019  {
3020  cat_write_lock write_lock(this);
3021  cat_sqlite_lock sqlite_lock(this);
3022  createForeignServerNoLocks(std::move(foreign_server), if_not_exists);
3023 }
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:3025
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:124
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:125
void Catalog_Namespace::Catalog::createForeignServerNoLocks ( std::unique_ptr< foreign_storage::ForeignServer foreign_server,
bool  if_not_exists 
)
private

Same as createForeignServer() but without acquiring locks. This should only be called from within a function/code block that already acquires appropriate locks.

Definition at line 3025 of file Catalog.cpp.

References CHECK, CHECK_EQ, setup::name, and to_string().

3027  {
3028  const auto& name = foreign_server->name;
3029 
3031  "SELECT name from omnisci_foreign_servers where name = ?",
3032  std::vector<std::string>{name});
3033 
3034  if (sqliteConnector_.getNumRows() == 0) {
3035  foreign_server->creation_time = std::time(nullptr);
3037  "INSERT INTO omnisci_foreign_servers (name, data_wrapper_type, owner_user_id, "
3038  "creation_time, "
3039  "options) "
3040  "VALUES (?, ?, ?, ?, ?)",
3041  std::vector<std::string>{name,
3042  foreign_server->data_wrapper_type,
3043  std::to_string(foreign_server->user_id),
3044  std::to_string(foreign_server->creation_time),
3045  foreign_server->getOptionsAsJsonString()});
3047  "SELECT id from omnisci_foreign_servers where name = ?",
3048  std::vector<std::string>{name});
3049  CHECK_EQ(sqliteConnector_.getNumRows(), size_t(1));
3050  foreign_server->id = sqliteConnector_.getData<int32_t>(0, 0);
3051  std::shared_ptr<foreign_storage::ForeignServer> foreign_server_shared =
3052  std::move(foreign_server);
3054  << "Attempting to insert a foreign server into foreign server map that already "
3055  "exists.";
3056  foreignServerMap_[name] = foreign_server_shared;
3057  foreignServerMapById_[foreign_server_shared->id] = foreign_server_shared;
3058  } else if (!if_not_exists) {
3059  throw std::runtime_error{"A foreign server with name \"" + foreign_server->name +
3060  "\" already exists."};
3061  }
3062 
3063  const auto& server_it = foreignServerMap_.find(name);
3064  CHECK(server_it != foreignServerMap_.end());
3065  CHECK(foreignServerMapById_.find(server_it->second->id) != foreignServerMapById_.end());
3066 }
#define CHECK_EQ(x, y)
Definition: Logger.h:230
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
std::string to_string(char const *&&v)
std::string name() const
Definition: Catalog.h:315
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:682
ForeignServerMap foreignServerMap_
Definition: Catalog.h:681
#define CHECK(condition)
Definition: Logger.h:222
virtual size_t getNumRows() const

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::createLink ( LinkDescriptor ld,
size_t  min_length 
)

Definition at line 4545 of file Catalog.cpp.

References LinkDescriptor::link, LinkDescriptor::linkId, to_string(), LinkDescriptor::updateTime, LinkDescriptor::userId, LinkDescriptor::viewMetadata, and LinkDescriptor::viewState.

4545  {
4546  cat_write_lock write_lock(this);
4547  cat_sqlite_lock sqlite_lock(this);
4548  sqliteConnector_.query("BEGIN TRANSACTION");
4549  try {
4551  .substr(0, 8);
4553  "SELECT linkid FROM mapd_links WHERE link = ? and userid = ?",
4554  std::vector<std::string>{ld.link, std::to_string(ld.userId)});
4555  if (sqliteConnector_.getNumRows() > 0) {
4557  "UPDATE mapd_links SET update_time = datetime('now') WHERE userid = ? AND "
4558  "link = ?",
4559  std::vector<std::string>{std::to_string(ld.userId), ld.link});
4560  } else {
4562  "INSERT INTO mapd_links (userid, link, view_state, view_metadata, "
4563  "update_time) VALUES (?,?,?,?, datetime('now'))",
4564  std::vector<std::string>{
4565  std::to_string(ld.userId), ld.link, ld.viewState, ld.viewMetadata});
4566  }
4567  // now get the auto generated dashid
4569  "SELECT linkid, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_links "
4570  "WHERE link = ?",
4571  ld.link);
4572  ld.linkId = sqliteConnector_.getData<int>(0, 0);
4573  ld.updateTime = sqliteConnector_.getData<std::string>(0, 1);
4574  } catch (std::exception& e) {
4575  sqliteConnector_.query("ROLLBACK TRANSACTION");
4576  throw;
4577  }
4578  sqliteConnector_.query("END TRANSACTION");
4579  addLinkToMap(ld);
4580  return ld.link;
4581 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::string to_string(char const *&&v)
std::string calculateSHA1(const std::string &data)
Definition: Catalog.cpp:4533
void addLinkToMap(LinkDescriptor &ld)
Definition: Catalog.cpp:1826
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:124
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:125
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createOrUpdateDashboardSystemRole ( const std::string &  view_meta,
const int32_t &  user_id,
const std::string &  dash_role_name 
)
protected

Definition at line 1780 of file Catalog.cpp.

References Grantee::findDbObject(), Grantee::getDbObjects(), TableDBObjectType, and ViewDBObjectType.

Referenced by createDashboardSystemRoles().

1782  {
1783  auto objects = parseDashboardObjects(view_meta, user_id);
1784  Role* rl = SysCatalog::instance().getRoleGrantee(dash_role_name);
1785  if (!rl) {
1786  // Dashboard role does not exist
1787  // create role and grant privileges
1788  // NOTE(wamsi): Transactionally unsafe
1790  dash_role_name, /*user_private_role=*/false, /*is_temporary=*/false);
1791  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1792  } else {
1793  // Dashboard system role already exists
1794  // Add/remove privileges on objects
1795  std::set<DBObjectKey> revoke_keys;
1796  auto ex_objects = rl->getDbObjects(true);
1797  for (auto key : *ex_objects | boost::adaptors::map_keys) {
1798  if (key.permissionType != TableDBObjectType &&
1799  key.permissionType != ViewDBObjectType) {
1800  continue;
1801  }
1802  bool found = false;
1803  for (auto obj : objects) {
1804  found = key == obj.getObjectKey() ? true : false;
1805  if (found) {
1806  break;
1807  }
1808  }
1809  if (!found) {
1810  revoke_keys.insert(key);
1811  }
1812  }
1813  for (auto& key : revoke_keys) {
1814  // revoke privs on object since the object is no
1815  // longer used by the dashboard as source
1816  // NOTE(wamsi): Transactionally unsafe
1818  dash_role_name, *rl->findDbObject(key, true), *this);
1819  }
1820  // Update privileges on remaining objects
1821  // NOTE(wamsi): Transactionally unsafe
1822  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1823  }
1824 }
void revokeDBObjectPrivileges(const std::string &grantee, const DBObject &object, const Catalog_Namespace::Catalog &catalog)
void createRole(const std::string &roleName, const bool user_private_role, const bool is_temporary=false)
Definition: Grantee.h:81
DBObject * findDbObject(const DBObjectKey &objectKey, bool only_direct) const
Definition: Grantee.cpp:85
static SysCatalog & instance()
Definition: SysCatalog.h:337
Role * getRoleGrantee(const std::string &name) const
void grantDBObjectPrivilegesBatch(const std::vector< std::string > &grantees, const std::vector< DBObject > &objects, const Catalog_Namespace::Catalog &catalog)
const DBObjectMap * getDbObjects(bool only_direct) const
Definition: Grantee.h:56
std::vector< DBObject > parseDashboardObjects(const std::string &view_meta, const int &user_id)
Definition: Catalog.cpp:1750

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createShardedTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs 
)

Definition at line 3833 of file Catalog.cpp.

References CHECK, TableDescriptor::nShards, TableDescriptor::shard, table_is_temporary(), TableDescriptor::tableId, and TableDescriptor::tableName.

3836  {
3837  /* create logical table */
3838  TableDescriptor* tdl = &td;
3839  createTable(*tdl, cols, shared_dict_defs, true); // create logical table
3840  int32_t logical_tb_id = tdl->tableId;
3841  std::string logical_table_name = tdl->tableName;
3842 
3843  /* create physical tables and link them to the logical table */
3844  std::vector<int32_t> physicalTables;
3845  for (int32_t i = 1; i <= td.nShards; i++) {
3846  TableDescriptor* tdp = &td;
3847  tdp->tableName = generatePhysicalTableName(logical_table_name, i);
3848  tdp->shard = i - 1;
3849  createTable(*tdp, cols, shared_dict_defs, false); // create physical table
3850  int32_t physical_tb_id = tdp->tableId;
3851 
3852  /* add physical table to the vector of physical tables */
3853  physicalTables.push_back(physical_tb_id);
3854  }
3855 
3856  if (!physicalTables.empty()) {
3857  cat_write_lock write_lock(this);
3858  /* add logical to physical tables correspondence to the map */
3859  const auto it_ok =
3860  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
3861  CHECK(it_ok.second);
3862  /* update sqlite mapd_logical_to_physical in sqlite database */
3863  if (!table_is_temporary(&td)) {
3864  updateLogicalToPhysicalTableMap(logical_tb_id);
3865  }
3866  }
3867 }
std::string tableName
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:2674
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::string generatePhysicalTableName(const std::string &logicalTableName, const int32_t &shardNumber)
Definition: Catalog.cpp:4817
bool table_is_temporary(const TableDescriptor *const td)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:124
#define CHECK(condition)
Definition: Logger.h:222
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:693
void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id)
Definition: Catalog.cpp:635

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createSystemTableServer ( const std::string &  server_name,
const std::string &  data_wrapper_type,
const foreign_storage::OptionsMap options = {} 
)
private

Definition at line 6349 of file Catalog.cpp.

References logger::INFO, shared::kRootUserId, LOG, and run_benchmark_import::tables.

6351  {
6352  auto server = std::make_unique<foreign_storage::ForeignServer>(
6353  server_name, data_wrapper_type, options, shared::kRootUserId);
6354  server->validate();
6355  auto stored_server = getForeignServer(server_name);
6356  if (stored_server && stored_server->options != server->options) {
6357  // Drop all tables for server before dropping server.
6358  auto tables = getAllForeignTablesForForeignServer(stored_server->id);
6359  for (const auto table : tables) {
6360  LOG(INFO) << "Dropping existing \"" << table->tableName << "\" system table for \""
6361  << server_name << "\" foreign server.";
6362  dropTable(table);
6363  }
6364  LOG(INFO) << "Dropping existing \"" << server_name
6365  << "\" system table foreign server.";
6366  dropForeignServer(server_name);
6367  stored_server = nullptr;
6368  }
6369  if (!stored_server) {
6370  LOG(INFO) << "Creating a new \"" << server_name << "\" system table foreign server.";
6371  createForeignServer(std::move(server), true);
6372  }
6373 }
const int kRootUserId
void createForeignServer(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:3017
const foreign_storage::ForeignServer * getForeignServer(const std::string &server_name) const
Definition: Catalog.cpp:3068
#define LOG(tag)
Definition: Logger.h:216
std::vector< const foreign_storage::ForeignTable * > getAllForeignTablesForForeignServer(const int32_t foreign_server_id)
Get all of the foreign tables for associated with a foreign server id.
Definition: Catalog.cpp:3365
void dropForeignServer(const std::string &server_name)
Definition: Catalog.cpp:3184
void dropTable(const TableDescriptor *td)
Definition: Catalog.cpp:3987
void Catalog_Namespace::Catalog::createTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs,
bool  isLogicalTable 
)

Definition at line 2674 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnId, TableDescriptor::columnIdBySpi_, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::default_value, dict_ref_t::dictId, Data_Namespace::DISK_LEVEL, StorageType::FOREIGN_TABLE, TableDescriptor::fragPageSize, TableDescriptor::fragType, g_enable_fsi, g_serialize_temp_tables, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_next_refresh_time(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), TableDescriptor::hasDeletedCol, SQLTypeInfo::is_array(), SQLTypeInfo::is_geometry(), TableDescriptor::is_system_table, ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, kBIGINT, kBOOLEAN, kENCODING_DICT, TableDescriptor::keyMetainfo, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nColumns, foreign_storage::ForeignTable::next_refresh_time, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_size(), TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::storageType, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), TableDescriptor::userId, TableDescriptor::viewSQL, and ColumnDescriptor::virtualExpr.

Referenced by Parser::CreateTableAsSelectStmt::execute(), and Parser::CreateViewStmt::execute().

2678  {
2679  cat_write_lock write_lock(this);
2680  list<ColumnDescriptor> cds = cols;
2681  list<DictDescriptor> dds;
2682  std::set<std::string> toplevel_column_names;
2683  list<ColumnDescriptor> columns;
2684 
2685  if (!td.storageType.empty() &&
2688  throw std::runtime_error("Only temporary tables can be backed by foreign storage.");
2689  }
2690  dataMgr_->getForeignStorageInterface()->prepareTable(getCurrentDB().dbId, td, cds);
2691  }
2692 
2693  for (auto cd : cds) {
2694  if (cd.columnName == "rowid") {
2695  throw std::runtime_error(
2696  "Cannot create column with name rowid. rowid is a system defined column.");
2697  }
2698  columns.push_back(cd);
2699  toplevel_column_names.insert(cd.columnName);
2700  if (cd.columnType.is_geometry()) {
2701  expandGeoColumn(cd, columns);
2702  }
2703  }
2704  cds.clear();
2705 
2706  ColumnDescriptor cd;
2707  // add row_id column -- Must be last column in the table
2708  cd.columnName = "rowid";
2709  cd.isSystemCol = true;
2710  cd.columnType = SQLTypeInfo(kBIGINT, true);
2711 #ifdef MATERIALIZED_ROWID
2712  cd.isVirtualCol = false;
2713 #else
2714  cd.isVirtualCol = true;
2715  cd.virtualExpr = "MAPD_FRAG_ID * MAPD_ROWS_PER_FRAG + MAPD_FRAG_ROW_ID";
2716 #endif
2717  columns.push_back(cd);
2718  toplevel_column_names.insert(cd.columnName);
2719 
2720  if (td.hasDeletedCol) {
2721  ColumnDescriptor cd_del;
2722  cd_del.columnName = "$deleted$";
2723  cd_del.isSystemCol = true;
2724  cd_del.isVirtualCol = false;
2725  cd_del.columnType = SQLTypeInfo(kBOOLEAN, true);
2726  cd_del.isDeletedCol = true;
2727 
2728  columns.push_back(cd_del);
2729  }
2730 
2731  td.nColumns = columns.size();
2732  // TODO(sy): don't take disk locks or touch sqlite connector for temporary tables
2733  cat_sqlite_lock sqlite_lock(this);
2734  sqliteConnector_.query("BEGIN TRANSACTION");
2736  try {
2738  R"(INSERT INTO mapd_tables (name, userid, ncolumns, isview, fragments, frag_type, max_frag_rows, max_chunk_size, frag_page_size, max_rows, partitions, shard_column_id, shard, num_shards, sort_column_id, storage_type, max_rollback_epochs, is_system_table, key_metainfo) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?))",
2739  std::vector<std::string>{td.tableName,
2740  std::to_string(td.userId),
2742  std::to_string(td.isView),
2743  "",
2748  std::to_string(td.maxRows),
2749  td.partitions,
2751  std::to_string(td.shard),
2752  std::to_string(td.nShards),
2754  td.storageType,
2757  td.keyMetainfo});
2758 
2759  // now get the auto generated tableid
2761  "SELECT tableid FROM mapd_tables WHERE name = ?", td.tableName);
2762  td.tableId = sqliteConnector_.getData<int>(0, 0);
2763  int colId = 1;
2764  for (auto cd : columns) {
2766  const bool is_foreign_col =
2767  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2768  if (!is_foreign_col) {
2769  // Ideally we would like to not persist string dictionaries for system tables,
2770  // since system table content can be highly dynamic and string dictionaries
2771  // are not currently vacuumed. However, in distributed this causes issues
2772  // when the cluster is out of sync (when the agg resets but leaves persist) so
2773  // for the sake of testing we need to leave this as normal dictionaries until
2774  // we solve the distributed issue.
2775  auto use_temp_dictionary = false; // td.is_system_table;
2776  setColumnDictionary(cd, dds, td, isLogicalTable, use_temp_dictionary);
2777  }
2778  }
2779 
2780  if (toplevel_column_names.count(cd.columnName)) {
2781  if (!cd.isGeoPhyCol) {
2782  td.columnIdBySpi_.push_back(colId);
2783  }
2784  }
2785 
2786  using BindType = SqliteConnector::BindType;
2787  std::vector<BindType> types(17, BindType::TEXT);
2788  if (!cd.default_value.has_value()) {
2789  types[16] = BindType::NULL_TYPE;
2790  }
2792  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, "
2793  "coldim, colscale, is_notnull, "
2794  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
2795  "virtual_expr, is_deletedcol, default_value) "
2796  "VALUES (?, ?, ?, ?, ?, "
2797  "?, "
2798  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2799  std::vector<std::string>{std::to_string(td.tableId),
2800  std::to_string(colId),
2801  cd.columnName,
2810  "",
2813  cd.virtualExpr,
2815  cd.default_value.value_or("NULL")},
2816  types);
2817  cd.tableId = td.tableId;
2818  cd.columnId = colId++;
2819  cds.push_back(cd);
2820  }
2821  if (td.isView) {
2823  "INSERT INTO mapd_views (tableid, sql) VALUES (?,?)",
2824  std::vector<std::string>{std::to_string(td.tableId), td.viewSQL});
2825  }
2827  auto& foreign_table = dynamic_cast<foreign_storage::ForeignTable&>(td);
2828  foreign_table.next_refresh_time = get_next_refresh_time(foreign_table);
2830  "INSERT INTO omnisci_foreign_tables (table_id, server_id, options, "
2831  "last_refresh_time, next_refresh_time) VALUES (?, ?, ?, ?, ?)",
2832  std::vector<std::string>{std::to_string(foreign_table.tableId),
2833  std::to_string(foreign_table.foreign_server->id),
2834  foreign_table.getOptionsAsJsonString(),
2835  std::to_string(foreign_table.last_refresh_time),
2836  std::to_string(foreign_table.next_refresh_time)});
2837  }
2838  } catch (std::exception& e) {
2839  sqliteConnector_.query("ROLLBACK TRANSACTION");
2840  throw;
2841  }
2842  } else { // Temporary table
2843  td.tableId = nextTempTableId_++;
2844  int colId = 1;
2845  for (auto cd : columns) {
2847  const bool is_foreign_col =
2848  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2849 
2850  if (!is_foreign_col) {
2851  // Create a new temporary dictionary
2852  std::string fileName("");
2853  std::string folderPath("");
2855  nextTempDictId_++;
2856  DictDescriptor dd(dict_ref,
2857  fileName,
2859  false,
2860  1,
2861  folderPath,
2862  true); // Is dictName (2nd argument) used?
2863  dds.push_back(dd);
2864  if (!cd.columnType.is_array()) {
2866  }
2867  cd.columnType.set_comp_param(dict_ref.dictId);
2868  }
2869  }
2870  if (toplevel_column_names.count(cd.columnName)) {
2871  if (!cd.isGeoPhyCol) {
2872  td.columnIdBySpi_.push_back(colId);
2873  }
2874  }
2875  cd.tableId = td.tableId;
2876  cd.columnId = colId++;
2877  cds.push_back(cd);
2878  }
2879 
2881  serializeTableJsonUnlocked(&td, cds);
2882  }
2883  }
2884 
2885  try {
2886  auto cache = dataMgr_->getPersistentStorageMgr()->getDiskCache();
2887  if (cache) {
2888  CHECK(!cache->hasCachedMetadataForKeyPrefix({getCurrentDB().dbId, td.tableId}))
2889  << "Disk cache at " + cache->getCacheDirectory()
2890  << " contains preexisting data for new table. Please "
2891  "delete or clear cache before continuing";
2892  }
2893 
2894  addTableToMap(&td, cds, dds);
2895  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
2896  if (!td.storageType.empty() && td.storageType != StorageType::FOREIGN_TABLE) {
2897  dataMgr_->getForeignStorageInterface()->registerTable(this, td, cds);
2898  }
2899  } catch (std::exception& e) {
2900  sqliteConnector_.query("ROLLBACK TRANSACTION");
2901  removeTableFromMap(td.tableName, td.tableId, true);
2902  throw;
2903  }
2904  sqliteConnector_.query("END TRANSACTION");
2905 
2907  write_lock.unlock();
2908  sqlite_lock.unlock();
2910  true); // cause instantiateFragmenter() to be called
2911  }
2912 }
int32_t maxRollbackEpochs
void serializeTableJsonUnlocked(const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
Definition: Catalog.cpp:2914
int64_t get_next_refresh_time(const foreign_storage::ForeignTable &foreign_table)
Definition: Catalog.cpp:2661
std::string virtualExpr
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:330
void set_size(int s)
Definition: sqltypes.h:437
std::string partitions
HOST DEVICE int get_size() const
Definition: sqltypes.h:339
T getData(const int row, const int col)
std::string tableName
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
const DBMetadata currentDB_
Definition: Catalog.h:687
HOST DEVICE int get_scale() const
Definition: sqltypes.h:334
std::string storageType
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, bool is_logical_table, bool use_temp_dictionary=false)
Definition: Catalog.cpp:3791
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:688
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:329
std::vector< int > columnIdBySpi_
std::string to_string(char const *&&v)
void addTableToMap(const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
Definition: Catalog.cpp:1598
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:242
specifies the content in-memory of a row in the column metadata table
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:2530
std::string keyMetainfo
bool g_serialize_temp_tables
Definition: Catalog.cpp:108
void set_comp_param(int p)
Definition: sqltypes.h:441
std::optional< std::string > default_value
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:691
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:337
Fragmenter_Namespace::FragmenterType fragType
Data_Namespace::MemoryLevel persistenceLevel
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:331
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1657
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:124
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:338
bool setColumnSharedDictionary(ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:3722
#define CHECK(condition)
Definition: Logger.h:222
Descriptor for a dictionary for a string columne.
std::string viewSQL
SQLTypeInfo columnType
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:336
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:125
static constexpr char const * FOREIGN_TABLE
bool g_enable_fsi
Definition: Catalog.cpp:96
std::string columnName
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
bool is_array() const
Definition: sqltypes.h:518

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::delDictionary ( const ColumnDescriptor cd)

Definition at line 2308 of file Catalog.cpp.

References CHECK, CHECK_GT, ColumnDescriptor::columnType, g_base_path, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::is_string(), SQLTypeInfo::is_string_array(), shared::kDataDirectoryName, kENCODING_DICT, File_Namespace::renameForDelete(), ColumnDescriptor::tableId, to_string(), and VLOG.

2308  {
2309  cat_write_lock write_lock(this);
2310  cat_sqlite_lock sqlite_lock(this);
2311  if (!(cd.columnType.is_string() || cd.columnType.is_string_array())) {
2312  return;
2313  }
2314  if (!(cd.columnType.get_compression() == kENCODING_DICT)) {
2315  return;
2316  }
2317  const auto dictId = cd.columnType.get_comp_param();
2318  CHECK_GT(dictId, 0);
2319  // decrement and zero check dict ref count
2320  const auto td = getMetadataForTable(cd.tableId, false);
2321  CHECK(td);
2323  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
2324  std::to_string(dictId));
2326  "SELECT refcount FROM mapd_dictionaries WHERE dictid = ?", std::to_string(dictId));
2327  const auto refcount = sqliteConnector_.getData<int>(0, 0);
2328  VLOG(3) << "Dictionary " << dictId << "from dropped table has reference count "
2329  << refcount;
2330  if (refcount > 0) {
2331  return;
2332  }
2333  const DictRef dictRef(currentDB_.dbId, dictId);
2334  sqliteConnector_.query_with_text_param("DELETE FROM mapd_dictionaries WHERE dictid = ?",
2335  std::to_string(dictId));
2337  "/DB_" + std::to_string(currentDB_.dbId) + "_DICT_" +
2338  std::to_string(dictId));
2339 
2340  std::unique_ptr<StringDictionaryClient> client;
2341  if (!string_dict_hosts_.empty()) {
2342  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dictRef, true));
2343  }
2344  if (client) {
2345  client->drop(dictRef);
2346  }
2347 
2348  dictDescriptorMapByRef_.erase(dictRef);
2349 }
const std::string kDataDirectoryName
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:690
const DBMetadata currentDB_
Definition: Catalog.h:687
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:677
heavyai::unique_lock< heavyai::shared_mutex > write_lock
#define CHECK_GT(x, y)
Definition: Logger.h:234
std::string to_string(char const *&&v)
std::string g_base_path
Definition: SysCatalog.cpp:62
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:337
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:124
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:338
#define CHECK(condition)
Definition: Logger.h:222
SQLTypeInfo columnType
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
bool is_string() const
Definition: sqltypes.h:510
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:227
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:125
bool is_string_array() const
Definition: sqltypes.h:511
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
#define VLOG(n)
Definition: Logger.h:316

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::deleteCustomExpressions ( const std::vector< int32_t > &  custom_expression_ids,
bool  do_soft_delete 
)

Deletes custom expressions with the given ids.

Parameters
custom_expression_ids- ids of custom expressions to delete
do_soft_delete- flag indicating whether or not to do a soft delete

Definition at line 5633 of file Catalog.cpp.

References CHECK, CHECK_EQ, join(), and to_string().

5634  {
5635  cat_write_lock write_lock(this);
5636  cat_sqlite_lock sqlite_lock(this);
5637 
5638  std::vector<int32_t> invalid_ids;
5639  for (const auto id : custom_expression_ids) {
5640  if (custom_expr_map_by_id_.find(id) == custom_expr_map_by_id_.end()) {
5641  invalid_ids.emplace_back(id);
5642  }
5643  }
5644  if (!invalid_ids.empty()) {
5645  throw std::runtime_error{"Custom expressions with ids: " + join(invalid_ids, ",") +
5646  " do not exist."};
5647  }
5648  sqliteConnector_.query("BEGIN TRANSACTION");
5649  try {
5650  for (const auto id : custom_expression_ids) {
5652  "SELECT id FROM omnisci_custom_expressions WHERE id = ?",
5653  std::vector<std::string>{std::to_string(id)});
5654  CHECK_EQ(sqliteConnector_.getNumRows(), static_cast<size_t>(1));
5655  if (do_soft_delete) {
5657  "UPDATE omnisci_custom_expressions SET is_deleted = ? WHERE id = ?",
5658  std::vector<std::string>{std::to_string(true), std::to_string(id)});
5659  } else {
5661  "DELETE FROM omnisci_custom_expressions WHERE id = ?",
5662  std::vector<std::string>{std::to_string(id)});
5663  }
5664  }
5665 
5666  for (const auto id : custom_expression_ids) {
5667  if (do_soft_delete) {
5668  auto it = custom_expr_map_by_id_.find(id);
5669  CHECK(it != custom_expr_map_by_id_.end());
5670  it->second->is_deleted = true;
5671  } else {
5672  custom_expr_map_by_id_.erase(id);
5673  }
5674  }
5675  } catch (std::exception& e) {
5676  sqliteConnector_.query("ROLLBACK TRANSACTION");
5677  throw;
5678  }
5679  sqliteConnector_.query("END TRANSACTION");
5680 }
#define CHECK_EQ(x, y)
Definition: Logger.h:230
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
std::string join(T const &container, std::string const &delim)
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:124
#define CHECK(condition)
Definition: Logger.h:222
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:683
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:125
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::deleteMetadataForDashboards ( const std::vector< int32_t >  ids,
const UserMetadata user 
)

Definition at line 2067 of file Catalog.cpp.

References DashboardDBObjectType, AccessPrivileges::DELETE_DASHBOARD, and to_string().

2068  {
2069  std::stringstream invalid_ids, restricted_ids;
2070 
2071  for (int32_t dashboard_id : dashboard_ids) {
2072  if (!getMetadataForDashboard(dashboard_id)) {
2073  invalid_ids << (!invalid_ids.str().empty() ? ", " : "") << dashboard_id;
2074  continue;
2075  }
2076  DBObject object(dashboard_id, DashboardDBObjectType);
2077  object.loadKey(*this);
2078  object.setPrivileges(AccessPrivileges::DELETE_DASHBOARD);
2079  std::vector<DBObject> privs = {object};
2080  if (!SysCatalog::instance().checkPrivileges(user, privs)) {
2081  restricted_ids << (!restricted_ids.str().empty() ? ", " : "") << dashboard_id;
2082  }
2083  }
2084 
2085  if (invalid_ids.str().size() > 0 || restricted_ids.str().size() > 0) {
2086  std::stringstream error_message;
2087  error_message << "Delete dashboard(s) failed with error(s):";
2088  if (invalid_ids.str().size() > 0) {
2089  error_message << "\nDashboard id: " << invalid_ids.str()
2090  << " - Dashboard id does not exist";
2091  }
2092  if (restricted_ids.str().size() > 0) {
2093  error_message
2094  << "\nDashboard id: " << restricted_ids.str()
2095  << " - User should be either owner of dashboard or super user to delete it";
2096  }
2097  throw std::runtime_error(error_message.str());
2098  }
2099  std::vector<DBObject> dash_objs;
2100 
2101  for (int32_t dashboard_id : dashboard_ids) {
2102  dash_objs.emplace_back(dashboard_id, DashboardDBObjectType);
2103  }
2104  // BE-5245: Transactionally unsafe (like other combined Catalog/Syscatalog operations)
2106  {
2107  cat_write_lock write_lock(this);
2108  cat_sqlite_lock sqlite_lock(this);
2109 
2110  sqliteConnector_.query("BEGIN TRANSACTION");
2111  try {
2112  for (int32_t dashboard_id : dashboard_ids) {
2113  auto dash = getMetadataForDashboard(dashboard_id);
2114  // Dash should still exist if revokeDBObjectPrivileges passed but throw and
2115  // rollback if already deleted
2116  if (!dash) {
2117  throw std::runtime_error(
2118  std::string("Delete dashboard(s) failed with error(s):\nDashboard id: ") +
2119  std::to_string(dashboard_id) + " - Dashboard id does not exist ");
2120  }
2121  std::string user_id = std::to_string(dash->userId);
2122  std::string dash_name = dash->dashboardName;
2123  auto viewDescIt = dashboardDescriptorMap_.find(user_id + ":" + dash_name);
2124  dashboardDescriptorMap_.erase(viewDescIt);
2126  "DELETE FROM mapd_dashboards WHERE name = ? and userid = ?",
2127  std::vector<std::string>{dash_name, user_id});
2128  }
2129  } catch (std::exception& e) {
2130  sqliteConnector_.query("ROLLBACK TRANSACTION");
2131  throw;
2132  }
2133  sqliteConnector_.query("END TRANSACTION");
2134  }
2135 }
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:678
std::string to_string(char const *&&v)
static SysCatalog & instance()
Definition: SysCatalog.h:337
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:124
static const AccessPrivileges DELETE_DASHBOARD
Definition: DBObject.h:173
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:125
void revokeDBObjectPrivilegesFromAllBatch(std::vector< DBObject > &objects, Catalog *catalog)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::deleteTableCatalogMetadata ( const TableDescriptor logical_table,
const std::vector< const TableDescriptor * > &  physical_tables 
)
private

Definition at line 4015 of file Catalog.cpp.

References TableDescriptor::tableId, and to_string().

4017  {
4018  cat_write_lock write_lock(this);
4019  cat_sqlite_lock sqlite_lock(this);
4020  sqliteConnector_.query("BEGIN TRANSACTION");
4021  try {
4022  // remove corresponding record from the logicalToPhysicalTableMap in sqlite database
4024  "DELETE FROM mapd_logical_to_physical WHERE logical_table_id = ?",
4025  std::to_string(logical_table->tableId));
4026  logicalToPhysicalTableMapById_.erase(logical_table->tableId);
4027  for (auto table : physical_tables) {
4028  eraseTableMetadata(table);
4029  }
4030  } catch (std::exception& e) {
4031  sqliteConnector_.query("ROLLBACK TRANSACTION");
4032  throw;
4033  }
4034  sqliteConnector_.query("END TRANSACTION");
4035 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:124
void eraseTableMetadata(const TableDescriptor *td)
Definition: Catalog.cpp:4037
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:125
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:693

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::doTruncateTable ( const TableDescriptor td)
protected

Definition at line 3877 of file Catalog.cpp.

References CHECK, CHECK_GE, ColumnDescriptor::columnType, Data_Namespace::CPU_LEVEL, dict_ref_t::dictId, DictDescriptor::dictIsTemp, DictDescriptor::dictRef, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), Data_Namespace::GPU_LEVEL, kENCODING_DICT, File_Namespace::renameForDelete(), ColumnDescriptor::tableId, and TableDescriptor::tableId.

3877  {
3878  // must destroy fragmenter before deleteChunks is called.
3880 
3881  const int tableId = td->tableId;
3882  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
3883  // assuming deleteChunksWithPrefix is atomic
3884  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
3885  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
3886 
3887  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
3888 
3889  cat_write_lock write_lock(this);
3890  std::unique_ptr<StringDictionaryClient> client;
3891  if (SysCatalog::instance().isAggregator()) {
3892  CHECK(!string_dict_hosts_.empty());
3893  DictRef dict_ref(currentDB_.dbId, -1);
3894  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
3895  }
3896  // clean up any dictionaries
3897  // delete all column descriptors for the table
3898  for (const auto& columnDescriptor : columnDescriptorMapById_) {
3899  auto cd = columnDescriptor.second;
3900  if (cd->tableId != td->tableId) {
3901  continue;
3902  }
3903  const int dict_id = cd->columnType.get_comp_param();
3904  // Dummy dictionaries created for a shard of a logical table have the id set to zero.
3905  if (cd->columnType.get_compression() == kENCODING_DICT && dict_id) {
3906  const DictRef dict_ref(currentDB_.dbId, dict_id);
3907  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
3908  CHECK(dictIt != dictDescriptorMapByRef_.end());
3909  const auto& dd = dictIt->second;
3910  CHECK_GE(dd->refcount, 1);
3911  // if this is the only table using this dict reset the dict
3912  if (dd->refcount == 1) {
3913  // close the dictionary
3914  dd->stringDict.reset();
3915  File_Namespace::renameForDelete(dd->dictFolderPath);
3916  if (client) {
3917  client->drop(dd->dictRef);
3918  }
3919  if (!dd->dictIsTemp) {
3920  boost::filesystem::create_directory(dd->dictFolderPath);
3921  }
3922  }
3923 
3924  DictDescriptor* new_dd = new DictDescriptor(dd->dictRef,
3925  dd->dictName,
3926  dd->dictNBits,
3927  dd->dictIsShared,
3928  dd->refcount,
3929  dd->dictFolderPath,
3930  dd->dictIsTemp);
3931  dictDescriptorMapByRef_.erase(dictIt);
3932  // now create new Dict -- need to figure out what to do here for temp tables
3933  if (client) {
3934  client->create(new_dd->dictRef, new_dd->dictIsTemp);
3935  }
3936  dictDescriptorMapByRef_[new_dd->dictRef].reset(new_dd);
3938  }
3939  }
3940 }
std::vector< int > ChunkKey
Definition: types.h:36
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:690
const DBMetadata currentDB_
Definition: Catalog.h:687
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:677
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:688
#define CHECK_GE(x, y)
Definition: Logger.h:235
heavyai::unique_lock< heavyai::shared_mutex > write_lock
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:676
static SysCatalog & instance()
Definition: SysCatalog.h:337
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1960
int32_t dictId
Definition: DictRef.h:11
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:124
void removeFragmenterForTable(const int table_id) const
Definition: Catalog.cpp:3965
#define CHECK(condition)
Definition: Logger.h:222
Descriptor for a dictionary for a string columne.
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:227

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropColumn ( const TableDescriptor td,
const ColumnDescriptor cd 
)

Definition at line 2441 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnId, ColumnDescriptor::columnName, TableDescriptor::nShards, TableDescriptor::shard, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), and to_upper().

2441  {
2442  {
2443  cat_write_lock write_lock(this);
2444  cat_sqlite_lock sqlite_lock(this);
2445  // caller must handle sqlite/chunk transaction TOGETHER
2447  "DELETE FROM mapd_columns where tableid = ? and columnid = ?",
2448  std::vector<std::string>{std::to_string(td.tableId),
2449  std::to_string(cd.columnId)});
2450 
2452  "UPDATE mapd_tables SET ncolumns = ncolumns - 1 WHERE tableid = ?",
2453  std::vector<std::string>{std::to_string(td.tableId)});
2454 
2455  ColumnDescriptorMap::iterator columnDescIt =
2457  CHECK(columnDescIt != columnDescriptorMap_.end());
2458 
2459  columnDescriptorsForRoll.emplace_back(columnDescIt->second, nullptr);
2460  removeFromColumnMap(columnDescIt->second);
2461  --tableDescriptorMapById_[td.tableId]->nColumns;
2462  }
2463 
2464  // for each shard
2465  if (td.nShards > 0 && td.shard < 0) {
2466  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
2467  const auto shard_cd = getMetadataForColumn(shard->tableId, cd.columnId);
2468  CHECK(shard_cd);
2469  dropColumn(*shard, *shard_cd);
2470  }
2471  }
2472 }
void removeFromColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6476
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:675
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void dropColumn(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:2441
std::string to_string(char const *&&v)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:674
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4601
std::string to_upper(const std::string &str)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:124
#define CHECK(condition)
Definition: Logger.h:222
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:125
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:705
std::string columnName

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropForeignServer ( const std::string &  server_name)

Drops/deletes a foreign server DB object.

Parameters
server_name- Name of foreign server that will be deleted

Definition at line 3184 of file Catalog.cpp.

References CHECK_EQ, and to_string().

3184  {
3185  cat_write_lock write_lock(this);
3186  cat_sqlite_lock sqlite_lock(this);
3187 
3189  "SELECT id from omnisci_foreign_servers where name = ?",
3190  std::vector<std::string>{server_name});
3191  auto num_rows = sqliteConnector_.getNumRows();
3192  if (num_rows > 0) {
3193  CHECK_EQ(size_t(1), num_rows);
3194  auto server_id = sqliteConnector_.getData<int32_t>(0, 0);
3196  "SELECT table_id from omnisci_foreign_tables where server_id = ?",
3197  std::to_string(server_id));
3198  if (sqliteConnector_.getNumRows() > 0) {
3199  throw std::runtime_error{"Foreign server \"" + server_name +
3200  "\" is referenced "
3201  "by existing foreign tables and cannot be dropped."};
3202  }
3203  sqliteConnector_.query("BEGIN TRANSACTION");
3204  try {
3206  "DELETE FROM omnisci_foreign_servers WHERE name = ?",
3207  std::vector<std::string>{server_name});
3208  } catch (const std::exception& e) {
3209  sqliteConnector_.query("ROLLBACK TRANSACTION");
3210  throw;
3211  }
3212  sqliteConnector_.query("END TRANSACTION");
3213  foreignServerMap_.erase(server_name);
3214  foreignServerMapById_.erase(server_id);
3215  }
3216 }
#define CHECK_EQ(x, y)
Definition: Logger.h:230
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::string to_string(char const *&&v)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:682
ForeignServerMap foreignServerMap_
Definition: Catalog.h:681
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:124
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:125
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropTable ( const TableDescriptor td)

Definition at line 3987 of file Catalog.cpp.

References CHECK, TableDescriptor::isView, TableDBObjectType, TableDescriptor::tableId, TableDescriptor::tableName, and ViewDBObjectType.

Referenced by Catalog_Namespace::anonymous_namespace{Catalog.cpp}::drop_tables(), Parser::CreateTableAsSelectStmt::execute(), and Parser::DropViewStmt::execute().

3987  {
3990  std::vector<const TableDescriptor*> tables_to_drop;
3991  {
3992  cat_read_lock read_lock(this);
3993  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
3994  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
3995  // remove all corresponding physical tables if this is a logical table
3996  const auto physicalTables = physicalTableIt->second;
3997  CHECK(!physicalTables.empty());
3998  for (size_t i = 0; i < physicalTables.size(); i++) {
3999  int32_t physical_tb_id = physicalTables[i];
4000  const TableDescriptor* phys_td =
4001  getMutableMetadataForTableUnlocked(physical_tb_id);
4002  CHECK(phys_td);
4003  tables_to_drop.emplace_back(phys_td);
4004  }
4005  }
4006  tables_to_drop.emplace_back(td);
4007  }
4008 
4009  for (auto table : tables_to_drop) {
4010  eraseTablePhysicalData(table);
4011  }
4012  deleteTableCatalogMetadata(td, tables_to_drop);
4013 }
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:4799
std::string tableName
heavyai::shared_lock< heavyai::shared_mutex > read_lock
void revokeDBObjectPrivilegesFromAll(DBObject object, Catalog *catalog)
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:123
void deleteTableCatalogMetadata(const TableDescriptor *logical_table, const std::vector< const TableDescriptor * > &physical_tables)
Definition: Catalog.cpp:4015
TableDescriptor * getMutableMetadataForTableUnlocked(int table_id) const
Definition: Catalog.cpp:1952
static SysCatalog & instance()
Definition: SysCatalog.h:337
#define CHECK(condition)
Definition: Logger.h:222
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:693

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::dropTableFromJsonUnlocked ( const std::string &  table_name) const
protected

Definition at line 2985 of file Catalog.cpp.

References CHECK, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and VLOG.

2985  {
2986  // relies on the catalog write lock
2987  using namespace rapidjson;
2988 
2989  VLOG(1) << "Dropping temporary table " << table_name << " to JSON for Calcite.";
2990 
2991  const auto db_name = currentDB_.dbName;
2992  const auto file_path = table_json_filepath(basePath_, db_name);
2993 
2994  CHECK(boost::filesystem::exists(file_path));
2995  Document d;
2996 
2997  std::ifstream reader(file_path.string());
2998  CHECK(reader.is_open());
2999  IStreamWrapper json_read_wrapper(reader);
3000  d.ParseStream(json_read_wrapper);
3001 
3002  CHECK(d.IsObject());
3003  auto table_name_ref = StringRef(table_name.c_str());
3004  CHECK(d.HasMember(table_name_ref));
3005  CHECK(d.RemoveMember(table_name_ref));
3006 
3007  // Overwrite the existing file
3008  std::ofstream writer(file_path.string(), std::ios::trunc | std::ios::out);
3009  CHECK(writer.is_open());
3010  OStreamWrapper json_wrapper(writer);
3011 
3012  Writer<OStreamWrapper> json_writer(json_wrapper);
3013  d.Accept(json_writer);
3014  writer.close();
3015 }
const DBMetadata currentDB_
Definition: Catalog.h:687
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:164
#define CHECK(condition)
Definition: Logger.h:222
#define VLOG(n)
Definition: Logger.h:316

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::dumpCreateServer ( const std::string &  name,
bool  multiline_formatting = true 
) const

Definition at line 5310 of file Catalog.cpp.

References join().

5311  {
5312  cat_read_lock read_lock(this);
5313  auto server_it = foreignServerMap_.find(name);
5314  if (server_it == foreignServerMap_.end()) {
5315  throw std::runtime_error("Foreign server " + name + " does not exist.");
5316  }
5317  auto server = server_it->second.get();
5318  std::ostringstream os;
5319  os << "CREATE SERVER " << name << " FOREIGN DATA WRAPPER " << server->data_wrapper_type;
5320  std::vector<std::string> with_options;
5321  for (const auto& [option, value] : server->options) {
5322  with_options.emplace_back(option + "='" + value + "'");
5323  }
5324  if (!with_options.empty()) {
5325  if (!multiline_formatting) {
5326  os << " ";
5327  } else {
5328  os << "\n";
5329  }
5330  os << "WITH (" + boost::algorithm::join(with_options, ", ") + ")";
5331  }
5332  os << ";";
5333  return os.str();
5334 }
heavyai::shared_lock< heavyai::shared_mutex > read_lock
std::string join(T const &container, std::string const &delim)
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:123
std::string name() const
Definition: Catalog.h:315
ForeignServerMap foreignServerMap_
Definition: Catalog.h:681

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::dumpCreateTable ( const TableDescriptor td,
bool  multiline_formatting = true,
bool  dump_defaults = false 
) const

Definition at line 5131 of file Catalog.cpp.

5133  {
5134  cat_read_lock read_lock(this);
5135  return dumpCreateTableUnlocked(td, multiline_formatting, dump_defaults);
5136 }
heavyai::shared_lock< heavyai::shared_mutex > read_lock
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:123
std::string dumpCreateTableUnlocked(const TableDescriptor *td, bool multiline_formatting, bool dump_defaults) const
Definition: Catalog.cpp:5149
std::optional< std::string > Catalog_Namespace::Catalog::dumpCreateTable ( int32_t  table_id,
bool  multiline_formatting = true,
bool  dump_defaults = false 
) const

Definition at line 5138 of file Catalog.cpp.

5140  {
5141  cat_read_lock read_lock(this);
5142  const auto td = getMutableMetadataForTableUnlocked(table_id);
5143  if (!td) {
5144  return {};
5145  }
5146  return dumpCreateTableUnlocked(td, multiline_formatting, dump_defaults);
5147 }
heavyai::shared_lock< heavyai::shared_mutex > read_lock
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:123
std::string dumpCreateTableUnlocked(const TableDescriptor *td, bool multiline_formatting, bool dump_defaults) const
Definition: Catalog.cpp:5149
TableDescriptor * getMutableMetadataForTableUnlocked(int table_id) const
Definition: Catalog.cpp:1952
std::string Catalog_Namespace::Catalog::dumpCreateTableUnlocked ( const TableDescriptor td,
bool  multiline_formatting,
bool  dump_defaults 
) const
private

Definition at line 5149 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, Data_Namespace::CPU_LEVEL, DEFAULT_FRAGMENT_ROWS, DEFAULT_MAX_CHUNK_SIZE, DEFAULT_MAX_ROLLBACK_EPOCHS, DEFAULT_MAX_ROWS, DEFAULT_PAGE_SIZE, ColumnDescriptor::default_value, TableDescriptor::fragPageSize, g_leaf_count, ColumnDescriptor::getDefaultValueLiteral(), TableDescriptor::hasDeletedCol, TableDescriptor::is_system_table, ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, join(), kCHAR, kENCODING_DICT, kENCODING_GEOINT, kTEXT, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and TableDescriptor::viewSQL.

5151  {
5152  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
5153  std::ostringstream os;
5154 
5155  if (foreign_table && !td->is_system_table) {
5156  os << "CREATE FOREIGN TABLE " << td->tableName << " (";
5157  } else if (!td->isView) {
5158  os << "CREATE ";
5160  os << "TEMPORARY ";
5161  }
5162  os << "TABLE " + td->tableName + " (";
5163  } else {
5164  os << "CREATE VIEW " + td->tableName + " AS " << td->viewSQL;
5165  return os.str();
5166  }
5167  // scan column defines
5168  std::vector<std::string> additional_info;
5169  std::set<std::string> shared_dict_column_names;
5170 
5171  gatherAdditionalInfo(additional_info, shared_dict_column_names, td);
5172 
5173  // gather column defines
5174  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
5175  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
5176  bool first = true;
5177  for (const auto cd : cds) {
5178  if (!(cd->isSystemCol || cd->isVirtualCol)) {
5179  const auto& ti = cd->columnType;
5180  if (!first) {
5181  os << ",";
5182  if (!multiline_formatting) {
5183  os << " ";
5184  }
5185  } else {
5186  first = false;
5187  }
5188  if (multiline_formatting) {
5189  os << "\n ";
5190  }
5191  // column name
5192  os << quoteIfRequired(cd->columnName);
5193  // CHAR is perculiar... better dump it as TEXT(32) like \d does
5194  if (ti.get_type() == SQLTypes::kCHAR) {
5195  os << " "
5196  << "TEXT";
5197  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
5198  os << " "
5199  << "TEXT[]";
5200  } else {
5201  os << " " << ti.get_type_name();
5202  }
5203  os << (ti.get_notnull() ? " NOT NULL" : "");
5204  if (cd->default_value.has_value()) {
5205  os << " DEFAULT " << cd->getDefaultValueLiteral();
5206  }
5207  if (shared_dict_column_names.find(cd->columnName) ==
5208  shared_dict_column_names.end()) {
5209  // avoids "Column ... shouldn't specify an encoding, it borrows it
5210  // from the referenced column"
5211  if (ti.is_string() || (ti.is_array() && ti.get_subtype() == kTEXT)) {
5212  auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
5213  if (ti.get_compression() == kENCODING_DICT) {
5214  os << " ENCODING " << ti.get_compression_name() << "(" << (size * 8) << ")";
5215  } else {
5216  os << " ENCODING NONE";
5217  }
5218  } else if (ti.is_date_in_days() ||
5219  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
5220  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
5221  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
5222  } else if (ti.is_geometry()) {
5223  if (ti.get_compression() == kENCODING_GEOINT) {
5224  os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
5225  << ")";
5226  } else {
5227  os << " ENCODING NONE";
5228  }
5229  }
5230  }
5231  }
5232  }
5233  // gather SHARED DICTIONARYs
5234  if (additional_info.size()) {
5235  std::string comma;
5236  if (!multiline_formatting) {
5237  comma = ", ";
5238  } else {
5239  comma = ",\n ";
5240  }
5241  os << comma;
5242  os << boost::algorithm::join(additional_info, comma);
5243  }
5244  os << ")";
5245 
5246  std::vector<std::string> with_options;
5247  if (foreign_table && !td->is_system_table) {
5248  if (multiline_formatting) {
5249  os << "\n";
5250  } else {
5251  os << " ";
5252  }
5253  os << "SERVER " << foreign_table->foreign_server->name;
5254 
5255  // gather WITH options ...
5256  for (const auto& [option, value] : foreign_table->options) {
5257  with_options.emplace_back(option + "='" + value + "'");
5258  }
5259  }
5260 
5261  if (dump_defaults || td->maxFragRows != DEFAULT_FRAGMENT_ROWS) {
5262  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
5263  }
5264  if (dump_defaults || td->maxChunkSize != DEFAULT_MAX_CHUNK_SIZE) {
5265  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
5266  }
5267  if (!foreign_table && (dump_defaults || td->fragPageSize != DEFAULT_PAGE_SIZE)) {
5268  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
5269  }
5270  if (!foreign_table && (dump_defaults || td->maxRows != DEFAULT_MAX_ROWS)) {
5271  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
5272  }
5273  if ((dump_defaults || td->maxRollbackEpochs != DEFAULT_MAX_ROLLBACK_EPOCHS) &&
5274  td->maxRollbackEpochs != -1) {
5275  with_options.push_back("MAX_ROLLBACK_EPOCHS=" +
5277  }
5278  if (!foreign_table && (dump_defaults || !td->hasDeletedCol)) {
5279  with_options.emplace_back(td->hasDeletedCol ? "VACUUM='DELAYED'"
5280  : "VACUUM='IMMEDIATE'");
5281  }
5282  if (!foreign_table && !td->partitions.empty()) {
5283  with_options.push_back("PARTITIONS='" + td->partitions + "'");
5284  }
5285  if (!foreign_table && td->nShards > 0) {
5286  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
5287  CHECK(shard_cd);
5288  with_options.push_back(
5289  "SHARD_COUNT=" +
5290  std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
5291  }
5292  if (!foreign_table && td->sortedColumnId > 0) {
5293  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
5294  CHECK(sort_cd);
5295  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
5296  }
5297 
5298  if (!with_options.empty()) {
5299  if (!multiline_formatting) {
5300  os << " ";
5301  } else {
5302  os << "\n";
5303  }
5304  os << "WITH (" + boost::algorithm::join(with_options, ", ") + ")";
5305  }
5306  os << ";";
5307  return os.str();
5308 }
int32_t maxRollbackEpochs
std::string partitions
std::string tableName
std::string join(T const &container, std::string const &delim)
#define DEFAULT_MAX_CHUNK_SIZE
std::string to_string(char const *&&v)
std::string quoteIfRequired(const std::string &column_name) const
Definition: Catalog.cpp:5431
#define DEFAULT_MAX_ROWS
void gatherAdditionalInfo(std::vector< std::string > &additional_info, std::set< std::string > &shared_dict_column_names, const TableDescriptor *td) const
Definition: Catalog.cpp:5442
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
#define DEFAULT_MAX_ROLLBACK_EPOCHS
specifies the content in-memory of a row in the table metadata table
#define DEFAULT_PAGE_SIZE
Definition: sqltypes.h:52
#define DEFAULT_FRAGMENT_ROWS
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:2228
Data_Namespace::MemoryLevel persistenceLevel
Definition: sqltypes.h:41
#define CHECK(condition)
Definition: Logger.h:222
std::string viewSQL
size_t g_leaf_count
Definition: ParserNode.cpp:76

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::dumpSchema ( const TableDescriptor td) const

Definition at line 5004 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, DEFAULT_MAX_ROLLBACK_EPOCHS, ColumnDescriptor::default_value, TableDescriptor::fragPageSize, g_leaf_count, ColumnDescriptor::getDefaultValueLiteral(), TableDescriptor::hasDeletedCol, TableDescriptor::is_system_table, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, join(), kCHAR, kENCODING_DICT, kENCODING_GEOINT, kTEXT, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::tableId, and to_string().

Referenced by TableArchiver::dumpTable().

5004  {
5005  CHECK(!td->is_system_table);
5006  cat_read_lock read_lock(this);
5007 
5008  std::ostringstream os;
5009  os << "CREATE TABLE @T (";
5010  // gather column defines
5011  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
5012  std::string comma;
5013  std::vector<std::string> shared_dicts;
5014  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
5015  for (const auto cd : cds) {
5016  if (!(cd->isSystemCol || cd->isVirtualCol)) {
5017  const auto& ti = cd->columnType;
5018  os << comma << cd->columnName;
5019  // CHAR is perculiar... better dump it as TEXT(32) like \d does
5020  if (ti.get_type() == SQLTypes::kCHAR) {
5021  os << " "
5022  << "TEXT";
5023  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
5024  os << " "
5025  << "TEXT[]";
5026  } else {
5027  os << " " << ti.get_type_name();
5028  }
5029  os << (ti.get_notnull() ? " NOT NULL" : "");
5030  if (cd->default_value.has_value()) {
5031  os << " DEFAULT " << cd->getDefaultValueLiteral();
5032  }
5033  if (ti.is_string() || (ti.is_array() && ti.get_subtype() == kTEXT)) {
5034  auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
5035  if (ti.get_compression() == kENCODING_DICT) {
5036  // if foreign reference, get referenced tab.col
5037  const auto dict_id = ti.get_comp_param();
5038  const DictRef dict_ref(currentDB_.dbId, dict_id);
5039  const auto dict_it = dictDescriptorMapByRef_.find(dict_ref);
5040  CHECK(dict_it != dictDescriptorMapByRef_.end());
5041  const auto dict_name = dict_it->second->dictName;
5042  // when migrating a table, any foreign dict ref will be dropped
5043  // and the first cd of a dict will become root of the dict
5044  if (dict_root_cds.end() == dict_root_cds.find(dict_name)) {
5045  dict_root_cds[dict_name] = cd;
5046  os << " ENCODING " << ti.get_compression_name() << "(" << (size * 8) << ")";
5047  } else {
5048  const auto dict_root_cd = dict_root_cds[dict_name];
5049  shared_dicts.push_back("SHARED DICTIONARY (" + cd->columnName +
5050  ") REFERENCES @T(" + dict_root_cd->columnName + ")");
5051  // "... shouldn't specify an encoding, it borrows from the referenced
5052  // column"
5053  }
5054  } else {
5055  os << " ENCODING NONE";
5056  }
5057  } else if (ti.is_date_in_days() ||
5058  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
5059  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
5060  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
5061  } else if (ti.is_geometry()) {
5062  if (ti.get_compression() == kENCODING_GEOINT) {
5063  os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
5064  << ")";
5065  } else {
5066  os << " ENCODING NONE";
5067  }
5068  }
5069  comma = ", ";
5070  }
5071  }
5072  // gather SHARED DICTIONARYs
5073  if (shared_dicts.size()) {
5074  os << ", " << boost::algorithm::join(shared_dicts, ", ");
5075  }
5076  // gather WITH options ...
5077  std::vector<std::string> with_options;
5078  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
5079  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
5080  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
5081  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
5082  with_options.emplace_back(td->hasDeletedCol ? "VACUUM='DELAYED'"
5083  : "VACUUM='IMMEDIATE'");
5084  if (!td->partitions.empty()) {
5085  with_options.push_back("PARTITIONS='" + td->partitions + "'");
5086  }
5087  if (td->nShards > 0) {
5088  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
5089  CHECK(shard_cd);
5090  os << ", SHARD KEY(" << shard_cd->columnName << ")";
5091  with_options.push_back(
5092  "SHARD_COUNT=" +
5093  std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
5094  }
5095  if (td->sortedColumnId > 0) {
5096  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
5097  CHECK(sort_cd);
5098  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
5099  }
5101  td->maxRollbackEpochs != -1) {
5102  with_options.push_back("MAX_ROLLBACK_EPOCHS=" +
5104  }
5105  os << ") WITH (" + boost::algorithm::join(with_options, ", ") + ");";
5106  return os.str();
5107 }
int32_t maxRollbackEpochs
std::string partitions
heavyai::shared_lock< heavyai::shared_mutex > read_lock
const DBMetadata currentDB_
Definition: Catalog.h:687
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:677
std::string join(T const &container, std::string const &delim)
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:123
std::string to_string(char const *&&v)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
#define DEFAULT_MAX_ROLLBACK_EPOCHS
specifies the content in-memory of a row in the table metadata table
Definition: sqltypes.h:52
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:2228
Definition: sqltypes.h:41
#define CHECK(condition)
Definition: Logger.h:222
size_t g_leaf_count
Definition: ParserNode.cpp:76

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::eraseDbMetadata ( )

Definition at line 4781 of file Catalog.cpp.

References shared::kCatalogDirectoryName, and run_benchmark_import::tables.

4781  {
4782  const auto tables = getAllTableMetadata();
4783  for (const auto table : tables) {
4784  eraseTableMetadata(table);
4785  }
4786  // Physically erase database metadata
4787  boost::filesystem::remove(basePath_ + "/" + shared::kCatalogDirectoryName + "/" +
4788  currentDB_.dbName);
4789  calciteMgr_->updateMetadata(currentDB_.dbName, "");
4790 }
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:2244
const DBMetadata currentDB_
Definition: Catalog.h:687
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:691
const std::string kCatalogDirectoryName
void eraseTableMetadata(const TableDescriptor *td)
Definition: Catalog.cpp:4037
void Catalog_Namespace::Catalog::eraseDbPhysicalData ( )

Definition at line 4792 of file Catalog.cpp.

References run_benchmark_import::tables.

4792  {
4793  const auto tables = getAllTableMetadata();
4794  for (const auto table : tables) {
4795  eraseTablePhysicalData(table);
4796  }
4797 }
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:4799
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:2244
void Catalog_Namespace::Catalog::eraseTableMetadata ( const TableDescriptor td)
protected

Definition at line 4037 of file Catalog.cpp.

References g_serialize_temp_tables, INJECT_TIMER, table_is_temporary(), TableDescriptor::tableId, and TableDescriptor::tableName.

4037  {
4041  }
4042  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
4043  {
4044  INJECT_TIMER(removeTableFromMap_);
4046  }
4047 }
void executeDropTableSqliteQueries(const TableDescriptor *td)
Definition: Catalog.cpp:4049
std::string tableName
const DBMetadata currentDB_
Definition: Catalog.h:687
void dropTableFromJsonUnlocked(const std::string &table_name) const
Definition: Catalog.cpp:2985
#define INJECT_TIMER(DESC)
Definition: measure.h:93
bool g_serialize_temp_tables
Definition: Catalog.cpp:108
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:691
bool table_is_temporary(const TableDescriptor *const td)
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1657

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::eraseTablePhysicalData ( const TableDescriptor td)

Definition at line 4799 of file Catalog.cpp.

References Data_Namespace::CPU_LEVEL, Data_Namespace::GPU_LEVEL, INJECT_TIMER, TableDescriptor::isView, and TableDescriptor::tableId.

4799  {
4800  const int tableId = td->tableId;
4801  // must destroy fragmenter before deleteChunks is called.
4802  removeFragmenterForTable(tableId);
4803 
4804  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
4805  {
4806  INJECT_TIMER(deleteChunksWithPrefix);
4807  // assuming deleteChunksWithPrefix is atomic
4808  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
4809  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
4810  }
4811  if (!td->isView) {
4812  INJECT_TIMER(Remove_Table);
4813  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
4814  }
4815 }
std::vector< int > ChunkKey
Definition: types.h:36
const DBMetadata currentDB_
Definition: Catalog.h:687