OmniSciDB  ca0c39ec8f
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Catalog_Namespace::Catalog Class Referencefinal

class for a per-database catalog. also includes metadata for the current database and the current user. More...

#include <Catalog.h>

+ Collaboration diagram for Catalog_Namespace::Catalog:

Public Member Functions

 Catalog (const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
 Constructor - takes basePath to already extant data directory for writing. More...
 
 ~Catalog ()
 Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite. More...
 
void createTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
 
void createShardedTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
int32_t createDashboard (DashboardDescriptor &vd)
 
void replaceDashboard (DashboardDescriptor &vd)
 
std::string createLink (LinkDescriptor &ld, size_t min_length)
 
void dropTable (const TableDescriptor *td)
 
void truncateTable (const TableDescriptor *td)
 
void renameTable (const TableDescriptor *td, const std::string &newTableName)
 
void renameTable (std::vector< std::pair< std::string, std::string >> &names)
 
void renameColumn (const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
 
void addColumn (const TableDescriptor &td, ColumnDescriptor &cd)
 
void dropColumn (const TableDescriptor &td, const ColumnDescriptor &cd)
 
void invalidateCachesForTable (const int table_id)
 
void removeFragmenterForTable (const int table_id) const
 
const std::map< int, const
ColumnDescriptor * > 
getDictionaryToColumnMapping ()
 
const TableDescriptorgetMetadataForTable (const std::string &tableName, const bool populateFragmenter=true) const
 Returns a pointer to a const TableDescriptor struct matching the provided tableName. More...
 
const TableDescriptorgetMetadataForTable (int tableId, bool populateFragmenter=true) const
 
std::optional< std::string > getTableName (int32_t table_id) const
 
std::optional< int32_t > getTableId (const std::string &table_name) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, const std::string &colName) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, int columnId) const
 
const std::optional< std::string > getColumnName (int table_id, int column_id) const
 
const int getColumnIdBySpi (const int tableId, const size_t spi) const
 
const ColumnDescriptorgetMetadataForColumnBySpi (const int tableId, const size_t spi) const
 
const DashboardDescriptorgetMetadataForDashboard (const std::string &userId, const std::string &dashName) const
 
const DashboardDescriptorgetMetadataForDashboard (const int32_t dashboard_id) const
 
void deleteMetadataForDashboards (const std::vector< int32_t > ids, const UserMetadata &user)
 
const LinkDescriptorgetMetadataForLink (const std::string &link) const
 
const LinkDescriptorgetMetadataForLink (int linkId) const
 
const
foreign_storage::ForeignTable
getForeignTable (const std::string &tableName) const
 
const
foreign_storage::ForeignTable
getForeignTable (int table_id) const
 
std::list< const
ColumnDescriptor * > 
getAllColumnMetadataForTable (const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular table specified by table id. More...
 
std::list< const
TableDescriptor * > 
getAllTableMetadata () const
 
std::vector< TableDescriptorgetAllTableMetadataCopy () const
 
std::list< const
DashboardDescriptor * > 
getAllDashboardsMetadata () const
 
std::vector< DashboardDescriptorgetAllDashboardsMetadataCopy () const
 
const DBMetadatagetCurrentDB () const
 
Data_Namespace::DataMgrgetDataMgr () const
 
std::shared_ptr< CalcitegetCalciteMgr () const
 
const std::string & getCatalogBasePath () const
 
const DictDescriptorgetMetadataForDict (int dict_ref, bool loadDict=true) const
 
const std::vector< LeafHostInfo > & getStringDictionaryHosts () const
 
const ColumnDescriptorgetShardColumnMetadataForTable (const TableDescriptor *td) const
 
std::vector< const
TableDescriptor * > 
getPhysicalTablesDescriptors (const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
 
std::vector< std::pair
< int32_t, int32_t > > 
getAllPersistedTableAndShardIds () const
 
std::vector< std::string > getTableNamesForUser (const UserMetadata &user, const GetTablesType get_tables_type) const
 
std::vector< TableMetadatagetTablesMetadataForUser (const UserMetadata &user_metadata, const GetTablesType get_tables_type, const std::string &filter_table_name) const
 
int32_t getTableEpoch (const int32_t db_id, const int32_t table_id) const
 
void setTableEpoch (const int db_id, const int table_id, const int new_epoch)
 
void setMaxRollbackEpochs (const int32_t table_id, const int32_t max_rollback_epochs)
 
void setMaxRows (const int32_t table_id, const int64_t max_rows)
 
std::vector< TableEpochInfogetTableEpochs (const int32_t db_id, const int32_t table_id) const
 
void setTableEpochs (const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
 
void setTableEpochsLogExceptions (const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
 
int getDatabaseId () const
 
SqliteConnectorgetSqliteConnector ()
 
void roll (const bool forward)
 
DictRef addDictionary (ColumnDescriptor &cd)
 
void delDictionary (const ColumnDescriptor &cd)
 
void getDictionary (const ColumnDescriptor &cd, std::map< int, StringDictionary * > &stringDicts)
 
const bool checkMetadataForDeletedRecs (const TableDescriptor *td, int column_id) const
 
const ColumnDescriptorgetDeletedColumn (const TableDescriptor *td) const
 
const ColumnDescriptorgetDeletedColumnIfRowsDeleted (const TableDescriptor *td) const
 
void setDeletedColumn (const TableDescriptor *td, const ColumnDescriptor *cd)
 
int getLogicalTableId (const int physicalTableId) const
 
void checkpoint (const int logicalTableId) const
 
void checkpointWithAutoRollback (const int logical_table_id) const
 
void resetTableEpochFloor (const int logicalTableId) const
 
std::string name () const
 
void eraseDbMetadata ()
 
void eraseDbPhysicalData ()
 
void eraseTablePhysicalData (const TableDescriptor *td)
 
void setForReload (const int32_t tableId)
 
std::vector< std::string > getTableDataDirectories (const TableDescriptor *td) const
 
std::vector< std::string > getTableDictDirectories (const TableDescriptor *td) const
 
std::set< std::string > getTableDictDirectoryPaths (int32_t table_id) const
 
std::string getColumnDictDirectory (const ColumnDescriptor *cd, bool file_name_only=true) const
 
std::string dumpSchema (const TableDescriptor *td) const
 
std::string dumpCreateTable (const TableDescriptor *td, bool multiline_formatting=true, bool dump_defaults=false) const
 
std::optional< std::string > dumpCreateTable (int32_t table_id, bool multiline_formatting=true, bool dump_defaults=false) const
 
std::string dumpCreateServer (const std::string &name, bool multiline_formatting=true) const
 
void createForeignServer (std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
 
const
foreign_storage::ForeignServer
getForeignServer (const std::string &server_name) const
 
const std::unique_ptr< const
foreign_storage::ForeignServer
getForeignServerFromStorage (const std::string &server_name)
 
const std::unique_ptr< const
foreign_storage::ForeignTable
getForeignTableFromStorage (int table_id)
 
void changeForeignServerOwner (const std::string &server_name, const int new_owner_id)
 
void setForeignServerDataWrapper (const std::string &server_name, const std::string &data_wrapper)
 
void setForeignServerOptions (const std::string &server_name, const std::string &options)
 
void renameForeignServer (const std::string &server_name, const std::string &name)
 
void dropForeignServer (const std::string &server_name)
 
std::vector< const
foreign_storage::ForeignTable * > 
getAllForeignTablesForForeignServer (const int32_t foreign_server_id)
 Get all of the foreign tables for associated with a foreign server id. More...
 
void getForeignServersForUser (const rapidjson::Value *filters, const UserMetadata &user, std::vector< const foreign_storage::ForeignServer * > &results)
 
void createDefaultServersIfNotExists ()
 
bool validateNonExistentTableOrView (const std::string &name, const bool if_not_exists)
 
std::vector< const
TableDescriptor * > 
getAllForeignTablesForRefresh () const
 
void updateForeignTableRefreshTimes (const int32_t table_id)
 
void setForeignTableOptions (const std::string &table_name, foreign_storage::OptionsMap &options_map, bool clear_existing_options=true)
 
void updateLeaf (const LeafHostInfo &string_dict_host)
 
void setUncappedTableEpoch (const std::string &table_name)
 
int32_t createCustomExpression (std::unique_ptr< CustomExpression > custom_expression)
 
const CustomExpressiongetCustomExpression (int32_t custom_expression_id) const
 
const std::unique_ptr< const
CustomExpression
getCustomExpressionFromStorage (int32_t custom_expression_id)
 
std::vector< const
CustomExpression * > 
getCustomExpressionsForUser (const UserMetadata &user) const
 
void updateCustomExpression (int32_t custom_expression_id, const std::string &expression_json)
 
void deleteCustomExpressions (const std::vector< int32_t > &custom_expression_ids, bool do_soft_delete)
 
void reassignOwners (const std::set< std::string > &old_owners, const std::string &new_owner)
 
bool isInfoSchemaDb () const
 

Static Public Member Functions

static void expandGeoColumn (const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
 
static const std::string getForeignTableSchema (bool if_not_exists=false)
 
static const std::string getForeignServerSchema (bool if_not_exists=false)
 
static const std::string getCustomExpressionsSchema (bool if_not_exists=false)
 

Public Attributes

std::unique_ptr
< heavyai::DistributedSharedMutex
dcatalogMutex_
 
std::unique_ptr
< heavyai::DistributedSharedMutex
dsqliteMutex_
 
std::mutex sqliteMutex_
 
heavyai::shared_mutex sharedMutex_
 
std::atomic< std::thread::id > thread_holding_sqlite_lock
 
std::atomic< std::thread::id > thread_holding_write_lock
 
bool initialized_ = false
 

Static Public Attributes

static thread_local bool thread_holds_read_lock = false
 

Protected Types

using ColumnDescriptorsForRoll = std::vector< std::pair< ColumnDescriptor *, ColumnDescriptor * >>
 

Protected Member Functions

void CheckAndExecuteMigrations ()
 
void CheckAndExecuteMigrationsPostBuildMaps ()
 
void updateDictionaryNames ()
 
void updateTableDescriptorSchema ()
 
void updateFixlenArrayColumns ()
 
void updateGeoColumns ()
 
void updateFrontendViewSchema ()
 
void updateLinkSchema ()
 
void updateFrontendViewAndLinkUsers ()
 
void updateLogicalToPhysicalTableLinkSchema ()
 
void updateLogicalToPhysicalTableMap (const int32_t logical_tb_id)
 
void updateDictionarySchema ()
 
void updatePageSize ()
 
void updateDeletedColumnIndicator ()
 
void updateDefaultColumnValues ()
 
void updateFrontendViewsToDashboards ()
 
void updateCustomExpressionsSchema ()
 
void updateFsiSchemas ()
 
void renameLegacyDataWrappers ()
 
void recordOwnershipOfObjectsInObjectPermissions ()
 
void checkDateInDaysColumnMigration ()
 
void createDashboardSystemRoles ()
 
void buildMaps ()
 
void addTableToMap (const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
 
void addReferenceToForeignDict (ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def, const bool persist_reference)
 
bool setColumnSharedDictionary (ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
void setColumnDictionary (ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, bool is_logical_table, bool use_temp_dictionary=false)
 
void addFrontendViewToMap (DashboardDescriptor &vd)
 
void addFrontendViewToMapNoLock (DashboardDescriptor &vd)
 
void addLinkToMap (LinkDescriptor &ld)
 
void removeTableFromMap (const std::string &tableName, const int tableId, const bool is_on_error=false)
 
void eraseTableMetadata (const TableDescriptor *td)
 
void executeDropTableSqliteQueries (const TableDescriptor *td)
 
void doTruncateTable (const TableDescriptor *td)
 
void renamePhysicalTable (const TableDescriptor *td, const std::string &newTableName)
 
void renamePhysicalTable (std::vector< std::pair< std::string, std::string >> &names, std::vector< int > &tableIds)
 
void instantiateFragmenter (TableDescriptor *td) const
 
void getAllColumnMetadataForTableImpl (const TableDescriptor *td, std::list< const ColumnDescriptor * > &colDescs, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 
std::string calculateSHA1 (const std::string &data)
 
std::string generatePhysicalTableName (const std::string &logicalTableName, const int32_t &shardNumber)
 
std::vector< DBObjectparseDashboardObjects (const std::string &view_meta, const int &user_id)
 
void createOrUpdateDashboardSystemRole (const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
 
const int getColumnIdBySpiUnlocked (const int table_id, const size_t spi) const
 
void serializeTableJsonUnlocked (const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
 
void dropTableFromJsonUnlocked (const std::string &table_name) const
 

Protected Attributes

std::string basePath_
 
TableDescriptorMap tableDescriptorMap_
 
TableDescriptorMapById tableDescriptorMapById_
 
ColumnDescriptorMap columnDescriptorMap_
 
ColumnDescriptorMapById columnDescriptorMapById_
 
DictDescriptorMapById dictDescriptorMapByRef_
 
DashboardDescriptorMap dashboardDescriptorMap_
 
LinkDescriptorMap linkDescriptorMap_
 
LinkDescriptorMapById linkDescriptorMapById_
 
ForeignServerMap foreignServerMap_
 
ForeignServerMapById foreignServerMapById_
 
CustomExpressionMapById custom_expr_map_by_id_
 
TableDictColumnsMap dict_columns_by_table_id_
 
SqliteConnector sqliteConnector_
 
const DBMetadata currentDB_
 
std::shared_ptr
< Data_Namespace::DataMgr
dataMgr_
 
const std::vector< LeafHostInfostring_dict_hosts_
 
std::shared_ptr< CalcitecalciteMgr_
 
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
 
int nextTempTableId_
 
int nextTempDictId_
 
ColumnDescriptorsForRoll columnDescriptorsForRoll
 

Static Protected Attributes

static const std::string physicalTableNameTag_
 

Private Member Functions

void buildDictionaryMapUnlocked ()
 
void reloadTableMetadata (int table_id)
 
void reloadTableMetadataUnlocked (int table_id)
 
void reloadCatalogMetadata (const std::map< int32_t, std::string > &user_name_by_user_id)
 
void reloadCatalogMetadataUnlocked (const std::map< int32_t, std::string > &user_name_by_user_id)
 
void buildTablesMapUnlocked ()
 
void buildColumnsMapUnlocked ()
 
void updateViewsInMapUnlocked ()
 
void buildDashboardsMapUnlocked (const std::map< int32_t, std::string > &user_name_by_user_id)
 
void buildLinksMapUnlocked ()
 
void buildLogicalToPhysicalMapUnlocked ()
 
void updateForeignTablesInMapUnlocked ()
 
void gatherAdditionalInfo (std::vector< std::string > &additional_info, std::set< std::string > &shared_dict_column_names, const TableDescriptor *td) const
 
std::string quoteIfRequired (const std::string &column_name) const
 
void adjustAlteredTableFiles (const std::string &temp_data_dir, const std::unordered_map< int, int > &all_column_ids_map) const
 
void renameTableDirectories (const std::string &temp_data_dir, const std::vector< std::string > &target_paths, const std::string &name_prefix) const
 
void buildForeignServerMapUnlocked ()
 
void setForeignServerProperty (const std::string &server_name, const std::string &property, const std::string &value)
 
void setForeignTableProperty (const foreign_storage::ForeignTable *table, const std::string &property, const std::string &value)
 
void alterPhysicalTableMetadata (const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
 
void alterTableMetadata (const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
 
void setTableFileMgrParams (const int table_id, const File_Namespace::FileMgrParams &file_mgr_params)
 
bool filterTableByTypeAndUser (const TableDescriptor *td, const UserMetadata &user_metadata, const GetTablesType get_tables_type) const
 
TableDescriptorgetMutableMetadataForTableUnlocked (int table_id) const
 
TableDescriptorgetMutableMetadataForTableUnlocked (const std::string &table_name) const
 
void createForeignServerNoLocks (std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
 
foreign_storage::ForeignTablegetForeignTableUnlocked (const std::string &tableName) const
 
void removeChunks (const int table_id) const
 
void buildCustomExpressionsMapUnlocked ()
 
std::unique_ptr< CustomExpressiongetCustomExpressionFromConnector (size_t row)
 
void restoreOldOwners (const std::map< int32_t, std::string > &old_owners_user_name_by_id, const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects, int32_t new_owner_id)
 
void restoreOldOwnersInMemory (const std::map< int32_t, std::string > &old_owners_user_name_by_id, const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects, int32_t new_owner_id)
 
void conditionallyInitializeSystemObjects ()
 
void initializeSystemServers ()
 
void initializeSystemTables ()
 
void initializeUsersSystemTable ()
 
void initializeDatabasesSystemTable ()
 
void initializePermissionsSystemTable ()
 
void initializeRolesSystemTable ()
 
void initializeTablesSystemTable ()
 
void initializeDashboardsSystemTable ()
 
void initializeRoleAssignmentsSystemTable ()
 
void initializeMemorySummarySystemTable ()
 
void initializeMemoryDetailsSystemTable ()
 
void initializeStorageDetailsSystemTable ()
 
void initializeServerLogsSystemTables ()
 
void initializeRequestLogsSystemTables ()
 
void initializeWebServerLogsSystemTables ()
 
void initializeWebServerAccessLogsSystemTables ()
 
void createSystemTableServer (const std::string &server_name, const std::string &data_wrapper_type, const foreign_storage::OptionsMap &options={})
 
std::pair
< foreign_storage::ForeignTable,
std::list< ColumnDescriptor > > 
getSystemTableSchema (const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name, bool is_in_memory_system_table)
 
bool recreateSystemTableIfUpdated (foreign_storage::ForeignTable &foreign_table, const std::list< ColumnDescriptor > &columns)
 
void setDeletedColumnUnlocked (const TableDescriptor *td, const ColumnDescriptor *cd)
 
void addToColumnMap (ColumnDescriptor *cd)
 
void removeFromColumnMap (ColumnDescriptor *cd)
 
void deleteTableCatalogMetadata (const TableDescriptor *logical_table, const std::vector< const TableDescriptor * > &physical_tables)
 
std::string dumpCreateTableUnlocked (const TableDescriptor *td, bool multiline_formatting, bool dump_defaults) const
 

Private Attributes

DeletedColumnPerTableMap deletedColumnPerTable_
 

Static Private Attributes

static constexpr const char * CATALOG_SERVER_NAME {"system_catalog_server"}
 
static constexpr const char * MEMORY_STATS_SERVER_NAME {"system_memory_stats_server"}
 
static constexpr const char * STORAGE_STATS_SERVER_NAME {"system_storage_stats_server"}
 
static constexpr const char * LOGS_SERVER_NAME {"system_logs_server"}
 
static constexpr std::array
< const char *, 4 > 
INTERNAL_SERVERS
 

Friends

template<class T >
class lockmgr::TableLockMgrImpl
 

Detailed Description

class for a per-database catalog. also includes metadata for the current database and the current user.

Catalog

Definition at line 132 of file Catalog.h.

Member Typedef Documentation

Definition at line 704 of file Catalog.h.

Constructor & Destructor Documentation

Catalog_Namespace::Catalog::Catalog ( const std::string &  basePath,
const DBMetadata curDB,
std::shared_ptr< Data_Namespace::DataMgr dataMgr,
const std::vector< LeafHostInfo > &  string_dict_hosts,
std::shared_ptr< Calcite calcite,
bool  is_new_db 
)

Constructor - takes basePath to already extant data directory for writing.

Parameters
basePathdirectory path for writing catalog
dbNamename of the database
fragmenterFragmenter object metadata - expects for this directory to already exist

Definition at line 171 of file Catalog.cpp.

References dsqliteMutex_, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_user_id_to_user_name_map(), initialized_, and reloadCatalogMetadataUnlocked().

177  : basePath_(basePath)
178  , sqliteConnector_(curDB.dbName, basePath + "/" + shared::kCatalogDirectoryName + "/")
179  , currentDB_(curDB)
180  , dataMgr_(dataMgr)
181  , string_dict_hosts_(string_dict_hosts)
182  , calciteMgr_(calcite)
185  , dcatalogMutex_(std::make_unique<heavyai::DistributedSharedMutex>(
186  std::filesystem::path(basePath_) / shared::kLockfilesDirectoryName /
188  [this](size_t) {
189  if (!initialized_) {
190  return;
191  }
192  const auto user_name_by_user_id = get_user_id_to_user_name_map();
194  *dsqliteMutex_);
195  reloadCatalogMetadataUnlocked(user_name_by_user_id);
196  }))
const int MAPD_TEMP_TABLE_START_ID
Definition: Catalog.cpp:111
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:690
const DBMetadata currentDB_
Definition: Catalog.h:687
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:688
const int MAPD_TEMP_DICT_START_ID
Definition: Catalog.cpp:113
void reloadCatalogMetadataUnlocked(const std::map< int32_t, std::string > &user_name_by_user_id)
Definition: Catalog.cpp:1269
std::unique_lock< T > unique_lock
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:691
const std::string kCatalogDirectoryName
std::unique_ptr< heavyai::DistributedSharedMutex > dsqliteMutex_
Definition: Catalog.h:840
const std::string kLockfilesDirectoryName
std::unique_ptr< heavyai::DistributedSharedMutex > dcatalogMutex_
Definition: Catalog.h:839
std::map< int32_t, std::string > get_user_id_to_user_name_map()
Definition: Catalog.cpp:1027

+ Here is the call graph for this function:

Catalog_Namespace::Catalog::~Catalog ( )

Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite.

Definition at line 229 of file Catalog.cpp.

References basePath_, columnDescriptorMap_, currentDB_, Catalog_Namespace::DBMetadata::dbName, g_serialize_temp_tables, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and tableDescriptorMap_.

229  {
230  // cat_write_lock write_lock(this);
231 
232  // must clean up heap-allocated TableDescriptor and ColumnDescriptor structs
233  for (TableDescriptorMap::iterator tableDescIt = tableDescriptorMap_.begin();
234  tableDescIt != tableDescriptorMap_.end();
235  ++tableDescIt) {
236  tableDescIt->second->fragmenter = nullptr;
237  delete tableDescIt->second;
238  }
239 
240  // TableDescriptorMapById points to the same descriptors. No need to delete
241 
242  for (ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.begin();
243  columnDescIt != columnDescriptorMap_.end();
244  ++columnDescIt) {
245  delete columnDescIt->second;
246  }
247 
248  // ColumnDescriptorMapById points to the same descriptors. No need to delete
249 
251  boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
252  }
253 }
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:675
const DBMetadata currentDB_
Definition: Catalog.h:687
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:162
bool g_serialize_temp_tables
Definition: Catalog.cpp:106
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:673

+ Here is the call graph for this function:

Member Function Documentation

void Catalog_Namespace::Catalog::addColumn ( const TableDescriptor td,
ColumnDescriptor cd 
)

Definition at line 2379 of file Catalog.cpp.

References ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::default_value, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, kENCODING_DICT, TableDescriptor::nShards, TableDescriptor::shard, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), and ColumnDescriptor::virtualExpr.

2379  {
2380  // caller must handle sqlite/chunk transaction TOGETHER
2381  cd.tableId = td.tableId;
2382  if (td.nShards > 0 && td.shard < 0) {
2383  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
2384  auto shard_cd = cd;
2385  addColumn(*shard, shard_cd);
2386  }
2387  }
2389  addDictionary(cd);
2390  }
2391 
2392  using BindType = SqliteConnector::BindType;
2393  std::vector<BindType> types(17, BindType::TEXT);
2394  if (!cd.default_value.has_value()) {
2395  types[16] = BindType::NULL_TYPE;
2396  }
2398  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
2399  "colscale, is_notnull, "
2400  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
2401  "is_deletedcol, default_value) "
2402  "VALUES (?, "
2403  "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
2404  "?, ?, ?, "
2405  "?, "
2406  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2407  std::vector<std::string>{std::to_string(td.tableId),
2408  std::to_string(td.tableId),
2409  cd.columnName,
2418  "",
2421  cd.virtualExpr,
2423  cd.default_value.value_or("NULL")},
2424  types);
2425 
2427  "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
2428  std::vector<std::string>{std::to_string(td.tableId)});
2429 
2431  "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
2432  std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
2433  cd.columnId = sqliteConnector_.getData<int>(0, 0);
2434 
2435  ++tableDescriptorMapById_[td.tableId]->nColumns;
2436  auto ncd = new ColumnDescriptor(cd);
2437  addToColumnMap(ncd);
2438  columnDescriptorsForRoll.emplace_back(nullptr, ncd);
2439 }
std::string virtualExpr
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:380
HOST DEVICE int get_size() const
Definition: sqltypes.h:389
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
void addColumn(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:2379
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
HOST DEVICE int get_scale() const
Definition: sqltypes.h:384
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:379
std::string to_string(char const *&&v)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:674
specifies the content in-memory of a row in the column metadata table
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4630
std::optional< std::string > default_value
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:387
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:381
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:388
SQLTypeInfo columnType
DictRef addDictionary(ColumnDescriptor &cd)
Definition: Catalog.cpp:2283
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:386
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:705
std::string columnName
void addToColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6491

+ Here is the call graph for this function:

DictRef Catalog_Namespace::Catalog::addDictionary ( ColumnDescriptor cd)

Definition at line 2283 of file Catalog.cpp.

References CHECK, DictDescriptor::dictFolderPath, and ColumnDescriptor::tableId.

2283  {
2284  cat_write_lock write_lock(this);
2285  const auto& td = *tableDescriptorMapById_[cd.tableId];
2286  list<DictDescriptor> dds;
2287  setColumnDictionary(cd, dds, td, true);
2288  auto& dd = dds.back();
2289  CHECK(dd.dictRef.dictId);
2290 
2291  std::unique_ptr<StringDictionaryClient> client;
2292  if (!string_dict_hosts_.empty()) {
2293  client.reset(new StringDictionaryClient(
2294  string_dict_hosts_.front(), DictRef(currentDB_.dbId, -1), true));
2295  }
2296  if (client) {
2297  client->create(dd.dictRef, dd.dictIsTemp);
2298  }
2299 
2300  DictDescriptor* new_dd = new DictDescriptor(dd);
2301  dictDescriptorMapByRef_[dd.dictRef].reset(new_dd);
2302  if (!dd.dictIsTemp) {
2303  boost::filesystem::create_directory(new_dd->dictFolderPath);
2304  }
2305  return dd.dictRef;
2306 }
std::string dictFolderPath
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:690
const DBMetadata currentDB_
Definition: Catalog.h:687
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:677
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, bool is_logical_table, bool use_temp_dictionary=false)
Definition: Catalog.cpp:3820
heavyai::unique_lock< heavyai::shared_mutex > write_lock
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:674
struct dict_ref_t DictRef
Definition: DictRef.h:49
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:222
Descriptor for a dictionary for a string columne.
void Catalog_Namespace::Catalog::addFrontendViewToMap ( DashboardDescriptor vd)
protected

Definition at line 1739 of file Catalog.cpp.

1739  {
1740  cat_write_lock write_lock(this);
1742 }
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1744
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
void Catalog_Namespace::Catalog::addFrontendViewToMapNoLock ( DashboardDescriptor vd)
protected

Definition at line 1744 of file Catalog.cpp.

References DashboardDescriptor::dashboardName, to_string(), and DashboardDescriptor::userId.

1744  {
1745  cat_write_lock write_lock(this);
1747  std::make_shared<DashboardDescriptor>(vd);
1748 }
heavyai::unique_lock< heavyai::shared_mutex > write_lock
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:678
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addLinkToMap ( LinkDescriptor ld)
protected

Definition at line 1826 of file Catalog.cpp.

References LinkDescriptor::link, LinkDescriptor::linkId, and to_string().

1826  {
1827  cat_write_lock write_lock(this);
1828  LinkDescriptor* new_ld = new LinkDescriptor();
1829  *new_ld = ld;
1831  linkDescriptorMapById_[ld.linkId] = new_ld;
1832 }
const DBMetadata currentDB_
Definition: Catalog.h:687
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:680
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:679

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addReferenceToForeignDict ( ColumnDescriptor referencing_column,
Parser::SharedDictionaryDef  shared_dict_def,
const bool  persist_reference 
)
protected

Definition at line 3729 of file Catalog.cpp.

References CHECK, CHECK_GE, ColumnDescriptor::columnType, SQLTypeInfo::get_comp_param(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_foreign_col(), and to_string().

3731  {
3732  cat_write_lock write_lock(this);
3733  const auto foreign_ref_col = get_foreign_col(*this, shared_dict_def);
3734  CHECK(foreign_ref_col);
3735  referencing_column.columnType = foreign_ref_col->columnType;
3736  const int dict_id = referencing_column.columnType.get_comp_param();
3737  const DictRef dict_ref(currentDB_.dbId, dict_id);
3738  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
3739  CHECK(dictIt != dictDescriptorMapByRef_.end());
3740  const auto& dd = dictIt->second;
3741  CHECK_GE(dd->refcount, 1);
3742  ++dd->refcount;
3743  if (persist_reference) {
3744  cat_sqlite_lock sqlite_lock(this);
3746  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
3747  {std::to_string(dict_id)});
3748  }
3749 }
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
const DBMetadata currentDB_
Definition: Catalog.h:687
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:677
#define CHECK_GE(x, y)
Definition: Logger.h:235
heavyai::unique_lock< heavyai::shared_mutex > write_lock
const ColumnDescriptor * get_foreign_col(const Catalog &cat, const Parser::SharedDictionaryDef &shared_dict_def)
Definition: Catalog.cpp:3717
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:388
#define CHECK(condition)
Definition: Logger.h:222
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addTableToMap ( const TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::list< DictDescriptor > &  dicts 
)
protected

Definition at line 1598 of file Catalog.cpp.

References anonymous_namespace{Utm.h}::a, CHECK, TableDescriptor::columnIdBySpi_, DictDescriptor::dictFolderPath, dict_ref_t::dictId, TableDescriptor::hasDeletedCol, ColumnDescriptor::isDeletedCol, TableDescriptor::mutex_, gpu_enabled::sort(), TableDescriptor::tableId, TableDescriptor::tableName, and to_upper().

1600  {
1601  cat_write_lock write_lock(this);
1602  TableDescriptor* new_td;
1603 
1604  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
1605  if (foreign_table) {
1606  auto new_foreign_table = new foreign_storage::ForeignTable();
1607  *new_foreign_table = *foreign_table;
1608  new_td = new_foreign_table;
1609  } else {
1610  new_td = new TableDescriptor();
1611  *new_td = *td;
1612  }
1613 
1614  new_td->mutex_ = std::make_shared<std::mutex>();
1615  tableDescriptorMap_[to_upper(td->tableName)] = new_td;
1616  tableDescriptorMapById_[td->tableId] = new_td;
1617  for (auto cd : columns) {
1618  ColumnDescriptor* new_cd = new ColumnDescriptor();
1619  *new_cd = cd;
1620  addToColumnMap(new_cd);
1621 
1622  // Add deleted column to the map
1623  if (cd.isDeletedCol) {
1624  CHECK(new_td->hasDeletedCol);
1625  setDeletedColumnUnlocked(new_td, new_cd);
1626  }
1627  }
1628 
1629  std::sort(new_td->columnIdBySpi_.begin(),
1630  new_td->columnIdBySpi_.end(),
1631  [](const size_t a, const size_t b) -> bool { return a < b; });
1632  // TODO(sy): Why does addTableToMap() sort columnIdBySpi_ but not insert into it while
1633  // buildColumnsMapUnlocked() does both?
1634 
1635  std::unique_ptr<StringDictionaryClient> client;
1636  DictRef dict_ref(currentDB_.dbId, -1);
1637  if (!string_dict_hosts_.empty()) {
1638  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1639  }
1640  for (auto dd : dicts) {
1641  if (!dd.dictRef.dictId) {
1642  // Dummy entry created for a shard of a logical table, nothing to do.
1643  continue;
1644  }
1645  dict_ref.dictId = dd.dictRef.dictId;
1646  if (client) {
1647  client->create(dict_ref, dd.dictIsTemp);
1648  }
1649  DictDescriptor* new_dd = new DictDescriptor(dd);
1650  dictDescriptorMapByRef_[dict_ref].reset(new_dd);
1651  if (!dd.dictIsTemp) {
1652  boost::filesystem::create_directory(new_dd->dictFolderPath);
1653  }
1654  }
1655 }
std::string dictFolderPath
std::string tableName
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:690
const DBMetadata currentDB_
Definition: Catalog.h:687
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:677
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::vector< int > columnIdBySpi_
std::shared_ptr< std::mutex > mutex_
constexpr double a
Definition: Utm.h:32
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3709
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:674
specifies the content in-memory of a row in the column metadata table
std::string to_upper(const std::string &str)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:222
Descriptor for a dictionary for a string columne.
void addToColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6491
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:673

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addToColumnMap ( ColumnDescriptor cd)
private

Definition at line 6491 of file Catalog.cpp.

References ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::is_dict_encoded_type(), ColumnDescriptor::tableId, and to_upper().

6491  {
6494  if (cd->columnType.is_dict_encoded_type()) {
6495  dict_columns_by_table_id_[cd->tableId].emplace(cd);
6496  }
6497 }
TableDictColumnsMap dict_columns_by_table_id_
Definition: Catalog.h:684
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:675
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:676
bool is_dict_encoded_type() const
Definition: sqltypes.h:639
std::string to_upper(const std::string &str)
SQLTypeInfo columnType
std::string columnName
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::adjustAlteredTableFiles ( const std::string &  temp_data_dir,
const std::unordered_map< int, int > &  all_column_ids_map 
) const
private
void Catalog_Namespace::Catalog::alterPhysicalTableMetadata ( const TableDescriptor td,
const TableDescriptorUpdateParams table_update_params 
)
private

Definition at line 3443 of file Catalog.cpp.

References CHECK, TableDescriptorUpdateParams::max_rollback_epochs, TableDescriptorUpdateParams::max_rows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::tableId, and to_string().

3445  {
3446  // Only called from parent alterTableParamMetadata, expect already to have catalog and
3447  // sqlite write locks
3448 
3449  // Sqlite transaction should have already been begun in parent alterTableCatalogMetadata
3450 
3452  CHECK(mutable_td);
3453  if (td->maxRollbackEpochs != table_update_params.max_rollback_epochs) {
3455  "UPDATE mapd_tables SET max_rollback_epochs = ? WHERE tableid = ?",
3456  std::vector<std::string>{std::to_string(table_update_params.max_rollback_epochs),
3457  std::to_string(td->tableId)});
3458  mutable_td->maxRollbackEpochs = table_update_params.max_rollback_epochs;
3459  }
3460 
3461  if (td->maxRows != table_update_params.max_rows) {
3463  "UPDATE mapd_tables SET max_rows = ? WHERE tableid = ?",
3464  std::vector<std::string>{std::to_string(table_update_params.max_rows),
3465  std::to_string(td->tableId)});
3466  mutable_td->maxRows = table_update_params.max_rows;
3467  }
3468 }
int32_t maxRollbackEpochs
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
TableDescriptor * getMutableMetadataForTableUnlocked(int table_id) const
Definition: Catalog.cpp:1952
std::string to_string(char const *&&v)
#define CHECK(condition)
Definition: Logger.h:222

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::alterTableMetadata ( const TableDescriptor td,
const TableDescriptorUpdateParams table_update_params 
)
private

Definition at line 3470 of file Catalog.cpp.

References CHECK, logger::FATAL, LOG, TableDescriptor::tableId, and TableDescriptor::tableName.

3471  {
3472  cat_write_lock write_lock(this);
3473  cat_sqlite_lock sqlite_lock(this);
3474  sqliteConnector_.query("BEGIN TRANSACTION");
3475  try {
3476  const auto physical_table_it = logicalToPhysicalTableMapById_.find(td->tableId);
3477  if (physical_table_it != logicalToPhysicalTableMapById_.end()) {
3478  const auto physical_tables = physical_table_it->second;
3479  CHECK(!physical_tables.empty());
3480  for (size_t i = 0; i < physical_tables.size(); i++) {
3481  int32_t physical_tb_id = physical_tables[i];
3482  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id, false);
3483  CHECK(phys_td);
3484  alterPhysicalTableMetadata(phys_td, table_update_params);
3485  }
3486  }
3487  alterPhysicalTableMetadata(td, table_update_params);
3488  } catch (std::exception& e) {
3489  sqliteConnector_.query("ROLLBACK TRANSACTION");
3490  LOG(FATAL) << "Table '" << td->tableName << "' catalog update failed";
3491  }
3492  sqliteConnector_.query("END TRANSACTION");
3493 }
std::string tableName
#define LOG(tag)
Definition: Logger.h:216
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void alterPhysicalTableMetadata(const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
Definition: Catalog.cpp:3443
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:222
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:693
void Catalog_Namespace::Catalog::buildColumnsMapUnlocked ( )
private

Definition at line 1401 of file Catalog.cpp.

References anonymous_namespace{Utm.h}::a, CHECK, ColumnDescriptor::chunks, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::default_value, SQLTypeInfo::get_physical_cols(), SQLTypeInfo::is_geometry(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_compression(), SQLTypeInfo::set_dimension(), SQLTypeInfo::set_notnull(), SQLTypeInfo::set_scale(), SQLTypeInfo::set_size(), SQLTypeInfo::set_subtype(), SQLTypeInfo::set_type(), gpu_enabled::sort(), ColumnDescriptor::tableId, and ColumnDescriptor::virtualExpr.

1401  {
1402  std::string columnQuery(
1403  "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
1404  "is_notnull, compression, comp_param, "
1405  "size, chunks, is_systemcol, is_virtualcol, virtual_expr, is_deletedcol, "
1406  "default_value from "
1407  "mapd_columns ORDER BY tableid, "
1408  "columnid");
1409  sqliteConnector_.query(columnQuery);
1410  auto numRows = sqliteConnector_.getNumRows();
1411  int32_t skip_physical_cols = 0;
1412  for (size_t r = 0; r < numRows; ++r) {
1413  ColumnDescriptor* cd = new ColumnDescriptor();
1414  cd->tableId = sqliteConnector_.getData<int>(r, 0);
1415  cd->columnId = sqliteConnector_.getData<int>(r, 1);
1416  cd->columnName = sqliteConnector_.getData<string>(r, 2);
1420  cd->columnType.set_scale(sqliteConnector_.getData<int>(r, 6));
1424  cd->columnType.set_size(sqliteConnector_.getData<int>(r, 10));
1425  cd->chunks = sqliteConnector_.getData<string>(r, 11);
1426  cd->isSystemCol = sqliteConnector_.getData<bool>(r, 12);
1427  cd->isVirtualCol = sqliteConnector_.getData<bool>(r, 13);
1428  cd->virtualExpr = sqliteConnector_.getData<string>(r, 14);
1429  cd->isDeletedCol = sqliteConnector_.getData<bool>(r, 15);
1430  if (sqliteConnector_.isNull(r, 16)) {
1431  cd->default_value = std::nullopt;
1432  } else {
1433  cd->default_value = std::make_optional(sqliteConnector_.getData<string>(r, 16));
1434  }
1435  cd->isGeoPhyCol = skip_physical_cols > 0;
1436  addToColumnMap(cd);
1437 
1438  if (skip_physical_cols <= 0) {
1439  skip_physical_cols = cd->columnType.get_physical_cols();
1440  }
1441 
1442  auto td_itr = tableDescriptorMapById_.find(cd->tableId);
1443  CHECK(td_itr != tableDescriptorMapById_.end());
1444 
1445  if (cd->isDeletedCol) {
1446  td_itr->second->hasDeletedCol = true;
1447  setDeletedColumnUnlocked(td_itr->second, cd);
1448  } else if (cd->columnType.is_geometry() || skip_physical_cols-- <= 0) {
1449  tableDescriptorMapById_[cd->tableId]->columnIdBySpi_.push_back(cd->columnId);
1450  }
1451  }
1452 
1453  // sort columnIdBySpi_ based on columnId
1454  for (auto& tit : tableDescriptorMapById_) {
1455  std::sort(tit.second->columnIdBySpi_.begin(),
1456  tit.second->columnIdBySpi_.end(),
1457  [](const size_t a, const size_t b) -> bool { return a < b; });
1458  }
1459 } // namespace Catalog_Namespace
std::string virtualExpr
void set_compression(EncodingType c)
Definition: sqltypes.h:500
void set_size(int s)
Definition: sqltypes.h:497
T getData(const int row, const int col)
SQLTypes
Definition: sqltypes.h:53
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:490
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
virtual void query(const std::string &queryString)
std::string chunks
constexpr double a
Definition: Utm.h:32
EncodingType
Definition: sqltypes.h:228
int get_physical_cols() const
Definition: sqltypes.h:410
void set_scale(int s)
Definition: sqltypes.h:494
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3709
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:674
specifies the content in-memory of a row in the column metadata table
void set_comp_param(int p)
Definition: sqltypes.h:501
std::optional< std::string > default_value
bool isNull(const int row, const int col) const
void set_dimension(int d)
Definition: sqltypes.h:491
void set_notnull(bool n)
Definition: sqltypes.h:496
#define CHECK(condition)
Definition: Logger.h:222
bool is_geometry() const
Definition: sqltypes.h:587
SQLTypeInfo columnType
std::string columnName
void addToColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6491
virtual size_t getNumRows() const
HOST DEVICE void set_type(SQLTypes t)
Definition: sqltypes.h:489

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildCustomExpressionsMapUnlocked ( )
private

Definition at line 1570 of file Catalog.cpp.

1570  {
1572  "SELECT id, name, expression_json, data_source_type, data_source_id, "
1573  "is_deleted "
1574  "FROM omnisci_custom_expressions");
1575  auto num_rows = sqliteConnector_.getNumRows();
1576  for (size_t row = 0; row < num_rows; row++) {
1577  auto custom_expr = getCustomExpressionFromConnector(row);
1578  custom_expr_map_by_id_[custom_expr->id] = std::move(custom_expr);
1579  }
1580 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
virtual void query(const std::string &queryString)
std::unique_ptr< CustomExpression > getCustomExpressionFromConnector(size_t row)
Definition: Catalog.cpp:1582
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:683
virtual size_t getNumRows() const
void Catalog_Namespace::Catalog::buildDashboardsMapUnlocked ( const std::map< int32_t, std::string > &  user_name_by_user_id)
private

Definition at line 1473 of file Catalog.cpp.

References generate_dashboard_system_rolename(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_user_name_from_id(), and to_string().

1474  {
1475  std::string frontendViewQuery(
1476  "SELECT id, state, name, image_hash, strftime('%Y-%m-%dT%H:%M:%SZ', update_time), "
1477  "userid, "
1478  "metadata "
1479  "FROM mapd_dashboards");
1480  sqliteConnector_.query(frontendViewQuery);
1481  auto numRows = sqliteConnector_.getNumRows();
1482  for (size_t r = 0; r < numRows; ++r) {
1483  auto vd = std::make_shared<DashboardDescriptor>();
1484  vd->dashboardId = sqliteConnector_.getData<int>(r, 0);
1485  vd->dashboardState = sqliteConnector_.getData<string>(r, 1);
1486  vd->dashboardName = sqliteConnector_.getData<string>(r, 2);
1487  vd->imageHash = sqliteConnector_.getData<string>(r, 3);
1488  vd->updateTime = sqliteConnector_.getData<string>(r, 4);
1489  vd->userId = sqliteConnector_.getData<int>(r, 5);
1490  vd->dashboardMetadata = sqliteConnector_.getData<string>(r, 6);
1491  vd->user = get_user_name_from_id(vd->userId, user_name_by_user_id);
1492  vd->dashboardSystemRoleName = generate_dashboard_system_rolename(
1494  dashboardDescriptorMap_[std::to_string(vd->userId) + ":" + vd->dashboardName] = vd;
1495  }
1496 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
const DBMetadata currentDB_
Definition: Catalog.h:687
virtual void query(const std::string &queryString)
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:678
std::string to_string(char const *&&v)
std::string get_user_name_from_id(int32_t id, const std::map< int32_t, std::string > &user_name_by_user_id)
Definition: Catalog.cpp:1036
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildDictionaryMapUnlocked ( )
private

Definition at line 1048 of file Catalog.cpp.

References currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, g_base_path, SqliteConnector::getData(), SqliteConnector::getNumRows(), shared::kDataDirectoryName, SqliteConnector::query(), sqliteConnector_, and to_string().

1048  {
1049  std::string dictQuery(
1050  "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
1051  sqliteConnector_.query(dictQuery);
1052  auto numRows = sqliteConnector_.getNumRows();
1053  for (size_t r = 0; r < numRows; ++r) {
1054  auto dictId = sqliteConnector_.getData<int>(r, 0);
1055  auto dictName = sqliteConnector_.getData<string>(r, 1);
1056  auto dictNBits = sqliteConnector_.getData<int>(r, 2);
1057  auto is_shared = sqliteConnector_.getData<bool>(r, 3);
1058  auto refcount = sqliteConnector_.getData<int>(r, 4);
1059  auto fname = g_base_path + "/" + shared::kDataDirectoryName + "/DB_" +
1060  std::to_string(currentDB_.dbId) + "_DICT_" + std::to_string(dictId);
1061  DictRef dict_ref(currentDB_.dbId, dictId);
1062  auto dd = new DictDescriptor(
1063  dict_ref, dictName, dictNBits, is_shared, refcount, fname, false);
1064  dictDescriptorMapByRef_[dict_ref].reset(dd);
1065  }
1066 }
const std::string kDataDirectoryName
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
const DBMetadata currentDB_
Definition: Catalog.h:687
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:677
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
std::string g_base_path
Definition: SysCatalog.cpp:62
Descriptor for a dictionary for a string columne.
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildForeignServerMapUnlocked ( )
private

Definition at line 4853 of file Catalog.cpp.

References CHECK, and g_enable_fsi.

4853  {
4856  "SELECT id, name, data_wrapper_type, options, owner_user_id, creation_time FROM "
4857  "omnisci_foreign_servers");
4858  auto num_rows = sqliteConnector_.getNumRows();
4859 
4860  for (size_t row = 0; row < num_rows; row++) {
4861  auto foreign_server = std::make_shared<foreign_storage::ForeignServer>(
4862  sqliteConnector_.getData<int>(row, 0),
4863  sqliteConnector_.getData<std::string>(row, 1),
4864  sqliteConnector_.getData<std::string>(row, 2),
4865  sqliteConnector_.getData<std::string>(row, 3),
4866  sqliteConnector_.getData<std::int32_t>(row, 4),
4867  sqliteConnector_.getData<std::int32_t>(row, 5));
4868  foreignServerMap_[foreign_server->name] = foreign_server;
4869  foreignServerMapById_[foreign_server->id] = foreign_server;
4870  }
4871 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
virtual void query(const std::string &queryString)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:682
ForeignServerMap foreignServerMap_
Definition: Catalog.h:681
#define CHECK(condition)
Definition: Logger.h:222
bool g_enable_fsi
Definition: Catalog.cpp:96
virtual size_t getNumRows() const
void Catalog_Namespace::Catalog::buildLinksMapUnlocked ( )
private

Definition at line 1498 of file Catalog.cpp.

References to_string().

1498  {
1499  std::string linkQuery(
1500  "SELECT linkid, userid, link, view_state, strftime('%Y-%m-%dT%H:%M:%SZ', "
1501  "update_time), view_metadata "
1502  "FROM mapd_links");
1503  sqliteConnector_.query(linkQuery);
1504  auto numRows = sqliteConnector_.getNumRows();
1505  for (size_t r = 0; r < numRows; ++r) {
1506  auto ld = new LinkDescriptor();
1507  ld->linkId = sqliteConnector_.getData<int>(r, 0);
1508  ld->userId = sqliteConnector_.getData<int>(r, 1);
1509  ld->link = sqliteConnector_.getData<string>(r, 2);
1510  ld->viewState = sqliteConnector_.getData<string>(r, 3);
1511  ld->updateTime = sqliteConnector_.getData<string>(r, 4);
1512  ld->viewMetadata = sqliteConnector_.getData<string>(r, 5);
1514  linkDescriptorMapById_[ld->linkId] = ld;
1515  }
1516 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
const DBMetadata currentDB_
Definition: Catalog.h:687
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:680
virtual size_t getNumRows() const
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:679

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildLogicalToPhysicalMapUnlocked ( )
private

Definition at line 1518 of file Catalog.cpp.

References CHECK.

1518  {
1519  /* rebuild map linking logical tables to corresponding physical ones */
1520  std::string logicalToPhysicalTableMapQuery(
1521  "SELECT logical_table_id, physical_table_id "
1522  "FROM mapd_logical_to_physical");
1523  sqliteConnector_.query(logicalToPhysicalTableMapQuery);
1524  auto numRows = sqliteConnector_.getNumRows();
1525  for (size_t r = 0; r < numRows; ++r) {
1526  auto logical_tb_id = sqliteConnector_.getData<int>(r, 0);
1527  auto physical_tb_id = sqliteConnector_.getData<int>(r, 1);
1528  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
1529  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
1530  /* add new entity to the map logicalToPhysicalTableMapById_ */
1531  std::vector<int32_t> physicalTables{physical_tb_id};
1532  const auto it_ok =
1533  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
1534  CHECK(it_ok.second);
1535  } else {
1536  /* update map logicalToPhysicalTableMapById_ */
1537  physicalTableIt->second.push_back(physical_tb_id);
1538  }
1539  }
1540 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
virtual void query(const std::string &queryString)
#define CHECK(condition)
Definition: Logger.h:222
virtual size_t getNumRows() const
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:693
void Catalog_Namespace::Catalog::buildMaps ( )
protected

Definition at line 1545 of file Catalog.cpp.

References g_enable_fsi, and Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_user_id_to_user_name_map().

1545  {
1546  // Get all user id to username mapping here in order to avoid making a call to
1547  // SysCatalog (and attempting to acquire SysCatalog locks) while holding locks for this
1548  // catalog.
1549  const auto user_name_by_user_id = get_user_id_to_user_name_map();
1550 
1551  cat_write_lock write_lock(this);
1552  cat_sqlite_lock sqlite_lock(this);
1553 
1556 
1557  if (g_enable_fsi) {
1560  }
1561 
1564  buildDashboardsMapUnlocked(user_name_by_user_id);
1568 }
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void buildLogicalToPhysicalMapUnlocked()
Definition: Catalog.cpp:1518
void buildDashboardsMapUnlocked(const std::map< int32_t, std::string > &user_name_by_user_id)
Definition: Catalog.cpp:1473
void buildCustomExpressionsMapUnlocked()
Definition: Catalog.cpp:1570
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
void updateForeignTablesInMapUnlocked()
Definition: Catalog.cpp:4873
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
bool g_enable_fsi
Definition: Catalog.cpp:96
std::map< int32_t, std::string > get_user_id_to_user_name_map()
Definition: Catalog.cpp:1027

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildTablesMapUnlocked ( )
private

Definition at line 1342 of file Catalog.cpp.

References logger::FATAL, StorageType::FOREIGN_TABLE, TableDescriptor::fragmenter, TableDescriptor::fragments, TableDescriptor::fragPageSize, TableDescriptor::fragType, TableDescriptor::hasDeletedCol, TableDescriptor::is_system_table, TableDescriptor::isView, TableDescriptor::keyMetainfo, LOG, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nColumns, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::storageType, TableDescriptor::tableId, TableDescriptor::tableName, to_upper(), and TableDescriptor::userId.

1342  {
1343  std::string tableQuery(
1344  "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
1345  "max_chunk_size, frag_page_size, "
1346  "max_rows, partitions, shard_column_id, shard, num_shards, key_metainfo, userid, "
1347  "sort_column_id, storage_type, max_rollback_epochs, is_system_table "
1348  "from mapd_tables");
1349  sqliteConnector_.query(tableQuery);
1350  auto numRows = sqliteConnector_.getNumRows();
1351  for (size_t r = 0; r < numRows; ++r) {
1352  TableDescriptor* td;
1353  const auto& storage_type = sqliteConnector_.getData<string>(r, 17);
1354  if (!storage_type.empty() && storage_type != StorageType::FOREIGN_TABLE) {
1355  const auto table_id = sqliteConnector_.getData<int>(r, 0);
1356  const auto& table_name = sqliteConnector_.getData<string>(r, 1);
1357  LOG(FATAL) << "Unable to read Catalog metadata: storage type is currently not a "
1358  "supported table option (table "
1359  << table_name << " [" << table_id << "] in database "
1360  << currentDB_.dbName << ").";
1361  }
1362 
1363  if (storage_type == StorageType::FOREIGN_TABLE) {
1364  td = new foreign_storage::ForeignTable();
1365  } else {
1366  td = new TableDescriptor();
1367  }
1368 
1369  td->storageType = storage_type;
1370  td->tableId = sqliteConnector_.getData<int>(r, 0);
1371  td->tableName = sqliteConnector_.getData<string>(r, 1);
1372  td->nColumns = sqliteConnector_.getData<int>(r, 2);
1373  td->isView = sqliteConnector_.getData<bool>(r, 3);
1374  td->fragments = sqliteConnector_.getData<string>(r, 4);
1375  td->fragType =
1377  td->maxFragRows = sqliteConnector_.getData<int>(r, 6);
1378  td->maxChunkSize = sqliteConnector_.getData<int64_t>(r, 7);
1379  td->fragPageSize = sqliteConnector_.getData<int>(r, 8);
1380  td->maxRows = sqliteConnector_.getData<int64_t>(r, 9);
1381  td->partitions = sqliteConnector_.getData<string>(r, 10);
1382  td->shardedColumnId = sqliteConnector_.getData<int>(r, 11);
1383  td->shard = sqliteConnector_.getData<int>(r, 12);
1384  td->nShards = sqliteConnector_.getData<int>(r, 13);
1385  td->keyMetainfo = sqliteConnector_.getData<string>(r, 14);
1386  td->userId = sqliteConnector_.getData<int>(r, 15);
1387  td->sortedColumnId =
1388  sqliteConnector_.isNull(r, 16) ? 0 : sqliteConnector_.getData<int>(r, 16);
1389  if (!td->isView) {
1390  td->fragmenter = nullptr;
1391  }
1392  td->maxRollbackEpochs = sqliteConnector_.getData<int>(r, 18);
1393  td->is_system_table = sqliteConnector_.getData<bool>(r, 19);
1394  td->hasDeletedCol = false;
1395 
1397  tableDescriptorMapById_[td->tableId] = td;
1398  }
1399 }
int32_t maxRollbackEpochs
std::string partitions
T getData(const int row, const int col)
std::string tableName
#define LOG(tag)
Definition: Logger.h:216
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
const DBMetadata currentDB_
Definition: Catalog.h:687
std::string storageType
virtual void query(const std::string &queryString)
std::string fragments
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:674
std::string keyMetainfo
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
std::string to_upper(const std::string &str)
bool isNull(const int row, const int col) const
Fragmenter_Namespace::FragmenterType fragType
static constexpr char const * FOREIGN_TABLE
virtual size_t getNumRows() const
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:673

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::calculateSHA1 ( const std::string &  data)
protected

Definition at line 4562 of file Catalog.cpp.

4562  {
4563  boost::uuids::detail::sha1 sha1;
4564  unsigned int digest[5];
4565  sha1.process_bytes(data.c_str(), data.length());
4566  sha1.get_digest(digest);
4567  std::stringstream ss;
4568  for (size_t i = 0; i < 5; i++) {
4569  ss << std::hex << digest[i];
4570  }
4571  return ss.str();
4572 }
void Catalog_Namespace::Catalog::changeForeignServerOwner ( const std::string &  server_name,
const int  new_owner_id 
)

Change the owner of a Foreign Server to a new owner.

Parameters
server_name- Name of the foreign server whose owner to change
new_owner_id- New owner's user id

Definition at line 3149 of file Catalog.cpp.

References CHECK, to_string(), and foreign_storage::ForeignServer::user_id.

3150  {
3151  cat_write_lock write_lock(this);
3152  foreign_storage::ForeignServer* foreign_server =
3153  foreignServerMap_.find(server_name)->second.get();
3154  CHECK(foreign_server);
3155  setForeignServerProperty(server_name, "owner_user_id", std::to_string(new_owner_id));
3156  // update in-memory server
3157  foreign_server->user_id = new_owner_id;
3158 }
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::string to_string(char const *&&v)
ForeignServerMap foreignServerMap_
Definition: Catalog.h:681
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:222
void setForeignServerProperty(const std::string &server_name, const std::string &property, const std::string &value)
Definition: Catalog.cpp:4903

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::CheckAndExecuteMigrations ( )
protected

Definition at line 997 of file Catalog.cpp.

References g_enable_fsi, recordOwnershipOfObjectsInObjectPermissions(), renameLegacyDataWrappers(), updateCustomExpressionsSchema(), updateDefaultColumnValues(), updateDeletedColumnIndicator(), updateDictionaryNames(), updateDictionarySchema(), updateFixlenArrayColumns(), updateFrontendViewAndLinkUsers(), updateFrontendViewSchema(), updateFrontendViewsToDashboards(), updateFsiSchemas(), updateGeoColumns(), updateLinkSchema(), updateLogicalToPhysicalTableLinkSchema(), updatePageSize(), and updateTableDescriptorSchema().

997  {
999  cat_sqlite_lock sqlite_lock(this);
1002  updateGeoColumns();
1005  updateLinkSchema();
1009  updatePageSize();
1013  if (g_enable_fsi) {
1014  updateFsiSchemas();
1016  }
1019 }
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:460
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:787
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:129
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
void updateCustomExpressionsSchema()
Definition: Catalog.cpp:753
void updateLogicalToPhysicalTableLinkSchema()
Definition: Catalog.cpp:621
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
bool g_enable_fsi
Definition: Catalog.cpp:96

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::CheckAndExecuteMigrationsPostBuildMaps ( )
protected

Definition at line 1021 of file Catalog.cpp.

References checkDateInDaysColumnMigration(), and createDashboardSystemRoles().

1021  {
1024 }
void checkDateInDaysColumnMigration()
Definition: Catalog.cpp:915

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::checkDateInDaysColumnMigration ( )
protected

Definition at line 915 of file Catalog.cpp.

References getCurrentDB(), migrations::MigrationMgr::migrateDateInDaysMetadata(), sqliteConnector_, and tableDescriptorMapById_.

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

915  {
916  cat_sqlite_lock sqlite_lock(this);
919 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:242
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:674
static void migrateDateInDaysMetadata(const Catalog_Namespace::TableDescriptorMapById &table_descriptors_by_id, const int database_id, Catalog_Namespace::Catalog *cat, SqliteConnector &sqlite)
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const bool Catalog_Namespace::Catalog::checkMetadataForDeletedRecs ( const TableDescriptor td,
int  column_id 
) const

Definition at line 3667 of file Catalog.cpp.

References CHECK, and TableDescriptor::fragmenter.

3668  {
3669  // check if there are rows deleted by examining the deletedColumn metadata
3670  CHECK(td);
3671  auto fragmenter = td->fragmenter;
3672  if (fragmenter) {
3673  return fragmenter->hasDeletedRows(delete_column_id);
3674  } else {
3675  return false;
3676  }
3677 }
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
#define CHECK(condition)
Definition: Logger.h:222
void Catalog_Namespace::Catalog::checkpoint ( const int  logicalTableId) const

Definition at line 4783 of file Catalog.cpp.

Referenced by UpdelRoll::commitUpdate(), TableOptimizer::vacuumDeletedRows(), and TableOptimizer::vacuumFragmentsAboveMinSelectivity().

4783  {
4784  const auto td = getMetadataForTable(logicalTableId);
4785  const auto shards = getPhysicalTablesDescriptors(td);
4786  for (const auto shard : shards) {
4787  getDataMgr().checkpoint(getCurrentDB().dbId, shard->tableId);
4788  }
4789 }
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:243
void checkpoint(const int db_id, const int tb_id)
Definition: DataMgr.cpp:545
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:242
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4630
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::checkpointWithAutoRollback ( const int  logical_table_id) const

Definition at line 4791 of file Catalog.cpp.

Referenced by Fragmenter_Namespace::LocalInsertConnector::checkpoint(), import_export::Loader::checkpoint(), StorageIOFacility::TransactionParameters::finalizeTransaction(), and TableOptimizer::vacuumFragmentsAboveMinSelectivity().

4791  {
4792  auto table_epochs = getTableEpochs(getDatabaseId(), logical_table_id);
4793  try {
4794  checkpoint(logical_table_id);
4795  } catch (...) {
4796  setTableEpochsLogExceptions(getDatabaseId(), table_epochs);
4797  throw;
4798  }
4799 }
int getDatabaseId() const
Definition: Catalog.h:298
void checkpoint(const int logicalTableId) const
Definition: Catalog.cpp:4783
void setTableEpochsLogExceptions(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3649
std::vector< TableEpochInfo > getTableEpochs(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:3585

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::conditionallyInitializeSystemObjects ( )
private

Definition at line 5975 of file Catalog.cpp.

References g_enable_system_tables.

5975  {
5979  }
5980 }
bool g_enable_system_tables
Definition: SysCatalog.cpp:64
bool isInfoSchemaDb() const
Definition: Catalog.cpp:5982
int32_t Catalog_Namespace::Catalog::createCustomExpression ( std::unique_ptr< CustomExpression custom_expression)

Creates a new custom expression.

Parameters
custom_expression- unique pointer to struct containing custom expression details.
Returns
id of created custom expression

Definition at line 5537 of file Catalog.cpp.

References CHECK, CHECK_EQ, CHECK_GT, and to_string().

5538  {
5539  cat_write_lock write_lock(this);
5540  cat_sqlite_lock sqlite_lock(this);
5541  sqliteConnector_.query("BEGIN TRANSACTION");
5542  int32_t custom_expression_id{-1};
5543  try {
5544  auto data_source_type_str =
5545  CustomExpression::dataSourceTypeToString(custom_expression->data_source_type);
5546  auto data_source_id_str = std::to_string(custom_expression->data_source_id);
5547  std::string custom_expr_select_query{
5548  "SELECT id FROM omnisci_custom_expressions WHERE name = ? and data_source_type = "
5549  "? and data_source_id = ? and is_deleted = ?"};
5550  std::vector<std::string> custom_expr_select_params{custom_expression->name,
5551  data_source_type_str,
5552  data_source_id_str,
5553  std::to_string(false)};
5554  sqliteConnector_.query_with_text_params(custom_expr_select_query,
5555  custom_expr_select_params);
5556  if (sqliteConnector_.getNumRows() > 0) {
5557  throw std::runtime_error{
5558  "A custom expression with the given "
5559  "name and data source already exists."};
5560  }
5562  "INSERT INTO omnisci_custom_expressions(name, expression_json, "
5563  "data_source_type, data_source_id, is_deleted) VALUES (?,?,?,?,?)",
5564  std::vector<std::string>{custom_expression->name,
5565  custom_expression->expression_json,
5566  data_source_type_str,
5567  data_source_id_str,
5568  std::to_string(false)});
5569  sqliteConnector_.query_with_text_params(custom_expr_select_query,
5570  custom_expr_select_params);
5571  CHECK_EQ(sqliteConnector_.getNumRows(), static_cast<size_t>(1));
5572  custom_expression->id = sqliteConnector_.getData<int32_t>(0, 0);
5573  custom_expression_id = custom_expression->id;
5574  CHECK(custom_expr_map_by_id_.find(custom_expression->id) ==
5575  custom_expr_map_by_id_.end());
5576  custom_expr_map_by_id_[custom_expression->id] = std::move(custom_expression);
5577  } catch (std::exception& e) {
5578  sqliteConnector_.query("ROLLBACK TRANSACTION");
5579  throw;
5580  }
5581  sqliteConnector_.query("END TRANSACTION");
5582  CHECK_GT(custom_expression_id, 0);
5583  return custom_expression_id;
5584 }
#define CHECK_EQ(x, y)
Definition: Logger.h:230
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
#define CHECK_GT(x, y)
Definition: Logger.h:234
std::string to_string(char const *&&v)
static std::string dataSourceTypeToString(DataSourceType type_enum)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:222
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:683
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

int32_t Catalog_Namespace::Catalog::createDashboard ( DashboardDescriptor vd)

Definition at line 4421 of file Catalog.cpp.

References DashboardDescriptor::dashboardId, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, DashboardDescriptor::dashboardSystemRoleName, generate_dashboard_system_rolename(), DashboardDescriptor::imageHash, to_string(), Catalog_Namespace::sqlite_lock< T >::unlock(), Catalog_Namespace::write_lock< T >::unlock(), DashboardDescriptor::updateTime, and DashboardDescriptor::userId.

4421  {
4422  cat_write_lock write_lock(this);
4423  cat_sqlite_lock sqlite_lock(this);
4424  sqliteConnector_.query("BEGIN TRANSACTION");
4425  try {
4426  // TODO(andrew): this should be an upsert
4428  "SELECT id FROM mapd_dashboards WHERE name = ? and userid = ?",
4429  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
4430  if (sqliteConnector_.getNumRows() > 0) {
4432  "UPDATE mapd_dashboards SET state = ?, image_hash = ?, metadata = ?, "
4433  "update_time = "
4434  "datetime('now') where name = ? "
4435  "and userid = ?",
4436  std::vector<std::string>{vd.dashboardState,
4437  vd.imageHash,
4438  vd.dashboardMetadata,
4439  vd.dashboardName,
4440  std::to_string(vd.userId)});
4441  } else {
4443  "INSERT INTO mapd_dashboards (name, state, image_hash, metadata, "
4444  "update_time, "
4445  "userid) "
4446  "VALUES "
4447  "(?,?,?,?, "
4448  "datetime('now'), ?)",
4449  std::vector<std::string>{vd.dashboardName,
4450  vd.dashboardState,
4451  vd.imageHash,
4452  vd.dashboardMetadata,
4453  std::to_string(vd.userId)});
4454  }
4455  } catch (std::exception& e) {
4456  sqliteConnector_.query("ROLLBACK TRANSACTION");
4457  throw;
4458  }
4459  sqliteConnector_.query("END TRANSACTION");
4460 
4461  // now get the auto generated dashboardId
4462  try {
4464  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_dashboards "
4465  "WHERE name = ? and userid = ?",
4466  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
4467  vd.dashboardId = sqliteConnector_.getData<int>(0, 0);
4468  vd.updateTime = sqliteConnector_.getData<std::string>(0, 1);
4469  } catch (std::exception& e) {
4470  throw;
4471  }
4475  sqlite_lock.unlock();
4476  write_lock.unlock();
4477  if (!isInfoSchemaDb()) {
4478  // NOTE(wamsi): Transactionally unsafe
4481  }
4482  return vd.dashboardId;
4483 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
const DBMetadata currentDB_
Definition: Catalog.h:687
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void addFrontendViewToMap(DashboardDescriptor &vd)
Definition: Catalog.cpp:1739
std::string to_string(char const *&&v)
std::string dashboardSystemRoleName
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1780
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
bool isInfoSchemaDb() const
Definition: Catalog.cpp:5982
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createDashboardSystemRoles ( )
protected

Definition at line 921 of file Catalog.cpp.

References createOrUpdateDashboardSystemRole(), currentDB_, Catalog_Namespace::DBMetadata::dbId, logger::ERROR, generate_dashboard_system_rolename(), SqliteConnector::getData(), Catalog_Namespace::SysCatalog::getGranteesOfSharedDashboards(), SqliteConnector::getNumRows(), Catalog_Namespace::SysCatalog::getRoleGrantee(), Catalog_Namespace::SysCatalog::grantRoleBatch(), logger::INFO, Catalog_Namespace::SysCatalog::instance(), LOG, MAPD_VERSION, SqliteConnector::query(), SqliteConnector::query_with_text_params(), run_benchmark_import::result, sqliteConnector_, and to_string().

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

921  {
922  std::unordered_map<std::string, std::pair<int, std::string>> dashboards;
923  std::vector<std::string> dashboard_ids;
924  static const std::string migration_name{"dashboard_roles_migration"};
925  {
926  cat_sqlite_lock sqlite_lock(this);
927  sqliteConnector_.query("BEGIN TRANSACTION");
928  try {
929  // migration_history should be present in all catalogs by now
930  // if not then would be created before this migration
932  "select * from mapd_version_history where migration_history = '" +
933  migration_name + "'");
934  if (sqliteConnector_.getNumRows() != 0) {
935  // no need for further execution
936  sqliteConnector_.query("END TRANSACTION");
937  return;
938  }
939  LOG(INFO) << "Performing dashboard internal roles Migration.";
940  sqliteConnector_.query("select id, userid, metadata from mapd_dashboards");
941  for (size_t i = 0; i < sqliteConnector_.getNumRows(); ++i) {
944  sqliteConnector_.getData<string>(i, 0)))) {
945  // Successfully created roles during previous migration/crash
946  // No need to include them
947  continue;
948  }
949  dashboards[sqliteConnector_.getData<string>(i, 0)] = std::make_pair(
950  sqliteConnector_.getData<int>(i, 1), sqliteConnector_.getData<string>(i, 2));
951  dashboard_ids.push_back(sqliteConnector_.getData<string>(i, 0));
952  }
953  } catch (const std::exception& e) {
954  sqliteConnector_.query("ROLLBACK TRANSACTION");
955  throw;
956  }
957  sqliteConnector_.query("END TRANSACTION");
958  }
959  // All current grantees with shared dashboards.
960  const auto active_grantees =
962 
963  try {
964  // NOTE(wamsi): Transactionally unsafe
965  for (auto dash : dashboards) {
966  createOrUpdateDashboardSystemRole(dash.second.second,
967  dash.second.first,
969  std::to_string(currentDB_.dbId), dash.first));
970  auto result = active_grantees.find(dash.first);
971  if (result != active_grantees.end()) {
974  dash.first)},
975  result->second);
976  }
977  }
978  cat_sqlite_lock sqlite_lock(this);
979  // check if this has already been completed
981  "select * from mapd_version_history where migration_history = '" +
982  migration_name + "'");
983  if (sqliteConnector_.getNumRows() != 0) {
984  return;
985  }
987  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
988  std::vector<std::string>{std::to_string(MAPD_VERSION), migration_name});
989  } catch (const std::exception& e) {
990  LOG(ERROR) << "Failed to create dashboard system roles during migration: "
991  << e.what();
992  throw;
993  }
994  LOG(INFO) << "Successfully created dashboard system roles during migration.";
995 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
#define LOG(tag)
Definition: Logger.h:216
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
const DBMetadata currentDB_
Definition: Catalog.h:687
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
void grantRoleBatch(const std::vector< std::string > &roles, const std::vector< std::string > &grantees)
static SysCatalog & instance()
Definition: SysCatalog.h:341
static const int32_t MAPD_VERSION
Definition: release.h:32
Role * getRoleGrantee(const std::string &name) const
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1780
std::unordered_map< std::string, std::vector< std::string > > getGranteesOfSharedDashboards(const std::vector< std::string > &dashboard_ids)
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createDefaultServersIfNotExists ( )

Creates default local file servers (if they don't already exist).

Definition at line 4924 of file Catalog.cpp.

References CHECK, foreign_storage::DataWrapperType::CSV, g_enable_fsi, shared::kRootUserId, foreign_storage::AbstractFileStorageDataWrapper::LOCAL_FILE_STORAGE_TYPE, foreign_storage::DataWrapperType::PARQUET, foreign_storage::DataWrapperType::REGEX_PARSER, and foreign_storage::AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY.

4924  {
4929 
4930  auto local_csv_server = std::make_unique<foreign_storage::ForeignServer>(
4931  "default_local_delimited",
4933  options,
4935  local_csv_server->validate();
4936  createForeignServerNoLocks(std::move(local_csv_server), true);
4937 
4938 #ifdef ENABLE_IMPORT_PARQUET
4939  auto local_parquet_server = std::make_unique<foreign_storage::ForeignServer>(
4940  "default_local_parquet",
4942  options,
4944  local_parquet_server->validate();
4945  createForeignServerNoLocks(std::move(local_parquet_server), true);
4946 #endif
4947 
4948  auto local_regex_parser_server = std::make_unique<foreign_storage::ForeignServer>(
4949  "default_local_regex_parsed",
4951  options,
4953  local_regex_parser_server->validate();
4954  createForeignServerNoLocks(std::move(local_regex_parser_server), true);
4955 }
const int kRootUserId
static constexpr char const * REGEX_PARSER
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:3054
#define CHECK(condition)
Definition: Logger.h:222
static constexpr char const * CSV
std::map< std::string, std::string, std::less<>> OptionsMap
bool g_enable_fsi
Definition: Catalog.cpp:96
static constexpr char const * PARQUET
void Catalog_Namespace::Catalog::createForeignServer ( std::unique_ptr< foreign_storage::ForeignServer foreign_server,
bool  if_not_exists 
)

Creates a new foreign server DB object.

Parameters
foreign_server- unique pointer to struct containing foreign server details
if_not_exists- flag indicating whether or not an attempt to create a new foreign server should occur if a server with the same name already exists. An exception is thrown if this flag is set to "false" and an attempt is made to create a pre-existing foreign server

Definition at line 3046 of file Catalog.cpp.

3048  {
3049  cat_write_lock write_lock(this);
3050  cat_sqlite_lock sqlite_lock(this);
3051  createForeignServerNoLocks(std::move(foreign_server), if_not_exists);
3052 }
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:3054
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
void Catalog_Namespace::Catalog::createForeignServerNoLocks ( std::unique_ptr< foreign_storage::ForeignServer foreign_server,
bool  if_not_exists 
)
private

Same as createForeignServer() but without acquiring locks. This should only be called from within a function/code block that already acquires appropriate locks.

Definition at line 3054 of file Catalog.cpp.

References CHECK, CHECK_EQ, setup::name, and to_string().

3056  {
3057  const auto& name = foreign_server->name;
3058 
3060  "SELECT name from omnisci_foreign_servers where name = ?",
3061  std::vector<std::string>{name});
3062 
3063  if (sqliteConnector_.getNumRows() == 0) {
3064  foreign_server->creation_time = std::time(nullptr);
3066  "INSERT INTO omnisci_foreign_servers (name, data_wrapper_type, owner_user_id, "
3067  "creation_time, "
3068  "options) "
3069  "VALUES (?, ?, ?, ?, ?)",
3070  std::vector<std::string>{name,
3071  foreign_server->data_wrapper_type,
3072  std::to_string(foreign_server->user_id),
3073  std::to_string(foreign_server->creation_time),
3074  foreign_server->getOptionsAsJsonString()});
3076  "SELECT id from omnisci_foreign_servers where name = ?",
3077  std::vector<std::string>{name});
3078  CHECK_EQ(sqliteConnector_.getNumRows(), size_t(1));
3079  foreign_server->id = sqliteConnector_.getData<int32_t>(0, 0);
3080  std::shared_ptr<foreign_storage::ForeignServer> foreign_server_shared =
3081  std::move(foreign_server);
3083  << "Attempting to insert a foreign server into foreign server map that already "
3084  "exists.";
3085  foreignServerMap_[name] = foreign_server_shared;
3086  foreignServerMapById_[foreign_server_shared->id] = foreign_server_shared;
3087  } else if (!if_not_exists) {
3088  throw std::runtime_error{"A foreign server with name \"" + foreign_server->name +
3089  "\" already exists."};
3090  }
3091 
3092  const auto& server_it = foreignServerMap_.find(name);
3093  CHECK(server_it != foreignServerMap_.end());
3094  CHECK(foreignServerMapById_.find(server_it->second->id) != foreignServerMapById_.end());
3095 }
#define CHECK_EQ(x, y)
Definition: Logger.h:230
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
std::string to_string(char const *&&v)
std::string name() const
Definition: Catalog.h:315
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:682
ForeignServerMap foreignServerMap_
Definition: Catalog.h:681
#define CHECK(condition)
Definition: Logger.h:222
virtual size_t getNumRows() const

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::createLink ( LinkDescriptor ld,
size_t  min_length 
)

Definition at line 4574 of file Catalog.cpp.

References LinkDescriptor::link, LinkDescriptor::linkId, to_string(), LinkDescriptor::updateTime, LinkDescriptor::userId, LinkDescriptor::viewMetadata, and LinkDescriptor::viewState.

4574  {
4575  cat_write_lock write_lock(this);
4576  cat_sqlite_lock sqlite_lock(this);
4577  sqliteConnector_.query("BEGIN TRANSACTION");
4578  try {
4580  .substr(0, 8);
4582  "SELECT linkid FROM mapd_links WHERE link = ? and userid = ?",
4583  std::vector<std::string>{ld.link, std::to_string(ld.userId)});
4584  if (sqliteConnector_.getNumRows() > 0) {
4586  "UPDATE mapd_links SET update_time = datetime('now') WHERE userid = ? AND "
4587  "link = ?",
4588  std::vector<std::string>{std::to_string(ld.userId), ld.link});
4589  } else {
4591  "INSERT INTO mapd_links (userid, link, view_state, view_metadata, "
4592  "update_time) VALUES (?,?,?,?, datetime('now'))",
4593  std::vector<std::string>{
4594  std::to_string(ld.userId), ld.link, ld.viewState, ld.viewMetadata});
4595  }
4596  // now get the auto generated dashid
4598  "SELECT linkid, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_links "
4599  "WHERE link = ?",
4600  ld.link);
4601  ld.linkId = sqliteConnector_.getData<int>(0, 0);
4602  ld.updateTime = sqliteConnector_.getData<std::string>(0, 1);
4603  } catch (std::exception& e) {
4604  sqliteConnector_.query("ROLLBACK TRANSACTION");
4605  throw;
4606  }
4607  sqliteConnector_.query("END TRANSACTION");
4608  addLinkToMap(ld);
4609  return ld.link;
4610 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::string to_string(char const *&&v)
std::string calculateSHA1(const std::string &data)
Definition: Catalog.cpp:4562
void addLinkToMap(LinkDescriptor &ld)
Definition: Catalog.cpp:1826
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createOrUpdateDashboardSystemRole ( const std::string &  view_meta,
const int32_t &  user_id,
const std::string &  dash_role_name 
)
protected

Definition at line 1780 of file Catalog.cpp.

References Grantee::findDbObject(), Grantee::getDbObjects(), TableDBObjectType, and ViewDBObjectType.

Referenced by createDashboardSystemRoles().

1782  {
1783  auto objects = parseDashboardObjects(view_meta, user_id);
1784  Role* rl = SysCatalog::instance().getRoleGrantee(dash_role_name);
1785  if (!rl) {
1786  // Dashboard role does not exist
1787  // create role and grant privileges
1788  // NOTE(wamsi): Transactionally unsafe
1790  dash_role_name, /*user_private_role=*/false, /*is_temporary=*/false);
1791  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1792  } else {
1793  // Dashboard system role already exists
1794  // Add/remove privileges on objects
1795  std::set<DBObjectKey> revoke_keys;
1796  auto ex_objects = rl->getDbObjects(true);
1797  for (auto key : *ex_objects | boost::adaptors::map_keys) {
1798  if (key.permissionType != TableDBObjectType &&
1799  key.permissionType != ViewDBObjectType) {
1800  continue;
1801  }
1802  bool found = false;
1803  for (auto obj : objects) {
1804  found = key == obj.getObjectKey() ? true : false;
1805  if (found) {
1806  break;
1807  }
1808  }
1809  if (!found) {
1810  revoke_keys.insert(key);
1811  }
1812  }
1813  for (auto& key : revoke_keys) {
1814  // revoke privs on object since the object is no
1815  // longer used by the dashboard as source
1816  // NOTE(wamsi): Transactionally unsafe
1818  dash_role_name, *rl->findDbObject(key, true), *this);
1819  }
1820  // Update privileges on remaining objects
1821  // NOTE(wamsi): Transactionally unsafe
1822  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1823  }
1824 }
void revokeDBObjectPrivileges(const std::string &grantee, const DBObject &object, const Catalog_Namespace::Catalog &catalog)
void createRole(const std::string &roleName, const bool user_private_role, const bool is_temporary=false)
Definition: Grantee.h:81
DBObject * findDbObject(const DBObjectKey &objectKey, bool only_direct) const
Definition: Grantee.cpp:85
static SysCatalog & instance()
Definition: SysCatalog.h:341
Role * getRoleGrantee(const std::string &name) const
void grantDBObjectPrivilegesBatch(const std::vector< std::string > &grantees, const std::vector< DBObject > &objects, const Catalog_Namespace::Catalog &catalog)
const DBObjectMap * getDbObjects(bool only_direct) const
Definition: Grantee.h:56
std::vector< DBObject > parseDashboardObjects(const std::string &view_meta, const int &user_id)
Definition: Catalog.cpp:1750

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createShardedTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs 
)

Definition at line 3862 of file Catalog.cpp.

References CHECK, TableDescriptor::nShards, TableDescriptor::shard, table_is_temporary(), TableDescriptor::tableId, and TableDescriptor::tableName.

3865  {
3866  /* create logical table */
3867  TableDescriptor* tdl = &td;
3868  createTable(*tdl, cols, shared_dict_defs, true); // create logical table
3869  int32_t logical_tb_id = tdl->tableId;
3870  std::string logical_table_name = tdl->tableName;
3871 
3872  /* create physical tables and link them to the logical table */
3873  std::vector<int32_t> physicalTables;
3874  for (int32_t i = 1; i <= td.nShards; i++) {
3875  TableDescriptor* tdp = &td;
3876  tdp->tableName = generatePhysicalTableName(logical_table_name, i);
3877  tdp->shard = i - 1;
3878  createTable(*tdp, cols, shared_dict_defs, false); // create physical table
3879  int32_t physical_tb_id = tdp->tableId;
3880 
3881  /* add physical table to the vector of physical tables */
3882  physicalTables.push_back(physical_tb_id);
3883  }
3884 
3885  if (!physicalTables.empty()) {
3886  cat_write_lock write_lock(this);
3887  /* add logical to physical tables correspondence to the map */
3888  const auto it_ok =
3889  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
3890  CHECK(it_ok.second);
3891  /* update sqlite mapd_logical_to_physical in sqlite database */
3892  if (!table_is_temporary(&td)) {
3893  updateLogicalToPhysicalTableMap(logical_tb_id);
3894  }
3895  }
3896 }
std::string tableName
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:2703
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::string generatePhysicalTableName(const std::string &logicalTableName, const int32_t &shardNumber)
Definition: Catalog.cpp:4846
bool table_is_temporary(const TableDescriptor *const td)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:222
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:693
void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id)
Definition: Catalog.cpp:635

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createSystemTableServer ( const std::string &  server_name,
const std::string &  data_wrapper_type,
const foreign_storage::OptionsMap options = {} 
)
private

Definition at line 6372 of file Catalog.cpp.

References logger::INFO, shared::kRootUserId, LOG, and run_benchmark_import::tables.

6374  {
6375  auto server = std::make_unique<foreign_storage::ForeignServer>(
6376  server_name, data_wrapper_type, options, shared::kRootUserId);
6377  server->validate();
6378  auto stored_server = getForeignServer(server_name);
6379  if (stored_server && stored_server->options != server->options) {
6380  // Drop all tables for server before dropping server.
6381  auto tables = getAllForeignTablesForForeignServer(stored_server->id);
6382  for (const auto table : tables) {
6383  LOG(INFO) << "Dropping existing \"" << table->tableName << "\" system table for \""
6384  << server_name << "\" foreign server.";
6385  dropTable(table);
6386  }
6387  LOG(INFO) << "Dropping existing \"" << server_name
6388  << "\" system table foreign server.";
6389  dropForeignServer(server_name);
6390  stored_server = nullptr;
6391  }
6392  if (!stored_server) {
6393  LOG(INFO) << "Creating a new \"" << server_name << "\" system table foreign server.";
6394  createForeignServer(std::move(server), true);
6395  }
6396 }
const int kRootUserId
void createForeignServer(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:3046
const foreign_storage::ForeignServer * getForeignServer(const std::string &server_name) const
Definition: Catalog.cpp:3097
#define LOG(tag)
Definition: Logger.h:216
std::vector< const foreign_storage::ForeignTable * > getAllForeignTablesForForeignServer(const int32_t foreign_server_id)
Get all of the foreign tables for associated with a foreign server id.
Definition: Catalog.cpp:3394
void dropForeignServer(const std::string &server_name)
Definition: Catalog.cpp:3213
void dropTable(const TableDescriptor *td)
Definition: Catalog.cpp:4016
void Catalog_Namespace::Catalog::createTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs,
bool  isLogicalTable 
)

Definition at line 2703 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnId, TableDescriptor::columnIdBySpi_, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::default_value, dict_ref_t::dictId, Data_Namespace::DISK_LEVEL, StorageType::FOREIGN_TABLE, TableDescriptor::fragPageSize, TableDescriptor::fragType, g_enable_fsi, g_serialize_temp_tables, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_next_refresh_time(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), TableDescriptor::hasDeletedCol, SQLTypeInfo::is_array(), SQLTypeInfo::is_geometry(), TableDescriptor::is_system_table, ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, kBIGINT, kBOOLEAN, kENCODING_DICT, TableDescriptor::keyMetainfo, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nColumns, foreign_storage::ForeignTable::next_refresh_time, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_size(), TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::storageType, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), TableDescriptor::userId, TableDescriptor::viewSQL, and ColumnDescriptor::virtualExpr.

Referenced by Parser::CreateTableAsSelectStmt::execute(), and Parser::CreateViewStmt::execute().

2707  {
2708  cat_write_lock write_lock(this);
2709  list<ColumnDescriptor> cds = cols;
2710  list<DictDescriptor> dds;
2711  std::set<std::string> toplevel_column_names;
2712  list<ColumnDescriptor> columns;
2713 
2714  if (!td.storageType.empty() &&
2717  throw std::runtime_error("Only temporary tables can be backed by foreign storage.");
2718  }
2719  dataMgr_->getForeignStorageInterface()->prepareTable(getCurrentDB().dbId, td, cds);
2720  }
2721 
2722  for (auto cd : cds) {
2723  if (cd.columnName == "rowid") {
2724  throw std::runtime_error(
2725  "Cannot create column with name rowid. rowid is a system defined column.");
2726  }
2727  columns.push_back(cd);
2728  toplevel_column_names.insert(cd.columnName);
2729  if (cd.columnType.is_geometry()) {
2730  expandGeoColumn(cd, columns);
2731  }
2732  }
2733  cds.clear();
2734 
2735  ColumnDescriptor cd;
2736  // add row_id column -- Must be last column in the table
2737  cd.columnName = "rowid";
2738  cd.isSystemCol = true;
2739  cd.columnType = SQLTypeInfo(kBIGINT, true);
2740 #ifdef MATERIALIZED_ROWID
2741  cd.isVirtualCol = false;
2742 #else
2743  cd.isVirtualCol = true;
2744  cd.virtualExpr = "MAPD_FRAG_ID * MAPD_ROWS_PER_FRAG + MAPD_FRAG_ROW_ID";
2745 #endif
2746  columns.push_back(cd);
2747  toplevel_column_names.insert(cd.columnName);
2748 
2749  if (td.hasDeletedCol) {
2750  ColumnDescriptor cd_del;
2751  cd_del.columnName = "$deleted$";
2752  cd_del.isSystemCol = true;
2753  cd_del.isVirtualCol = false;
2754  cd_del.columnType = SQLTypeInfo(kBOOLEAN, true);
2755  cd_del.isDeletedCol = true;
2756 
2757  columns.push_back(cd_del);
2758  }
2759 
2760  td.nColumns = columns.size();
2761  // TODO(sy): don't take disk locks or touch sqlite connector for temporary tables
2762  cat_sqlite_lock sqlite_lock(this);
2763  sqliteConnector_.query("BEGIN TRANSACTION");
2765  try {
2767  R"(INSERT INTO mapd_tables (name, userid, ncolumns, isview, fragments, frag_type, max_frag_rows, max_chunk_size, frag_page_size, max_rows, partitions, shard_column_id, shard, num_shards, sort_column_id, storage_type, max_rollback_epochs, is_system_table, key_metainfo) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?))",
2768  std::vector<std::string>{td.tableName,
2769  std::to_string(td.userId),
2771  std::to_string(td.isView),
2772  "",
2777  std::to_string(td.maxRows),
2778  td.partitions,
2780  std::to_string(td.shard),
2781  std::to_string(td.nShards),
2783  td.storageType,
2786  td.keyMetainfo});
2787 
2788  // now get the auto generated tableid
2790  "SELECT tableid FROM mapd_tables WHERE name = ?", td.tableName);
2791  td.tableId = sqliteConnector_.getData<int>(0, 0);
2792  int colId = 1;
2793  for (auto cd : columns) {
2795  const bool is_foreign_col =
2796  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2797  if (!is_foreign_col) {
2798  // Ideally we would like to not persist string dictionaries for system tables,
2799  // since system table content can be highly dynamic and string dictionaries
2800  // are not currently vacuumed. However, in distributed this causes issues
2801  // when the cluster is out of sync (when the agg resets but leaves persist) so
2802  // for the sake of testing we need to leave this as normal dictionaries until
2803  // we solve the distributed issue.
2804  auto use_temp_dictionary = false; // td.is_system_table;
2805  setColumnDictionary(cd, dds, td, isLogicalTable, use_temp_dictionary);
2806  }
2807  }
2808 
2809  if (toplevel_column_names.count(cd.columnName)) {
2810  if (!cd.isGeoPhyCol) {
2811  td.columnIdBySpi_.push_back(colId);
2812  }
2813  }
2814 
2815  using BindType = SqliteConnector::BindType;
2816  std::vector<BindType> types(17, BindType::TEXT);
2817  if (!cd.default_value.has_value()) {
2818  types[16] = BindType::NULL_TYPE;
2819  }
2821  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, "
2822  "coldim, colscale, is_notnull, "
2823  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
2824  "virtual_expr, is_deletedcol, default_value) "
2825  "VALUES (?, ?, ?, ?, ?, "
2826  "?, "
2827  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2828  std::vector<std::string>{std::to_string(td.tableId),
2829  std::to_string(colId),
2830  cd.columnName,
2839  "",
2842  cd.virtualExpr,
2844  cd.default_value.value_or("NULL")},
2845  types);
2846  cd.tableId = td.tableId;
2847  cd.columnId = colId++;
2848  cds.push_back(cd);
2849  }
2850  if (td.isView) {
2852  "INSERT INTO mapd_views (tableid, sql) VALUES (?,?)",
2853  std::vector<std::string>{std::to_string(td.tableId), td.viewSQL});
2854  }
2856  auto& foreign_table = dynamic_cast<foreign_storage::ForeignTable&>(td);
2857  foreign_table.next_refresh_time = get_next_refresh_time(foreign_table);
2859  "INSERT INTO omnisci_foreign_tables (table_id, server_id, options, "
2860  "last_refresh_time, next_refresh_time) VALUES (?, ?, ?, ?, ?)",
2861  std::vector<std::string>{std::to_string(foreign_table.tableId),
2862  std::to_string(foreign_table.foreign_server->id),
2863  foreign_table.getOptionsAsJsonString(),
2864  std::to_string(foreign_table.last_refresh_time),
2865  std::to_string(foreign_table.next_refresh_time)});
2866  }
2867  } catch (std::exception& e) {
2868  sqliteConnector_.query("ROLLBACK TRANSACTION");
2869  throw;
2870  }
2871  } else { // Temporary table
2872  td.tableId = nextTempTableId_++;
2873  int colId = 1;
2874  for (auto cd : columns) {
2876  const bool is_foreign_col =
2877  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2878 
2879  if (!is_foreign_col) {
2880  // Create a new temporary dictionary
2881  std::string fileName("");
2882  std::string folderPath("");
2884  nextTempDictId_++;
2885  DictDescriptor dd(dict_ref,
2886  fileName,
2888  false,
2889  1,
2890  folderPath,
2891  true); // Is dictName (2nd argument) used?
2892  dds.push_back(dd);
2893  if (!cd.columnType.is_array()) {
2895  }
2896  cd.columnType.set_comp_param(dict_ref.dictId);
2897  }
2898  }
2899  if (toplevel_column_names.count(cd.columnName)) {
2900  if (!cd.isGeoPhyCol) {
2901  td.columnIdBySpi_.push_back(colId);
2902  }
2903  }
2904  cd.tableId = td.tableId;
2905  cd.columnId = colId++;
2906  cds.push_back(cd);
2907  }
2908 
2910  serializeTableJsonUnlocked(&td, cds);
2911  }
2912  }
2913 
2914  try {
2915  auto cache = dataMgr_->getPersistentStorageMgr()->getDiskCache();
2916  if (cache) {
2917  CHECK(!cache->hasCachedMetadataForKeyPrefix({getCurrentDB().dbId, td.tableId}))
2918  << "Disk cache at " + cache->getCacheDirectory()
2919  << " contains preexisting data for new table. Please "
2920  "delete or clear cache before continuing";
2921  }
2922 
2923  addTableToMap(&td, cds, dds);
2924  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
2925  if (!td.storageType.empty() && td.storageType != StorageType::FOREIGN_TABLE) {
2926  dataMgr_->getForeignStorageInterface()->registerTable(this, td, cds);
2927  }
2928  } catch (std::exception& e) {
2929  sqliteConnector_.query("ROLLBACK TRANSACTION");
2930  removeTableFromMap(td.tableName, td.tableId, true);
2931  throw;
2932  }
2933  sqliteConnector_.query("END TRANSACTION");
2934 
2936  write_lock.unlock();
2937  sqlite_lock.unlock();
2939  true); // cause instantiateFragmenter() to be called
2940  }
2941 }
int32_t maxRollbackEpochs
void serializeTableJsonUnlocked(const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
Definition: Catalog.cpp:2943
int64_t get_next_refresh_time(const foreign_storage::ForeignTable &foreign_table)
Definition: Catalog.cpp:2690
std::string virtualExpr
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:380
void set_size(int s)
Definition: sqltypes.h:497
std::string partitions
HOST DEVICE int get_size() const
Definition: sqltypes.h:389
T getData(const int row, const int col)
std::string tableName
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
const DBMetadata currentDB_
Definition: Catalog.h:687
HOST DEVICE int get_scale() const
Definition: sqltypes.h:384
std::string storageType
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, bool is_logical_table, bool use_temp_dictionary=false)
Definition: Catalog.cpp:3820
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:688
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:379
std::vector< int > columnIdBySpi_
std::string to_string(char const *&&v)
void addTableToMap(const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
Definition: Catalog.cpp:1598
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:242
specifies the content in-memory of a row in the column metadata table
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:2530
std::string keyMetainfo
bool g_serialize_temp_tables
Definition: Catalog.cpp:106
void set_comp_param(int p)
Definition: sqltypes.h:501
std::optional< std::string > default_value
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:691
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:387
Fragmenter_Namespace::FragmenterType fragType
Data_Namespace::MemoryLevel persistenceLevel
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:381
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1657
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:388
bool setColumnSharedDictionary(ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:3751
#define CHECK(condition)
Definition: Logger.h:222
Descriptor for a dictionary for a string columne.
std::string viewSQL
SQLTypeInfo columnType
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:386
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
static constexpr char const * FOREIGN_TABLE
bool g_enable_fsi
Definition: Catalog.cpp:96
std::string columnName
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
bool is_array() const
Definition: sqltypes.h:583

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::delDictionary ( const ColumnDescriptor cd)

Definition at line 2308 of file Catalog.cpp.

References CHECK, CHECK_GT, ColumnDescriptor::columnType, g_base_path, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::is_string(), SQLTypeInfo::is_string_array(), shared::kDataDirectoryName, kENCODING_DICT, File_Namespace::renameForDelete(), ColumnDescriptor::tableId, to_string(), and VLOG.

2308  {
2309  cat_write_lock write_lock(this);
2310  cat_sqlite_lock sqlite_lock(this);
2311  if (!(cd.columnType.is_string() || cd.columnType.is_string_array())) {
2312  return;
2313  }
2314  if (!(cd.columnType.get_compression() == kENCODING_DICT)) {
2315  return;
2316  }
2317  const auto dictId = cd.columnType.get_comp_param();
2318  CHECK_GT(dictId, 0);
2319  // decrement and zero check dict ref count
2320  const auto td = getMetadataForTable(cd.tableId, false);
2321  CHECK(td);
2323  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
2324  std::to_string(dictId));
2326  "SELECT refcount FROM mapd_dictionaries WHERE dictid = ?", std::to_string(dictId));
2327  const auto refcount = sqliteConnector_.getData<int>(0, 0);
2328  VLOG(3) << "Dictionary " << dictId << "from dropped table has reference count "
2329  << refcount;
2330  if (refcount > 0) {
2331  return;
2332  }
2333  const DictRef dictRef(currentDB_.dbId, dictId);
2334  sqliteConnector_.query_with_text_param("DELETE FROM mapd_dictionaries WHERE dictid = ?",
2335  std::to_string(dictId));
2337  "/DB_" + std::to_string(currentDB_.dbId) + "_DICT_" +
2338  std::to_string(dictId));
2339 
2340  std::unique_ptr<StringDictionaryClient> client;
2341  if (!string_dict_hosts_.empty()) {
2342  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dictRef, true));
2343  }
2344  if (client) {
2345  client->drop(dictRef);
2346  }
2347 
2348  dictDescriptorMapByRef_.erase(dictRef);
2349 }
const std::string kDataDirectoryName
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:690
const DBMetadata currentDB_
Definition: Catalog.h:687
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:677
heavyai::unique_lock< heavyai::shared_mutex > write_lock
#define CHECK_GT(x, y)
Definition: Logger.h:234
std::string to_string(char const *&&v)
std::string g_base_path
Definition: SysCatalog.cpp:62
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:387
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:388
#define CHECK(condition)
Definition: Logger.h:222
SQLTypeInfo columnType
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
bool is_string() const
Definition: sqltypes.h:575
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:227
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
bool is_string_array() const
Definition: sqltypes.h:576
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
#define VLOG(n)
Definition: Logger.h:316

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::deleteCustomExpressions ( const std::vector< int32_t > &  custom_expression_ids,
bool  do_soft_delete 
)

Deletes custom expressions with the given ids.

Parameters
custom_expression_ids- ids of custom expressions to delete
do_soft_delete- flag indicating whether or not to do a soft delete

Definition at line 5662 of file Catalog.cpp.

References CHECK, CHECK_EQ, join(), and to_string().

5663  {
5664  cat_write_lock write_lock(this);
5665  cat_sqlite_lock sqlite_lock(this);
5666 
5667  std::vector<int32_t> invalid_ids;
5668  for (const auto id : custom_expression_ids) {
5669  if (custom_expr_map_by_id_.find(id) == custom_expr_map_by_id_.end()) {
5670  invalid_ids.emplace_back(id);
5671  }
5672  }
5673  if (!invalid_ids.empty()) {
5674  throw std::runtime_error{"Custom expressions with ids: " + join(invalid_ids, ",") +
5675  " do not exist."};
5676  }
5677  sqliteConnector_.query("BEGIN TRANSACTION");
5678  try {
5679  for (const auto id : custom_expression_ids) {
5681  "SELECT id FROM omnisci_custom_expressions WHERE id = ?",
5682  std::vector<std::string>{std::to_string(id)});
5683  CHECK_EQ(sqliteConnector_.getNumRows(), static_cast<size_t>(1));
5684  if (do_soft_delete) {
5686  "UPDATE omnisci_custom_expressions SET is_deleted = ? WHERE id = ?",
5687  std::vector<std::string>{std::to_string(true), std::to_string(id)});
5688  } else {
5690  "DELETE FROM omnisci_custom_expressions WHERE id = ?",
5691  std::vector<std::string>{std::to_string(id)});
5692  }
5693  }
5694 
5695  for (const auto id : custom_expression_ids) {
5696  if (do_soft_delete) {
5697  auto it = custom_expr_map_by_id_.find(id);
5698  CHECK(it != custom_expr_map_by_id_.end());
5699  it->second->is_deleted = true;
5700  } else {
5701  custom_expr_map_by_id_.erase(id);
5702  }
5703  }
5704  } catch (std::exception& e) {
5705  sqliteConnector_.query("ROLLBACK TRANSACTION");
5706  throw;
5707  }
5708  sqliteConnector_.query("END TRANSACTION");
5709 }
#define CHECK_EQ(x, y)
Definition: Logger.h:230
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
std::string join(T const &container, std::string const &delim)
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:222
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:683
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::deleteMetadataForDashboards ( const std::vector< int32_t >  ids,
const UserMetadata user 
)

Definition at line 2067 of file Catalog.cpp.

References DashboardDBObjectType, AccessPrivileges::DELETE_DASHBOARD, and to_string().

2068  {
2069  std::stringstream invalid_ids, restricted_ids;
2070 
2071  for (int32_t dashboard_id : dashboard_ids) {
2072  if (!getMetadataForDashboard(dashboard_id)) {
2073  invalid_ids << (!invalid_ids.str().empty() ? ", " : "") << dashboard_id;
2074  continue;
2075  }
2076  DBObject object(dashboard_id, DashboardDBObjectType);
2077  object.loadKey(*this);
2078  object.setPrivileges(AccessPrivileges::DELETE_DASHBOARD);
2079  std::vector<DBObject> privs = {object};
2080  if (!SysCatalog::instance().checkPrivileges(user, privs)) {
2081  restricted_ids << (!restricted_ids.str().empty() ? ", " : "") << dashboard_id;
2082  }
2083  }
2084 
2085  if (invalid_ids.str().size() > 0 || restricted_ids.str().size() > 0) {
2086  std::stringstream error_message;
2087  error_message << "Delete dashboard(s) failed with error(s):";
2088  if (invalid_ids.str().size() > 0) {
2089  error_message << "\nDashboard id: " << invalid_ids.str()
2090  << " - Dashboard id does not exist";
2091  }
2092  if (restricted_ids.str().size() > 0) {
2093  error_message
2094  << "\nDashboard id: " << restricted_ids.str()
2095  << " - User should be either owner of dashboard or super user to delete it";
2096  }
2097  throw std::runtime_error(error_message.str());
2098  }
2099  std::vector<DBObject> dash_objs;
2100 
2101  for (int32_t dashboard_id : dashboard_ids) {
2102  dash_objs.emplace_back(dashboard_id, DashboardDBObjectType);
2103  }
2104  // BE-5245: Transactionally unsafe (like other combined Catalog/Syscatalog operations)
2106  {
2107  cat_write_lock write_lock(this);
2108  cat_sqlite_lock sqlite_lock(this);
2109 
2110  sqliteConnector_.query("BEGIN TRANSACTION");
2111  try {
2112  for (int32_t dashboard_id : dashboard_ids) {
2113  auto dash = getMetadataForDashboard(dashboard_id);
2114  // Dash should still exist if revokeDBObjectPrivileges passed but throw and
2115  // rollback if already deleted
2116  if (!dash) {
2117  throw std::runtime_error(
2118  std::string("Delete dashboard(s) failed with error(s):\nDashboard id: ") +
2119  std::to_string(dashboard_id) + " - Dashboard id does not exist ");
2120  }
2121  std::string user_id = std::to_string(dash->userId);
2122  std::string dash_name = dash->dashboardName;
2123  auto viewDescIt = dashboardDescriptorMap_.find(user_id + ":" + dash_name);
2124  dashboardDescriptorMap_.erase(viewDescIt);
2126  "DELETE FROM mapd_dashboards WHERE name = ? and userid = ?",
2127  std::vector<std::string>{dash_name, user_id});
2128  }
2129  } catch (std::exception& e) {
2130  sqliteConnector_.query("ROLLBACK TRANSACTION");
2131  throw;
2132  }
2133  sqliteConnector_.query("END TRANSACTION");
2134  }
2135 }
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:678
std::string to_string(char const *&&v)
static SysCatalog & instance()
Definition: SysCatalog.h:341
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
static const AccessPrivileges DELETE_DASHBOARD
Definition: DBObject.h:173
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
void revokeDBObjectPrivilegesFromAllBatch(std::vector< DBObject > &objects, Catalog *catalog)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::deleteTableCatalogMetadata ( const TableDescriptor logical_table,
const std::vector< const TableDescriptor * > &  physical_tables 
)
private

Definition at line 4044 of file Catalog.cpp.

References TableDescriptor::tableId, and to_string().

4046  {
4047  cat_write_lock write_lock(this);
4048  cat_sqlite_lock sqlite_lock(this);
4049  sqliteConnector_.query("BEGIN TRANSACTION");
4050  try {
4051  // remove corresponding record from the logicalToPhysicalTableMap in sqlite database
4053  "DELETE FROM mapd_logical_to_physical WHERE logical_table_id = ?",
4054  std::to_string(logical_table->tableId));
4055  logicalToPhysicalTableMapById_.erase(logical_table->tableId);
4056  for (auto table : physical_tables) {
4057  eraseTableMetadata(table);
4058  }
4059  } catch (std::exception& e) {
4060  sqliteConnector_.query("ROLLBACK TRANSACTION");
4061  throw;
4062  }
4063  sqliteConnector_.query("END TRANSACTION");
4064 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
void eraseTableMetadata(const TableDescriptor *td)
Definition: Catalog.cpp:4066
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:693

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::doTruncateTable ( const TableDescriptor td)
protected

Definition at line 3906 of file Catalog.cpp.

References CHECK, CHECK_GE, ColumnDescriptor::columnType, Data_Namespace::CPU_LEVEL, dict_ref_t::dictId, DictDescriptor::dictIsTemp, DictDescriptor::dictRef, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), Data_Namespace::GPU_LEVEL, kENCODING_DICT, File_Namespace::renameForDelete(), ColumnDescriptor::tableId, and TableDescriptor::tableId.

3906  {
3907  // must destroy fragmenter before deleteChunks is called.
3909 
3910  const int tableId = td->tableId;
3911  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
3912  // assuming deleteChunksWithPrefix is atomic
3913  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
3914  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
3915 
3916  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
3917 
3918  cat_write_lock write_lock(this);
3919  std::unique_ptr<StringDictionaryClient> client;
3920  if (SysCatalog::instance().isAggregator()) {
3921  CHECK(!string_dict_hosts_.empty());
3922  DictRef dict_ref(currentDB_.dbId, -1);
3923  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
3924  }
3925  // clean up any dictionaries
3926  // delete all column descriptors for the table
3927  for (const auto& columnDescriptor : columnDescriptorMapById_) {
3928  auto cd = columnDescriptor.second;
3929  if (cd->tableId != td->tableId) {
3930  continue;
3931  }
3932  const int dict_id = cd->columnType.get_comp_param();
3933  // Dummy dictionaries created for a shard of a logical table have the id set to zero.
3934  if (cd->columnType.get_compression() == kENCODING_DICT && dict_id) {
3935  const DictRef dict_ref(currentDB_.dbId, dict_id);
3936  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
3937  CHECK(dictIt != dictDescriptorMapByRef_.end());
3938  const auto& dd = dictIt->second;
3939  CHECK_GE(dd->refcount, 1);
3940  // if this is the only table using this dict reset the dict
3941  if (dd->refcount == 1) {
3942  // close the dictionary
3943  dd->stringDict.reset();
3944  File_Namespace::renameForDelete(dd->dictFolderPath);
3945  if (client) {
3946  client->drop(dd->dictRef);
3947  }
3948  if (!dd->dictIsTemp) {
3949  boost::filesystem::create_directory(dd->dictFolderPath);
3950  }
3951  }
3952 
3953  DictDescriptor* new_dd = new DictDescriptor(dd->dictRef,
3954  dd->dictName,
3955  dd->dictNBits,
3956  dd->dictIsShared,
3957  dd->refcount,
3958  dd->dictFolderPath,
3959  dd->dictIsTemp);
3960  dictDescriptorMapByRef_.erase(dictIt);
3961  // now create new Dict -- need to figure out what to do here for temp tables
3962  if (client) {
3963  client->create(new_dd->dictRef, new_dd->dictIsTemp);
3964  }
3965  dictDescriptorMapByRef_[new_dd->dictRef].reset(new_dd);
3967  }
3968  }
3969 }
std::vector< int > ChunkKey
Definition: types.h:36
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:690
const DBMetadata currentDB_
Definition: Catalog.h:687
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:677
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:688
#define CHECK_GE(x, y)
Definition: Logger.h:235
heavyai::unique_lock< heavyai::shared_mutex > write_lock
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:676
static SysCatalog & instance()
Definition: SysCatalog.h:341
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1960
int32_t dictId
Definition: DictRef.h:11
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
void removeFragmenterForTable(const int table_id) const
Definition: Catalog.cpp:3994
#define CHECK(condition)
Definition: Logger.h:222
Descriptor for a dictionary for a string columne.
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:227

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropColumn ( const TableDescriptor td,
const ColumnDescriptor cd 
)

Definition at line 2441 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnId, ColumnDescriptor::columnName, TableDescriptor::nShards, TableDescriptor::shard, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), and to_upper().

2441  {
2442  {
2443  cat_write_lock write_lock(this);
2444  cat_sqlite_lock sqlite_lock(this);
2445  // caller must handle sqlite/chunk transaction TOGETHER
2447  "DELETE FROM mapd_columns where tableid = ? and columnid = ?",
2448  std::vector<std::string>{std::to_string(td.tableId),
2449  std::to_string(cd.columnId)});
2450 
2452  "UPDATE mapd_tables SET ncolumns = ncolumns - 1 WHERE tableid = ?",
2453  std::vector<std::string>{std::to_string(td.tableId)});
2454 
2455  ColumnDescriptorMap::iterator columnDescIt =
2457  CHECK(columnDescIt != columnDescriptorMap_.end());
2458 
2459  columnDescriptorsForRoll.emplace_back(columnDescIt->second, nullptr);
2460  removeFromColumnMap(columnDescIt->second);
2461  --tableDescriptorMapById_[td.tableId]->nColumns;
2462  }
2463 
2464  // for each shard
2465  if (td.nShards > 0 && td.shard < 0) {
2466  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
2467  const auto shard_cd = getMetadataForColumn(shard->tableId, cd.columnId);
2468  CHECK(shard_cd);
2469  dropColumn(*shard, *shard_cd);
2470  }
2471  }
2472 }
void removeFromColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6499
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:675
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void dropColumn(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:2441
std::string to_string(char const *&&v)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:674
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4630
std::string to_upper(const std::string &str)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:222
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:705
std::string columnName

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropForeignServer ( const std::string &  server_name)

Drops/deletes a foreign server DB object.

Parameters
server_name- Name of foreign server that will be deleted

Definition at line 3213 of file Catalog.cpp.

References CHECK_EQ, and to_string().

3213  {
3214  cat_write_lock write_lock(this);
3215  cat_sqlite_lock sqlite_lock(this);
3216 
3218  "SELECT id from omnisci_foreign_servers where name = ?",
3219  std::vector<std::string>{server_name});
3220  auto num_rows = sqliteConnector_.getNumRows();
3221  if (num_rows > 0) {
3222  CHECK_EQ(size_t(1), num_rows);
3223  auto server_id = sqliteConnector_.getData<int32_t>(0, 0);
3225  "SELECT table_id from omnisci_foreign_tables where server_id = ?",
3226  std::to_string(server_id));
3227  if (sqliteConnector_.getNumRows() > 0) {
3228  throw std::runtime_error{"Foreign server \"" + server_name +
3229  "\" is referenced "
3230  "by existing foreign tables and cannot be dropped."};
3231  }
3232  sqliteConnector_.query("BEGIN TRANSACTION");
3233  try {
3235  "DELETE FROM omnisci_foreign_servers WHERE name = ?",
3236  std::vector<std::string>{server_name});
3237  } catch (const std::exception& e) {
3238  sqliteConnector_.query("ROLLBACK TRANSACTION");
3239  throw;
3240  }
3241  sqliteConnector_.query("END TRANSACTION");
3242  foreignServerMap_.erase(server_name);
3243  foreignServerMapById_.erase(server_id);
3244  }
3245 }
#define CHECK_EQ(x, y)
Definition: Logger.h:230
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:686
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::string to_string(char const *&&v)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:682
ForeignServerMap foreignServerMap_
Definition: Catalog.h:681
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropTable ( const TableDescriptor td)

Definition at line 4016 of file Catalog.cpp.

References CHECK, TableDescriptor::isView, TableDBObjectType, TableDescriptor::tableId, TableDescriptor::tableName, and ViewDBObjectType.

Referenced by Catalog_Namespace::anonymous_namespace{Catalog.cpp}::drop_tables(), Parser::CreateTableAsSelectStmt::execute(), and Parser::DropViewStmt::execute().

4016  {
4019  std::vector<const TableDescriptor*> tables_to_drop;
4020  {
4021  cat_read_lock read_lock(this);
4022  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
4023  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
4024  // remove all corresponding physical tables if this is a logical table
4025  const auto physicalTables = physicalTableIt->second;
4026  CHECK(!physicalTables.empty());
4027  for (size_t i = 0; i < physicalTables.size(); i++) {
4028  int32_t physical_tb_id = physicalTables[i];
4029  const TableDescriptor* phys_td =
4030  getMutableMetadataForTableUnlocked(physical_tb_id);
4031  CHECK(phys_td);
4032  tables_to_drop.emplace_back(phys_td);
4033  }
4034  }
4035  tables_to_drop.emplace_back(td);
4036  }
4037 
4038  for (auto table : tables_to_drop) {
4039  eraseTablePhysicalData(table);
4040  }
4041  deleteTableCatalogMetadata(td, tables_to_drop);
4042 }
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:4828
std::string tableName
heavyai::shared_lock< heavyai::shared_mutex > read_lock
void revokeDBObjectPrivilegesFromAll(DBObject object, Catalog *catalog)
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
void deleteTableCatalogMetadata(const TableDescriptor *logical_table, const std::vector< const TableDescriptor * > &physical_tables)
Definition: Catalog.cpp:4044
TableDescriptor * getMutableMetadataForTableUnlocked(int table_id) const
Definition: Catalog.cpp:1952
static SysCatalog & instance()
Definition: SysCatalog.h:341
#define CHECK(condition)
Definition: Logger.h:222
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:693

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::dropTableFromJsonUnlocked ( const std::string &  table_name) const
protected

Definition at line 3014 of file Catalog.cpp.

References CHECK, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and VLOG.

3014  {
3015  // relies on the catalog write lock
3016  using namespace rapidjson;
3017 
3018  VLOG(1) << "Dropping temporary table " << table_name << " to JSON for Calcite.";
3019 
3020  const auto db_name = currentDB_.dbName;
3021  const auto file_path = table_json_filepath(basePath_, db_name);
3022 
3023  CHECK(boost::filesystem::exists(file_path));
3024  Document d;
3025 
3026  std::ifstream reader(file_path.string());
3027  CHECK(reader.is_open());
3028  IStreamWrapper json_read_wrapper(reader);
3029  d.ParseStream(json_read_wrapper);
3030 
3031  CHECK(d.IsObject());
3032  auto table_name_ref = StringRef(table_name.c_str());
3033  CHECK(d.HasMember(table_name_ref));
3034  CHECK(d.RemoveMember(table_name_ref));
3035 
3036  // Overwrite the existing file
3037  std::ofstream writer(file_path.string(), std::ios::trunc | std::ios::out);
3038  CHECK(writer.is_open());
3039  OStreamWrapper json_wrapper(writer);
3040 
3041  Writer<OStreamWrapper> json_writer(json_wrapper);
3042  d.Accept(json_writer);
3043  writer.close();
3044 }
const DBMetadata currentDB_
Definition: Catalog.h:687
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:162
#define CHECK(condition)
Definition: Logger.h:222
#define VLOG(n)
Definition: Logger.h:316

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::dumpCreateServer ( const std::string &  name,
bool  multiline_formatting = true 
) const

Definition at line 5339 of file Catalog.cpp.

References join().

5340  {
5341  cat_read_lock read_lock(this);
5342  auto server_it = foreignServerMap_.find(name);
5343  if (server_it == foreignServerMap_.end()) {
5344  throw std::runtime_error("Foreign server " + name + " does not exist.");
5345  }
5346  auto server = server_it->second.get();
5347  std::ostringstream os;
5348  os << "CREATE SERVER " << name << " FOREIGN DATA WRAPPER " << server->data_wrapper_type;
5349  std::vector<std::string> with_options;
5350  for (const auto& [option, value] : server->options) {
5351  with_options.emplace_back(option + "='" + value + "'");
5352  }
5353  if (!with_options.empty()) {
5354  if (!multiline_formatting) {
5355  os << " ";
5356  } else {
5357  os << "\n";
5358  }
5359  os << "WITH (" + boost::algorithm::join(with_options, ", ") + ")";
5360  }
5361  os << ";";
5362  return os.str();
5363 }
heavyai::shared_lock< heavyai::shared_mutex > read_lock
std::string join(T const &container, std::string const &delim)
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
std::string name() const
Definition: Catalog.h:315
ForeignServerMap foreignServerMap_
Definition: Catalog.h:681

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::dumpCreateTable ( const TableDescriptor td,
bool  multiline_formatting = true,
bool  dump_defaults = false 
) const

Definition at line 5160 of file Catalog.cpp.

5162  {
5163  cat_read_lock read_lock(this);
5164  return dumpCreateTableUnlocked(td, multiline_formatting, dump_defaults);
5165 }
heavyai::shared_lock< heavyai::shared_mutex > read_lock
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
std::string dumpCreateTableUnlocked(const TableDescriptor *td, bool multiline_formatting, bool dump_defaults) const
Definition: Catalog.cpp:5178
std::optional< std::string > Catalog_Namespace::Catalog::dumpCreateTable ( int32_t  table_id,
bool  multiline_formatting = true,
bool  dump_defaults = false 
) const

Definition at line 5167 of file Catalog.cpp.

5169  {
5170  cat_read_lock read_lock(this);
5171  const auto td = getMutableMetadataForTableUnlocked(table_id);
5172  if (!td) {
5173  return {};
5174  }
5175  return dumpCreateTableUnlocked(td, multiline_formatting, dump_defaults);
5176 }
heavyai::shared_lock< heavyai::shared_mutex > read_lock
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
std::string dumpCreateTableUnlocked(const TableDescriptor *td, bool multiline_formatting, bool dump_defaults) const
Definition: Catalog.cpp:5178
TableDescriptor * getMutableMetadataForTableUnlocked(int table_id) const
Definition: Catalog.cpp:1952
std::string Catalog_Namespace::Catalog::dumpCreateTableUnlocked ( const TableDescriptor td,
bool  multiline_formatting,
bool  dump_defaults 
) const
private

Definition at line 5178 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, Data_Namespace::CPU_LEVEL, DEFAULT_FRAGMENT_ROWS, DEFAULT_MAX_CHUNK_SIZE, DEFAULT_MAX_ROLLBACK_EPOCHS, DEFAULT_MAX_ROWS, DEFAULT_PAGE_SIZE, ColumnDescriptor::default_value, TableDescriptor::fragPageSize, g_leaf_count, ColumnDescriptor::getDefaultValueLiteral(), TableDescriptor::hasDeletedCol, TableDescriptor::is_system_table, ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, join(), kCHAR, kENCODING_DICT, kENCODING_GEOINT, kTEXT, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and TableDescriptor::viewSQL.

5180  {
5181  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
5182  std::ostringstream os;
5183 
5184  if (foreign_table && !td->is_system_table) {
5185  os << "CREATE FOREIGN TABLE " << td->tableName << " (";
5186  } else if (!td->isView) {
5187  os << "CREATE ";
5189  os << "TEMPORARY ";
5190  }
5191  os << "TABLE " + td->tableName + " (";
5192  } else {
5193  os << "CREATE VIEW " + td->tableName + " AS " << td->viewSQL;
5194  return os.str();
5195  }
5196  // scan column defines
5197  std::vector<std::string> additional_info;
5198  std::set<std::string> shared_dict_column_names;
5199 
5200  gatherAdditionalInfo(additional_info, shared_dict_column_names, td);
5201 
5202  // gather column defines
5203  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
5204  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
5205  bool first = true;
5206  for (const auto cd : cds) {
5207  if (!(cd->isSystemCol || cd->isVirtualCol)) {
5208  const auto& ti = cd->columnType;
5209  if (!first) {
5210  os << ",";
5211  if (!multiline_formatting) {
5212  os << " ";
5213  }
5214  } else {
5215  first = false;
5216  }
5217  if (multiline_formatting) {
5218  os << "\n ";
5219  }
5220  // column name
5221  os << quoteIfRequired(cd->columnName);
5222  // CHAR is perculiar... better dump it as TEXT(32) like \d does
5223  if (ti.get_type() == SQLTypes::kCHAR) {
5224  os << " "
5225  << "TEXT";
5226  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
5227  os << " "
5228  << "TEXT[]";
5229  } else {
5230  os << " " << ti.get_type_name();
5231  }
5232  os << (ti.get_notnull() ? " NOT NULL" : "");
5233  if (cd->default_value.has_value()) {
5234  os << " DEFAULT " << cd->getDefaultValueLiteral();
5235  }
5236  if (shared_dict_column_names.find(cd->columnName) ==
5237  shared_dict_column_names.end()) {
5238  // avoids "Column ... shouldn't specify an encoding, it borrows it
5239  // from the referenced column"
5240  if (ti.is_string() || (ti.is_array() && ti.get_subtype() == kTEXT)) {
5241  auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
5242  if (ti.get_compression() == kENCODING_DICT) {
5243  os << " ENCODING " << ti.get_compression_name() << "(" << (size * 8) << ")";
5244  } else {
5245  os << " ENCODING NONE";
5246  }
5247  } else if (ti.is_date_in_days() ||
5248  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
5249  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
5250  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
5251  } else if (ti.is_geometry()) {
5252  if (ti.get_compression() == kENCODING_GEOINT) {
5253  os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
5254  << ")";
5255  } else {
5256  os << " ENCODING NONE";
5257  }
5258  }
5259  }
5260  }
5261  }
5262  // gather SHARED DICTIONARYs
5263  if (additional_info.size()) {
5264  std::string comma;
5265  if (!multiline_formatting) {
5266  comma = ", ";
5267  } else {
5268  comma = ",\n ";
5269  }
5270  os << comma;
5271  os << boost::algorithm::join(additional_info, comma);
5272  }
5273  os << ")";
5274 
5275  std::vector<std::string> with_options;
5276  if (foreign_table && !td->is_system_table) {
5277  if (multiline_formatting) {
5278  os << "\n";
5279  } else {
5280  os << " ";
5281  }
5282  os << "SERVER " << foreign_table->foreign_server->name;
5283 
5284  // gather WITH options ...
5285  for (const auto& [option, value] : foreign_table->options) {
5286  with_options.emplace_back(option + "='" + value + "'");
5287  }
5288  }
5289 
5290  if (dump_defaults || td->maxFragRows != DEFAULT_FRAGMENT_ROWS) {
5291  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
5292  }
5293  if (dump_defaults || td->maxChunkSize != DEFAULT_MAX_CHUNK_SIZE) {
5294  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
5295  }
5296  if (!foreign_table && (dump_defaults || td->fragPageSize != DEFAULT_PAGE_SIZE)) {
5297  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
5298  }
5299  if (!foreign_table && (dump_defaults || td->maxRows != DEFAULT_MAX_ROWS)) {
5300  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
5301  }
5302  if ((dump_defaults || td->maxRollbackEpochs != DEFAULT_MAX_ROLLBACK_EPOCHS) &&
5303  td->maxRollbackEpochs != -1) {
5304  with_options.push_back("MAX_ROLLBACK_EPOCHS=" +
5306  }
5307  if (!foreign_table && (dump_defaults || !td->hasDeletedCol)) {
5308  with_options.emplace_back(td->hasDeletedCol ? "VACUUM='DELAYED'"
5309  : "VACUUM='IMMEDIATE'");
5310  }
5311  if (!foreign_table && !td->partitions.empty()) {
5312  with_options.push_back("PARTITIONS='" + td->partitions + "'");
5313  }
5314  if (!foreign_table && td->nShards > 0) {
5315  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
5316  CHECK(shard_cd);
5317  with_options.push_back(
5318  "SHARD_COUNT=" +
5319  std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
5320  }
5321  if (!foreign_table && td->sortedColumnId > 0) {
5322  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
5323  CHECK(sort_cd);
5324  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
5325  }
5326 
5327  if (!with_options.empty()) {
5328  if (!multiline_formatting) {
5329  os << " ";
5330  } else {
5331  os << "\n";
5332  }
5333  os << "WITH (" + boost::algorithm::join(with_options, ", ") + ")";
5334  }
5335  os << ";";
5336  return os.str();
5337 }
int32_t maxRollbackEpochs
std::string partitions
std::string tableName
std::string join(T const &container, std::string const &delim)
#define DEFAULT_MAX_CHUNK_SIZE
std::string to_string(char const *&&v)
std::string quoteIfRequired(const std::string &column_name) const
Definition: Catalog.cpp:5460
#define DEFAULT_MAX_ROWS
void gatherAdditionalInfo(std::vector< std::string > &additional_info, std::set< std::string > &shared_dict_column_names, const TableDescriptor *td) const
Definition: Catalog.cpp:5471
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
#define DEFAULT_MAX_ROLLBACK_EPOCHS
specifies the content in-memory of a row in the table metadata table
#define DEFAULT_PAGE_SIZE
Definition: sqltypes.h:67
#define DEFAULT_FRAGMENT_ROWS
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:2228
Data_Namespace::MemoryLevel persistenceLevel
Definition: sqltypes.h:56
#define CHECK(condition)
Definition: Logger.h:222
std::string viewSQL
size_t g_leaf_count
Definition: ParserNode.cpp:76

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::dumpSchema ( const TableDescriptor td) const

Definition at line 5033 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, DEFAULT_MAX_ROLLBACK_EPOCHS, ColumnDescriptor::default_value, TableDescriptor::fragPageSize, g_leaf_count, ColumnDescriptor::getDefaultValueLiteral(), TableDescriptor::hasDeletedCol, TableDescriptor::is_system_table, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, join(), kCHAR, kENCODING_DICT, kENCODING_GEOINT, kTEXT, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::tableId, and to_string().

Referenced by TableArchiver::dumpTable().

5033  {
5034  CHECK(!td->is_system_table);
5035  cat_read_lock read_lock(this);
5036 
5037  std::ostringstream os;
5038  os << "CREATE TABLE @T (";
5039  // gather column defines
5040  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
5041  std::string comma;
5042  std::vector<std::string> shared_dicts;
5043  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
5044  for (const auto cd : cds) {
5045  if (!(cd->isSystemCol || cd->isVirtualCol)) {
5046  const auto& ti = cd->columnType;
5047  os << comma << cd->columnName;
5048  // CHAR is perculiar... better dump it as TEXT(32) like \d does
5049  if (ti.get_type() == SQLTypes::kCHAR) {
5050  os << " "
5051  << "TEXT";
5052  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
5053  os << " "
5054  << "TEXT[]";
5055  } else {
5056  os << " " << ti.get_type_name();
5057  }
5058  os << (ti.get_notnull() ? " NOT NULL" : "");
5059  if (cd->default_value.has_value()) {
5060  os << " DEFAULT " << cd->getDefaultValueLiteral();
5061  }
5062  if (ti.is_string() || (ti.is_array() && ti.get_subtype() == kTEXT)) {
5063  auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
5064  if (ti.get_compression() == kENCODING_DICT) {
5065  // if foreign reference, get referenced tab.col
5066  const auto dict_id = ti.get_comp_param();
5067  const DictRef dict_ref(currentDB_.dbId, dict_id);
5068  const auto dict_it = dictDescriptorMapByRef_.find(dict_ref);
5069  CHECK(dict_it != dictDescriptorMapByRef_.end());
5070  const auto dict_name = dict_it->second->dictName;
5071  // when migrating a table, any foreign dict ref will be dropped
5072  // and the first cd of a dict will become root of the dict
5073  if (dict_root_cds.end() == dict_root_cds.find(dict_name)) {
5074  dict_root_cds[dict_name] = cd;
5075  os << " ENCODING " << ti.get_compression_name() << "(" << (size * 8) << ")";
5076  } else {
5077  const auto dict_root_cd = dict_root_cds[dict_name];
5078  shared_dicts.push_back("SHARED DICTIONARY (" + cd->columnName +
5079  ") REFERENCES @T(" + dict_root_cd->columnName + ")");
5080  // "... shouldn't specify an encoding, it borrows from the referenced
5081  // column"
5082  }
5083  } else {
5084  os << " ENCODING NONE";
5085  }
5086  } else if (ti.is_date_in_days() ||
5087  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
5088  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
5089  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
5090  } else if (ti.is_geometry()) {
5091  if (ti.get_compression() == kENCODING_GEOINT) {
5092  os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
5093  << ")";
5094  } else {
5095  os << " ENCODING NONE";
5096  }
5097  }
5098  comma = ", ";
5099  }
5100  }
5101  // gather SHARED DICTIONARYs
5102  if (shared_dicts.size()) {
5103  os << ", " << boost::algorithm::join(shared_dicts, ", ");
5104  }
5105  // gather WITH options ...
5106  std::vector<std::string> with_options;
5107  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
5108  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
5109  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
5110  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
5111  with_options.emplace_back(td->hasDeletedCol ? "VACUUM='DELAYED'"
5112  : "VACUUM='IMMEDIATE'");
5113  if (!td->partitions.empty()) {
5114  with_options.push_back("PARTITIONS='" + td->partitions + "'");
5115  }
5116  if (td->nShards > 0) {
5117  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
5118  CHECK(shard_cd);
5119  os << ", SHARD KEY(" << shard_cd->columnName << ")";
5120  with_options.push_back(
5121  "SHARD_COUNT=" +
5122  std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
5123  }
5124  if (td->sortedColumnId > 0) {
5125  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
5126  CHECK(sort_cd);
5127  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
5128  }
5130  td->maxRollbackEpochs != -1) {
5131  with_options.push_back("MAX_ROLLBACK_EPOCHS=" +
5133  }
5134  os << ") WITH (" + boost::algorithm::join(with_options, ", ") + ");";
5135  return os.str();
5136 }
int32_t maxRollbackEpochs
std::string partitions
heavyai::shared_lock< heavyai::shared_mutex > read_lock
const DBMetadata currentDB_
Definition: Catalog.h:687
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:677
std::string join(T const &container, std::string const &delim)
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
std::string to_string(char const *&&v)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
#define DEFAULT_MAX_ROLLBACK_EPOCHS
specifies the content in-memory of a row in the table metadata table
Definition: sqltypes.h:67
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:2228
Definition: sqltypes.h:56
#define CHECK(condition)
Definition: Logger.h:222
size_t g_leaf_count
Definition: ParserNode.cpp:76

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::eraseDbMetadata ( )

Definition at line 4810 of file Catalog.cpp.

References shared::kCatalogDirectoryName, and run_benchmark_import::tables.

4810  {
4811  const auto tables = getAllTableMetadata();
4812  for (const auto table : tables) {
4813  eraseTableMetadata(table);
4814  }
4815  // Physically erase database metadata
4816  boost::filesystem::remove(basePath_ + "/" + shared::kCatalogDirectoryName + "/" +
4817  currentDB_.dbName);
4818  calciteMgr_->updateMetadata(currentDB_.dbName, "");
4819 }
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:2244
const DBMetadata currentDB_
Definition: Catalog.h:687
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:691
const std::string kCatalogDirectoryName
void eraseTableMetadata(const TableDescriptor *td)
Definition: Catalog.cpp:4066
void Catalog_Namespace::Catalog::eraseDbPhysicalData ( )

Definition at line 4821 of file Catalog.cpp.

References run_benchmark_import::tables.

4821  {
4822  const auto tables = getAllTableMetadata();
4823  for (const auto table : tables) {
4824  eraseTablePhysicalData(table);
4825  }
4826 }
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:4828
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:2244
void Catalog_Namespace::Catalog::eraseTableMetadata ( const TableDescriptor td)
protected

Definition at line 4066 of file Catalog.cpp.

References g_serialize_temp_tables, INJECT_TIMER, table_is_temporary(), TableDescriptor::tableId, and TableDescriptor::tableName.

4066  {
4070  }
4071  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
4072  {
4073  INJECT_TIMER(removeTableFromMap_);
4075  }
4076 }
void executeDropTableSqliteQueries(const TableDescriptor *td)
Definition: Catalog.cpp:4078
std::string tableName
const DBMetadata currentDB_
Definition: Catalog.h:687
void dropTableFromJsonUnlocked(const std::string &table_name) const
Definition: Catalog.cpp:3014
#define INJECT_TIMER(DESC)
Definition: measure.h:93
bool g_serialize_temp_tables
Definition: Catalog.cpp:106
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:691
bool table_is_temporary(const TableDescriptor *const td)
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1657

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::eraseTablePhysicalData ( const TableDescriptor td)

Definition at line 4828 of file Catalog.cpp.

References Data_Namespace::CPU_LEVEL, Data_Namespace::GPU_LEVEL, INJECT_TIMER, TableDescriptor::isView, and TableDescriptor::tableId.

4828  {
4829  const int tableId = td->tableId;
4830  // must destroy fragmenter before deleteChunks is called.
4831  removeFragmenterForTable(tableId);
4832 
4833  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
4834  {
4835  INJECT_TIMER(deleteChunksWithPrefix);
4836  // assuming deleteChunksWithPrefix is atomic
4837  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
4838  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
4839  }
4840  if (!td->isView) {
4841  INJECT_TIMER(Remove_Table);
4842  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
4843  }
4844 }
std::vector< int > ChunkKey
Definition: types.h:36
const DBMetadata currentDB_
Definition: Catalog.h:687