OmniSciDB  c1a53651b2
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Catalog_Namespace::Catalog Class Referencefinal

class for a per-database catalog. also includes metadata for the current database and the current user. More...

#include <Catalog.h>

+ Collaboration diagram for Catalog_Namespace::Catalog:

Public Member Functions

 Catalog (const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
 Constructor - takes basePath to already extant data directory for writing. More...
 
 Catalog ()
 Constructor builds a hollow catalog used during constructor of other catalogs. More...
 
 ~Catalog ()
 Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite. More...
 
void createTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
 
void createShardedTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
int32_t createDashboard (DashboardDescriptor &vd)
 
void replaceDashboard (DashboardDescriptor &vd)
 
std::string createLink (LinkDescriptor &ld, size_t min_length)
 
void dropTable (const TableDescriptor *td)
 
void truncateTable (const TableDescriptor *td)
 
void renameTable (const TableDescriptor *td, const std::string &newTableName)
 
void renameTable (std::vector< std::pair< std::string, std::string >> &names)
 
void renameColumn (const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
 
void addColumn (const TableDescriptor &td, ColumnDescriptor &cd)
 
void dropColumn (const TableDescriptor &td, const ColumnDescriptor &cd)
 
void invalidateCachesForTable (const int table_id)
 
void removeFragmenterForTable (const int table_id) const
 
const std::map< int, const
ColumnDescriptor * > 
getDictionaryToColumnMapping ()
 
const TableDescriptorgetMetadataForTable (const std::string &tableName, const bool populateFragmenter=true) const
 Returns a pointer to a const TableDescriptor struct matching the provided tableName. More...
 
const TableDescriptorgetMetadataForTable (int tableId, bool populateFragmenter=true) const
 
std::optional< std::string > getTableName (int32_t table_id) const
 
std::optional< int32_t > getTableId (const std::string &table_name) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, const std::string &colName) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, int columnId) const
 
const std::optional< std::string > getColumnName (int table_id, int column_id) const
 
const int getColumnIdBySpi (const int tableId, const size_t spi) const
 
const ColumnDescriptorgetMetadataForColumnBySpi (const int tableId, const size_t spi) const
 
const DashboardDescriptorgetMetadataForDashboard (const std::string &userId, const std::string &dashName) const
 
const DashboardDescriptorgetMetadataForDashboard (const int32_t dashboard_id) const
 
void deleteMetadataForDashboards (const std::vector< int32_t > ids, const UserMetadata &user)
 
const LinkDescriptorgetMetadataForLink (const std::string &link) const
 
const LinkDescriptorgetMetadataForLink (int linkId) const
 
const
foreign_storage::ForeignTable
getForeignTable (const std::string &tableName) const
 
const
foreign_storage::ForeignTable
getForeignTable (int table_id) const
 
std::list< const
ColumnDescriptor * > 
getAllColumnMetadataForTable (const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular table specified by table id. More...
 
std::list< const
TableDescriptor * > 
getAllTableMetadata () const
 
std::vector< TableDescriptorgetAllTableMetadataCopy () const
 
std::list< const
DashboardDescriptor * > 
getAllDashboardsMetadata () const
 
std::vector< DashboardDescriptorgetAllDashboardsMetadataCopy () const
 
const DBMetadatagetCurrentDB () const
 
Data_Namespace::DataMgrgetDataMgr () const
 
std::shared_ptr< CalcitegetCalciteMgr () const
 
const std::string & getCatalogBasePath () const
 
const DictDescriptorgetMetadataForDict (int dict_ref, bool loadDict=true) const
 
const std::vector< LeafHostInfo > & getStringDictionaryHosts () const
 
const ColumnDescriptorgetShardColumnMetadataForTable (const TableDescriptor *td) const
 
std::vector< const
TableDescriptor * > 
getPhysicalTablesDescriptors (const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
 
std::vector< std::pair
< int32_t, int32_t > > 
getAllPersistedTableAndShardIds () const
 
std::vector< std::string > getTableNamesForUser (const UserMetadata &user, const GetTablesType get_tables_type) const
 
std::vector< TableMetadatagetTablesMetadataForUser (const UserMetadata &user_metadata, const GetTablesType get_tables_type, const std::string &filter_table_name) const
 
int32_t getTableEpoch (const int32_t db_id, const int32_t table_id) const
 
void setTableEpoch (const int db_id, const int table_id, const int new_epoch)
 
void setMaxRollbackEpochs (const int32_t table_id, const int32_t max_rollback_epochs)
 
void setMaxRows (const int32_t table_id, const int64_t max_rows)
 
std::vector< TableEpochInfogetTableEpochs (const int32_t db_id, const int32_t table_id) const
 
void setTableEpochs (const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
 
void setTableEpochsLogExceptions (const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
 
int getDatabaseId () const
 
SqliteConnectorgetSqliteConnector ()
 
void roll (const bool forward)
 
DictRef addDictionary (ColumnDescriptor &cd)
 
void delDictionary (const ColumnDescriptor &cd)
 
void getDictionary (const ColumnDescriptor &cd, std::map< int, StringDictionary * > &stringDicts)
 
const bool checkMetadataForDeletedRecs (const TableDescriptor *td, int column_id) const
 
const ColumnDescriptorgetDeletedColumn (const TableDescriptor *td) const
 
const ColumnDescriptorgetDeletedColumnIfRowsDeleted (const TableDescriptor *td) const
 
void setDeletedColumn (const TableDescriptor *td, const ColumnDescriptor *cd)
 
int getLogicalTableId (const int physicalTableId) const
 
void checkpoint (const int logicalTableId) const
 
void checkpointWithAutoRollback (const int logical_table_id) const
 
void resetTableEpochFloor (const int logicalTableId) const
 
std::string name () const
 
void eraseDbMetadata ()
 
void eraseDbPhysicalData ()
 
void eraseTablePhysicalData (const TableDescriptor *td)
 
void setForReload (const int32_t tableId)
 
std::vector< std::string > getTableDataDirectories (const TableDescriptor *td) const
 
std::vector< std::string > getTableDictDirectories (const TableDescriptor *td) const
 
std::set< std::string > getTableDictDirectoryPaths (int32_t table_id) const
 
std::string getColumnDictDirectory (const ColumnDescriptor *cd, bool file_name_only=true) const
 
std::string dumpSchema (const TableDescriptor *td) const
 
std::string dumpCreateTable (const TableDescriptor *td, bool multiline_formatting=true, bool dump_defaults=false) const
 
std::optional< std::string > dumpCreateTable (int32_t table_id, bool multiline_formatting=true, bool dump_defaults=false) const
 
std::string dumpCreateServer (const std::string &name, bool multiline_formatting=true) const
 
void createForeignServer (std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
 
const
foreign_storage::ForeignServer
getForeignServer (const std::string &server_name) const
 
const std::unique_ptr< const
foreign_storage::ForeignServer
getForeignServerFromStorage (const std::string &server_name)
 
const std::unique_ptr< const
foreign_storage::ForeignTable
getForeignTableFromStorage (int table_id)
 
void changeForeignServerOwner (const std::string &server_name, const int new_owner_id)
 
void setForeignServerDataWrapper (const std::string &server_name, const std::string &data_wrapper)
 
void setForeignServerOptions (const std::string &server_name, const std::string &options)
 
void renameForeignServer (const std::string &server_name, const std::string &name)
 
void dropForeignServer (const std::string &server_name)
 
std::vector< const
foreign_storage::ForeignTable * > 
getAllForeignTablesForForeignServer (const int32_t foreign_server_id)
 Get all of the foreign tables for associated with a foreign server id. More...
 
void getForeignServersForUser (const rapidjson::Value *filters, const UserMetadata &user, std::vector< const foreign_storage::ForeignServer * > &results)
 
void createDefaultServersIfNotExists ()
 
bool validateNonExistentTableOrView (const std::string &name, const bool if_not_exists)
 
std::vector< const
TableDescriptor * > 
getAllForeignTablesForRefresh () const
 
void updateForeignTableRefreshTimes (const int32_t table_id)
 
void setForeignTableOptions (const std::string &table_name, foreign_storage::OptionsMap &options_map, bool clear_existing_options=true)
 
void updateLeaf (const LeafHostInfo &string_dict_host)
 
void setUncappedTableEpoch (const std::string &table_name)
 
int32_t createCustomExpression (std::unique_ptr< CustomExpression > custom_expression)
 
const CustomExpressiongetCustomExpression (int32_t custom_expression_id) const
 
const std::unique_ptr< const
CustomExpression
getCustomExpressionFromStorage (int32_t custom_expression_id)
 
std::vector< const
CustomExpression * > 
getCustomExpressionsForUser (const UserMetadata &user) const
 
void updateCustomExpression (int32_t custom_expression_id, const std::string &expression_json)
 
void deleteCustomExpressions (const std::vector< int32_t > &custom_expression_ids, bool do_soft_delete)
 
void reassignOwners (const std::set< std::string > &old_owners, const std::string &new_owner)
 
bool isInfoSchemaDb () const
 

Static Public Member Functions

static void expandGeoColumn (const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
 
static const std::string getForeignTableSchema (bool if_not_exists=false)
 
static const std::string getForeignServerSchema (bool if_not_exists=false)
 
static const std::string getCustomExpressionsSchema (bool if_not_exists=false)
 

Public Attributes

std::unique_ptr
< heavyai::DistributedSharedMutex
dcatalogMutex_
 
std::unique_ptr
< heavyai::DistributedSharedMutex
dsqliteMutex_
 
std::mutex sqliteMutex_
 
heavyai::shared_mutex sharedMutex_
 
std::atomic< std::thread::id > thread_holding_sqlite_lock
 
std::atomic< std::thread::id > thread_holding_write_lock
 
bool initialized_ = false
 

Static Public Attributes

static thread_local bool thread_holds_read_lock = false
 

Protected Types

using ColumnDescriptorsForRoll = std::vector< std::pair< ColumnDescriptor *, ColumnDescriptor * >>
 

Protected Member Functions

void CheckAndExecuteMigrations ()
 
void CheckAndExecuteMigrationsPostBuildMaps ()
 
void updateDictionaryNames ()
 
void updateTableDescriptorSchema ()
 
void updateFixlenArrayColumns ()
 
void updateGeoColumns ()
 
void updateFrontendViewSchema ()
 
void updateLinkSchema ()
 
void updateFrontendViewAndLinkUsers ()
 
void updateLogicalToPhysicalTableLinkSchema ()
 
void updateLogicalToPhysicalTableMap (const int32_t logical_tb_id)
 
void updateDictionarySchema ()
 
void updatePageSize ()
 
void updateDeletedColumnIndicator ()
 
void updateDefaultColumnValues ()
 
void updateFrontendViewsToDashboards ()
 
void updateCustomExpressionsSchema ()
 
void updateFsiSchemas ()
 
void renameLegacyDataWrappers ()
 
void recordOwnershipOfObjectsInObjectPermissions ()
 
void checkDateInDaysColumnMigration ()
 
void createDashboardSystemRoles ()
 
void buildMaps ()
 
void addTableToMap (const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
 
void addReferenceToForeignDict (ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def, const bool persist_reference)
 
bool setColumnSharedDictionary (ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
void setColumnDictionary (ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, bool is_logical_table, bool use_temp_dictionary=false)
 
void addFrontendViewToMap (DashboardDescriptor &vd)
 
void addFrontendViewToMapNoLock (DashboardDescriptor &vd)
 
void addLinkToMap (LinkDescriptor &ld)
 
void removeTableFromMap (const std::string &tableName, const int tableId, const bool is_on_error=false)
 
void eraseTableMetadata (const TableDescriptor *td)
 
void executeDropTableSqliteQueries (const TableDescriptor *td)
 
void doTruncateTable (const TableDescriptor *td)
 
void renamePhysicalTable (const TableDescriptor *td, const std::string &newTableName)
 
void renamePhysicalTable (std::vector< std::pair< std::string, std::string >> &names, std::vector< int > &tableIds)
 
void instantiateFragmenter (TableDescriptor *td) const
 
void getAllColumnMetadataForTableImpl (const TableDescriptor *td, std::list< const ColumnDescriptor * > &colDescs, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 
std::string calculateSHA1 (const std::string &data)
 
std::string generatePhysicalTableName (const std::string &logicalTableName, const int32_t &shardNumber)
 
std::vector< DBObjectparseDashboardObjects (const std::string &view_meta, const int &user_id)
 
void createOrUpdateDashboardSystemRole (const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
 
const int getColumnIdBySpiUnlocked (const int table_id, const size_t spi) const
 
void serializeTableJsonUnlocked (const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
 
void dropTableFromJsonUnlocked (const std::string &table_name) const
 

Protected Attributes

std::string basePath_
 
TableDescriptorMap tableDescriptorMap_
 
TableDescriptorMapById tableDescriptorMapById_
 
ColumnDescriptorMap columnDescriptorMap_
 
ColumnDescriptorMapById columnDescriptorMapById_
 
DictDescriptorMapById dictDescriptorMapByRef_
 
DashboardDescriptorMap dashboardDescriptorMap_
 
LinkDescriptorMap linkDescriptorMap_
 
LinkDescriptorMapById linkDescriptorMapById_
 
ForeignServerMap foreignServerMap_
 
ForeignServerMapById foreignServerMapById_
 
CustomExpressionMapById custom_expr_map_by_id_
 
TableDictColumnsMap dict_columns_by_table_id_
 
SqliteConnector sqliteConnector_
 
const DBMetadata currentDB_
 
std::shared_ptr
< Data_Namespace::DataMgr
dataMgr_
 
const std::vector< LeafHostInfostring_dict_hosts_
 
std::shared_ptr< CalcitecalciteMgr_
 
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
 
int nextTempTableId_
 
int nextTempDictId_
 
ColumnDescriptorsForRoll columnDescriptorsForRoll
 

Static Protected Attributes

static const std::string physicalTableNameTag_
 

Private Member Functions

void buildDictionaryMapUnlocked ()
 
void reloadTableMetadata (int table_id)
 
void reloadTableMetadataUnlocked (int table_id)
 
void reloadCatalogMetadata (const std::map< int32_t, std::string > &user_name_by_user_id)
 
void reloadCatalogMetadataUnlocked (const std::map< int32_t, std::string > &user_name_by_user_id)
 
void refreshDictionaryCachesForTableUnlocked (const TableDescriptor &td)
 
void buildTablesMapUnlocked ()
 
void buildColumnsMapUnlocked ()
 
void updateViewsInMapUnlocked ()
 
void updateViewUnlocked (TableDescriptor &)
 
void buildDashboardsMapUnlocked (const std::map< int32_t, std::string > &user_name_by_user_id)
 
void buildLinksMapUnlocked ()
 
void buildLogicalToPhysicalMapUnlocked ()
 
void updateForeignTablesInMapUnlocked ()
 
void gatherAdditionalInfo (std::vector< std::string > &additional_info, std::set< std::string > &shared_dict_column_names, const TableDescriptor *td) const
 
std::string quoteIfRequired (const std::string &column_name) const
 
void adjustAlteredTableFiles (const std::string &temp_data_dir, const std::unordered_map< int, int > &all_column_ids_map) const
 
void renameTableDirectories (const std::string &temp_data_dir, const std::vector< std::string > &target_paths, const std::string &name_prefix) const
 
void buildForeignServerMapUnlocked ()
 
void setForeignServerProperty (const std::string &server_name, const std::string &property, const std::string &value)
 
void setForeignTableProperty (const foreign_storage::ForeignTable *table, const std::string &property, const std::string &value)
 
void alterPhysicalTableMetadata (const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
 
void alterTableMetadata (const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
 
void setTableFileMgrParams (const int table_id, const File_Namespace::FileMgrParams &file_mgr_params)
 
bool filterTableByTypeAndUser (const TableDescriptor *td, const UserMetadata &user_metadata, const GetTablesType get_tables_type) const
 
TableDescriptorgetMutableMetadataForTableUnlocked (int table_id) const
 
TableDescriptorgetMutableMetadataForTableUnlocked (const std::string &table_name) const
 
void createForeignServerNoLocks (std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
 
foreign_storage::ForeignTablegetForeignTableUnlocked (const std::string &tableName) const
 
const CataloggetObjForLock ()
 
void removeChunks (const int table_id) const
 
void buildCustomExpressionsMapUnlocked ()
 
std::unique_ptr< CustomExpressiongetCustomExpressionFromConnector (size_t row)
 
void restoreOldOwners (const std::map< int32_t, std::string > &old_owners_user_name_by_id, const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects, int32_t new_owner_id)
 
void restoreOldOwnersInMemory (const std::map< int32_t, std::string > &old_owners_user_name_by_id, const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects, int32_t new_owner_id)
 
void conditionallyInitializeSystemObjects ()
 
void initializeSystemServers ()
 
void initializeSystemTables ()
 
void initializeUsersSystemTable ()
 
void initializeDatabasesSystemTable ()
 
void initializePermissionsSystemTable ()
 
void initializeRolesSystemTable ()
 
void initializeTablesSystemTable ()
 
void initializeDashboardsSystemTable ()
 
void initializeRoleAssignmentsSystemTable ()
 
void initializeMemorySummarySystemTable ()
 
void initializeMemoryDetailsSystemTable ()
 
void initializeStorageDetailsSystemTable ()
 
void initializeServerLogsSystemTables ()
 
void initializeRequestLogsSystemTables ()
 
void initializeWebServerLogsSystemTables ()
 
void initializeWebServerAccessLogsSystemTables ()
 
void createSystemTableServer (const std::string &server_name, const std::string &data_wrapper_type, const foreign_storage::OptionsMap &options={})
 
std::pair
< foreign_storage::ForeignTable,
std::list< ColumnDescriptor > > 
getSystemTableSchema (const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name, bool is_in_memory_system_table)
 
bool recreateSystemTableIfUpdated (foreign_storage::ForeignTable &foreign_table, const std::list< ColumnDescriptor > &columns)
 
void setDeletedColumnUnlocked (const TableDescriptor *td, const ColumnDescriptor *cd)
 
void addToColumnMap (ColumnDescriptor *cd)
 
void removeFromColumnMap (ColumnDescriptor *cd)
 
void deleteTableCatalogMetadata (const TableDescriptor *logical_table, const std::vector< const TableDescriptor * > &physical_tables)
 
std::string dumpCreateTableUnlocked (const TableDescriptor *td, bool multiline_formatting, bool dump_defaults) const
 

Private Attributes

DeletedColumnPerTableMap deletedColumnPerTable_
 

Static Private Attributes

static constexpr const char * CATALOG_SERVER_NAME {"system_catalog_server"}
 
static constexpr const char * MEMORY_STATS_SERVER_NAME {"system_memory_stats_server"}
 
static constexpr const char * STORAGE_STATS_SERVER_NAME {"system_storage_stats_server"}
 
static constexpr const char * LOGS_SERVER_NAME {"system_logs_server"}
 
static constexpr std::array
< const char *, 4 > 
INTERNAL_SERVERS
 

Friends

template<class T >
class lockmgr::TableLockMgrImpl
 

Detailed Description

class for a per-database catalog. also includes metadata for the current database and the current user.

Catalog

Definition at line 132 of file Catalog.h.

Member Typedef Documentation

Definition at line 710 of file Catalog.h.

Constructor & Destructor Documentation

Catalog_Namespace::Catalog::Catalog ( const std::string &  basePath,
const DBMetadata curDB,
std::shared_ptr< Data_Namespace::DataMgr dataMgr,
const std::vector< LeafHostInfo > &  string_dict_hosts,
std::shared_ptr< Calcite calcite,
bool  is_new_db 
)

Constructor - takes basePath to already extant data directory for writing.

Parameters
basePathdirectory path for writing catalog
dbNamename of the database
fragmenterFragmenter object metadata - expects for this directory to already exist
Catalog_Namespace::Catalog::Catalog ( )

Constructor builds a hollow catalog used during constructor of other catalogs.

Definition at line 171 of file Catalog.cpp.

171 {}
Catalog_Namespace::Catalog::~Catalog ( )

Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite.

Definition at line 231 of file Catalog.cpp.

References basePath_, columnDescriptorMap_, currentDB_, Catalog_Namespace::DBMetadata::dbName, g_serialize_temp_tables, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and tableDescriptorMap_.

231  {
232  // cat_write_lock write_lock(this);
233 
234  // must clean up heap-allocated TableDescriptor and ColumnDescriptor structs
235  for (TableDescriptorMap::iterator tableDescIt = tableDescriptorMap_.begin();
236  tableDescIt != tableDescriptorMap_.end();
237  ++tableDescIt) {
238  tableDescIt->second->fragmenter = nullptr;
239  delete tableDescIt->second;
240  }
241 
242  // TableDescriptorMapById points to the same descriptors. No need to delete
243 
244  for (ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.begin();
245  columnDescIt != columnDescriptorMap_.end();
246  ++columnDescIt) {
247  delete columnDescIt->second;
248  }
249 
250  // ColumnDescriptorMapById points to the same descriptors. No need to delete
251 
253  boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
254  }
255 }
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:681
const DBMetadata currentDB_
Definition: Catalog.h:693
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:162
bool g_serialize_temp_tables
Definition: Catalog.cpp:106
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:679

+ Here is the call graph for this function:

Member Function Documentation

void Catalog_Namespace::Catalog::addColumn ( const TableDescriptor td,
ColumnDescriptor cd 
)

Definition at line 2418 of file Catalog.cpp.

References ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::db_id, ColumnDescriptor::default_value, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, kENCODING_DICT, TableDescriptor::nShards, TableDescriptor::shard, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), and ColumnDescriptor::virtualExpr.

2418  {
2419  // caller must handle sqlite/chunk transaction TOGETHER
2420  cd.tableId = td.tableId;
2421  cd.db_id = getDatabaseId();
2422  if (td.nShards > 0 && td.shard < 0) {
2423  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
2424  auto shard_cd = cd;
2425  addColumn(*shard, shard_cd);
2426  }
2427  }
2429  addDictionary(cd);
2430  }
2431 
2432  using BindType = SqliteConnector::BindType;
2433  std::vector<BindType> types(17, BindType::TEXT);
2434  if (!cd.default_value.has_value()) {
2435  types[16] = BindType::NULL_TYPE;
2436  }
2438  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
2439  "colscale, is_notnull, "
2440  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
2441  "is_deletedcol, default_value) "
2442  "VALUES (?, "
2443  "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
2444  "?, ?, ?, "
2445  "?, "
2446  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2447  std::vector<std::string>{std::to_string(td.tableId),
2448  std::to_string(td.tableId),
2449  cd.columnName,
2458  "",
2461  cd.virtualExpr,
2463  cd.default_value.value_or("NULL")},
2464  types);
2465 
2467  "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
2468  std::vector<std::string>{std::to_string(td.tableId)});
2469 
2471  "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
2472  std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
2473  cd.columnId = sqliteConnector_.getData<int>(0, 0);
2474 
2475  ++tableDescriptorMapById_[td.tableId]->nColumns;
2476  auto ncd = new ColumnDescriptor(cd);
2477  addToColumnMap(ncd);
2478  columnDescriptorsForRoll.emplace_back(nullptr, ncd);
2479 }
std::string virtualExpr
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:382
HOST DEVICE int get_size() const
Definition: sqltypes.h:393
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
void addColumn(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:2418
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
HOST DEVICE int get_scale() const
Definition: sqltypes.h:386
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:381
std::string to_string(char const *&&v)
int getDatabaseId() const
Definition: Catalog.h:304
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:680
specifies the content in-memory of a row in the column metadata table
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4700
std::optional< std::string > default_value
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:389
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:383
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:392
SQLTypeInfo columnType
DictRef addDictionary(ColumnDescriptor &cd)
Definition: Catalog.cpp:2322
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:388
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:711
std::string columnName
void addToColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6561

+ Here is the call graph for this function:

DictRef Catalog_Namespace::Catalog::addDictionary ( ColumnDescriptor cd)

Definition at line 2322 of file Catalog.cpp.

References CHECK, DictDescriptor::dictFolderPath, and ColumnDescriptor::tableId.

2322  {
2323  cat_write_lock write_lock(this);
2324  const auto& td = *tableDescriptorMapById_[cd.tableId];
2325  list<DictDescriptor> dds;
2326  setColumnDictionary(cd, dds, td, true);
2327  auto& dd = dds.back();
2328  CHECK(dd.dictRef.dictId);
2329 
2330  std::unique_ptr<StringDictionaryClient> client;
2331  if (!string_dict_hosts_.empty()) {
2332  client.reset(new StringDictionaryClient(
2333  string_dict_hosts_.front(), DictRef(currentDB_.dbId, -1), true));
2334  }
2335  if (client) {
2336  client->create(dd.dictRef, dd.dictIsTemp);
2337  }
2338 
2339  DictDescriptor* new_dd = new DictDescriptor(dd);
2340  dictDescriptorMapByRef_[dd.dictRef].reset(new_dd);
2341  if (!dd.dictIsTemp) {
2342  boost::filesystem::create_directory(new_dd->dictFolderPath);
2343  }
2344  return dd.dictRef;
2345 }
std::string dictFolderPath
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:696
const DBMetadata currentDB_
Definition: Catalog.h:693
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:683
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, bool is_logical_table, bool use_temp_dictionary=false)
Definition: Catalog.cpp:3865
heavyai::unique_lock< heavyai::shared_mutex > write_lock
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:680
struct dict_ref_t DictRef
Definition: DictRef.h:54
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
Descriptor for a dictionary for a string columne.
void Catalog_Namespace::Catalog::addFrontendViewToMap ( DashboardDescriptor vd)
protected

Definition at line 1778 of file Catalog.cpp.

1778  {
1779  cat_write_lock write_lock(this);
1781 }
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1783
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
void Catalog_Namespace::Catalog::addFrontendViewToMapNoLock ( DashboardDescriptor vd)
protected

Definition at line 1783 of file Catalog.cpp.

References DashboardDescriptor::dashboardName, to_string(), and DashboardDescriptor::userId.

1783  {
1784  cat_write_lock write_lock(this);
1786  std::make_shared<DashboardDescriptor>(vd);
1787 }
heavyai::unique_lock< heavyai::shared_mutex > write_lock
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:684
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addLinkToMap ( LinkDescriptor ld)
protected

Definition at line 1865 of file Catalog.cpp.

References LinkDescriptor::link, LinkDescriptor::linkId, and to_string().

1865  {
1866  cat_write_lock write_lock(this);
1867  LinkDescriptor* new_ld = new LinkDescriptor();
1868  *new_ld = ld;
1870  linkDescriptorMapById_[ld.linkId] = new_ld;
1871 }
const DBMetadata currentDB_
Definition: Catalog.h:693
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:686
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:685

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addReferenceToForeignDict ( ColumnDescriptor referencing_column,
Parser::SharedDictionaryDef  shared_dict_def,
const bool  persist_reference 
)
protected

Definition at line 3774 of file Catalog.cpp.

References CHECK, CHECK_GE, ColumnDescriptor::columnType, SQLTypeInfo::get_comp_param(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_foreign_col(), and to_string().

3776  {
3777  cat_write_lock write_lock(this);
3778  const auto foreign_ref_col = get_foreign_col(*this, shared_dict_def);
3779  CHECK(foreign_ref_col);
3780  referencing_column.columnType = foreign_ref_col->columnType;
3781  const int dict_id = referencing_column.columnType.get_comp_param();
3782  const DictRef dict_ref(currentDB_.dbId, dict_id);
3783  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
3784  CHECK(dictIt != dictDescriptorMapByRef_.end());
3785  const auto& dd = dictIt->second;
3786  CHECK_GE(dd->refcount, 1);
3787  ++dd->refcount;
3788  if (persist_reference) {
3789  cat_sqlite_lock sqlite_lock(getObjForLock());
3791  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
3792  {std::to_string(dict_id)});
3793  }
3794 }
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
const DBMetadata currentDB_
Definition: Catalog.h:693
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:683
#define CHECK_GE(x, y)
Definition: Logger.h:306
heavyai::unique_lock< heavyai::shared_mutex > write_lock
const ColumnDescriptor * get_foreign_col(const Catalog &cat, const Parser::SharedDictionaryDef &shared_dict_def)
Definition: Catalog.cpp:3762
std::string to_string(char const *&&v)
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:392
#define CHECK(condition)
Definition: Logger.h:291
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addTableToMap ( const TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::list< DictDescriptor > &  dicts 
)
protected

Definition at line 1637 of file Catalog.cpp.

References anonymous_namespace{Utm.h}::a, CHECK, TableDescriptor::columnIdBySpi_, DictDescriptor::dictFolderPath, dict_ref_t::dictId, TableDescriptor::hasDeletedCol, ColumnDescriptor::isDeletedCol, TableDescriptor::mutex_, gpu_enabled::sort(), TableDescriptor::tableId, TableDescriptor::tableName, and to_upper().

1639  {
1640  cat_write_lock write_lock(this);
1641  TableDescriptor* new_td;
1642 
1643  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
1644  if (foreign_table) {
1645  auto new_foreign_table = new foreign_storage::ForeignTable();
1646  *new_foreign_table = *foreign_table;
1647  new_td = new_foreign_table;
1648  } else {
1649  new_td = new TableDescriptor();
1650  *new_td = *td;
1651  }
1652 
1653  new_td->mutex_ = std::make_shared<std::mutex>();
1654  tableDescriptorMap_[to_upper(td->tableName)] = new_td;
1655  tableDescriptorMapById_[td->tableId] = new_td;
1656  for (auto cd : columns) {
1657  ColumnDescriptor* new_cd = new ColumnDescriptor();
1658  *new_cd = cd;
1659  addToColumnMap(new_cd);
1660 
1661  // Add deleted column to the map
1662  if (cd.isDeletedCol) {
1663  CHECK(new_td->hasDeletedCol);
1664  setDeletedColumnUnlocked(new_td, new_cd);
1665  }
1666  }
1667 
1668  std::sort(new_td->columnIdBySpi_.begin(),
1669  new_td->columnIdBySpi_.end(),
1670  [](const size_t a, const size_t b) -> bool { return a < b; });
1671  // TODO(sy): Why does addTableToMap() sort columnIdBySpi_ but not insert into it while
1672  // buildColumnsMapUnlocked() does both?
1673 
1674  std::unique_ptr<StringDictionaryClient> client;
1675  DictRef dict_ref(currentDB_.dbId, -1);
1676  if (!string_dict_hosts_.empty()) {
1677  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1678  }
1679  for (auto dd : dicts) {
1680  if (!dd.dictRef.dictId) {
1681  // Dummy entry created for a shard of a logical table, nothing to do.
1682  continue;
1683  }
1684  dict_ref.dictId = dd.dictRef.dictId;
1685  if (client) {
1686  client->create(dict_ref, dd.dictIsTemp);
1687  }
1688  DictDescriptor* new_dd = new DictDescriptor(dd);
1689  dictDescriptorMapByRef_[dict_ref].reset(new_dd);
1690  if (!dd.dictIsTemp) {
1691  boost::filesystem::create_directory(new_dd->dictFolderPath);
1692  }
1693  }
1694 }
std::string dictFolderPath
std::string tableName
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:696
const DBMetadata currentDB_
Definition: Catalog.h:693
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:683
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::vector< int > columnIdBySpi_
std::shared_ptr< std::mutex > mutex_
constexpr double a
Definition: Utm.h:32
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3754
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:680
specifies the content in-memory of a row in the column metadata table
std::string to_upper(const std::string &str)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
Descriptor for a dictionary for a string columne.
void addToColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6561
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:679

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addToColumnMap ( ColumnDescriptor cd)
private

Definition at line 6561 of file Catalog.cpp.

References ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::is_dict_encoded_type(), ColumnDescriptor::tableId, and to_upper().

6561  {
6564  if (cd->columnType.is_dict_encoded_type()) {
6565  dict_columns_by_table_id_[cd->tableId].emplace(cd);
6566  }
6567 }
TableDictColumnsMap dict_columns_by_table_id_
Definition: Catalog.h:690
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:681
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:682
bool is_dict_encoded_type() const
Definition: sqltypes.h:644
std::string to_upper(const std::string &str)
SQLTypeInfo columnType
std::string columnName
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::adjustAlteredTableFiles ( const std::string &  temp_data_dir,
const std::unordered_map< int, int > &  all_column_ids_map 
) const
private
void Catalog_Namespace::Catalog::alterPhysicalTableMetadata ( const TableDescriptor td,
const TableDescriptorUpdateParams table_update_params 
)
private

Definition at line 3488 of file Catalog.cpp.

References CHECK, TableDescriptorUpdateParams::max_rollback_epochs, TableDescriptorUpdateParams::max_rows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::tableId, and to_string().

3490  {
3491  // Only called from parent alterTableParamMetadata, expect already to have catalog and
3492  // sqlite write locks
3493 
3494  // Sqlite transaction should have already been begun in parent alterTableCatalogMetadata
3495 
3497  CHECK(mutable_td);
3498  if (td->maxRollbackEpochs != table_update_params.max_rollback_epochs) {
3500  "UPDATE mapd_tables SET max_rollback_epochs = ? WHERE tableid = ?",
3501  std::vector<std::string>{std::to_string(table_update_params.max_rollback_epochs),
3502  std::to_string(td->tableId)});
3503  mutable_td->maxRollbackEpochs = table_update_params.max_rollback_epochs;
3504  }
3505 
3506  if (td->maxRows != table_update_params.max_rows) {
3508  "UPDATE mapd_tables SET max_rows = ? WHERE tableid = ?",
3509  std::vector<std::string>{std::to_string(table_update_params.max_rows),
3510  std::to_string(td->tableId)});
3511  mutable_td->maxRows = table_update_params.max_rows;
3512  }
3513 }
int32_t maxRollbackEpochs
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
TableDescriptor * getMutableMetadataForTableUnlocked(int table_id) const
Definition: Catalog.cpp:1991
std::string to_string(char const *&&v)
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::alterTableMetadata ( const TableDescriptor td,
const TableDescriptorUpdateParams table_update_params 
)
private

Definition at line 3515 of file Catalog.cpp.

References CHECK, logger::FATAL, LOG, TableDescriptor::tableId, and TableDescriptor::tableName.

3516  {
3517  cat_write_lock write_lock(this);
3518  cat_sqlite_lock sqlite_lock(getObjForLock());
3519  sqliteConnector_.query("BEGIN TRANSACTION");
3520  try {
3521  const auto physical_table_it = logicalToPhysicalTableMapById_.find(td->tableId);
3522  if (physical_table_it != logicalToPhysicalTableMapById_.end()) {
3523  const auto physical_tables = physical_table_it->second;
3524  CHECK(!physical_tables.empty());
3525  for (size_t i = 0; i < physical_tables.size(); i++) {
3526  int32_t physical_tb_id = physical_tables[i];
3527  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id, false);
3528  CHECK(phys_td);
3529  alterPhysicalTableMetadata(phys_td, table_update_params);
3530  }
3531  }
3532  alterPhysicalTableMetadata(td, table_update_params);
3533  } catch (std::exception& e) {
3534  sqliteConnector_.query("ROLLBACK TRANSACTION");
3535  LOG(FATAL) << "Table '" << td->tableName << "' catalog update failed";
3536  }
3537  sqliteConnector_.query("END TRANSACTION");
3538 }
std::string tableName
#define LOG(tag)
Definition: Logger.h:285
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void alterPhysicalTableMetadata(const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
Definition: Catalog.cpp:3488
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:699
void Catalog_Namespace::Catalog::buildColumnsMapUnlocked ( )
private

Definition at line 1428 of file Catalog.cpp.

References anonymous_namespace{Utm.h}::a, CHECK, ColumnDescriptor::chunks, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::db_id, ColumnDescriptor::default_value, SQLTypeInfo::get_physical_cols(), SQLTypeInfo::is_geometry(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_compression(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::set_dict_key(), SQLTypeInfo::set_dimension(), SQLTypeInfo::set_notnull(), SQLTypeInfo::set_scale(), SQLTypeInfo::set_size(), SQLTypeInfo::set_subtype(), SQLTypeInfo::set_type(), gpu_enabled::sort(), ColumnDescriptor::tableId, and ColumnDescriptor::virtualExpr.

1428  {
1429  std::string columnQuery(
1430  "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
1431  "is_notnull, compression, comp_param, "
1432  "size, chunks, is_systemcol, is_virtualcol, virtual_expr, is_deletedcol, "
1433  "default_value from "
1434  "mapd_columns ORDER BY tableid, "
1435  "columnid");
1436  sqliteConnector_.query(columnQuery);
1437  auto numRows = sqliteConnector_.getNumRows();
1438  int32_t skip_physical_cols = 0;
1439  for (size_t r = 0; r < numRows; ++r) {
1440  ColumnDescriptor* cd = new ColumnDescriptor();
1441  cd->tableId = sqliteConnector_.getData<int>(r, 0);
1442  cd->columnId = sqliteConnector_.getData<int>(r, 1);
1443  cd->columnName = sqliteConnector_.getData<string>(r, 2);
1447  cd->columnType.set_scale(sqliteConnector_.getData<int>(r, 6));
1451  cd->columnType.set_size(sqliteConnector_.getData<int>(r, 10));
1452  cd->chunks = sqliteConnector_.getData<string>(r, 11);
1453  cd->isSystemCol = sqliteConnector_.getData<bool>(r, 12);
1454  cd->isVirtualCol = sqliteConnector_.getData<bool>(r, 13);
1455  cd->virtualExpr = sqliteConnector_.getData<string>(r, 14);
1456  cd->isDeletedCol = sqliteConnector_.getData<bool>(r, 15);
1457  if (sqliteConnector_.isNull(r, 16)) {
1458  cd->default_value = std::nullopt;
1459  } else {
1460  cd->default_value = std::make_optional(sqliteConnector_.getData<string>(r, 16));
1461  }
1462  cd->isGeoPhyCol = skip_physical_cols > 0;
1463  cd->db_id = getDatabaseId();
1464  set_dict_key(*cd);
1465  addToColumnMap(cd);
1466 
1467  if (skip_physical_cols <= 0) {
1468  skip_physical_cols = cd->columnType.get_physical_cols();
1469  }
1470 
1471  auto td_itr = tableDescriptorMapById_.find(cd->tableId);
1472  CHECK(td_itr != tableDescriptorMapById_.end());
1473 
1474  if (cd->isDeletedCol) {
1475  td_itr->second->hasDeletedCol = true;
1476  setDeletedColumnUnlocked(td_itr->second, cd);
1477  } else if (cd->columnType.is_geometry() || skip_physical_cols-- <= 0) {
1478  tableDescriptorMapById_[cd->tableId]->columnIdBySpi_.push_back(cd->columnId);
1479  }
1480  }
1481 
1482  // sort columnIdBySpi_ based on columnId
1483  for (auto& tit : tableDescriptorMapById_) {
1484  std::sort(tit.second->columnIdBySpi_.begin(),
1485  tit.second->columnIdBySpi_.end(),
1486  [](const size_t a, const size_t b) -> bool { return a < b; });
1487  }
1488 }
std::string virtualExpr
void set_compression(EncodingType c)
Definition: sqltypes.h:504
void set_size(int s)
Definition: sqltypes.h:501
T getData(const int row, const int col)
SQLTypes
Definition: sqltypes.h:55
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:494
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
virtual void query(const std::string &queryString)
std::string chunks
constexpr double a
Definition: Utm.h:32
EncodingType
Definition: sqltypes.h:230
int get_physical_cols() const
Definition: sqltypes.h:414
void set_scale(int s)
Definition: sqltypes.h:498
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3754
int getDatabaseId() const
Definition: Catalog.h:304
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:680
specifies the content in-memory of a row in the column metadata table
void set_comp_param(int p)
Definition: sqltypes.h:505
std::optional< std::string > default_value
bool isNull(const int row, const int col) const
void set_dimension(int d)
Definition: sqltypes.h:495
void set_notnull(bool n)
Definition: sqltypes.h:500
#define CHECK(condition)
Definition: Logger.h:291
bool is_geometry() const
Definition: sqltypes.h:592
SQLTypeInfo columnType
std::string columnName
void addToColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6561
virtual size_t getNumRows() const
HOST DEVICE void set_type(SQLTypes t)
Definition: sqltypes.h:493

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildCustomExpressionsMapUnlocked ( )
private

Definition at line 1609 of file Catalog.cpp.

1609  {
1611  "SELECT id, name, expression_json, data_source_type, data_source_id, "
1612  "is_deleted "
1613  "FROM omnisci_custom_expressions");
1614  auto num_rows = sqliteConnector_.getNumRows();
1615  for (size_t row = 0; row < num_rows; row++) {
1616  auto custom_expr = getCustomExpressionFromConnector(row);
1617  custom_expr_map_by_id_[custom_expr->id] = std::move(custom_expr);
1618  }
1619 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
virtual void query(const std::string &queryString)
std::unique_ptr< CustomExpression > getCustomExpressionFromConnector(size_t row)
Definition: Catalog.cpp:1621
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:689
virtual size_t getNumRows() const
void Catalog_Namespace::Catalog::buildDashboardsMapUnlocked ( const std::map< int32_t, std::string > &  user_name_by_user_id)
private

Definition at line 1512 of file Catalog.cpp.

References generate_dashboard_system_rolename(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_user_name_from_id(), and to_string().

1513  {
1514  std::string frontendViewQuery(
1515  "SELECT id, state, name, image_hash, strftime('%Y-%m-%dT%H:%M:%SZ', update_time), "
1516  "userid, "
1517  "metadata "
1518  "FROM mapd_dashboards");
1519  sqliteConnector_.query(frontendViewQuery);
1520  auto numRows = sqliteConnector_.getNumRows();
1521  for (size_t r = 0; r < numRows; ++r) {
1522  auto vd = std::make_shared<DashboardDescriptor>();
1523  vd->dashboardId = sqliteConnector_.getData<int>(r, 0);
1524  vd->dashboardState = sqliteConnector_.getData<string>(r, 1);
1525  vd->dashboardName = sqliteConnector_.getData<string>(r, 2);
1526  vd->imageHash = sqliteConnector_.getData<string>(r, 3);
1527  vd->updateTime = sqliteConnector_.getData<string>(r, 4);
1528  vd->userId = sqliteConnector_.getData<int>(r, 5);
1529  vd->dashboardMetadata = sqliteConnector_.getData<string>(r, 6);
1530  vd->user = get_user_name_from_id(vd->userId, user_name_by_user_id);
1531  vd->dashboardSystemRoleName = generate_dashboard_system_rolename(
1533  dashboardDescriptorMap_[std::to_string(vd->userId) + ":" + vd->dashboardName] = vd;
1534  }
1535 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
const DBMetadata currentDB_
Definition: Catalog.h:693
virtual void query(const std::string &queryString)
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:684
std::string to_string(char const *&&v)
std::string get_user_name_from_id(int32_t id, const std::map< int32_t, std::string > &user_name_by_user_id)
Definition: Catalog.cpp:1046
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildDictionaryMapUnlocked ( )
private

Definition at line 1067 of file Catalog.cpp.

References currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, g_base_path, SqliteConnector::getData(), SqliteConnector::getNumRows(), shared::kDataDirectoryName, SqliteConnector::query(), sqliteConnector_, and to_string().

1067  {
1068  std::string dictQuery(
1069  "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
1070  sqliteConnector_.query(dictQuery);
1071  auto numRows = sqliteConnector_.getNumRows();
1072  for (size_t r = 0; r < numRows; ++r) {
1073  auto dictId = sqliteConnector_.getData<int>(r, 0);
1074  auto dictName = sqliteConnector_.getData<string>(r, 1);
1075  auto dictNBits = sqliteConnector_.getData<int>(r, 2);
1076  auto is_shared = sqliteConnector_.getData<bool>(r, 3);
1077  auto refcount = sqliteConnector_.getData<int>(r, 4);
1078  auto fname = g_base_path + "/" + shared::kDataDirectoryName + "/DB_" +
1079  std::to_string(currentDB_.dbId) + "_DICT_" + std::to_string(dictId);
1080  DictRef dict_ref(currentDB_.dbId, dictId);
1081  auto dd = new DictDescriptor(
1082  dict_ref, dictName, dictNBits, is_shared, refcount, fname, false);
1083  dictDescriptorMapByRef_[dict_ref].reset(dd);
1084  }
1085 }
const std::string kDataDirectoryName
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
const DBMetadata currentDB_
Definition: Catalog.h:693
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:683
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
std::string g_base_path
Definition: SysCatalog.cpp:62
Descriptor for a dictionary for a string columne.
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildForeignServerMapUnlocked ( )
private

Definition at line 4923 of file Catalog.cpp.

References CHECK, and g_enable_fsi.

4923  {
4926  "SELECT id, name, data_wrapper_type, options, owner_user_id, creation_time FROM "
4927  "omnisci_foreign_servers");
4928  auto num_rows = sqliteConnector_.getNumRows();
4929 
4930  for (size_t row = 0; row < num_rows; row++) {
4931  auto foreign_server = std::make_shared<foreign_storage::ForeignServer>(
4932  sqliteConnector_.getData<int>(row, 0),
4933  sqliteConnector_.getData<std::string>(row, 1),
4934  sqliteConnector_.getData<std::string>(row, 2),
4935  sqliteConnector_.getData<std::string>(row, 3),
4936  sqliteConnector_.getData<std::int32_t>(row, 4),
4937  sqliteConnector_.getData<std::int32_t>(row, 5));
4938  foreignServerMap_[foreign_server->name] = foreign_server;
4939  foreignServerMapById_[foreign_server->id] = foreign_server;
4940  }
4941 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
virtual void query(const std::string &queryString)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:688
ForeignServerMap foreignServerMap_
Definition: Catalog.h:687
#define CHECK(condition)
Definition: Logger.h:291
bool g_enable_fsi
Definition: Catalog.cpp:96
virtual size_t getNumRows() const
void Catalog_Namespace::Catalog::buildLinksMapUnlocked ( )
private

Definition at line 1537 of file Catalog.cpp.

References to_string().

1537  {
1538  std::string linkQuery(
1539  "SELECT linkid, userid, link, view_state, strftime('%Y-%m-%dT%H:%M:%SZ', "
1540  "update_time), view_metadata "
1541  "FROM mapd_links");
1542  sqliteConnector_.query(linkQuery);
1543  auto numRows = sqliteConnector_.getNumRows();
1544  for (size_t r = 0; r < numRows; ++r) {
1545  auto ld = new LinkDescriptor();
1546  ld->linkId = sqliteConnector_.getData<int>(r, 0);
1547  ld->userId = sqliteConnector_.getData<int>(r, 1);
1548  ld->link = sqliteConnector_.getData<string>(r, 2);
1549  ld->viewState = sqliteConnector_.getData<string>(r, 3);
1550  ld->updateTime = sqliteConnector_.getData<string>(r, 4);
1551  ld->viewMetadata = sqliteConnector_.getData<string>(r, 5);
1553  linkDescriptorMapById_[ld->linkId] = ld;
1554  }
1555 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
const DBMetadata currentDB_
Definition: Catalog.h:693
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:686
virtual size_t getNumRows() const
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:685

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildLogicalToPhysicalMapUnlocked ( )
private

Definition at line 1557 of file Catalog.cpp.

References CHECK.

1557  {
1558  /* rebuild map linking logical tables to corresponding physical ones */
1559  std::string logicalToPhysicalTableMapQuery(
1560  "SELECT logical_table_id, physical_table_id "
1561  "FROM mapd_logical_to_physical");
1562  sqliteConnector_.query(logicalToPhysicalTableMapQuery);
1563  auto numRows = sqliteConnector_.getNumRows();
1564  for (size_t r = 0; r < numRows; ++r) {
1565  auto logical_tb_id = sqliteConnector_.getData<int>(r, 0);
1566  auto physical_tb_id = sqliteConnector_.getData<int>(r, 1);
1567  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
1568  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
1569  /* add new entity to the map logicalToPhysicalTableMapById_ */
1570  std::vector<int32_t> physicalTables{physical_tb_id};
1571  const auto it_ok =
1572  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
1573  CHECK(it_ok.second);
1574  } else {
1575  /* update map logicalToPhysicalTableMapById_ */
1576  physicalTableIt->second.push_back(physical_tb_id);
1577  }
1578  }
1579 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
virtual void query(const std::string &queryString)
#define CHECK(condition)
Definition: Logger.h:291
virtual size_t getNumRows() const
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:699
void Catalog_Namespace::Catalog::buildMaps ( )
protected

Definition at line 1584 of file Catalog.cpp.

References g_enable_fsi, and Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_user_id_to_user_name_map().

1584  {
1585  // Get all user id to username mapping here in order to avoid making a call to
1586  // SysCatalog (and attempting to acquire SysCatalog locks) while holding locks for this
1587  // catalog.
1588  const auto user_name_by_user_id = get_user_id_to_user_name_map();
1589 
1590  cat_write_lock write_lock(this);
1591  cat_sqlite_lock sqlite_lock(getObjForLock());
1592 
1595 
1596  if (g_enable_fsi) {
1599  }
1600 
1603  buildDashboardsMapUnlocked(user_name_by_user_id);
1607 }
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void buildLogicalToPhysicalMapUnlocked()
Definition: Catalog.cpp:1557
void buildDashboardsMapUnlocked(const std::map< int32_t, std::string > &user_name_by_user_id)
Definition: Catalog.cpp:1512
void buildCustomExpressionsMapUnlocked()
Definition: Catalog.cpp:1609
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
void updateForeignTablesInMapUnlocked()
Definition: Catalog.cpp:4943
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
bool g_enable_fsi
Definition: Catalog.cpp:96
std::map< int32_t, std::string > get_user_id_to_user_name_map()
Definition: Catalog.cpp:1037

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildTablesMapUnlocked ( )
private

Definition at line 1369 of file Catalog.cpp.

References logger::FATAL, StorageType::FOREIGN_TABLE, TableDescriptor::fragmenter, TableDescriptor::fragments, TableDescriptor::fragPageSize, TableDescriptor::fragType, TableDescriptor::hasDeletedCol, TableDescriptor::is_system_table, TableDescriptor::isView, TableDescriptor::keyMetainfo, LOG, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nColumns, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::storageType, TableDescriptor::tableId, TableDescriptor::tableName, to_upper(), and TableDescriptor::userId.

1369  {
1370  std::string tableQuery(
1371  "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
1372  "max_chunk_size, frag_page_size, "
1373  "max_rows, partitions, shard_column_id, shard, num_shards, key_metainfo, userid, "
1374  "sort_column_id, storage_type, max_rollback_epochs, is_system_table "
1375  "from mapd_tables");
1376  sqliteConnector_.query(tableQuery);
1377  auto numRows = sqliteConnector_.getNumRows();
1378  for (size_t r = 0; r < numRows; ++r) {
1379  TableDescriptor* td;
1380  const auto& storage_type = sqliteConnector_.getData<string>(r, 17);
1381  if (!storage_type.empty() && storage_type != StorageType::FOREIGN_TABLE) {
1382  const auto table_id = sqliteConnector_.getData<int>(r, 0);
1383  const auto& table_name = sqliteConnector_.getData<string>(r, 1);
1384  LOG(FATAL) << "Unable to read Catalog metadata: storage type is currently not a "
1385  "supported table option (table "
1386  << table_name << " [" << table_id << "] in database "
1387  << currentDB_.dbName << ").";
1388  }
1389 
1390  if (storage_type == StorageType::FOREIGN_TABLE) {
1391  td = new foreign_storage::ForeignTable();
1392  } else {
1393  td = new TableDescriptor();
1394  }
1395 
1396  td->storageType = storage_type;
1397  td->tableId = sqliteConnector_.getData<int>(r, 0);
1398  td->tableName = sqliteConnector_.getData<string>(r, 1);
1399  td->nColumns = sqliteConnector_.getData<int>(r, 2);
1400  td->isView = sqliteConnector_.getData<bool>(r, 3);
1401  td->fragments = sqliteConnector_.getData<string>(r, 4);
1402  td->fragType =
1404  td->maxFragRows = sqliteConnector_.getData<int>(r, 6);
1405  td->maxChunkSize = sqliteConnector_.getData<int64_t>(r, 7);
1406  td->fragPageSize = sqliteConnector_.getData<int>(r, 8);
1407  td->maxRows = sqliteConnector_.getData<int64_t>(r, 9);
1408  td->partitions = sqliteConnector_.getData<string>(r, 10);
1409  td->shardedColumnId = sqliteConnector_.getData<int>(r, 11);
1410  td->shard = sqliteConnector_.getData<int>(r, 12);
1411  td->nShards = sqliteConnector_.getData<int>(r, 13);
1412  td->keyMetainfo = sqliteConnector_.getData<string>(r, 14);
1413  td->userId = sqliteConnector_.getData<int>(r, 15);
1414  td->sortedColumnId =
1415  sqliteConnector_.isNull(r, 16) ? 0 : sqliteConnector_.getData<int>(r, 16);
1416  if (!td->isView) {
1417  td->fragmenter = nullptr;
1418  }
1419  td->maxRollbackEpochs = sqliteConnector_.getData<int>(r, 18);
1420  td->is_system_table = sqliteConnector_.getData<bool>(r, 19);
1421  td->hasDeletedCol = false;
1422 
1424  tableDescriptorMapById_[td->tableId] = td;
1425  }
1426 }
int32_t maxRollbackEpochs
std::string partitions
T getData(const int row, const int col)
std::string tableName
#define LOG(tag)
Definition: Logger.h:285
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
const DBMetadata currentDB_
Definition: Catalog.h:693
std::string storageType
virtual void query(const std::string &queryString)
std::string fragments
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:680
std::string keyMetainfo
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
std::string to_upper(const std::string &str)
bool isNull(const int row, const int col) const
Fragmenter_Namespace::FragmenterType fragType
static constexpr char const * FOREIGN_TABLE
virtual size_t getNumRows() const
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:679

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::calculateSHA1 ( const std::string &  data)
protected

Definition at line 4632 of file Catalog.cpp.

4632  {
4633  boost::uuids::detail::sha1 sha1;
4634  unsigned int digest[5];
4635  sha1.process_bytes(data.c_str(), data.length());
4636  sha1.get_digest(digest);
4637  std::stringstream ss;
4638  for (size_t i = 0; i < 5; i++) {
4639  ss << std::hex << digest[i];
4640  }
4641  return ss.str();
4642 }
void Catalog_Namespace::Catalog::changeForeignServerOwner ( const std::string &  server_name,
const int  new_owner_id 
)

Change the owner of a Foreign Server to a new owner.

Parameters
server_name- Name of the foreign server whose owner to change
new_owner_id- New owner's user id

Definition at line 3194 of file Catalog.cpp.

References CHECK, to_string(), and foreign_storage::ForeignServer::user_id.

3195  {
3196  cat_write_lock write_lock(this);
3197  foreign_storage::ForeignServer* foreign_server =
3198  foreignServerMap_.find(server_name)->second.get();
3199  CHECK(foreign_server);
3200  setForeignServerProperty(server_name, "owner_user_id", std::to_string(new_owner_id));
3201  // update in-memory server
3202  foreign_server->user_id = new_owner_id;
3203 }
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::string to_string(char const *&&v)
ForeignServerMap foreignServerMap_
Definition: Catalog.h:687
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
void setForeignServerProperty(const std::string &server_name, const std::string &property, const std::string &value)
Definition: Catalog.cpp:4973

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::CheckAndExecuteMigrations ( )
protected

Definition at line 1007 of file Catalog.cpp.

References g_enable_fsi, getObjForLock(), recordOwnershipOfObjectsInObjectPermissions(), renameLegacyDataWrappers(), updateCustomExpressionsSchema(), updateDefaultColumnValues(), updateDeletedColumnIndicator(), updateDictionaryNames(), updateDictionarySchema(), updateFixlenArrayColumns(), updateFrontendViewAndLinkUsers(), updateFrontendViewSchema(), updateFrontendViewsToDashboards(), updateFsiSchemas(), updateGeoColumns(), updateLinkSchema(), updateLogicalToPhysicalTableLinkSchema(), updatePageSize(), and updateTableDescriptorSchema().

1007  {
1008  cat_write_lock write_lock(this);
1009  cat_sqlite_lock sqlite_lock(getObjForLock());
1012  updateGeoColumns();
1015  updateLinkSchema();
1019  updatePageSize();
1023  if (g_enable_fsi) {
1024  updateFsiSchemas();
1026  }
1029 }
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:470
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:797
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:129
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
void updateCustomExpressionsSchema()
Definition: Catalog.cpp:763
void updateLogicalToPhysicalTableLinkSchema()
Definition: Catalog.cpp:631
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
bool g_enable_fsi
Definition: Catalog.cpp:96

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::CheckAndExecuteMigrationsPostBuildMaps ( )
protected

Definition at line 1031 of file Catalog.cpp.

References checkDateInDaysColumnMigration(), and createDashboardSystemRoles().

1031  {
1034 }
void checkDateInDaysColumnMigration()
Definition: Catalog.cpp:925

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::checkDateInDaysColumnMigration ( )
protected

Definition at line 925 of file Catalog.cpp.

References getCurrentDB(), getObjForLock(), migrations::MigrationMgr::migrateDateInDaysMetadata(), sqliteConnector_, and tableDescriptorMapById_.

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

925  {
926  cat_sqlite_lock sqlite_lock(getObjForLock());
929 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:248
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:680
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
static void migrateDateInDaysMetadata(const Catalog_Namespace::TableDescriptorMapById &table_descriptors_by_id, const int database_id, Catalog_Namespace::Catalog *cat, SqliteConnector &sqlite)
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const bool Catalog_Namespace::Catalog::checkMetadataForDeletedRecs ( const TableDescriptor td,
int  column_id 
) const

Definition at line 3712 of file Catalog.cpp.

References CHECK, and TableDescriptor::fragmenter.

3713  {
3714  // check if there are rows deleted by examining the deletedColumn metadata
3715  CHECK(td);
3716  auto fragmenter = td->fragmenter;
3717  if (fragmenter) {
3718  return fragmenter->hasDeletedRows(delete_column_id);
3719  } else {
3720  return false;
3721  }
3722 }
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
#define CHECK(condition)
Definition: Logger.h:291
void Catalog_Namespace::Catalog::checkpoint ( const int  logicalTableId) const

Definition at line 4853 of file Catalog.cpp.

Referenced by UpdelRoll::commitUpdate(), TableOptimizer::vacuumDeletedRows(), and TableOptimizer::vacuumFragmentsAboveMinSelectivity().

4853  {
4854  const auto td = getMetadataForTable(logicalTableId);
4855  const auto shards = getPhysicalTablesDescriptors(td);
4856  for (const auto shard : shards) {
4857  getDataMgr().checkpoint(getCurrentDB().dbId, shard->tableId);
4858  }
4859 }
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:249
void checkpoint(const int db_id, const int tb_id)
Definition: DataMgr.cpp:545
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:248
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4700
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::checkpointWithAutoRollback ( const int  logical_table_id) const

Definition at line 4861 of file Catalog.cpp.

Referenced by Fragmenter_Namespace::LocalInsertConnector::checkpoint(), import_export::Loader::checkpoint(), StorageIOFacility::TransactionParameters::finalizeTransaction(), and TableOptimizer::vacuumFragmentsAboveMinSelectivity().

4861  {
4862  auto table_epochs = getTableEpochs(getDatabaseId(), logical_table_id);
4863  try {
4864  checkpoint(logical_table_id);
4865  } catch (...) {
4866  setTableEpochsLogExceptions(getDatabaseId(), table_epochs);
4867  throw;
4868  }
4869 }
int getDatabaseId() const
Definition: Catalog.h:304
void checkpoint(const int logicalTableId) const
Definition: Catalog.cpp:4853
void setTableEpochsLogExceptions(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3694
std::vector< TableEpochInfo > getTableEpochs(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:3630

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::conditionallyInitializeSystemObjects ( )
private

Definition at line 6045 of file Catalog.cpp.

References g_enable_system_tables.

6045  {
6049  }
6050 }
bool g_enable_system_tables
Definition: SysCatalog.cpp:64
bool isInfoSchemaDb() const
Definition: Catalog.cpp:6052
int32_t Catalog_Namespace::Catalog::createCustomExpression ( std::unique_ptr< CustomExpression custom_expression)

Creates a new custom expression.

Parameters
custom_expression- unique pointer to struct containing custom expression details.
Returns
id of created custom expression

Definition at line 5607 of file Catalog.cpp.

References CHECK, CHECK_EQ, CHECK_GT, and to_string().

5608  {
5609  cat_write_lock write_lock(this);
5610  cat_sqlite_lock sqlite_lock(getObjForLock());
5611  sqliteConnector_.query("BEGIN TRANSACTION");
5612  int32_t custom_expression_id{-1};
5613  try {
5614  auto data_source_type_str =
5615  CustomExpression::dataSourceTypeToString(custom_expression->data_source_type);
5616  auto data_source_id_str = std::to_string(custom_expression->data_source_id);
5617  std::string custom_expr_select_query{
5618  "SELECT id FROM omnisci_custom_expressions WHERE name = ? and data_source_type = "
5619  "? and data_source_id = ? and is_deleted = ?"};
5620  std::vector<std::string> custom_expr_select_params{custom_expression->name,
5621  data_source_type_str,
5622  data_source_id_str,
5623  std::to_string(false)};
5624  sqliteConnector_.query_with_text_params(custom_expr_select_query,
5625  custom_expr_select_params);
5626  if (sqliteConnector_.getNumRows() > 0) {
5627  throw std::runtime_error{
5628  "A custom expression with the given "
5629  "name and data source already exists."};
5630  }
5632  "INSERT INTO omnisci_custom_expressions(name, expression_json, "
5633  "data_source_type, data_source_id, is_deleted) VALUES (?,?,?,?,?)",
5634  std::vector<std::string>{custom_expression->name,
5635  custom_expression->expression_json,
5636  data_source_type_str,
5637  data_source_id_str,
5638  std::to_string(false)});
5639  sqliteConnector_.query_with_text_params(custom_expr_select_query,
5640  custom_expr_select_params);
5641  CHECK_EQ(sqliteConnector_.getNumRows(), static_cast<size_t>(1));
5642  custom_expression->id = sqliteConnector_.getData<int32_t>(0, 0);
5643  custom_expression_id = custom_expression->id;
5644  CHECK(custom_expr_map_by_id_.find(custom_expression->id) ==
5645  custom_expr_map_by_id_.end());
5646  custom_expr_map_by_id_[custom_expression->id] = std::move(custom_expression);
5647  } catch (std::exception& e) {
5648  sqliteConnector_.query("ROLLBACK TRANSACTION");
5649  throw;
5650  }
5651  sqliteConnector_.query("END TRANSACTION");
5652  CHECK_GT(custom_expression_id, 0);
5653  return custom_expression_id;
5654 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
#define CHECK_GT(x, y)
Definition: Logger.h:305
std::string to_string(char const *&&v)
static std::string dataSourceTypeToString(DataSourceType type_enum)
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:689
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

int32_t Catalog_Namespace::Catalog::createDashboard ( DashboardDescriptor vd)

Definition at line 4491 of file Catalog.cpp.

References DashboardDescriptor::dashboardId, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, DashboardDescriptor::dashboardSystemRoleName, generate_dashboard_system_rolename(), DashboardDescriptor::imageHash, to_string(), Catalog_Namespace::sqlite_lock< T >::unlock(), Catalog_Namespace::write_lock< T >::unlock(), DashboardDescriptor::updateTime, and DashboardDescriptor::userId.

4491  {
4492  cat_write_lock write_lock(this);
4493  cat_sqlite_lock sqlite_lock(getObjForLock());
4494  sqliteConnector_.query("BEGIN TRANSACTION");
4495  try {
4496  // TODO(andrew): this should be an upsert
4498  "SELECT id FROM mapd_dashboards WHERE name = ? and userid = ?",
4499  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
4500  if (sqliteConnector_.getNumRows() > 0) {
4502  "UPDATE mapd_dashboards SET state = ?, image_hash = ?, metadata = ?, "
4503  "update_time = "
4504  "datetime('now') where name = ? "
4505  "and userid = ?",
4506  std::vector<std::string>{vd.dashboardState,
4507  vd.imageHash,
4508  vd.dashboardMetadata,
4509  vd.dashboardName,
4510  std::to_string(vd.userId)});
4511  } else {
4513  "INSERT INTO mapd_dashboards (name, state, image_hash, metadata, "
4514  "update_time, "
4515  "userid) "
4516  "VALUES "
4517  "(?,?,?,?, "
4518  "datetime('now'), ?)",
4519  std::vector<std::string>{vd.dashboardName,
4520  vd.dashboardState,
4521  vd.imageHash,
4522  vd.dashboardMetadata,
4523  std::to_string(vd.userId)});
4524  }
4525  } catch (std::exception& e) {
4526  sqliteConnector_.query("ROLLBACK TRANSACTION");
4527  throw;
4528  }
4529  sqliteConnector_.query("END TRANSACTION");
4530 
4531  // now get the auto generated dashboardId
4532  try {
4534  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_dashboards "
4535  "WHERE name = ? and userid = ?",
4536  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
4537  vd.dashboardId = sqliteConnector_.getData<int>(0, 0);
4538  vd.updateTime = sqliteConnector_.getData<std::string>(0, 1);
4539  } catch (std::exception& e) {
4540  throw;
4541  }
4545  sqlite_lock.unlock();
4546  write_lock.unlock();
4547  if (!isInfoSchemaDb()) {
4548  // NOTE(wamsi): Transactionally unsafe
4551  }
4552  return vd.dashboardId;
4553 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
const DBMetadata currentDB_
Definition: Catalog.h:693
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void addFrontendViewToMap(DashboardDescriptor &vd)
Definition: Catalog.cpp:1778
std::string to_string(char const *&&v)
std::string dashboardSystemRoleName
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1819
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
bool isInfoSchemaDb() const
Definition: Catalog.cpp:6052
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createDashboardSystemRoles ( )
protected

Definition at line 931 of file Catalog.cpp.

References createOrUpdateDashboardSystemRole(), currentDB_, Catalog_Namespace::DBMetadata::dbId, logger::ERROR, generate_dashboard_system_rolename(), SqliteConnector::getData(), Catalog_Namespace::SysCatalog::getGranteesOfSharedDashboards(), SqliteConnector::getNumRows(), getObjForLock(), Catalog_Namespace::SysCatalog::getRoleGrantee(), Catalog_Namespace::SysCatalog::grantRoleBatch(), logger::INFO, Catalog_Namespace::SysCatalog::instance(), LOG, MAPD_VERSION, SqliteConnector::query(), SqliteConnector::query_with_text_params(), run_benchmark_import::result, sqliteConnector_, and to_string().

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

931  {
932  std::unordered_map<std::string, std::pair<int, std::string>> dashboards;
933  std::vector<std::string> dashboard_ids;
934  static const std::string migration_name{"dashboard_roles_migration"};
935  {
936  cat_sqlite_lock sqlite_lock(getObjForLock());
937  sqliteConnector_.query("BEGIN TRANSACTION");
938  try {
939  // migration_history should be present in all catalogs by now
940  // if not then would be created before this migration
942  "select * from mapd_version_history where migration_history = '" +
943  migration_name + "'");
944  if (sqliteConnector_.getNumRows() != 0) {
945  // no need for further execution
946  sqliteConnector_.query("END TRANSACTION");
947  return;
948  }
949  LOG(INFO) << "Performing dashboard internal roles Migration.";
950  sqliteConnector_.query("select id, userid, metadata from mapd_dashboards");
951  for (size_t i = 0; i < sqliteConnector_.getNumRows(); ++i) {
954  sqliteConnector_.getData<string>(i, 0)))) {
955  // Successfully created roles during previous migration/crash
956  // No need to include them
957  continue;
958  }
959  dashboards[sqliteConnector_.getData<string>(i, 0)] = std::make_pair(
960  sqliteConnector_.getData<int>(i, 1), sqliteConnector_.getData<string>(i, 2));
961  dashboard_ids.push_back(sqliteConnector_.getData<string>(i, 0));
962  }
963  } catch (const std::exception& e) {
964  sqliteConnector_.query("ROLLBACK TRANSACTION");
965  throw;
966  }
967  sqliteConnector_.query("END TRANSACTION");
968  }
969  // All current grantees with shared dashboards.
970  const auto active_grantees =
972 
973  try {
974  // NOTE(wamsi): Transactionally unsafe
975  for (auto dash : dashboards) {
976  createOrUpdateDashboardSystemRole(dash.second.second,
977  dash.second.first,
979  std::to_string(currentDB_.dbId), dash.first));
980  auto result = active_grantees.find(dash.first);
981  if (result != active_grantees.end()) {
984  dash.first)},
985  result->second);
986  }
987  }
988  cat_sqlite_lock sqlite_lock(getObjForLock());
989  // check if this has already been completed
991  "select * from mapd_version_history where migration_history = '" +
992  migration_name + "'");
993  if (sqliteConnector_.getNumRows() != 0) {
994  return;
995  }
997  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
998  std::vector<std::string>{std::to_string(MAPD_VERSION), migration_name});
999  } catch (const std::exception& e) {
1000  LOG(ERROR) << "Failed to create dashboard system roles during migration: "
1001  << e.what();
1002  throw;
1003  }
1004  LOG(INFO) << "Successfully created dashboard system roles during migration.";
1005 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
#define LOG(tag)
Definition: Logger.h:285
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
const DBMetadata currentDB_
Definition: Catalog.h:693
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
void grantRoleBatch(const std::vector< std::string > &roles, const std::vector< std::string > &grantees)
static SysCatalog & instance()
Definition: SysCatalog.h:343
static const int32_t MAPD_VERSION
Definition: release.h:32
Role * getRoleGrantee(const std::string &name) const
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1819
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
std::unordered_map< std::string, std::vector< std::string > > getGranteesOfSharedDashboards(const std::vector< std::string > &dashboard_ids)
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createDefaultServersIfNotExists ( )

Creates default local file servers (if they don't already exist).

Definition at line 4994 of file Catalog.cpp.

References CHECK, foreign_storage::DataWrapperType::CSV, g_enable_fsi, shared::kRootUserId, foreign_storage::AbstractFileStorageDataWrapper::LOCAL_FILE_STORAGE_TYPE, foreign_storage::DataWrapperType::PARQUET, foreign_storage::DataWrapperType::REGEX_PARSER, and foreign_storage::AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY.

4994  {
4999 
5000  auto local_csv_server = std::make_unique<foreign_storage::ForeignServer>(
5001  "default_local_delimited",
5003  options,
5005  local_csv_server->validate();
5006  createForeignServerNoLocks(std::move(local_csv_server), true);
5007 
5008 #ifdef ENABLE_IMPORT_PARQUET
5009  auto local_parquet_server = std::make_unique<foreign_storage::ForeignServer>(
5010  "default_local_parquet",
5012  options,
5014  local_parquet_server->validate();
5015  createForeignServerNoLocks(std::move(local_parquet_server), true);
5016 #endif
5017 
5018  auto local_regex_parser_server = std::make_unique<foreign_storage::ForeignServer>(
5019  "default_local_regex_parsed",
5021  options,
5023  local_regex_parser_server->validate();
5024  createForeignServerNoLocks(std::move(local_regex_parser_server), true);
5025 }
const int kRootUserId
static constexpr char const * REGEX_PARSER
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:3099
#define CHECK(condition)
Definition: Logger.h:291
static constexpr char const * CSV
std::map< std::string, std::string, std::less<>> OptionsMap
bool g_enable_fsi
Definition: Catalog.cpp:96
static constexpr char const * PARQUET
void Catalog_Namespace::Catalog::createForeignServer ( std::unique_ptr< foreign_storage::ForeignServer foreign_server,
bool  if_not_exists 
)

Creates a new foreign server DB object.

Parameters
foreign_server- unique pointer to struct containing foreign server details
if_not_exists- flag indicating whether or not an attempt to create a new foreign server should occur if a server with the same name already exists. An exception is thrown if this flag is set to "false" and an attempt is made to create a pre-existing foreign server

Definition at line 3091 of file Catalog.cpp.

3093  {
3094  cat_write_lock write_lock(this);
3095  cat_sqlite_lock sqlite_lock(getObjForLock());
3096  createForeignServerNoLocks(std::move(foreign_server), if_not_exists);
3097 }
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:3099
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
void Catalog_Namespace::Catalog::createForeignServerNoLocks ( std::unique_ptr< foreign_storage::ForeignServer foreign_server,
bool  if_not_exists 
)
private

Same as createForeignServer() but without acquiring locks. This should only be called from within a function/code block that already acquires appropriate locks.

Definition at line 3099 of file Catalog.cpp.

References CHECK, CHECK_EQ, setup::name, and to_string().

3101  {
3102  const auto& name = foreign_server->name;
3103 
3105  "SELECT name from omnisci_foreign_servers where name = ?",
3106  std::vector<std::string>{name});
3107 
3108  if (sqliteConnector_.getNumRows() == 0) {
3109  foreign_server->creation_time = std::time(nullptr);
3111  "INSERT INTO omnisci_foreign_servers (name, data_wrapper_type, owner_user_id, "
3112  "creation_time, "
3113  "options) "
3114  "VALUES (?, ?, ?, ?, ?)",
3115  std::vector<std::string>{name,
3116  foreign_server->data_wrapper_type,
3117  std::to_string(foreign_server->user_id),
3118  std::to_string(foreign_server->creation_time),
3119  foreign_server->getOptionsAsJsonString()});
3121  "SELECT id from omnisci_foreign_servers where name = ?",
3122  std::vector<std::string>{name});
3123  CHECK_EQ(sqliteConnector_.getNumRows(), size_t(1));
3124  foreign_server->id = sqliteConnector_.getData<int32_t>(0, 0);
3125  std::shared_ptr<foreign_storage::ForeignServer> foreign_server_shared =
3126  std::move(foreign_server);
3128  << "Attempting to insert a foreign server into foreign server map that already "
3129  "exists.";
3130  foreignServerMap_[name] = foreign_server_shared;
3131  foreignServerMapById_[foreign_server_shared->id] = foreign_server_shared;
3132  } else if (!if_not_exists) {
3133  throw std::runtime_error{"A foreign server with name \"" + foreign_server->name +
3134  "\" already exists."};
3135  }
3136 
3137  const auto& server_it = foreignServerMap_.find(name);
3138  CHECK(server_it != foreignServerMap_.end());
3139  CHECK(foreignServerMapById_.find(server_it->second->id) != foreignServerMapById_.end());
3140 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
std::string to_string(char const *&&v)
std::string name() const
Definition: Catalog.h:321
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:688
ForeignServerMap foreignServerMap_
Definition: Catalog.h:687
#define CHECK(condition)
Definition: Logger.h:291
virtual size_t getNumRows() const

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::createLink ( LinkDescriptor ld,
size_t  min_length 
)

Definition at line 4644 of file Catalog.cpp.

References LinkDescriptor::link, LinkDescriptor::linkId, to_string(), LinkDescriptor::updateTime, LinkDescriptor::userId, LinkDescriptor::viewMetadata, and LinkDescriptor::viewState.

4644  {
4645  cat_write_lock write_lock(this);
4646  cat_sqlite_lock sqlite_lock(getObjForLock());
4647  sqliteConnector_.query("BEGIN TRANSACTION");
4648  try {
4650  .substr(0, 8);
4652  "SELECT linkid FROM mapd_links WHERE link = ? and userid = ?",
4653  std::vector<std::string>{ld.link, std::to_string(ld.userId)});
4654  if (sqliteConnector_.getNumRows() > 0) {
4656  "UPDATE mapd_links SET update_time = datetime('now') WHERE userid = ? AND "
4657  "link = ?",
4658  std::vector<std::string>{std::to_string(ld.userId), ld.link});
4659  } else {
4661  "INSERT INTO mapd_links (userid, link, view_state, view_metadata, "
4662  "update_time) VALUES (?,?,?,?, datetime('now'))",
4663  std::vector<std::string>{
4664  std::to_string(ld.userId), ld.link, ld.viewState, ld.viewMetadata});
4665  }
4666  // now get the auto generated dashid
4668  "SELECT linkid, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_links "
4669  "WHERE link = ?",
4670  ld.link);
4671  ld.linkId = sqliteConnector_.getData<int>(0, 0);
4672  ld.updateTime = sqliteConnector_.getData<std::string>(0, 1);
4673  } catch (std::exception& e) {
4674  sqliteConnector_.query("ROLLBACK TRANSACTION");
4675  throw;
4676  }
4677  sqliteConnector_.query("END TRANSACTION");
4678  addLinkToMap(ld);
4679  return ld.link;
4680 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::string to_string(char const *&&v)
std::string calculateSHA1(const std::string &data)
Definition: Catalog.cpp:4632
void addLinkToMap(LinkDescriptor &ld)
Definition: Catalog.cpp:1865
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createOrUpdateDashboardSystemRole ( const std::string &  view_meta,
const int32_t &  user_id,
const std::string &  dash_role_name 
)
protected

Definition at line 1819 of file Catalog.cpp.

References Grantee::findDbObject(), Grantee::getDbObjects(), TableDBObjectType, and ViewDBObjectType.

Referenced by createDashboardSystemRoles().

1821  {
1822  auto objects = parseDashboardObjects(view_meta, user_id);
1823  Role* rl = SysCatalog::instance().getRoleGrantee(dash_role_name);
1824  if (!rl) {
1825  // Dashboard role does not exist
1826  // create role and grant privileges
1827  // NOTE(wamsi): Transactionally unsafe
1829  dash_role_name, /*user_private_role=*/false, /*is_temporary=*/false);
1830  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1831  } else {
1832  // Dashboard system role already exists
1833  // Add/remove privileges on objects
1834  std::set<DBObjectKey> revoke_keys;
1835  auto ex_objects = rl->getDbObjects(true);
1836  for (auto key : *ex_objects | boost::adaptors::map_keys) {
1837  if (key.permissionType != TableDBObjectType &&
1838  key.permissionType != ViewDBObjectType) {
1839  continue;
1840  }
1841  bool found = false;
1842  for (auto obj : objects) {
1843  found = key == obj.getObjectKey() ? true : false;
1844  if (found) {
1845  break;
1846  }
1847  }
1848  if (!found) {
1849  revoke_keys.insert(key);
1850  }
1851  }
1852  for (auto& key : revoke_keys) {
1853  // revoke privs on object since the object is no
1854  // longer used by the dashboard as source
1855  // NOTE(wamsi): Transactionally unsafe
1857  dash_role_name, *rl->findDbObject(key, true), *this);
1858  }
1859  // Update privileges on remaining objects
1860  // NOTE(wamsi): Transactionally unsafe
1861  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1862  }
1863 }
void revokeDBObjectPrivileges(const std::string &grantee, const DBObject &object, const Catalog_Namespace::Catalog &catalog)
void createRole(const std::string &roleName, const bool user_private_role, const bool is_temporary=false)
Definition: Grantee.h:81
DBObject * findDbObject(const DBObjectKey &objectKey, bool only_direct) const
Definition: Grantee.cpp:85
static SysCatalog & instance()
Definition: SysCatalog.h:343
Role * getRoleGrantee(const std::string &name) const
void grantDBObjectPrivilegesBatch(const std::vector< std::string > &grantees, const std::vector< DBObject > &objects, const Catalog_Namespace::Catalog &catalog)
const DBObjectMap * getDbObjects(bool only_direct) const
Definition: Grantee.h:56
std::vector< DBObject > parseDashboardObjects(const std::string &view_meta, const int &user_id)
Definition: Catalog.cpp:1789

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createShardedTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs 
)

Definition at line 3908 of file Catalog.cpp.

References CHECK, TableDescriptor::nShards, TableDescriptor::shard, table_is_temporary(), TableDescriptor::tableId, and TableDescriptor::tableName.

3911  {
3912  /* create logical table */
3913  TableDescriptor* tdl = &td;
3914  createTable(*tdl, cols, shared_dict_defs, true); // create logical table
3915  int32_t logical_tb_id = tdl->tableId;
3916  std::string logical_table_name = tdl->tableName;
3917 
3918  /* create physical tables and link them to the logical table */
3919  std::vector<int32_t> physicalTables;
3920  for (int32_t i = 1; i <= td.nShards; i++) {
3921  TableDescriptor* tdp = &td;
3922  tdp->tableName = generatePhysicalTableName(logical_table_name, i);
3923  tdp->shard = i - 1;
3924  createTable(*tdp, cols, shared_dict_defs, false); // create physical table
3925  int32_t physical_tb_id = tdp->tableId;
3926 
3927  /* add physical table to the vector of physical tables */
3928  physicalTables.push_back(physical_tb_id);
3929  }
3930 
3931  if (!physicalTables.empty()) {
3932  cat_write_lock write_lock(this);
3933  /* add logical to physical tables correspondence to the map */
3934  const auto it_ok =
3935  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
3936  CHECK(it_ok.second);
3937  /* update sqlite mapd_logical_to_physical in sqlite database */
3938  if (!table_is_temporary(&td)) {
3939  updateLogicalToPhysicalTableMap(logical_tb_id);
3940  }
3941  }
3942 }
std::string tableName
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:2743
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::string generatePhysicalTableName(const std::string &logicalTableName, const int32_t &shardNumber)
Definition: Catalog.cpp:4916
bool table_is_temporary(const TableDescriptor *const td)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:699
void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id)
Definition: Catalog.cpp:645

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createSystemTableServer ( const std::string &  server_name,
const std::string &  data_wrapper_type,
const foreign_storage::OptionsMap options = {} 
)
private

Definition at line 6442 of file Catalog.cpp.

References logger::INFO, shared::kRootUserId, LOG, and run_benchmark_import::tables.

6444  {
6445  auto server = std::make_unique<foreign_storage::ForeignServer>(
6446  server_name, data_wrapper_type, options, shared::kRootUserId);
6447  server->validate();
6448  auto stored_server = getForeignServer(server_name);
6449  if (stored_server && stored_server->options != server->options) {
6450  // Drop all tables for server before dropping server.
6451  auto tables = getAllForeignTablesForForeignServer(stored_server->id);
6452  for (const auto table : tables) {
6453  LOG(INFO) << "Dropping existing \"" << table->tableName << "\" system table for \""
6454  << server_name << "\" foreign server.";
6455  dropTable(table);
6456  }
6457  LOG(INFO) << "Dropping existing \"" << server_name
6458  << "\" system table foreign server.";
6459  dropForeignServer(server_name);
6460  stored_server = nullptr;
6461  }
6462  if (!stored_server) {
6463  LOG(INFO) << "Creating a new \"" << server_name << "\" system table foreign server.";
6464  createForeignServer(std::move(server), true);
6465  }
6466 }
const int kRootUserId
void createForeignServer(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:3091
const foreign_storage::ForeignServer * getForeignServer(const std::string &server_name) const
Definition: Catalog.cpp:3142
#define LOG(tag)
Definition: Logger.h:285
std::vector< const foreign_storage::ForeignTable * > getAllForeignTablesForForeignServer(const int32_t foreign_server_id)
Get all of the foreign tables for associated with a foreign server id.
Definition: Catalog.cpp:3439
void dropForeignServer(const std::string &server_name)
Definition: Catalog.cpp:3258
void dropTable(const TableDescriptor *td)
Definition: Catalog.cpp:4086
void Catalog_Namespace::Catalog::createTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs,
bool  isLogicalTable 
)

Definition at line 2743 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnId, TableDescriptor::columnIdBySpi_, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::default_value, dict_ref_t::dictId, Data_Namespace::DISK_LEVEL, StorageType::FOREIGN_TABLE, TableDescriptor::fragPageSize, TableDescriptor::fragType, g_enable_fsi, g_serialize_temp_tables, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_next_refresh_time(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), TableDescriptor::hasDeletedCol, SQLTypeInfo::is_array(), SQLTypeInfo::is_geometry(), TableDescriptor::is_system_table, ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, kBIGINT, kBOOLEAN, kENCODING_DICT, TableDescriptor::keyMetainfo, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nColumns, foreign_storage::ForeignTable::next_refresh_time, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, SQLTypeInfo::set_comp_param(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::set_dict_key(), SQLTypeInfo::set_size(), TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::storageType, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), TableDescriptor::userId, TableDescriptor::viewSQL, and ColumnDescriptor::virtualExpr.

2747  {
2748  cat_write_lock write_lock(this);
2749  list<ColumnDescriptor> cds = cols;
2750  list<DictDescriptor> dds;
2751  std::set<std::string> toplevel_column_names;
2752  list<ColumnDescriptor> columns;
2753 
2754  if (!td.storageType.empty() &&
2757  throw std::runtime_error("Only temporary tables can be backed by foreign storage.");
2758  }
2759  dataMgr_->getForeignStorageInterface()->prepareTable(getCurrentDB().dbId, td, cds);
2760  }
2761 
2762  for (auto cd : cds) {
2763  if (cd.columnName == "rowid") {
2764  throw std::runtime_error(
2765  "Cannot create column with name rowid. rowid is a system defined column.");
2766  }
2767  columns.push_back(cd);
2768  toplevel_column_names.insert(cd.columnName);
2769  if (cd.columnType.is_geometry()) {
2770  expandGeoColumn(cd, columns);
2771  }
2772  }
2773  cds.clear();
2774 
2775  ColumnDescriptor cd;
2776  // add row_id column -- Must be last column in the table
2777  cd.columnName = "rowid";
2778  cd.isSystemCol = true;
2779  cd.columnType = SQLTypeInfo(kBIGINT, true);
2780 #ifdef MATERIALIZED_ROWID
2781  cd.isVirtualCol = false;
2782 #else
2783  cd.isVirtualCol = true;
2784  cd.virtualExpr = "MAPD_FRAG_ID * MAPD_ROWS_PER_FRAG + MAPD_FRAG_ROW_ID";
2785 #endif
2786  columns.push_back(cd);
2787  toplevel_column_names.insert(cd.columnName);
2788 
2789  if (td.hasDeletedCol) {
2790  ColumnDescriptor cd_del;
2791  cd_del.columnName = "$deleted$";
2792  cd_del.isSystemCol = true;
2793  cd_del.isVirtualCol = false;
2794  cd_del.columnType = SQLTypeInfo(kBOOLEAN, true);
2795  cd_del.isDeletedCol = true;
2796 
2797  columns.push_back(cd_del);
2798  }
2799 
2800  for (auto& column : columns) {
2801  column.db_id = getDatabaseId();
2802  }
2803 
2804  td.nColumns = columns.size();
2805  // TODO(sy): don't take disk locks or touch sqlite connector for temporary tables
2806  cat_sqlite_lock sqlite_lock(getObjForLock());
2807  sqliteConnector_.query("BEGIN TRANSACTION");
2809  try {
2811  R"(INSERT INTO mapd_tables (name, userid, ncolumns, isview, fragments, frag_type, max_frag_rows, max_chunk_size, frag_page_size, max_rows, partitions, shard_column_id, shard, num_shards, sort_column_id, storage_type, max_rollback_epochs, is_system_table, key_metainfo) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?))",
2812  std::vector<std::string>{td.tableName,
2813  std::to_string(td.userId),
2815  std::to_string(td.isView),
2816  "",
2821  std::to_string(td.maxRows),
2822  td.partitions,
2824  std::to_string(td.shard),
2825  std::to_string(td.nShards),
2827  td.storageType,
2830  td.keyMetainfo});
2831 
2832  // now get the auto generated tableid
2834  "SELECT tableid FROM mapd_tables WHERE name = ?", td.tableName);
2835  td.tableId = sqliteConnector_.getData<int>(0, 0);
2836  int colId = 1;
2837  for (auto cd : columns) {
2839  const bool is_foreign_col =
2840  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2841  if (!is_foreign_col) {
2842  // Ideally we would like to not persist string dictionaries for system tables,
2843  // since system table content can be highly dynamic and string dictionaries
2844  // are not currently vacuumed. However, in distributed this causes issues
2845  // when the cluster is out of sync (when the agg resets but leaves persist) so
2846  // for the sake of testing we need to leave this as normal dictionaries until
2847  // we solve the distributed issue.
2848  auto use_temp_dictionary = false; // td.is_system_table;
2849  setColumnDictionary(cd, dds, td, isLogicalTable, use_temp_dictionary);
2850  }
2851  }
2852 
2853  if (toplevel_column_names.count(cd.columnName)) {
2854  if (!cd.isGeoPhyCol) {
2855  td.columnIdBySpi_.push_back(colId);
2856  }
2857  }
2858 
2859  using BindType = SqliteConnector::BindType;
2860  std::vector<BindType> types(17, BindType::TEXT);
2861  if (!cd.default_value.has_value()) {
2862  types[16] = BindType::NULL_TYPE;
2863  }
2865  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, "
2866  "coldim, colscale, is_notnull, "
2867  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
2868  "virtual_expr, is_deletedcol, default_value) "
2869  "VALUES (?, ?, ?, ?, ?, "
2870  "?, "
2871  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2872  std::vector<std::string>{std::to_string(td.tableId),
2873  std::to_string(colId),
2874  cd.columnName,
2883  "",
2886  cd.virtualExpr,
2888  cd.default_value.value_or("NULL")},
2889  types);
2890  cd.tableId = td.tableId;
2891  cd.columnId = colId++;
2892  cds.push_back(cd);
2893  }
2894  if (td.isView) {
2896  "INSERT INTO mapd_views (tableid, sql) VALUES (?,?)",
2897  std::vector<std::string>{std::to_string(td.tableId), td.viewSQL});
2898  }
2900  auto& foreign_table = dynamic_cast<foreign_storage::ForeignTable&>(td);
2901  foreign_table.next_refresh_time = get_next_refresh_time(foreign_table);
2903  "INSERT INTO omnisci_foreign_tables (table_id, server_id, options, "
2904  "last_refresh_time, next_refresh_time) VALUES (?, ?, ?, ?, ?)",
2905  std::vector<std::string>{std::to_string(foreign_table.tableId),
2906  std::to_string(foreign_table.foreign_server->id),
2907  foreign_table.getOptionsAsJsonString(),
2908  std::to_string(foreign_table.last_refresh_time),
2909  std::to_string(foreign_table.next_refresh_time)});
2910  }
2911  } catch (std::exception& e) {
2912  sqliteConnector_.query("ROLLBACK TRANSACTION");
2913  throw;
2914  }
2915  } else { // Temporary table
2916  td.tableId = nextTempTableId_++;
2917  int colId = 1;
2918  for (auto cd : columns) {
2920  const bool is_foreign_col =
2921  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2922 
2923  if (!is_foreign_col) {
2924  // Create a new temporary dictionary
2925  std::string fileName("");
2926  std::string folderPath("");
2928  nextTempDictId_++;
2929  DictDescriptor dd(dict_ref,
2930  fileName,
2932  false,
2933  1,
2934  folderPath,
2935  true); // Is dictName (2nd argument) used?
2936  dds.push_back(dd);
2937  if (!cd.columnType.is_array()) {
2939  }
2940  cd.columnType.set_comp_param(dict_ref.dictId);
2941  set_dict_key(cd);
2942  }
2943  }
2944  if (toplevel_column_names.count(cd.columnName)) {
2945  if (!cd.isGeoPhyCol) {
2946  td.columnIdBySpi_.push_back(colId);
2947  }
2948  }
2949  cd.tableId = td.tableId;
2950  cd.columnId = colId++;
2951  cds.push_back(cd);
2952  }
2953 
2955  serializeTableJsonUnlocked(&td, cds);
2956  }
2957  }
2958 
2959  try {
2960  auto cache = dataMgr_->getPersistentStorageMgr()->getDiskCache();
2961  if (cache) {
2962  CHECK(!cache->hasCachedMetadataForKeyPrefix({getCurrentDB().dbId, td.tableId}))
2963  << "Disk cache at " + cache->getCacheDirectory()
2964  << " contains preexisting data for new table. Please "
2965  "delete or clear cache before continuing";
2966  }
2967 
2968  addTableToMap(&td, cds, dds);
2969  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
2970  if (!td.storageType.empty() && td.storageType != StorageType::FOREIGN_TABLE) {
2971  dataMgr_->getForeignStorageInterface()->registerTable(this, td, cds);
2972  }
2973  } catch (std::exception& e) {
2974  sqliteConnector_.query("ROLLBACK TRANSACTION");
2975  removeTableFromMap(td.tableName, td.tableId, true);
2976  throw;
2977  }
2978  sqliteConnector_.query("END TRANSACTION");
2979 
2981  write_lock.unlock();
2982  sqlite_lock.unlock();
2984  true); // cause instantiateFragmenter() to be called
2985  }
2986 }
int32_t maxRollbackEpochs
void serializeTableJsonUnlocked(const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
Definition: Catalog.cpp:2988
int64_t get_next_refresh_time(const foreign_storage::ForeignTable &foreign_table)
Definition: Catalog.cpp:2730
std::string virtualExpr
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:382
void set_size(int s)
Definition: sqltypes.h:501
std::string partitions
HOST DEVICE int get_size() const
Definition: sqltypes.h:393
T getData(const int row, const int col)
std::string tableName
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
const DBMetadata currentDB_
Definition: Catalog.h:693
HOST DEVICE int get_scale() const
Definition: sqltypes.h:386
std::string storageType
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, bool is_logical_table, bool use_temp_dictionary=false)
Definition: Catalog.cpp:3865
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:694
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:381
std::vector< int > columnIdBySpi_
std::string to_string(char const *&&v)
void addTableToMap(const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
Definition: Catalog.cpp:1637
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:248
int getDatabaseId() const
Definition: Catalog.h:304
specifies the content in-memory of a row in the column metadata table
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:2570
std::string keyMetainfo
bool g_serialize_temp_tables
Definition: Catalog.cpp:106
void set_comp_param(int p)
Definition: sqltypes.h:505
std::optional< std::string > default_value
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:697
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:389
Fragmenter_Namespace::FragmenterType fragType
Data_Namespace::MemoryLevel persistenceLevel
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:383
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1696
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:392
bool setColumnSharedDictionary(ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:3796
#define CHECK(condition)
Definition: Logger.h:291
Descriptor for a dictionary for a string columne.
std::string viewSQL
SQLTypeInfo columnType
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:388
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
static constexpr char const * FOREIGN_TABLE
bool g_enable_fsi
Definition: Catalog.cpp:96
std::string columnName
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
bool is_array() const
Definition: sqltypes.h:588

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::delDictionary ( const ColumnDescriptor cd)

Definition at line 2347 of file Catalog.cpp.

References CHECK, CHECK_GT, ColumnDescriptor::columnType, g_base_path, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::is_string(), SQLTypeInfo::is_string_array(), shared::kDataDirectoryName, kENCODING_DICT, File_Namespace::renameForDelete(), ColumnDescriptor::tableId, to_string(), and VLOG.

2347  {
2348  cat_write_lock write_lock(this);
2349  cat_sqlite_lock sqlite_lock(getObjForLock());
2350  if (!(cd.columnType.is_string() || cd.columnType.is_string_array())) {
2351  return;
2352  }
2353  if (!(cd.columnType.get_compression() == kENCODING_DICT)) {
2354  return;
2355  }
2356  const auto dictId = cd.columnType.get_comp_param();
2357  CHECK_GT(dictId, 0);
2358  // decrement and zero check dict ref count
2359  const auto td = getMetadataForTable(cd.tableId, false);
2360  CHECK(td);
2362  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
2363  std::to_string(dictId));
2365  "SELECT refcount FROM mapd_dictionaries WHERE dictid = ?", std::to_string(dictId));
2366  const auto refcount = sqliteConnector_.getData<int>(0, 0);
2367  VLOG(3) << "Dictionary " << dictId << "from dropped table has reference count "
2368  << refcount;
2369  if (refcount > 0) {
2370  return;
2371  }
2372  const DictRef dictRef(currentDB_.dbId, dictId);
2373  sqliteConnector_.query_with_text_param("DELETE FROM mapd_dictionaries WHERE dictid = ?",
2374  std::to_string(dictId));
2376  "/DB_" + std::to_string(currentDB_.dbId) + "_DICT_" +
2377  std::to_string(dictId));
2378 
2379  std::unique_ptr<StringDictionaryClient> client;
2380  if (!string_dict_hosts_.empty()) {
2381  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dictRef, true));
2382  }
2383  if (client) {
2384  client->drop(dictRef);
2385  }
2386 
2387  dictDescriptorMapByRef_.erase(dictRef);
2388 }
const std::string kDataDirectoryName
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:696
const DBMetadata currentDB_
Definition: Catalog.h:693
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:683
heavyai::unique_lock< heavyai::shared_mutex > write_lock
#define CHECK_GT(x, y)
Definition: Logger.h:305
std::string to_string(char const *&&v)
std::string g_base_path
Definition: SysCatalog.cpp:62
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:389
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:392
#define CHECK(condition)
Definition: Logger.h:291
SQLTypeInfo columnType
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
bool is_string() const
Definition: sqltypes.h:580
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:242
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
bool is_string_array() const
Definition: sqltypes.h:581
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
#define VLOG(n)
Definition: Logger.h:387

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::deleteCustomExpressions ( const std::vector< int32_t > &  custom_expression_ids,
bool  do_soft_delete 
)

Deletes custom expressions with the given ids.

Parameters
custom_expression_ids- ids of custom expressions to delete
do_soft_delete- flag indicating whether or not to do a soft delete

Definition at line 5732 of file Catalog.cpp.

References CHECK, CHECK_EQ, join(), and to_string().

5733  {
5734  cat_write_lock write_lock(this);
5735  cat_sqlite_lock sqlite_lock(getObjForLock());
5736 
5737  std::vector<int32_t> invalid_ids;
5738  for (const auto id : custom_expression_ids) {
5739  if (custom_expr_map_by_id_.find(id) == custom_expr_map_by_id_.end()) {
5740  invalid_ids.emplace_back(id);
5741  }
5742  }
5743  if (!invalid_ids.empty()) {
5744  throw std::runtime_error{"Custom expressions with ids: " + join(invalid_ids, ",") +
5745  " do not exist."};
5746  }
5747  sqliteConnector_.query("BEGIN TRANSACTION");
5748  try {
5749  for (const auto id : custom_expression_ids) {
5751  "SELECT id FROM omnisci_custom_expressions WHERE id = ?",
5752  std::vector<std::string>{std::to_string(id)});
5753  CHECK_EQ(sqliteConnector_.getNumRows(), static_cast<size_t>(1));
5754  if (do_soft_delete) {
5756  "UPDATE omnisci_custom_expressions SET is_deleted = ? WHERE id = ?",
5757  std::vector<std::string>{std::to_string(true), std::to_string(id)});
5758  } else {
5760  "DELETE FROM omnisci_custom_expressions WHERE id = ?",
5761  std::vector<std::string>{std::to_string(id)});
5762  }
5763  }
5764 
5765  for (const auto id : custom_expression_ids) {
5766  if (do_soft_delete) {
5767  auto it = custom_expr_map_by_id_.find(id);
5768  CHECK(it != custom_expr_map_by_id_.end());
5769  it->second->is_deleted = true;
5770  } else {
5771  custom_expr_map_by_id_.erase(id);
5772  }
5773  }
5774  } catch (std::exception& e) {
5775  sqliteConnector_.query("ROLLBACK TRANSACTION");
5776  throw;
5777  }
5778  sqliteConnector_.query("END TRANSACTION");
5779 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
std::string join(T const &container, std::string const &delim)
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::string to_string(char const *&&v)
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:689
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::deleteMetadataForDashboards ( const std::vector< int32_t >  ids,
const UserMetadata user 
)

Definition at line 2106 of file Catalog.cpp.

References DashboardDBObjectType, AccessPrivileges::DELETE_DASHBOARD, and to_string().

2107  {
2108  std::stringstream invalid_ids, restricted_ids;
2109 
2110  for (int32_t dashboard_id : dashboard_ids) {
2111  if (!getMetadataForDashboard(dashboard_id)) {
2112  invalid_ids << (!invalid_ids.str().empty() ? ", " : "") << dashboard_id;
2113  continue;
2114  }
2115  DBObject object(dashboard_id, DashboardDBObjectType);
2116  object.loadKey(*this);
2117  object.setPrivileges(AccessPrivileges::DELETE_DASHBOARD);
2118  std::vector<DBObject> privs = {object};
2119  if (!SysCatalog::instance().checkPrivileges(user, privs)) {
2120  restricted_ids << (!restricted_ids.str().empty() ? ", " : "") << dashboard_id;
2121  }
2122  }
2123 
2124  if (invalid_ids.str().size() > 0 || restricted_ids.str().size() > 0) {
2125  std::stringstream error_message;
2126  error_message << "Delete dashboard(s) failed with error(s):";
2127  if (invalid_ids.str().size() > 0) {
2128  error_message << "\nDashboard id: " << invalid_ids.str()
2129  << " - Dashboard id does not exist";
2130  }
2131  if (restricted_ids.str().size() > 0) {
2132  error_message
2133  << "\nDashboard id: " << restricted_ids.str()
2134  << " - User should be either owner of dashboard or super user to delete it";
2135  }
2136  throw std::runtime_error(error_message.str());
2137  }
2138  std::vector<DBObject> dash_objs;
2139 
2140  for (int32_t dashboard_id : dashboard_ids) {
2141  dash_objs.emplace_back(dashboard_id, DashboardDBObjectType);
2142  }
2143  // BE-5245: Transactionally unsafe (like other combined Catalog/Syscatalog operations)
2145  {
2146  cat_write_lock write_lock(this);
2147  cat_sqlite_lock sqlite_lock(getObjForLock());
2148 
2149  sqliteConnector_.query("BEGIN TRANSACTION");
2150  try {
2151  for (int32_t dashboard_id : dashboard_ids) {
2152  auto dash = getMetadataForDashboard(dashboard_id);
2153  // Dash should still exist if revokeDBObjectPrivileges passed but throw and
2154  // rollback if already deleted
2155  if (!dash) {
2156  throw std::runtime_error(
2157  std::string("Delete dashboard(s) failed with error(s):\nDashboard id: ") +
2158  std::to_string(dashboard_id) + " - Dashboard id does not exist ");
2159  }
2160  std::string user_id = std::to_string(dash->userId);
2161  std::string dash_name = dash->dashboardName;
2162  auto viewDescIt = dashboardDescriptorMap_.find(user_id + ":" + dash_name);
2163  dashboardDescriptorMap_.erase(viewDescIt);
2165  "DELETE FROM mapd_dashboards WHERE name = ? and userid = ?",
2166  std::vector<std::string>{dash_name, user_id});
2167  }
2168  } catch (std::exception& e) {
2169  sqliteConnector_.query("ROLLBACK TRANSACTION");
2170  throw;
2171  }
2172  sqliteConnector_.query("END TRANSACTION");
2173  }
2174 }
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:684
std::string to_string(char const *&&v)
static SysCatalog & instance()
Definition: SysCatalog.h:343
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
static const AccessPrivileges DELETE_DASHBOARD
Definition: DBObject.h:173
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
void revokeDBObjectPrivilegesFromAllBatch(std::vector< DBObject > &objects, Catalog *catalog)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::deleteTableCatalogMetadata ( const TableDescriptor logical_table,
const std::vector< const TableDescriptor * > &  physical_tables 
)
private

Definition at line 4114 of file Catalog.cpp.

References TableDescriptor::tableId, and to_string().

4116  {
4117  cat_write_lock write_lock(this);
4118  cat_sqlite_lock sqlite_lock(getObjForLock());
4119  sqliteConnector_.query("BEGIN TRANSACTION");
4120  try {
4121  // remove corresponding record from the logicalToPhysicalTableMap in sqlite database
4123  "DELETE FROM mapd_logical_to_physical WHERE logical_table_id = ?",
4124  std::to_string(logical_table->tableId));
4125  logicalToPhysicalTableMapById_.erase(logical_table->tableId);
4126  for (auto table : physical_tables) {
4127  eraseTableMetadata(table);
4128  }
4129  } catch (std::exception& e) {
4130  sqliteConnector_.query("ROLLBACK TRANSACTION");
4131  throw;
4132  }
4133  sqliteConnector_.query("END TRANSACTION");
4134 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::string to_string(char const *&&v)
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
void eraseTableMetadata(const TableDescriptor *td)
Definition: Catalog.cpp:4136
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:699

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::doTruncateTable ( const TableDescriptor td)
protected

Definition at line 3952 of file Catalog.cpp.

References CHECK, CHECK_GE, ColumnDescriptor::columnType, Data_Namespace::CPU_LEVEL, dict_ref_t::dictId, DictDescriptor::dictIsTemp, DictDescriptor::dictRef, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), Data_Namespace::GPU_LEVEL, kENCODING_DICT, File_Namespace::renameForDelete(), ColumnDescriptor::tableId, and TableDescriptor::tableId.

3952  {
3953  // must destroy fragmenter before deleteChunks is called.
3955 
3956  const int tableId = td->tableId;
3957  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
3958  // assuming deleteChunksWithPrefix is atomic
3959  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
3960  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
3961 
3962  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
3963 
3964  cat_write_lock write_lock(this);
3965  std::unique_ptr<StringDictionaryClient> client;
3966  if (SysCatalog::instance().isAggregator()) {
3967  CHECK(!string_dict_hosts_.empty());
3968  DictRef dict_ref(currentDB_.dbId, -1);
3969  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
3970  }
3971  // clean up any dictionaries
3972  // delete all column descriptors for the table
3973  for (const auto& columnDescriptor : columnDescriptorMapById_) {
3974  auto cd = columnDescriptor.second;
3975  if (cd->tableId != td->tableId) {
3976  continue;
3977  }
3978  const int dict_id = cd->columnType.get_comp_param();
3979  // Dummy dictionaries created for a shard of a logical table have the id set to zero.
3980  if (cd->columnType.get_compression() == kENCODING_DICT && dict_id) {
3981  const DictRef dict_ref(currentDB_.dbId, dict_id);
3982  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
3983  CHECK(dictIt != dictDescriptorMapByRef_.end());
3984  const auto& dd = dictIt->second;
3985  CHECK_GE(dd->refcount, 1);
3986  // if this is the only table using this dict reset the dict
3987  if (dd->refcount == 1) {
3988  // close the dictionary
3989  dd->stringDict.reset();
3990  File_Namespace::renameForDelete(dd->dictFolderPath);
3991  if (client) {
3992  client->drop(dd->dictRef);
3993  }
3994  if (!dd->dictIsTemp) {
3995  boost::filesystem::create_directory(dd->dictFolderPath);
3996  }
3997  }
3998 
3999  DictDescriptor* new_dd = new DictDescriptor(dd->dictRef,
4000  dd->dictName,
4001  dd->dictNBits,
4002  dd->dictIsShared,
4003  dd->refcount,
4004  dd->dictFolderPath,
4005  dd->dictIsTemp);
4006  dictDescriptorMapByRef_.erase(dictIt);
4007  // now create new Dict -- need to figure out what to do here for temp tables
4008  if (client) {
4009  client->create(new_dd->dictRef, new_dd->dictIsTemp);
4010  }
4011  dictDescriptorMapByRef_[new_dd->dictRef].reset(new_dd);
4013  }
4014  }
4015 }
std::vector< int > ChunkKey
Definition: types.h:36
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:696
const DBMetadata currentDB_
Definition: Catalog.h:693
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:683
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:694
#define CHECK_GE(x, y)
Definition: Logger.h:306
heavyai::unique_lock< heavyai::shared_mutex > write_lock
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:682
static SysCatalog & instance()
Definition: SysCatalog.h:343
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1999
int32_t dictId
Definition: DictRef.h:14
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
void removeFragmenterForTable(const int table_id) const
Definition: Catalog.cpp:4064
#define CHECK(condition)
Definition: Logger.h:291
Descriptor for a dictionary for a string columne.
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:242

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropColumn ( const TableDescriptor td,
const ColumnDescriptor cd 
)

Definition at line 2481 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnId, ColumnDescriptor::columnName, TableDescriptor::nShards, TableDescriptor::shard, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), and to_upper().

2481  {
2482  {
2483  cat_write_lock write_lock(this);
2484  cat_sqlite_lock sqlite_lock(getObjForLock());
2485  // caller must handle sqlite/chunk transaction TOGETHER
2487  "DELETE FROM mapd_columns where tableid = ? and columnid = ?",
2488  std::vector<std::string>{std::to_string(td.tableId),
2489  std::to_string(cd.columnId)});
2490 
2492  "UPDATE mapd_tables SET ncolumns = ncolumns - 1 WHERE tableid = ?",
2493  std::vector<std::string>{std::to_string(td.tableId)});
2494 
2495  ColumnDescriptorMap::iterator columnDescIt =
2497  CHECK(columnDescIt != columnDescriptorMap_.end());
2498 
2499  columnDescriptorsForRoll.emplace_back(columnDescIt->second, nullptr);
2500  removeFromColumnMap(columnDescIt->second);
2501  --tableDescriptorMapById_[td.tableId]->nColumns;
2502  }
2503 
2504  // for each shard
2505  if (td.nShards > 0 && td.shard < 0) {
2506  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
2507  const auto shard_cd = getMetadataForColumn(shard->tableId, cd.columnId);
2508  CHECK(shard_cd);
2509  dropColumn(*shard, *shard_cd);
2510  }
2511  }
2512 }
void removeFromColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6569
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:681
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
heavyai::unique_lock< heavyai::shared_mutex > write_lock
void dropColumn(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:2481
std::string to_string(char const *&&v)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:680
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4700
std::string to_upper(const std::string &str)
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:711
std::string columnName

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropForeignServer ( const std::string &  server_name)

Drops/deletes a foreign server DB object.

Parameters
server_name- Name of foreign server that will be deleted

Definition at line 3258 of file Catalog.cpp.

References CHECK_EQ, and to_string().

3258  {
3259  cat_write_lock write_lock(this);
3260  cat_sqlite_lock sqlite_lock(getObjForLock());
3261 
3263  "SELECT id from omnisci_foreign_servers where name = ?",
3264  std::vector<std::string>{server_name});
3265  auto num_rows = sqliteConnector_.getNumRows();
3266  if (num_rows > 0) {
3267  CHECK_EQ(size_t(1), num_rows);
3268  auto server_id = sqliteConnector_.getData<int32_t>(0, 0);
3270  "SELECT table_id from omnisci_foreign_tables where server_id = ?",
3271  std::to_string(server_id));
3272  if (sqliteConnector_.getNumRows() > 0) {
3273  throw std::runtime_error{"Foreign server \"" + server_name +
3274  "\" is referenced "
3275  "by existing foreign tables and cannot be dropped."};
3276  }
3277  sqliteConnector_.query("BEGIN TRANSACTION");
3278  try {
3280  "DELETE FROM omnisci_foreign_servers WHERE name = ?",
3281  std::vector<std::string>{server_name});
3282  } catch (const std::exception& e) {
3283  sqliteConnector_.query("ROLLBACK TRANSACTION");
3284  throw;
3285  }
3286  sqliteConnector_.query("END TRANSACTION");
3287  foreignServerMap_.erase(server_name);
3288  foreignServerMapById_.erase(server_id);
3289  }
3290 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:692
virtual void query(const std::string &queryString)
heavyai::unique_lock< heavyai::shared_mutex > write_lock
std::string to_string(char const *&&v)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:688
ForeignServerMap foreignServerMap_
Definition: Catalog.h:687
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropTable ( const TableDescriptor td)

Definition at line 4086 of file Catalog.cpp.

References CHECK, TableDescriptor::isView, TableDBObjectType, TableDescriptor::tableId, TableDescriptor::tableName, and ViewDBObjectType.

Referenced by Catalog_Namespace::anonymous_namespace{Catalog.cpp}::drop_tables().

4086  {
4089  std::vector<const TableDescriptor*> tables_to_drop;
4090  {
4091  cat_read_lock read_lock(this);
4092  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
4093  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
4094  // remove all corresponding physical tables if this is a logical table
4095  const auto physicalTables = physicalTableIt->second;
4096  CHECK(!physicalTables.empty());
4097  for (size_t i = 0; i < physicalTables.size(); i++) {
4098  int32_t physical_tb_id = physicalTables[i];
4099  const TableDescriptor* phys_td =
4100  getMutableMetadataForTableUnlocked(physical_tb_id);
4101  CHECK(phys_td);
4102  tables_to_drop.emplace_back(phys_td);
4103  }
4104  }
4105  tables_to_drop.emplace_back(td);
4106  }
4107 
4108  for (auto table : tables_to_drop) {
4109  eraseTablePhysicalData(table);
4110  }
4111  deleteTableCatalogMetadata(td, tables_to_drop);
4112 }
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:4898
std::string tableName
heavyai::shared_lock< heavyai::shared_mutex > read_lock
void revokeDBObjectPrivilegesFromAll(DBObject object, Catalog *catalog)
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
void deleteTableCatalogMetadata(const TableDescriptor *logical_table, const std::vector< const TableDescriptor * > &physical_tables)
Definition: Catalog.cpp:4114
TableDescriptor * getMutableMetadataForTableUnlocked(int table_id) const
Definition: Catalog.cpp:1991
static SysCatalog & instance()
Definition: SysCatalog.h:343
#define CHECK(condition)
Definition: Logger.h:291
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:699

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::dropTableFromJsonUnlocked ( const std::string &  table_name) const
protected

Definition at line 3059 of file Catalog.cpp.

References CHECK, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and VLOG.

3059  {
3060  // relies on the catalog write lock
3061  using namespace rapidjson;
3062 
3063  VLOG(1) << "Dropping temporary table " << table_name << " to JSON for Calcite.";
3064 
3065  const auto db_name = currentDB_.dbName;
3066  const auto file_path = table_json_filepath(basePath_, db_name);
3067 
3068  CHECK(boost::filesystem::exists(file_path));
3069  Document d;
3070 
3071  std::ifstream reader(file_path.string());
3072  CHECK(reader.is_open());
3073  IStreamWrapper json_read_wrapper(reader);
3074  d.ParseStream(json_read_wrapper);
3075 
3076  CHECK(d.IsObject());
3077  auto table_name_ref = StringRef(table_name.c_str());
3078  CHECK(d.HasMember(table_name_ref));
3079  CHECK(d.RemoveMember(table_name_ref));
3080 
3081  // Overwrite the existing file
3082  std::ofstream writer(file_path.string(), std::ios::trunc | std::ios::out);
3083  CHECK(writer.is_open());
3084  OStreamWrapper json_wrapper(writer);
3085 
3086  Writer<OStreamWrapper> json_writer(json_wrapper);
3087  d.Accept(json_writer);
3088  writer.close();
3089 }
const DBMetadata currentDB_
Definition: Catalog.h:693
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:162
#define CHECK(condition)
Definition: Logger.h:291
#define VLOG(n)
Definition: Logger.h:387

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::dumpCreateServer ( const std::string &  name,
bool  multiline_formatting = true 
) const

Definition at line 5409 of file Catalog.cpp.

References join().

5410  {
5411  cat_read_lock read_lock(this);
5412  auto server_it = foreignServerMap_.find(name);
5413  if (server_it == foreignServerMap_.end()) {
5414  throw std::runtime_error("Foreign server " + name + " does not exist.");
5415  }
5416  auto server = server_it->second.get();
5417  std::ostringstream os;
5418  os << "CREATE SERVER " << name << " FOREIGN DATA WRAPPER " << server->data_wrapper_type;
5419  std::vector<std::string> with_options;
5420  for (const auto& [option, value] : server->options) {
5421  with_options.emplace_back(option + "='" + value + "'");
5422  }
5423  if (!with_options.empty()) {
5424  if (!multiline_formatting) {
5425  os << " ";
5426  } else {
5427  os << "\n";
5428  }
5429  os << "WITH (" + boost::algorithm::join(with_options, ", ") + ")";
5430  }
5431  os << ";";
5432  return os.str();
5433 }
heavyai::shared_lock< heavyai::shared_mutex > read_lock
std::string join(T const &container, std::string const &delim)
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
std::string name() const
Definition: Catalog.h:321
ForeignServerMap foreignServerMap_
Definition: Catalog.h:687

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::dumpCreateTable ( const TableDescriptor td,
bool  multiline_formatting = true,
bool  dump_defaults = false 
) const

Definition at line 5230 of file Catalog.cpp.

5232  {
5233  cat_read_lock read_lock(this);
5234  return dumpCreateTableUnlocked(td, multiline_formatting, dump_defaults);
5235 }
heavyai::shared_lock< heavyai::shared_mutex > read_lock
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
std::string dumpCreateTableUnlocked(const TableDescriptor *td, bool multiline_formatting, bool dump_defaults) const
Definition: Catalog.cpp:5248
std::optional< std::string > Catalog_Namespace::Catalog::dumpCreateTable ( int32_t  table_id,
bool  multiline_formatting = true,
bool  dump_defaults = false 
) const

Definition at line 5237 of file Catalog.cpp.

5239  {
5240  cat_read_lock read_lock(this);
5241  const auto td = getMutableMetadataForTableUnlocked(table_id);
5242  if (!td) {
5243  return {};
5244  }
5245  return dumpCreateTableUnlocked(td, multiline_formatting, dump_defaults);
5246 }
heavyai::shared_lock< heavyai::shared_mutex > read_lock
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
std::string dumpCreateTableUnlocked(const TableDescriptor *td, bool multiline_formatting, bool dump_defaults) const
Definition: Catalog.cpp:5248
TableDescriptor * getMutableMetadataForTableUnlocked(int table_id) const
Definition: Catalog.cpp:1991
std::string Catalog_Namespace::Catalog::dumpCreateTableUnlocked ( const TableDescriptor td,
bool  multiline_formatting,
bool  dump_defaults 
) const
private

Definition at line 5248 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, Data_Namespace::CPU_LEVEL, DEFAULT_FRAGMENT_ROWS, DEFAULT_MAX_CHUNK_SIZE, DEFAULT_MAX_ROLLBACK_EPOCHS, DEFAULT_MAX_ROWS, DEFAULT_PAGE_SIZE, ColumnDescriptor::default_value, TableDescriptor::fragPageSize, g_leaf_count, ColumnDescriptor::getDefaultValueLiteral(), TableDescriptor::hasDeletedCol, TableDescriptor::is_system_table, ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, join(), kCHAR, kENCODING_DICT, kENCODING_GEOINT, kTEXT, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and TableDescriptor::viewSQL.

5250  {
5251  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
5252  std::ostringstream os;
5253 
5254  if (foreign_table && !td->is_system_table) {
5255  os << "CREATE FOREIGN TABLE " << td->tableName << " (";
5256  } else if (!td->isView) {
5257  os << "CREATE ";
5259  os << "TEMPORARY ";
5260  }
5261  os << "TABLE " + td->tableName + " (";
5262  } else {
5263  os << "CREATE VIEW " + td->tableName + " AS " << td->viewSQL;
5264  return os.str();
5265  }
5266  // scan column defines
5267  std::vector<std::string> additional_info;
5268  std::set<std::string> shared_dict_column_names;
5269 
5270  gatherAdditionalInfo(additional_info, shared_dict_column_names, td);
5271 
5272  // gather column defines
5273  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
5274  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
5275  bool first = true;
5276  for (const auto cd : cds) {
5277  if (!(cd->isSystemCol || cd->isVirtualCol)) {
5278  const auto& ti = cd->columnType;
5279  if (!first) {
5280  os << ",";
5281  if (!multiline_formatting) {
5282  os << " ";
5283  }
5284  } else {
5285  first = false;
5286  }
5287  if (multiline_formatting) {
5288  os << "\n ";
5289  }
5290  // column name
5291  os << quoteIfRequired(cd->columnName);
5292  // CHAR is perculiar... better dump it as TEXT(32) like \d does
5293  if (ti.get_type() == SQLTypes::kCHAR) {
5294  os << " "
5295  << "TEXT";
5296  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
5297  os << " "
5298  << "TEXT[]";
5299  } else {
5300  os << " " << ti.get_type_name();
5301  }
5302  os << (ti.get_notnull() ? " NOT NULL" : "");
5303  if (cd->default_value.has_value()) {
5304  os << " DEFAULT " << cd->getDefaultValueLiteral();
5305  }
5306  if (shared_dict_column_names.find(cd->columnName) ==
5307  shared_dict_column_names.end()) {
5308  // avoids "Column ... shouldn't specify an encoding, it borrows it
5309  // from the referenced column"
5310  if (ti.is_string() || (ti.is_array() && ti.get_subtype() == kTEXT)) {
5311  auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
5312  if (ti.get_compression() == kENCODING_DICT) {
5313  os << " ENCODING " << ti.get_compression_name() << "(" << (size * 8) << ")";
5314  } else {
5315  os << " ENCODING NONE";
5316  }
5317  } else if (ti.is_date_in_days() ||
5318  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
5319  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
5320  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
5321  } else if (ti.is_geometry()) {
5322  if (ti.get_compression() == kENCODING_GEOINT) {
5323  os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
5324  << ")";
5325  } else {
5326  os << " ENCODING NONE";
5327  }
5328  }
5329  }
5330  }
5331  }
5332  // gather SHARED DICTIONARYs
5333  if (additional_info.size()) {
5334  std::string comma;
5335  if (!multiline_formatting) {
5336  comma = ", ";
5337  } else {
5338  comma = ",\n ";
5339  }
5340  os << comma;
5341  os << boost::algorithm::join(additional_info, comma);
5342  }
5343  os << ")";
5344 
5345  std::vector<std::string> with_options;
5346  if (foreign_table && !td->is_system_table) {
5347  if (multiline_formatting) {
5348  os << "\n";
5349  } else {
5350  os << " ";
5351  }
5352  os << "SERVER " << foreign_table->foreign_server->name;
5353 
5354  // gather WITH options ...
5355  for (const auto& [option, value] : foreign_table->options) {
5356  with_options.emplace_back(option + "='" + value + "'");
5357  }
5358  }
5359 
5360  if (dump_defaults || td->maxFragRows != DEFAULT_FRAGMENT_ROWS) {
5361  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
5362  }
5363  if (dump_defaults || td->maxChunkSize != DEFAULT_MAX_CHUNK_SIZE) {
5364  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
5365  }
5366  if (!foreign_table && (dump_defaults || td->fragPageSize != DEFAULT_PAGE_SIZE)) {
5367  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
5368  }
5369  if (!foreign_table && (dump_defaults || td->maxRows != DEFAULT_MAX_ROWS)) {
5370  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
5371  }
5372  if ((dump_defaults || td->maxRollbackEpochs != DEFAULT_MAX_ROLLBACK_EPOCHS) &&
5373  td->maxRollbackEpochs != -1) {
5374  with_options.push_back("MAX_ROLLBACK_EPOCHS=" +
5376  }
5377  if (!foreign_table && (dump_defaults || !td->hasDeletedCol)) {
5378  with_options.emplace_back(td->hasDeletedCol ? "VACUUM='DELAYED'"
5379  : "VACUUM='IMMEDIATE'");
5380  }
5381  if (!foreign_table && !td->partitions.empty()) {
5382  with_options.push_back("PARTITIONS='" + td->partitions + "'");
5383  }
5384  if (!foreign_table && td->nShards > 0) {
5385  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
5386  CHECK(shard_cd);
5387  with_options.push_back(
5388  "SHARD_COUNT=" +
5389  std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
5390  }
5391  if (!foreign_table && td->sortedColumnId > 0) {
5392  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
5393  CHECK(sort_cd);
5394  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
5395  }
5396 
5397  if (!with_options.empty()) {
5398  if (!multiline_formatting) {
5399  os << " ";
5400  } else {
5401  os << "\n";
5402  }
5403  os << "WITH (" + boost::algorithm::join(with_options, ", ") + ")";
5404  }
5405  os << ";";
5406  return os.str();
5407 }
int32_t maxRollbackEpochs
std::string partitions
std::string tableName
std::string join(T const &container, std::string const &delim)
#define DEFAULT_MAX_CHUNK_SIZE
std::string to_string(char const *&&v)
std::string quoteIfRequired(const std::string &column_name) const
Definition: Catalog.cpp:5530
#define DEFAULT_MAX_ROWS
void gatherAdditionalInfo(std::vector< std::string > &additional_info, std::set< std::string > &shared_dict_column_names, const TableDescriptor *td) const
Definition: Catalog.cpp:5541
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
#define DEFAULT_MAX_ROLLBACK_EPOCHS
specifies the content in-memory of a row in the table metadata table
#define DEFAULT_PAGE_SIZE
Definition: sqltypes.h:69
#define DEFAULT_FRAGMENT_ROWS
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:2267
Data_Namespace::MemoryLevel persistenceLevel
Definition: sqltypes.h:58
#define CHECK(condition)
Definition: Logger.h:291
std::string viewSQL
size_t g_leaf_count
Definition: ParserNode.cpp:77

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::dumpSchema ( const TableDescriptor td) const

Definition at line 5103 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, DEFAULT_MAX_ROLLBACK_EPOCHS, ColumnDescriptor::default_value, TableDescriptor::fragPageSize, g_leaf_count, ColumnDescriptor::getDefaultValueLiteral(), TableDescriptor::hasDeletedCol, TableDescriptor::is_system_table, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, join(), kCHAR, kENCODING_DICT, kENCODING_GEOINT, kTEXT, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::tableId, and to_string().

Referenced by TableArchiver::dumpTable().

5103  {
5104  CHECK(!td->is_system_table);
5105  cat_read_lock read_lock(this);
5106 
5107  std::ostringstream os;
5108  os << "CREATE TABLE @T (";
5109  // gather column defines
5110  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
5111  std::string comma;
5112  std::vector<std::string> shared_dicts;
5113  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
5114  for (const auto cd : cds) {
5115  if (!(cd->isSystemCol || cd->isVirtualCol)) {
5116  const auto& ti = cd->columnType;
5117  os << comma << cd->columnName;
5118  // CHAR is perculiar... better dump it as TEXT(32) like \d does
5119  if (ti.get_type() == SQLTypes::kCHAR) {
5120  os << " "
5121  << "TEXT";
5122  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
5123  os << " "
5124  << "TEXT[]";
5125  } else {
5126  os << " " << ti.get_type_name();
5127  }
5128  os << (ti.get_notnull() ? " NOT NULL" : "");
5129  if (cd->default_value.has_value()) {
5130  os << " DEFAULT " << cd->getDefaultValueLiteral();
5131  }
5132  if (ti.is_string() || (ti.is_array() && ti.get_subtype() == kTEXT)) {
5133  auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
5134  if (ti.get_compression() == kENCODING_DICT) {
5135  // if foreign reference, get referenced tab.col
5136  const auto dict_id = ti.get_comp_param();
5137  const DictRef dict_ref(currentDB_.dbId, dict_id);
5138  const auto dict_it = dictDescriptorMapByRef_.find(dict_ref);
5139  CHECK(dict_it != dictDescriptorMapByRef_.end());
5140  const auto dict_name = dict_it->second->dictName;
5141  // when migrating a table, any foreign dict ref will be dropped
5142  // and the first cd of a dict will become root of the dict
5143  if (dict_root_cds.end() == dict_root_cds.find(dict_name)) {
5144  dict_root_cds[dict_name] = cd;
5145  os << " ENCODING " << ti.get_compression_name() << "(" << (size * 8) << ")";
5146  } else {
5147  const auto dict_root_cd = dict_root_cds[dict_name];
5148  shared_dicts.push_back("SHARED DICTIONARY (" + cd->columnName +
5149  ") REFERENCES @T(" + dict_root_cd->columnName + ")");
5150  // "... shouldn't specify an encoding, it borrows from the referenced
5151  // column"
5152  }
5153  } else {
5154  os << " ENCODING NONE";
5155  }
5156  } else if (ti.is_date_in_days() ||
5157  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
5158  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
5159  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
5160  } else if (ti.is_geometry()) {
5161  if (ti.get_compression() == kENCODING_GEOINT) {
5162  os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
5163  << ")";
5164  } else {
5165  os << " ENCODING NONE";
5166  }
5167  }
5168  comma = ", ";
5169  }
5170  }
5171  // gather SHARED DICTIONARYs
5172  if (shared_dicts.size()) {
5173  os << ", " << boost::algorithm::join(shared_dicts, ", ");
5174  }
5175  // gather WITH options ...
5176  std::vector<std::string> with_options;
5177  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
5178  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
5179  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
5180  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
5181  with_options.emplace_back(td->hasDeletedCol ? "VACUUM='DELAYED'"
5182  : "VACUUM='IMMEDIATE'");
5183  if (!td->partitions.empty()) {
5184  with_options.push_back("PARTITIONS='" + td->partitions + "'");
5185  }
5186  if (td->nShards > 0) {
5187  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
5188  CHECK(shard_cd);
5189  os << ", SHARD KEY(" << shard_cd->columnName << ")";
5190  with_options.push_back(
5191  "SHARD_COUNT=" +
5192  std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
5193  }
5194  if (td->sortedColumnId > 0) {
5195  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
5196  CHECK(sort_cd);
5197  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
5198  }
5200  td->maxRollbackEpochs != -1) {
5201  with_options.push_back("MAX_ROLLBACK_EPOCHS=" +
5203  }
5204  os << ") WITH (" + boost::algorithm::join(with_options, ", ") + ");";
5205  return os.str();
5206 }
int32_t maxRollbackEpochs
std::string partitions
heavyai::shared_lock< heavyai::shared_mutex > read_lock
const DBMetadata currentDB_
Definition: Catalog.h:693
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:683
std::string join(T const &container, std::string const &delim)
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
std::string to_string(char const *&&v)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
#define DEFAULT_MAX_ROLLBACK_EPOCHS
specifies the content in-memory of a row in the table metadata table
Definition: sqltypes.h:69
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:2267
Definition: sqltypes.h:58
#define CHECK(condition)
Definition: Logger.h:291
size_t g_leaf_count
Definition: ParserNode.cpp:77

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::eraseDbMetadata ( )

Definition at line 4880 of file Catalog.cpp.

References shared::kCatalogDirectoryName, and run_benchmark_import::tables.

4880  {
4881  const auto tables = getAllTableMetadata();
4882  for (const auto table : tables) {
4883  eraseTableMetadata(table);
4884  }
4885  // Physically erase database metadata
4886  boost::filesystem::remove(basePath_ + "/" + shared::kCatalogDirectoryName + "/" +
4887  currentDB_.dbName);
4888  calciteMgr_->updateMetadata(currentDB_.dbName, "");
4889 }
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:2283
const DBMetadata currentDB_
Definition: Catalog.h:693
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:697
const std::string kCatalogDirectoryName
void eraseTableMetadata(const TableDescriptor *td)
Definition: Catalog.cpp:4136
void Catalog_Namespace::Catalog::eraseDbPhysicalData ( )

Definition at line 4891 of file Catalog.cpp.

References run_benchmark_import::tables.

4891  {
4892  const auto tables = getAllTableMetadata();
4893  for (const auto table : tables) {
4894  eraseTablePhysicalData(table);
4895  }
4896 }
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:4898
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:2283
void Catalog_Namespace::Catalog::eraseTableMetadata ( const TableDescriptor td)
protected

Definition at line 4136 of file Catalog.cpp.

References g_serialize_temp_tables, INJECT_TIMER, table_is_temporary(), TableDescriptor::tableId, and TableDescriptor::tableName.

4136  {
4140  }
4141  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
4142  {
4143  INJECT_TIMER(removeTableFromMap_);
4145  }
4146 }
void executeDropTableSqliteQueries(const TableDescriptor *td)
Definition: Catalog.cpp:4148
std::string tableName
const DBMetadata currentDB_
Definition: Catalog.h:693
void dropTableFromJsonUnlocked(const std::string &table_name) const
Definition: Catalog.cpp:3059
#define INJECT_TIMER(DESC)
Definition: measure.h:93
bool g_serialize_temp_tables
Definition: Catalog.cpp:106
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:697
bool table_is_temporary(const TableDescriptor *const td)
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1696

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::eraseTablePhysicalData ( const TableDescriptor td)

Definition at line 4898 of file Catalog.cpp.

References Data_Namespace::CPU_LEVEL, Data_Namespace::GPU_LEVEL, INJECT_TIMER, TableDescriptor::isView, and TableDescriptor::tableId.

4898  {
4899  const int tableId = td->tableId;
4900  // must destroy fragmenter before deleteChunks is called.
4901  removeFragmenterForTable(tableId);
4902 
4903  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
4904  {
4905  INJECT_TIMER(deleteChunksWithPrefix);
4906  // assuming deleteChunksWithPrefix is atomic
4907  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
4908  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
4909  }
4910  if (!td->isView) {
4911  INJECT_TIMER(Remove_Table);
4912  dataMgr_->removeTableRelatedDS(