OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Catalog_Namespace::Catalog Class Referencefinal

class for a per-database catalog. also includes metadata for the current database and the current user. More...

#include <Catalog.h>

+ Collaboration diagram for Catalog_Namespace::Catalog:

Classes

struct  NoTableFoundException
 

Public Member Functions

 Catalog (const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
 Constructor - takes basePath to already extant data directory for writing. More...
 
 Catalog ()
 Constructor builds a hollow catalog used during constructor of other catalogs. More...
 
 ~Catalog ()
 Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite. More...
 
void createTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
 
void createShardedTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
int32_t createDashboard (DashboardDescriptor &vd)
 
void replaceDashboard (DashboardDescriptor &vd)
 
std::string createLink (LinkDescriptor &ld, size_t min_length)
 
void dropTable (const TableDescriptor *td)
 
void truncateTable (const TableDescriptor *td)
 
void renameTable (const TableDescriptor *td, const std::string &newTableName)
 
void renameTables (const std::vector< std::pair< std::string, std::string >> &names)
 
void renameColumn (const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
 
int getNextAddedColumnId (const TableDescriptor &td)
 
void addColumn (const TableDescriptor &td, ColumnDescriptor &cd)
 
void addColumnTransactional (const TableDescriptor &td, ColumnDescriptor &cd)
 
void alterColumnTypeTransactional (const ColumnDescriptor &cd)
 
void dropColumnTransactional (const TableDescriptor &td, const ColumnDescriptor &cd)
 
void dropColumnNontransactional (const TableDescriptor &td, const ColumnDescriptor &cd)
 
void dropColumn (const TableDescriptor &td, const ColumnDescriptor &cd)
 
void dropColumnPolicies (const TableDescriptor &td, const ColumnDescriptor &cd)
 
void invalidateCachesForTable (const int table_id)
 
void removeFragmenterForTable (const int table_id) const
 
const std::map< int, const
ColumnDescriptor * > 
getDictionaryToColumnMapping ()
 
const TableDescriptorgetMetadataForTable (const std::string &tableName, const bool populateFragmenter=true) const
 Returns a pointer to a const TableDescriptor struct matching the provided tableName. More...
 
const TableDescriptorgetMetadataForTable (int tableId, bool populateFragmenter=true) const
 
std::optional< std::string > getTableName (int32_t table_id) const
 
std::optional< int32_t > getTableId (const std::string &table_name) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, const std::string &colName) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, int columnId) const
 
const std::optional< std::string > getColumnName (int table_id, int column_id) const
 
const int getColumnIdBySpi (const int tableId, const size_t spi) const
 
const ColumnDescriptorgetMetadataForColumnBySpi (const int tableId, const size_t spi) const
 
const DashboardDescriptorgetMetadataForDashboard (const std::string &userId, const std::string &dashName) const
 
const DashboardDescriptorgetMetadataForDashboard (const int32_t dashboard_id) const
 
void deleteMetadataForDashboards (const std::vector< int32_t > ids, const UserMetadata &user)
 
const LinkDescriptorgetMetadataForLink (const std::string &link) const
 
const LinkDescriptorgetMetadataForLink (int linkId) const
 
const
foreign_storage::ForeignTable
getForeignTable (const std::string &tableName) const
 
const
foreign_storage::ForeignTable
getForeignTable (int table_id) const
 
std::list< const
ColumnDescriptor * > 
getAllColumnMetadataForTable (const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular table specified by table id. More...
 
std::list< const
TableDescriptor * > 
getAllTableMetadata () const
 
std::vector< TableDescriptorgetAllTableMetadataCopy () const
 
std::list< const
DashboardDescriptor * > 
getAllDashboardsMetadata () const
 
std::vector< DashboardDescriptorgetAllDashboardsMetadataCopy () const
 
const DBMetadatagetCurrentDB () const
 
Data_Namespace::DataMgrgetDataMgr () const
 
std::shared_ptr< CalcitegetCalciteMgr () const
 
void setCalciteMgr (const std::shared_ptr< Calcite > &new_calcite_mgr)
 
const std::string & getCatalogBasePath () const
 
const DictDescriptorgetMetadataForDict (int dict_ref, bool loadDict=true) const
 
const std::vector< LeafHostInfo > & getStringDictionaryHosts () const
 
const ColumnDescriptorgetShardColumnMetadataForTable (const TableDescriptor *td) const
 
std::vector< const
TableDescriptor * > 
getPhysicalTablesDescriptors (const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
 
std::vector< std::pair
< int32_t, int32_t > > 
getAllPersistedTableAndShardIds () const
 
std::vector< std::string > getTableNamesForUser (const UserMetadata &user, const GetTablesType get_tables_type) const
 
std::vector< TableMetadatagetTablesMetadataForUser (const UserMetadata &user_metadata, const GetTablesType get_tables_type, const std::string &filter_table_name) const
 
int32_t getTableEpoch (const int32_t db_id, const int32_t table_id) const
 
void setTableEpoch (const int db_id, const int table_id, const int new_epoch)
 
void setMaxRollbackEpochs (const int32_t table_id, const int32_t max_rollback_epochs)
 
void setMaxRows (const int32_t table_id, const int64_t max_rows)
 
std::vector< TableEpochInfogetTableEpochs (const int32_t db_id, const int32_t table_id) const
 
void setTableEpochs (const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
 
void setTableEpochsLogExceptions (const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
 
int getDatabaseId () const
 
SqliteConnectorgetSqliteConnector ()
 
void rollLegacy (const bool forward)
 
void getDictionary (const ColumnDescriptor &cd, std::map< int, StringDictionary * > &stringDicts)
 
size_t getTotalMemorySizeForDictionariesForDatabase () const
 
DictRef addDictionaryTransactional (ColumnDescriptor &cd)
 
void delDictionaryTransactional (const ColumnDescriptor &cd)
 
std::list< const DictDescriptor * > getAllDictionariesWithColumnInName (const ColumnDescriptor *cd)
 
const bool checkMetadataForDeletedRecs (const TableDescriptor *td, int column_id) const
 
const ColumnDescriptorgetDeletedColumn (const TableDescriptor *td) const
 
const ColumnDescriptorgetDeletedColumnIfRowsDeleted (const TableDescriptor *td) const
 
void setDeletedColumn (const TableDescriptor *td, const ColumnDescriptor *cd)
 
int getLogicalTableId (const int physicalTableId) const
 
void checkpoint (const int logicalTableId) const
 
void checkpointWithAutoRollback (const int logical_table_id) const
 
void resetTableEpochFloor (const int logicalTableId) const
 
std::string name () const
 
void eraseDbMetadata ()
 
void eraseDbPhysicalData ()
 
void eraseTablePhysicalData (const TableDescriptor *td)
 
void setForReload (const int32_t tableId)
 
std::vector< std::string > getTableDataDirectories (const TableDescriptor *td) const
 
std::vector< std::string > getTableDictDirectories (const TableDescriptor *td) const
 
std::set< std::string > getTableDictDirectoryPaths (int32_t table_id) const
 
std::string getColumnDictDirectory (const ColumnDescriptor *cd, bool file_name_only=true) const
 
std::string dumpSchema (const TableDescriptor *td) const
 
std::string dumpCreateTable (const TableDescriptor *td, bool multiline_formatting=true, bool dump_defaults=false) const
 
std::optional< std::string > dumpCreateTable (int32_t table_id, bool multiline_formatting=true, bool dump_defaults=false) const
 
std::string dumpCreateServer (const std::string &name, bool multiline_formatting=true) const
 
void createForeignServer (std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
 
const
foreign_storage::ForeignServer
getForeignServer (const std::string &server_name) const
 
const std::unique_ptr< const
foreign_storage::ForeignServer
getForeignServerFromStorage (const std::string &server_name)
 
const std::unique_ptr< const
foreign_storage::ForeignTable
getForeignTableFromStorage (int table_id)
 
void changeForeignServerOwner (const std::string &server_name, const int new_owner_id)
 
void setForeignServerDataWrapper (const std::string &server_name, const std::string &data_wrapper)
 
void setForeignServerOptions (const std::string &server_name, const std::string &options)
 
void renameForeignServer (const std::string &server_name, const std::string &name)
 
void dropForeignServer (const std::string &server_name)
 
std::vector< const
foreign_storage::ForeignTable * > 
getAllForeignTablesForForeignServer (const int32_t foreign_server_id)
 Get all of the foreign tables for associated with a foreign server id. More...
 
void getForeignServersForUser (const rapidjson::Value *filters, const UserMetadata &user, std::vector< const foreign_storage::ForeignServer * > &results)
 
void createDefaultServersIfNotExists ()
 
bool validateNonExistentTableOrView (const std::string &name, const bool if_not_exists)
 
std::vector< const
TableDescriptor * > 
getAllForeignTablesForRefresh () const
 
void updateForeignTableRefreshTimes (const int32_t table_id)
 
void setForeignTableOptions (const std::string &table_name, foreign_storage::OptionsMap &options_map, bool clear_existing_options=true)
 
void updateLeaf (const LeafHostInfo &string_dict_host)
 
void setUncappedTableEpoch (const std::string &table_name)
 
int32_t createCustomExpression (std::unique_ptr< CustomExpression > custom_expression)
 
const CustomExpressiongetCustomExpression (int32_t custom_expression_id) const
 
const std::unique_ptr< const
CustomExpression
getCustomExpressionFromStorage (int32_t custom_expression_id)
 
std::vector< const
CustomExpression * > 
getCustomExpressionsForUser (const UserMetadata &user) const
 
void updateCustomExpression (int32_t custom_expression_id, const std::string &expression_json)
 
void deleteCustomExpressions (const std::vector< int32_t > &custom_expression_ids, bool do_soft_delete)
 
void reassignOwners (const std::set< std::string > &old_owners, const std::string &new_owner)
 
bool isInfoSchemaDb () const
 
bool checkDropRenderGroupColumnsMigration ()
 

Static Public Member Functions

static void expandGeoColumn (const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
 
static const std::string getForeignTableSchema (bool if_not_exists=false)
 
static const std::string getForeignServerSchema (bool if_not_exists=false)
 
static const std::string getCustomExpressionsSchema (bool if_not_exists=false)
 

Public Attributes

std::unique_ptr
< heavyai::DistributedSharedMutex
dcatalogMutex_
 
std::unique_ptr
< heavyai::DistributedSharedMutex
dsqliteMutex_
 
std::mutex sqliteMutex_
 
heavyai::shared_mutex sharedMutex_
 
std::atomic< std::thread::id > thread_holding_sqlite_lock
 
std::atomic< std::thread::id > thread_holding_write_lock
 
bool initialized_ = false
 

Static Public Attributes

static thread_local bool thread_holds_read_lock = false
 

Protected Types

using ColumnDescriptorsForRoll = std::vector< std::pair< ColumnDescriptor *, ColumnDescriptor * >>
 

Protected Member Functions

void CheckAndExecuteMigrations ()
 
void CheckAndExecuteMigrationsPostBuildMaps ()
 
void updateDictionaryNames ()
 
void updateTableDescriptorSchema ()
 
void updateFixlenArrayColumns ()
 
void updateGeoColumns ()
 
void updateFrontendViewSchema ()
 
void updateLinkSchema ()
 
void updateFrontendViewAndLinkUsers ()
 
void updateLogicalToPhysicalTableLinkSchema ()
 
void updateLogicalToPhysicalTableMap (const int32_t logical_tb_id)
 
void updateDictionarySchema ()
 
void updatePageSize ()
 
void updateDeletedColumnIndicator ()
 
void updateDefaultColumnValues ()
 
void updateFrontendViewsToDashboards ()
 
void updateCustomExpressionsSchema ()
 
void updateFsiSchemas ()
 
void renameLegacyDataWrappers ()
 
void recordOwnershipOfObjectsInObjectPermissions ()
 
void checkDateInDaysColumnMigration ()
 
void createDashboardSystemRoles ()
 
void buildMaps ()
 
void addTableToMap (const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
 
void addReferenceToForeignDict (ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def, const bool persist_reference)
 
bool setColumnSharedDictionary (ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
void setColumnDictionary (ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, bool is_logical_table, bool use_temp_dictionary=false)
 
void addFrontendViewToMap (DashboardDescriptor &vd)
 
void addFrontendViewToMapNoLock (DashboardDescriptor &vd)
 
void addLinkToMap (LinkDescriptor &ld)
 
void removeTableFromMap (const std::string &tableName, const int tableId, const bool is_on_error=false)
 
void eraseTableMetadata (const TableDescriptor *td)
 
void executeDropTableSqliteQueries (const TableDescriptor *td)
 
void doTruncateTable (const TableDescriptor *td)
 
void renamePhysicalTable (const TableDescriptor *td, const std::string &newTableName)
 
void renamePhysicalTables (std::vector< std::pair< std::string, std::string >> &names, std::vector< int > &tableIds)
 
void instantiateFragmenter (TableDescriptor *td) const
 
void getAllColumnMetadataForTableImpl (const TableDescriptor *td, std::list< const ColumnDescriptor * > &colDescs, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 
std::string calculateSHA1 (const std::string &data)
 
std::string generatePhysicalTableName (const std::string &logicalTableName, const size_t shardNumber)
 
std::vector< DBObjectparseDashboardObjects (const std::string &view_meta, const int &user_id)
 
void createOrUpdateDashboardSystemRole (const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
 
const int getColumnIdBySpiUnlocked (const int table_id, const size_t spi) const
 
void serializeTableJsonUnlocked (const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
 
void dropTableFromJsonUnlocked (const std::string &table_name) const
 

Protected Attributes

std::string basePath_
 
TableDescriptorMap tableDescriptorMap_
 
TableDescriptorMapById tableDescriptorMapById_
 
ColumnDescriptorMap columnDescriptorMap_
 
ColumnDescriptorMapById columnDescriptorMapById_
 
DictDescriptorMapById dictDescriptorMapByRef_
 
DashboardDescriptorMap dashboardDescriptorMap_
 
LinkDescriptorMap linkDescriptorMap_
 
LinkDescriptorMapById linkDescriptorMapById_
 
ForeignServerMap foreignServerMap_
 
ForeignServerMapById foreignServerMapById_
 
CustomExpressionMapById custom_expr_map_by_id_
 
TableDictColumnsMap dict_columns_by_table_id_
 
SqliteConnector sqliteConnector_
 
const DBMetadata currentDB_
 
std::shared_ptr
< Data_Namespace::DataMgr
dataMgr_
 
const std::vector< LeafHostInfostring_dict_hosts_
 
std::shared_ptr< CalcitecalciteMgr_
 
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
 
int nextTempTableId_
 
int nextTempDictId_
 
ColumnDescriptorsForRoll columnDescriptorsForRoll
 

Static Protected Attributes

static const std::string physicalTableNameTag_
 

Private Member Functions

void addColumnDescriptor (const ColumnDescriptor *cd)
 
void removeColumnDescriptor (const ColumnDescriptor *cd)
 
void delDictionaryNontransactional (const ColumnDescriptor &cd)
 
void addColumnNontransactional (const TableDescriptor &td, ColumnDescriptor &cd)
 
DictRef addDictionaryNontransactional (ColumnDescriptor &cd)
 
void buildDictionaryMapUnlocked ()
 
void reloadTableMetadata (int table_id)
 
void reloadTableMetadataUnlocked (int table_id)
 
TableDescriptorcreateTableFromDiskUnlocked (int32_t table_id)
 
void reloadDictionariesFromDiskUnlocked ()
 
std::list< ColumnDescriptor * > sqliteGetColumnsForTableUnlocked (int32_t table_id)
 
void reloadForeignTableUnlocked (foreign_storage::ForeignTable &foreign_table)
 
void reloadCatalogMetadata (const std::map< int32_t, std::string > &user_name_by_user_id)
 
void reloadCatalogMetadataUnlocked (const std::map< int32_t, std::string > &user_name_by_user_id)
 
void refreshDictionaryCachesForTableUnlocked (const TableDescriptor &td)
 
void buildTablesMapUnlocked ()
 
void buildColumnsMapUnlocked ()
 
void updateViewsInMapUnlocked ()
 
void updateViewUnlocked (TableDescriptor &)
 
void buildDashboardsMapUnlocked (const std::map< int32_t, std::string > &user_name_by_user_id)
 
void buildLinksMapUnlocked ()
 
void buildLogicalToPhysicalMapUnlocked ()
 
void updateForeignTablesInMapUnlocked ()
 
void gatherAdditionalInfo (std::vector< std::string > &additional_info, std::set< std::string > &shared_dict_column_names, const TableDescriptor *td) const
 
std::string quoteIfRequired (const std::string &column_name) const
 
void adjustAlteredTableFiles (const std::string &temp_data_dir, const std::unordered_map< int, int > &all_column_ids_map) const
 
void renameTableDirectories (const std::string &temp_data_dir, const std::vector< std::string > &target_paths, const std::string &name_prefix) const
 
void buildForeignServerMapUnlocked ()
 
void setForeignServerProperty (const std::string &server_name, const std::string &property, const std::string &value)
 
void setForeignTableProperty (const foreign_storage::ForeignTable *table, const std::string &property, const std::string &value)
 
void alterPhysicalTableMetadata (const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
 
void alterTableMetadata (const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
 
void setTableFileMgrParams (const int table_id, const File_Namespace::FileMgrParams &file_mgr_params)
 
bool filterTableByTypeAndUser (const TableDescriptor *td, const UserMetadata &user_metadata, const GetTablesType get_tables_type) const
 
TableDescriptorgetMutableMetadataForTableUnlocked (int table_id) const
 
TableDescriptorgetMutableMetadataForTableUnlocked (const std::string &table_name) const
 
void createForeignServerNoLocks (std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
 
foreign_storage::ForeignTablegetForeignTableUnlocked (const std::string &tableName) const
 
const CataloggetObjForLock ()
 
void removeChunks (const int table_id) const
 
void buildCustomExpressionsMapUnlocked ()
 
std::unique_ptr< CustomExpressiongetCustomExpressionFromConnector (size_t row)
 
void restoreOldOwners (const std::map< int32_t, std::string > &old_owners_user_name_by_id, const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects, int32_t new_owner_id)
 
void restoreOldOwnersInMemory (const std::map< int32_t, std::string > &old_owners_user_name_by_id, const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects, int32_t new_owner_id)
 
void conditionallyInitializeSystemObjects ()
 
void initializeSystemServers ()
 
void initializeSystemTables ()
 
void initializeUsersSystemTable ()
 
void initializeDatabasesSystemTable ()
 
void initializePermissionsSystemTable ()
 
void initializeRolesSystemTable ()
 
void initializeTablesSystemTable ()
 
void initializeDashboardsSystemTable ()
 
void initializeRoleAssignmentsSystemTable ()
 
void initializeMemorySummarySystemTable ()
 
void initializeMemoryDetailsSystemTable ()
 
void initializeStorageDetailsSystemTable ()
 
void initializeExecutorResourcePoolSummarySystemTable ()
 
void initializeMLModelMetadataSystemTable ()
 
void initializeServerLogsSystemTables ()
 
void initializeRequestLogsSystemTables ()
 
void initializeWebServerLogsSystemTables ()
 
void initializeWebServerAccessLogsSystemTables ()
 
void createSystemTableServer (const std::string &server_name, const std::string &data_wrapper_type, const foreign_storage::OptionsMap &options={})
 
std::pair
< foreign_storage::ForeignTable,
std::list< ColumnDescriptor > > 
getSystemTableSchema (const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name, bool is_in_memory_system_table)
 
bool recreateSystemTableIfUpdated (foreign_storage::ForeignTable &foreign_table, const std::list< ColumnDescriptor > &columns)
 
void setDeletedColumnUnlocked (const TableDescriptor *td, const ColumnDescriptor *cd)
 
void updateInColumnMap (ColumnDescriptor *cd, ColumnDescriptor *old_cd)
 
void addToColumnMap (ColumnDescriptor *cd)
 
void removeFromColumnMap (ColumnDescriptor *cd)
 
void deleteTableCatalogMetadata (const TableDescriptor *logical_table, const std::vector< const TableDescriptor * > &physical_tables)
 
std::string dumpCreateTableUnlocked (const TableDescriptor *td, bool multiline_formatting, bool dump_defaults) const
 
void removeDiskCachedDataForMutableTable (int32_t table_id) const
 
template<typename F , typename... Args>
void execInTransaction (F &&f, Args &&...args)
 
const TableDescriptorgetCachedTableDescriptor (const std::map< std::string, int > &cached_table_map, const std::string &cur_table_name)
 

Private Attributes

DeletedColumnPerTableMap deletedColumnPerTable_
 

Static Private Attributes

static constexpr const char * CATALOG_SERVER_NAME {"system_catalog_server"}
 
static constexpr const char * MEMORY_STATS_SERVER_NAME {"system_memory_stats_server"}
 
static constexpr const char * STORAGE_STATS_SERVER_NAME {"system_storage_stats_server"}
 
static constexpr const char * EXECUTOR_STATS_SERVER_NAME {"system_executor_stats_server"}
 
static constexpr const char * ML_METADATA_SERVER_NAME {"ml_metadata_server"}
 
static constexpr const char * LOGS_SERVER_NAME {"system_logs_server"}
 
static constexpr std::array
< const char *, 6 > 
INTERNAL_SERVERS
 

Friends

template<class T >
class lockmgr::TableLockMgrImpl
 

Detailed Description

class for a per-database catalog. also includes metadata for the current database and the current user.

Catalog

Definition at line 143 of file Catalog.h.

Member Typedef Documentation

Definition at line 738 of file Catalog.h.

Constructor & Destructor Documentation

Catalog_Namespace::Catalog::Catalog ( const std::string &  basePath,
const DBMetadata curDB,
std::shared_ptr< Data_Namespace::DataMgr dataMgr,
const std::vector< LeafHostInfo > &  string_dict_hosts,
std::shared_ptr< Calcite calcite,
bool  is_new_db 
)

Constructor - takes basePath to already extant data directory for writing.

Parameters
basePathdirectory path for writing catalog
dbNamename of the database
fragmenterFragmenter object metadata - expects for this directory to already exist
Catalog_Namespace::Catalog::Catalog ( )

Constructor builds a hollow catalog used during constructor of other catalogs.

Definition at line 171 of file Catalog.cpp.

171 {}
Catalog_Namespace::Catalog::~Catalog ( )

Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite.

Definition at line 231 of file Catalog.cpp.

References basePath_, columnDescriptorMap_, currentDB_, Catalog_Namespace::DBMetadata::dbName, g_serialize_temp_tables, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and tableDescriptorMap_.

231  {
232  // cat_write_lock write_lock(this);
233 
234  // must clean up heap-allocated TableDescriptor and ColumnDescriptor structs
235  for (TableDescriptorMap::iterator tableDescIt = tableDescriptorMap_.begin();
236  tableDescIt != tableDescriptorMap_.end();
237  ++tableDescIt) {
238  tableDescIt->second->fragmenter = nullptr;
239  delete tableDescIt->second;
240  }
241 
242  // TableDescriptorMapById points to the same descriptors. No need to delete
243 
244  for (ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.begin();
245  columnDescIt != columnDescriptorMap_.end();
246  ++columnDescIt) {
247  delete columnDescIt->second;
248  }
249 
250  // ColumnDescriptorMapById points to the same descriptors. No need to delete
251 
253  boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
254  }
255 }
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:709
const DBMetadata currentDB_
Definition: Catalog.h:721
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:162
bool g_serialize_temp_tables
Definition: Catalog.cpp:106
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:707

+ Here is the call graph for this function:

Member Function Documentation

void Catalog_Namespace::Catalog::addColumn ( const TableDescriptor td,
ColumnDescriptor cd 
)

Definition at line 2528 of file Catalog.cpp.

References ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::db_id, ColumnDescriptor::default_value, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, kENCODING_DICT, TableDescriptor::nShards, TableDescriptor::shard, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), and ColumnDescriptor::virtualExpr.

2528  {
2529  // caller must handle sqlite/chunk transaction TOGETHER
2530  cd.tableId = td.tableId;
2531  cd.db_id = getDatabaseId();
2532  if (td.nShards > 0 && td.shard < 0) {
2533  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
2534  auto shard_cd = cd;
2535  addColumn(*shard, shard_cd);
2536  }
2537  }
2540  }
2541 
2542  using BindType = SqliteConnector::BindType;
2543  std::vector<BindType> types(17, BindType::TEXT);
2544  if (!cd.default_value.has_value()) {
2545  types[16] = BindType::NULL_TYPE;
2546  }
2548  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
2549  "colscale, is_notnull, "
2550  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
2551  "is_deletedcol, default_value) "
2552  "VALUES (?, "
2553  "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
2554  "?, ?, ?, "
2555  "?, "
2556  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2557  std::vector<std::string>{std::to_string(td.tableId),
2558  std::to_string(td.tableId),
2559  cd.columnName,
2568  "",
2571  cd.virtualExpr,
2573  cd.default_value.value_or("NULL")},
2574  types);
2575 
2577  "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
2578  std::vector<std::string>{std::to_string(td.tableId)});
2579 
2581  "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
2582  std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
2583  cd.columnId = sqliteConnector_.getData<int>(0, 0);
2584 
2585  ++tableDescriptorMapById_[td.tableId]->nColumns;
2586  auto ncd = new ColumnDescriptor(cd);
2587  addToColumnMap(ncd);
2588  columnDescriptorsForRoll.emplace_back(nullptr, ncd);
2589 }
std::string virtualExpr
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:392
DictRef addDictionaryNontransactional(ColumnDescriptor &cd)
Definition: Catalog.cpp:2242
HOST DEVICE int get_size() const
Definition: sqltypes.h:403
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
void addColumn(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:2528
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
HOST DEVICE int get_scale() const
Definition: sqltypes.h:396
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:391
std::string to_string(char const *&&v)
int getDatabaseId() const
Definition: Catalog.h:326
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
specifies the content in-memory of a row in the column metadata table
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4869
std::optional< std::string > default_value
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:399
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:393
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:402
SQLTypeInfo columnType
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:398
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:739
std::string columnName
void addToColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6984

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addColumnDescriptor ( const ColumnDescriptor cd)
private

Definition at line 2701 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnId, ColumnDescriptor::isGeoPhyCol, gpu_enabled::sort(), and ColumnDescriptor::tableId.

2701  {
2702  if (!cd || cd->isGeoPhyCol) {
2703  return;
2704  }
2705 
2706  auto tabDescIt = tableDescriptorMapById_.find(cd->tableId);
2707  CHECK(tableDescriptorMapById_.end() != tabDescIt);
2708  auto td = tabDescIt->second;
2709  auto& cd_by_spi = td->columnIdBySpi_;
2710 
2711  if (cd_by_spi.end() == std::find(cd_by_spi.begin(), cd_by_spi.end(), cd->columnId)) {
2712  cd_by_spi.push_back(cd->columnId);
2713  }
2714  std::sort(cd_by_spi.begin(), cd_by_spi.end());
2715 }
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addColumnNontransactional ( const TableDescriptor td,
ColumnDescriptor cd 
)
private

Definition at line 2462 of file Catalog.cpp.

References ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::db_id, ColumnDescriptor::default_value, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, kENCODING_DICT, TableDescriptor::nShards, TableDescriptor::shard, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and ColumnDescriptor::virtualExpr.

2462  {
2463  cat_write_lock write_lock(this);
2464  cat_sqlite_lock sqlite_lock(this);
2465  cd.tableId = td.tableId;
2466  cd.db_id = getDatabaseId();
2467  if (td.nShards > 0 && td.shard < 0) {
2468  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
2469  auto shard_cd = cd;
2470  addColumnNontransactional(*shard, shard_cd);
2471  }
2472  }
2475  }
2476 
2477  using BindType = SqliteConnector::BindType;
2478  std::vector<BindType> types(17, BindType::TEXT);
2479  if (!cd.default_value.has_value()) {
2480  types[16] = BindType::NULL_TYPE;
2481  }
2483  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
2484  "colscale, is_notnull, "
2485  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
2486  "is_deletedcol, default_value) "
2487  "VALUES (?, "
2488  "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
2489  "?, ?, ?, "
2490  "?, "
2491  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2492  std::vector<std::string>{std::to_string(td.tableId),
2493  std::to_string(td.tableId),
2494  cd.columnName,
2503  "",
2506  cd.virtualExpr,
2508  cd.default_value.value_or("NULL")},
2509  types);
2510 
2512  "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
2513  std::vector<std::string>{std::to_string(td.tableId)});
2514 
2516  "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
2517  std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
2518  cd.columnId = sqliteConnector_.getData<int>(0, 0);
2519 
2520  ++tableDescriptorMapById_[td.tableId]->nColumns;
2521  auto ncd = new ColumnDescriptor(cd);
2522  addToColumnMap(ncd);
2523  addColumnDescriptor(ncd);
2524  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
2525 }
std::string virtualExpr
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:392
DictRef addDictionaryNontransactional(ColumnDescriptor &cd)
Definition: Catalog.cpp:2242
HOST DEVICE int get_size() const
Definition: sqltypes.h:403
T getData(const int row, const int col)
std::string tableName
virtual void query_with_text_params(std::string const &query_only)
void addColumnNontransactional(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:2462
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
HOST DEVICE int get_scale() const
Definition: sqltypes.h:396
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:391
std::string to_string(char const *&&v)
int getDatabaseId() const
Definition: Catalog.h:326
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
specifies the content in-memory of a row in the column metadata table
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4869
std::optional< std::string > default_value
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:725
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:399
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:393
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:402
void addColumnDescriptor(const ColumnDescriptor *cd)
Definition: Catalog.cpp:2701
SQLTypeInfo columnType
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:398
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
std::string columnName
void addToColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6984

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addColumnTransactional ( const TableDescriptor td,
ColumnDescriptor cd 
)

Definition at line 2449 of file Catalog.cpp.

2449  {
2450  cat_write_lock write_lock(this);
2451  cat_sqlite_lock sqlite_lock(this);
2452  sqliteConnector_.query("BEGIN TRANSACTION");
2453  try {
2454  addColumnNontransactional(td, cd);
2455  } catch (std::exception& e) {
2456  sqliteConnector_.query("ROLLBACK TRANSACTION");
2457  throw;
2458  }
2459  sqliteConnector_.query("END TRANSACTION");
2460 }
void addColumnNontransactional(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:2462
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
DictRef Catalog_Namespace::Catalog::addDictionaryNontransactional ( ColumnDescriptor cd)
private

Definition at line 2242 of file Catalog.cpp.

References CHECK, DictDescriptor::dictFolderPath, and ColumnDescriptor::tableId.

2242  {
2243  cat_write_lock write_lock(this);
2244  const auto& td = *tableDescriptorMapById_[cd.tableId];
2245  list<DictDescriptor> dds;
2246  setColumnDictionary(cd, dds, td, true);
2247  auto& dd = dds.back();
2248  CHECK(dd.dictRef.dictId);
2249 
2250  std::unique_ptr<StringDictionaryClient> client;
2251  if (!string_dict_hosts_.empty()) {
2252  client.reset(new StringDictionaryClient(
2253  string_dict_hosts_.front(), DictRef(currentDB_.dbId, -1), true));
2254  }
2255  if (client) {
2256  client->create(dd.dictRef, dd.dictIsTemp);
2257  }
2258 
2259  DictDescriptor* new_dd = new DictDescriptor(dd);
2260  dictDescriptorMapByRef_[dd.dictRef].reset(new_dd);
2261  if (!dd.dictIsTemp) {
2262  boost::filesystem::create_directory(new_dd->dictFolderPath);
2263  }
2264  return dd.dictRef;
2265 }
std::string dictFolderPath
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:724
const DBMetadata currentDB_
Definition: Catalog.h:721
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:711
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, bool is_logical_table, bool use_temp_dictionary=false)
Definition: Catalog.cpp:4056
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
struct dict_ref_t DictRef
Definition: DictRef.h:54
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
Descriptor for a dictionary for a string columne.
DictRef Catalog_Namespace::Catalog::addDictionaryTransactional ( ColumnDescriptor cd)

Definition at line 2227 of file Catalog.cpp.

2227  {
2228  cat_write_lock write_lock(this);
2229  cat_sqlite_lock sqlite_lock(this);
2230  DictRef ref{};
2231  sqliteConnector_.query("BEGIN TRANSACTION");
2232  try {
2234  } catch (std::exception& e) {
2235  sqliteConnector_.query("ROLLBACK TRANSACTION");
2236  throw;
2237  }
2238  sqliteConnector_.query("END TRANSACTION");
2239  return ref;
2240 }
DictRef addDictionaryNontransactional(ColumnDescriptor &cd)
Definition: Catalog.cpp:2242
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
void Catalog_Namespace::Catalog::addFrontendViewToMap ( DashboardDescriptor vd)
protected

Definition at line 1683 of file Catalog.cpp.

1683  {
1684  cat_write_lock write_lock(this);
1686 }
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1688
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
void Catalog_Namespace::Catalog::addFrontendViewToMapNoLock ( DashboardDescriptor vd)
protected

Definition at line 1688 of file Catalog.cpp.

References DashboardDescriptor::dashboardName, to_string(), and DashboardDescriptor::userId.

1688  {
1689  cat_write_lock write_lock(this);
1691  std::make_shared<DashboardDescriptor>(vd);
1692 }
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:712
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addLinkToMap ( LinkDescriptor ld)
protected

Definition at line 1770 of file Catalog.cpp.

References LinkDescriptor::link, LinkDescriptor::linkId, and to_string().

1770  {
1771  cat_write_lock write_lock(this);
1772  LinkDescriptor* new_ld = new LinkDescriptor();
1773  *new_ld = ld;
1775  linkDescriptorMapById_[ld.linkId] = new_ld;
1776 }
const DBMetadata currentDB_
Definition: Catalog.h:721
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:714
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:713

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addReferenceToForeignDict ( ColumnDescriptor referencing_column,
Parser::SharedDictionaryDef  shared_dict_def,
const bool  persist_reference 
)
protected

Definition at line 3965 of file Catalog.cpp.

References CHECK, CHECK_GE, ColumnDescriptor::columnType, SQLTypeInfo::get_comp_param(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_foreign_col(), and to_string().

3967  {
3968  cat_write_lock write_lock(this);
3969  const auto foreign_ref_col = get_foreign_col(*this, shared_dict_def);
3970  CHECK(foreign_ref_col);
3971  referencing_column.columnType = foreign_ref_col->columnType;
3972  const int dict_id = referencing_column.columnType.get_comp_param();
3973  const DictRef dict_ref(currentDB_.dbId, dict_id);
3974  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
3975  CHECK(dictIt != dictDescriptorMapByRef_.end());
3976  const auto& dd = dictIt->second;
3977  CHECK_GE(dd->refcount, 1);
3978  ++dd->refcount;
3979  if (persist_reference) {
3980  cat_sqlite_lock sqlite_lock(getObjForLock());
3982  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
3983  {std::to_string(dict_id)});
3984  }
3985 }
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:711
#define CHECK_GE(x, y)
Definition: Logger.h:306
const ColumnDescriptor * get_foreign_col(const Catalog &cat, const Parser::SharedDictionaryDef &shared_dict_def)
Definition: Catalog.cpp:3953
std::string to_string(char const *&&v)
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:402
#define CHECK(condition)
Definition: Logger.h:291
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addTableToMap ( const TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::list< DictDescriptor > &  dicts 
)
protected

Definition at line 1542 of file Catalog.cpp.

References anonymous_namespace{Utm.h}::a, CHECK, TableDescriptor::columnIdBySpi_, DictDescriptor::dictFolderPath, dict_ref_t::dictId, TableDescriptor::hasDeletedCol, TableDescriptor::mutex_, gpu_enabled::sort(), TableDescriptor::tableId, TableDescriptor::tableName, and to_upper().

1544  {
1545  cat_write_lock write_lock(this);
1546  TableDescriptor* new_td;
1547 
1548  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
1549  if (foreign_table) {
1550  auto new_foreign_table = new foreign_storage::ForeignTable();
1551  *new_foreign_table = *foreign_table;
1552  new_td = new_foreign_table;
1553  } else {
1554  new_td = new TableDescriptor();
1555  *new_td = *td;
1556  }
1557 
1558  new_td->mutex_ = std::make_shared<std::mutex>();
1559  tableDescriptorMap_[to_upper(td->tableName)] = new_td;
1560  tableDescriptorMapById_[td->tableId] = new_td;
1561  for (auto cd : columns) {
1562  ColumnDescriptor* new_cd = new ColumnDescriptor();
1563  *new_cd = cd;
1564  addToColumnMap(new_cd);
1565 
1566  // Add deleted column to the map
1567  if (cd.isDeletedCol) {
1568  CHECK(new_td->hasDeletedCol);
1569  setDeletedColumnUnlocked(new_td, new_cd);
1570  }
1571  }
1572 
1573  std::sort(new_td->columnIdBySpi_.begin(),
1574  new_td->columnIdBySpi_.end(),
1575  [](const size_t a, const size_t b) -> bool { return a < b; });
1576  // TODO(sy): Why does addTableToMap() sort columnIdBySpi_ but not insert into it while
1577  // buildColumnsMapUnlocked() does both?
1578 
1579  std::unique_ptr<StringDictionaryClient> client;
1580  DictRef dict_ref(currentDB_.dbId, -1);
1581  if (!string_dict_hosts_.empty()) {
1582  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1583  }
1584  for (auto dd : dicts) {
1585  if (!dd.dictRef.dictId) {
1586  // Dummy entry created for a shard of a logical table, nothing to do.
1587  continue;
1588  }
1589  dict_ref.dictId = dd.dictRef.dictId;
1590  if (client) {
1591  client->create(dict_ref, dd.dictIsTemp);
1592  }
1593  DictDescriptor* new_dd = new DictDescriptor(dd);
1594  dictDescriptorMapByRef_[dict_ref].reset(new_dd);
1595  if (!dd.dictIsTemp) {
1596  boost::filesystem::create_directory(new_dd->dictFolderPath);
1597  }
1598  }
1599 }
std::string dictFolderPath
std::string tableName
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:724
const DBMetadata currentDB_
Definition: Catalog.h:721
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:711
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
std::vector< int > columnIdBySpi_
std::shared_ptr< std::mutex > mutex_
constexpr double a
Definition: Utm.h:32
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3945
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
specifies the content in-memory of a row in the column metadata table
std::string to_upper(const std::string &str)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
Descriptor for a dictionary for a string columne.
void addToColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6984
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:707

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addToColumnMap ( ColumnDescriptor cd)
private

Definition at line 6984 of file Catalog.cpp.

References ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::is_dict_encoded_type(), ColumnDescriptor::tableId, and to_upper().

6984  {
6987  if (cd->columnType.is_dict_encoded_type()) {
6988  dict_columns_by_table_id_[cd->tableId].emplace(cd);
6989  }
6990 }
TableDictColumnsMap dict_columns_by_table_id_
Definition: Catalog.h:718
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:709
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:710
bool is_dict_encoded_type() const
Definition: sqltypes.h:653
std::string to_upper(const std::string &str)
SQLTypeInfo columnType
std::string columnName
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::adjustAlteredTableFiles ( const std::string &  temp_data_dir,
const std::unordered_map< int, int > &  all_column_ids_map 
) const
private
void Catalog_Namespace::Catalog::alterColumnTypeTransactional ( const ColumnDescriptor cd)

Definition at line 2383 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::default_value, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), ColumnDescriptor::tableId, to_string(), and to_upper().

Referenced by AlterTableAlterColumnCommandRecoveryMgr::rollback().

2383  {
2384  cat_write_lock write_lock(this);
2385  cat_sqlite_lock sqlite_lock(this);
2386 
2387  sqliteConnector_.query("BEGIN TRANSACTION");
2388  try {
2389  const auto table_id = cd.tableId;
2390 
2391  auto catalog_cd = getMetadataForColumn(table_id, cd.columnId);
2392 
2393  CHECK(catalog_cd) << " can not alter non existing column";
2394 
2395  using BindType = SqliteConnector::BindType;
2396  std::vector<BindType> types(11, BindType::TEXT);
2397  if (!cd.default_value.has_value()) {
2398  types[8] = BindType::NULL_TYPE;
2399  }
2401  "UPDATE mapd_columns SET "
2402  "coltype = ?,"
2403  "colsubtype = ?,"
2404  "coldim = ?,"
2405  "colscale = ?,"
2406  "is_notnull = ?,"
2407  "compression = ?,"
2408  "comp_param = ?,"
2409  "size = ?,"
2410  "default_value = ? "
2411  "WHERE tableid = ? and columnid = ?",
2412  std::vector<std::string>{std::to_string(cd.columnType.get_type()),
2420  cd.default_value.value_or("NULL"),
2421  std::to_string(table_id),
2422  std::to_string(cd.columnId)},
2423 
2424  types);
2425 
2426  auto ncd = new ColumnDescriptor(cd);
2427 
2428  ColumnDescriptorMap::iterator columnDescIt =
2430  CHECK(columnDescIt != columnDescriptorMap_.end());
2431  auto ocd = columnDescIt->second;
2432 
2433  updateInColumnMap(ncd, ocd);
2434  } catch (std::exception& e) {
2435  sqliteConnector_.query("ROLLBACK TRANSACTION");
2436  throw;
2437  }
2438  sqliteConnector_.query("END TRANSACTION");
2439 }
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:392
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
HOST DEVICE int get_size() const
Definition: sqltypes.h:403
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:709
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
HOST DEVICE int get_scale() const
Definition: sqltypes.h:396
virtual void query(const std::string &queryString)
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:391
std::string to_string(char const *&&v)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
specifies the content in-memory of a row in the column metadata table
void updateInColumnMap(ColumnDescriptor *cd, ColumnDescriptor *old_cd)
Definition: Catalog.cpp:6960
std::string to_upper(const std::string &str)
std::optional< std::string > default_value
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:399
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:393
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:402
#define CHECK(condition)
Definition: Logger.h:291
SQLTypeInfo columnType
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:398
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
std::string columnName

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::alterPhysicalTableMetadata ( const TableDescriptor td,
const TableDescriptorUpdateParams table_update_params 
)
private

Definition at line 3680 of file Catalog.cpp.

References CHECK, TableDescriptorUpdateParams::max_rollback_epochs, TableDescriptorUpdateParams::max_rows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::tableId, and to_string().

3682  {
3683  // Only called from parent alterTableParamMetadata, expect already to have catalog and
3684  // sqlite write locks
3685 
3686  // Sqlite transaction should have already been begun in parent alterTableCatalogMetadata
3687 
3689  CHECK(mutable_td);
3690  if (td->maxRollbackEpochs != table_update_params.max_rollback_epochs) {
3692  "UPDATE mapd_tables SET max_rollback_epochs = ? WHERE tableid = ?",
3693  std::vector<std::string>{std::to_string(table_update_params.max_rollback_epochs),
3694  std::to_string(td->tableId)});
3695  mutable_td->maxRollbackEpochs = table_update_params.max_rollback_epochs;
3696  }
3697 
3698  if (td->maxRows != table_update_params.max_rows) {
3700  "UPDATE mapd_tables SET max_rows = ? WHERE tableid = ?",
3701  std::vector<std::string>{std::to_string(table_update_params.max_rows),
3702  std::to_string(td->tableId)});
3703  mutable_td->maxRows = table_update_params.max_rows;
3704  }
3705 }
int32_t maxRollbackEpochs
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
TableDescriptor * getMutableMetadataForTableUnlocked(int table_id) const
Definition: Catalog.cpp:1896
std::string to_string(char const *&&v)
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::alterTableMetadata ( const TableDescriptor td,
const TableDescriptorUpdateParams table_update_params 
)
private

Definition at line 3707 of file Catalog.cpp.

References CHECK, logger::FATAL, LOG, TableDescriptor::tableId, and TableDescriptor::tableName.

3708  {
3709  cat_write_lock write_lock(this);
3710  cat_sqlite_lock sqlite_lock(getObjForLock());
3711  sqliteConnector_.query("BEGIN TRANSACTION");
3712  try {
3713  const auto physical_table_it = logicalToPhysicalTableMapById_.find(td->tableId);
3714  if (physical_table_it != logicalToPhysicalTableMapById_.end()) {
3715  const auto physical_tables = physical_table_it->second;
3716  CHECK(!physical_tables.empty());
3717  for (size_t i = 0; i < physical_tables.size(); i++) {
3718  int32_t physical_tb_id = physical_tables[i];
3719  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id, false);
3720  CHECK(phys_td);
3721  alterPhysicalTableMetadata(phys_td, table_update_params);
3722  }
3723  }
3724  alterPhysicalTableMetadata(td, table_update_params);
3725  } catch (std::exception& e) {
3726  sqliteConnector_.query("ROLLBACK TRANSACTION");
3727  LOG(FATAL) << "Table '" << td->tableName << "' catalog update failed";
3728  }
3729  sqliteConnector_.query("END TRANSACTION");
3730 }
std::string tableName
#define LOG(tag)
Definition: Logger.h:285
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
void alterPhysicalTableMetadata(const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
Definition: Catalog.cpp:3680
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:727
void Catalog_Namespace::Catalog::buildColumnsMapUnlocked ( )
private

Definition at line 1333 of file Catalog.cpp.

References anonymous_namespace{Utm.h}::a, CHECK, ColumnDescriptor::chunks, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::db_id, ColumnDescriptor::default_value, SQLTypeInfo::get_physical_cols(), SQLTypeInfo::is_geometry(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_compression(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::set_dict_key(), SQLTypeInfo::set_dimension(), SQLTypeInfo::set_notnull(), SQLTypeInfo::set_scale(), SQLTypeInfo::set_size(), SQLTypeInfo::set_subtype(), SQLTypeInfo::set_type(), gpu_enabled::sort(), ColumnDescriptor::tableId, and ColumnDescriptor::virtualExpr.

1333  {
1334  std::string columnQuery(
1335  "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
1336  "is_notnull, compression, comp_param, "
1337  "size, chunks, is_systemcol, is_virtualcol, virtual_expr, is_deletedcol, "
1338  "default_value from "
1339  "mapd_columns ORDER BY tableid, "
1340  "columnid");
1341  sqliteConnector_.query(columnQuery);
1342  auto numRows = sqliteConnector_.getNumRows();
1343  int32_t skip_physical_cols = 0;
1344  for (size_t r = 0; r < numRows; ++r) {
1345  ColumnDescriptor* cd = new ColumnDescriptor();
1346  cd->tableId = sqliteConnector_.getData<int>(r, 0);
1347  cd->columnId = sqliteConnector_.getData<int>(r, 1);
1348  cd->columnName = sqliteConnector_.getData<string>(r, 2);
1352  cd->columnType.set_scale(sqliteConnector_.getData<int>(r, 6));
1356  cd->columnType.set_size(sqliteConnector_.getData<int>(r, 10));
1357  cd->chunks = sqliteConnector_.getData<string>(r, 11);
1358  cd->isSystemCol = sqliteConnector_.getData<bool>(r, 12);
1359  cd->isVirtualCol = sqliteConnector_.getData<bool>(r, 13);
1360  cd->virtualExpr = sqliteConnector_.getData<string>(r, 14);
1361  cd->isDeletedCol = sqliteConnector_.getData<bool>(r, 15);
1362  if (sqliteConnector_.isNull(r, 16)) {
1363  cd->default_value = std::nullopt;
1364  } else {
1365  cd->default_value = std::make_optional(sqliteConnector_.getData<string>(r, 16));
1366  }
1367  cd->isGeoPhyCol = skip_physical_cols > 0;
1368  cd->db_id = getDatabaseId();
1369  set_dict_key(*cd);
1370  addToColumnMap(cd);
1371 
1372  if (skip_physical_cols <= 0) {
1373  skip_physical_cols = cd->columnType.get_physical_cols();
1374  }
1375 
1376  auto td_itr = tableDescriptorMapById_.find(cd->tableId);
1377  CHECK(td_itr != tableDescriptorMapById_.end());
1378 
1379  if (cd->isDeletedCol) {
1380  td_itr->second->hasDeletedCol = true;
1381  setDeletedColumnUnlocked(td_itr->second, cd);
1382  } else if (cd->columnType.is_geometry() || skip_physical_cols-- <= 0) {
1383  tableDescriptorMapById_[cd->tableId]->columnIdBySpi_.push_back(cd->columnId);
1384  }
1385  }
1386 
1387  // sort columnIdBySpi_ based on columnId
1388  for (auto& tit : tableDescriptorMapById_) {
1389  std::sort(tit.second->columnIdBySpi_.begin(),
1390  tit.second->columnIdBySpi_.end(),
1391  [](const size_t a, const size_t b) -> bool { return a < b; });
1392  }
1393 }
std::string virtualExpr
void set_compression(EncodingType c)
Definition: sqltypes.h:479
void set_size(int s)
Definition: sqltypes.h:476
T getData(const int row, const int col)
SQLTypes
Definition: sqltypes.h:65
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:469
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
virtual void query(const std::string &queryString)
std::string chunks
constexpr double a
Definition: Utm.h:32
EncodingType
Definition: sqltypes.h:240
int get_physical_cols() const
Definition: sqltypes.h:430
void set_scale(int s)
Definition: sqltypes.h:473
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3945
int getDatabaseId() const
Definition: Catalog.h:326
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
specifies the content in-memory of a row in the column metadata table
void set_comp_param(int p)
Definition: sqltypes.h:480
std::optional< std::string > default_value
bool isNull(const int row, const int col) const
void set_dimension(int d)
Definition: sqltypes.h:470
void set_notnull(bool n)
Definition: sqltypes.h:475
#define CHECK(condition)
Definition: Logger.h:291
bool is_geometry() const
Definition: sqltypes.h:595
SQLTypeInfo columnType
std::string columnName
void addToColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6984
virtual size_t getNumRows() const
HOST DEVICE void set_type(SQLTypes t)
Definition: sqltypes.h:468

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildCustomExpressionsMapUnlocked ( )
private

Definition at line 1514 of file Catalog.cpp.

1514  {
1516  "SELECT id, name, expression_json, data_source_type, data_source_id, "
1517  "is_deleted "
1518  "FROM omnisci_custom_expressions");
1519  auto num_rows = sqliteConnector_.getNumRows();
1520  for (size_t row = 0; row < num_rows; row++) {
1521  auto custom_expr = getCustomExpressionFromConnector(row);
1522  custom_expr_map_by_id_[custom_expr->id] = std::move(custom_expr);
1523  }
1524 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
std::unique_ptr< CustomExpression > getCustomExpressionFromConnector(size_t row)
Definition: Catalog.cpp:1526
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:717
virtual size_t getNumRows() const
void Catalog_Namespace::Catalog::buildDashboardsMapUnlocked ( const std::map< int32_t, std::string > &  user_name_by_user_id)
private

Definition at line 1417 of file Catalog.cpp.

References generate_dashboard_system_rolename(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_user_name_from_id(), and to_string().

1418  {
1419  std::string frontendViewQuery(
1420  "SELECT id, state, name, image_hash, strftime('%Y-%m-%dT%H:%M:%SZ', update_time), "
1421  "userid, "
1422  "metadata "
1423  "FROM mapd_dashboards");
1424  sqliteConnector_.query(frontendViewQuery);
1425  auto numRows = sqliteConnector_.getNumRows();
1426  for (size_t r = 0; r < numRows; ++r) {
1427  auto vd = std::make_shared<DashboardDescriptor>();
1428  vd->dashboardId = sqliteConnector_.getData<int>(r, 0);
1429  vd->dashboardState = sqliteConnector_.getData<string>(r, 1);
1430  vd->dashboardName = sqliteConnector_.getData<string>(r, 2);
1431  vd->imageHash = sqliteConnector_.getData<string>(r, 3);
1432  vd->updateTime = sqliteConnector_.getData<string>(r, 4);
1433  vd->userId = sqliteConnector_.getData<int>(r, 5);
1434  vd->dashboardMetadata = sqliteConnector_.getData<string>(r, 6);
1435  vd->user = get_user_name_from_id(vd->userId, user_name_by_user_id);
1436  vd->dashboardSystemRoleName = generate_dashboard_system_rolename(
1438  dashboardDescriptorMap_[std::to_string(vd->userId) + ":" + vd->dashboardName] = vd;
1439  }
1440 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
virtual void query(const std::string &queryString)
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:712
std::string to_string(char const *&&v)
std::string get_user_name_from_id(int32_t id, const std::map< int32_t, std::string > &user_name_by_user_id)
Definition: Catalog.cpp:1051
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildDictionaryMapUnlocked ( )
private

Definition at line 1072 of file Catalog.cpp.

References currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, g_base_path, SqliteConnector::getData(), SqliteConnector::getNumRows(), shared::kDataDirectoryName, SqliteConnector::query(), sqliteConnector_, and to_string().

1072  {
1073  std::string dictQuery(
1074  "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
1075  sqliteConnector_.query(dictQuery);
1076  auto numRows = sqliteConnector_.getNumRows();
1077  for (size_t r = 0; r < numRows; ++r) {
1078  auto dictId = sqliteConnector_.getData<int>(r, 0);
1079  auto dictName = sqliteConnector_.getData<string>(r, 1);
1080  auto dictNBits = sqliteConnector_.getData<int>(r, 2);
1081  auto is_shared = sqliteConnector_.getData<bool>(r, 3);
1082  auto refcount = sqliteConnector_.getData<int>(r, 4);
1083  auto fname = g_base_path + "/" + shared::kDataDirectoryName + "/DB_" +
1084  std::to_string(currentDB_.dbId) + "_DICT_" + std::to_string(dictId);
1085  DictRef dict_ref(currentDB_.dbId, dictId);
1086  auto dd = new DictDescriptor(
1087  dict_ref, dictName, dictNBits, is_shared, refcount, fname, false);
1088  dictDescriptorMapByRef_[dict_ref].reset(dd);
1089  }
1090 }
const std::string kDataDirectoryName
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:711
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
std::string g_base_path
Definition: SysCatalog.cpp:62
Descriptor for a dictionary for a string columne.
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildForeignServerMapUnlocked ( )
private

Definition at line 5092 of file Catalog.cpp.

References CHECK, and g_enable_fsi.

5092  {
5095  "SELECT id, name, data_wrapper_type, options, owner_user_id, creation_time FROM "
5096  "omnisci_foreign_servers");
5097  auto num_rows = sqliteConnector_.getNumRows();
5098 
5099  for (size_t row = 0; row < num_rows; row++) {
5100  auto foreign_server = std::make_shared<foreign_storage::ForeignServer>(
5101  sqliteConnector_.getData<int>(row, 0),
5102  sqliteConnector_.getData<std::string>(row, 1),
5103  sqliteConnector_.getData<std::string>(row, 2),
5104  sqliteConnector_.getData<std::string>(row, 3),
5105  sqliteConnector_.getData<std::int32_t>(row, 4),
5106  sqliteConnector_.getData<std::int32_t>(row, 5));
5107  foreignServerMap_[foreign_server->name] = foreign_server;
5108  foreignServerMapById_[foreign_server->id] = foreign_server;
5109  }
5110 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:716
ForeignServerMap foreignServerMap_
Definition: Catalog.h:715
#define CHECK(condition)
Definition: Logger.h:291
bool g_enable_fsi
Definition: Catalog.cpp:96
virtual size_t getNumRows() const
void Catalog_Namespace::Catalog::buildLinksMapUnlocked ( )
private

Definition at line 1442 of file Catalog.cpp.

References to_string().

1442  {
1443  std::string linkQuery(
1444  "SELECT linkid, userid, link, view_state, strftime('%Y-%m-%dT%H:%M:%SZ', "
1445  "update_time), view_metadata "
1446  "FROM mapd_links");
1447  sqliteConnector_.query(linkQuery);
1448  auto numRows = sqliteConnector_.getNumRows();
1449  for (size_t r = 0; r < numRows; ++r) {
1450  auto ld = new LinkDescriptor();
1451  ld->linkId = sqliteConnector_.getData<int>(r, 0);
1452  ld->userId = sqliteConnector_.getData<int>(r, 1);
1453  ld->link = sqliteConnector_.getData<string>(r, 2);
1454  ld->viewState = sqliteConnector_.getData<string>(r, 3);
1455  ld->updateTime = sqliteConnector_.getData<string>(r, 4);
1456  ld->viewMetadata = sqliteConnector_.getData<string>(r, 5);
1458  linkDescriptorMapById_[ld->linkId] = ld;
1459  }
1460 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:714
virtual size_t getNumRows() const
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:713

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildLogicalToPhysicalMapUnlocked ( )
private

Definition at line 1462 of file Catalog.cpp.

References CHECK.

1462  {
1463  /* rebuild map linking logical tables to corresponding physical ones */
1464  std::string logicalToPhysicalTableMapQuery(
1465  "SELECT logical_table_id, physical_table_id "
1466  "FROM mapd_logical_to_physical");
1467  sqliteConnector_.query(logicalToPhysicalTableMapQuery);
1468  auto numRows = sqliteConnector_.getNumRows();
1469  for (size_t r = 0; r < numRows; ++r) {
1470  auto logical_tb_id = sqliteConnector_.getData<int>(r, 0);
1471  auto physical_tb_id = sqliteConnector_.getData<int>(r, 1);
1472  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
1473  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
1474  /* add new entity to the map logicalToPhysicalTableMapById_ */
1475  std::vector<int32_t> physicalTables{physical_tb_id};
1476  const auto it_ok =
1477  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
1478  CHECK(it_ok.second);
1479  } else {
1480  /* update map logicalToPhysicalTableMapById_ */
1481  physicalTableIt->second.push_back(physical_tb_id);
1482  }
1483  }
1484 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
#define CHECK(condition)
Definition: Logger.h:291
virtual size_t getNumRows() const
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:727
void Catalog_Namespace::Catalog::buildMaps ( )
protected

Definition at line 1489 of file Catalog.cpp.

References g_enable_fsi, and Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_user_id_to_user_name_map().

1489  {
1490  // Get all user id to username mapping here in order to avoid making a call to
1491  // SysCatalog (and attempting to acquire SysCatalog locks) while holding locks for this
1492  // catalog.
1493  const auto user_name_by_user_id = get_user_id_to_user_name_map();
1494 
1495  cat_write_lock write_lock(this);
1496  cat_sqlite_lock sqlite_lock(getObjForLock());
1497 
1500 
1501  if (g_enable_fsi) {
1504  }
1505 
1508  buildDashboardsMapUnlocked(user_name_by_user_id);
1512 }
void buildLogicalToPhysicalMapUnlocked()
Definition: Catalog.cpp:1462
void buildDashboardsMapUnlocked(const std::map< int32_t, std::string > &user_name_by_user_id)
Definition: Catalog.cpp:1417
void buildCustomExpressionsMapUnlocked()
Definition: Catalog.cpp:1514
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
void updateForeignTablesInMapUnlocked()
Definition: Catalog.cpp:5112
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
bool g_enable_fsi
Definition: Catalog.cpp:96
std::map< int32_t, std::string > get_user_id_to_user_name_map()
Definition: Catalog.cpp:1042

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildTablesMapUnlocked ( )
private

Definition at line 1274 of file Catalog.cpp.

References logger::FATAL, StorageType::FOREIGN_TABLE, TableDescriptor::fragmenter, TableDescriptor::fragments, TableDescriptor::fragPageSize, TableDescriptor::fragType, TableDescriptor::hasDeletedCol, TableDescriptor::is_system_table, TableDescriptor::isView, TableDescriptor::keyMetainfo, LOG, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nColumns, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::storageType, TableDescriptor::tableId, TableDescriptor::tableName, to_upper(), and TableDescriptor::userId.

1274  {
1275  std::string tableQuery(
1276  "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
1277  "max_chunk_size, frag_page_size, "
1278  "max_rows, partitions, shard_column_id, shard, num_shards, key_metainfo, userid, "
1279  "sort_column_id, storage_type, max_rollback_epochs, is_system_table "
1280  "from mapd_tables");
1281  sqliteConnector_.query(tableQuery);
1282  auto numRows = sqliteConnector_.getNumRows();
1283  for (size_t r = 0; r < numRows; ++r) {
1284  TableDescriptor* td;
1285  const auto& storage_type = sqliteConnector_.getData<string>(r, 17);
1286  if (!storage_type.empty() && storage_type != StorageType::FOREIGN_TABLE) {
1287  const auto table_id = sqliteConnector_.getData<int>(r, 0);
1288  const auto& table_name = sqliteConnector_.getData<string>(r, 1);
1289  LOG(FATAL) << "Unable to read Catalog metadata: storage type is currently not a "
1290  "supported table option (table "
1291  << table_name << " [" << table_id << "] in database "
1292  << currentDB_.dbName << ").";
1293  }
1294 
1295  if (storage_type == StorageType::FOREIGN_TABLE) {
1296  td = new foreign_storage::ForeignTable();
1297  } else {
1298  td = new TableDescriptor();
1299  }
1300 
1301  td->storageType = storage_type;
1302  td->tableId = sqliteConnector_.getData<int>(r, 0);
1303  td->tableName = sqliteConnector_.getData<string>(r, 1);
1304  td->nColumns = sqliteConnector_.getData<int>(r, 2);
1305  td->isView = sqliteConnector_.getData<bool>(r, 3);
1306  td->fragments = sqliteConnector_.getData<string>(r, 4);
1307  td->fragType =
1309  td->maxFragRows = sqliteConnector_.getData<int>(r, 6);
1310  td->maxChunkSize = sqliteConnector_.getData<int64_t>(r, 7);
1311  td->fragPageSize = sqliteConnector_.getData<int>(r, 8);
1312  td->maxRows = sqliteConnector_.getData<int64_t>(r, 9);
1313  td->partitions = sqliteConnector_.getData<string>(r, 10);
1314  td->shardedColumnId = sqliteConnector_.getData<int>(r, 11);
1315  td->shard = sqliteConnector_.getData<int>(r, 12);
1316  td->nShards = sqliteConnector_.getData<int>(r, 13);
1317  td->keyMetainfo = sqliteConnector_.getData<string>(r, 14);
1318  td->userId = sqliteConnector_.getData<int>(r, 15);
1319  td->sortedColumnId =
1320  sqliteConnector_.isNull(r, 16) ? 0 : sqliteConnector_.getData<int>(r, 16);
1321  if (!td->isView) {
1322  td->fragmenter = nullptr;
1323  }
1324  td->maxRollbackEpochs = sqliteConnector_.getData<int>(r, 18);
1325  td->is_system_table = sqliteConnector_.getData<bool>(r, 19);
1326  td->hasDeletedCol = false;
1327 
1329  tableDescriptorMapById_[td->tableId] = td;
1330  }
1331 }
int32_t maxRollbackEpochs
std::string partitions
T getData(const int row, const int col)
std::string tableName
#define LOG(tag)
Definition: Logger.h:285
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
std::string storageType
virtual void query(const std::string &queryString)
std::string fragments
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
std::string keyMetainfo
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
std::string to_upper(const std::string &str)
bool isNull(const int row, const int col) const
Fragmenter_Namespace::FragmenterType fragType
static constexpr char const * FOREIGN_TABLE
virtual size_t getNumRows() const
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:707

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::calculateSHA1 ( const std::string &  data)
protected

Definition at line 4801 of file Catalog.cpp.

4801  {
4802  boost::uuids::detail::sha1 sha1;
4803  unsigned int digest[5];
4804  sha1.process_bytes(data.c_str(), data.length());
4805  sha1.get_digest(digest);
4806  std::stringstream ss;
4807  for (size_t i = 0; i < 5; i++) {
4808  ss << std::hex << digest[i];
4809  }
4810  return ss.str();
4811 }
void Catalog_Namespace::Catalog::changeForeignServerOwner ( const std::string &  server_name,
const int  new_owner_id 
)

Change the owner of a Foreign Server to a new owner.

Parameters
server_name- Name of the foreign server whose owner to change
new_owner_id- New owner's user id

Definition at line 3386 of file Catalog.cpp.

References CHECK, to_string(), and foreign_storage::ForeignServer::user_id.

3387  {
3388  cat_write_lock write_lock(this);
3389  foreign_storage::ForeignServer* foreign_server =
3390  foreignServerMap_.find(server_name)->second.get();
3391  CHECK(foreign_server);
3392  setForeignServerProperty(server_name, "owner_user_id", std::to_string(new_owner_id));
3393  // update in-memory server
3394  foreign_server->user_id = new_owner_id;
3395 }
std::string to_string(char const *&&v)
ForeignServerMap foreignServerMap_
Definition: Catalog.h:715
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
void setForeignServerProperty(const std::string &server_name, const std::string &property, const std::string &value)
Definition: Catalog.cpp:5310

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::CheckAndExecuteMigrations ( )
protected

Definition at line 1012 of file Catalog.cpp.

References g_enable_fsi, getObjForLock(), recordOwnershipOfObjectsInObjectPermissions(), renameLegacyDataWrappers(), updateCustomExpressionsSchema(), updateDefaultColumnValues(), updateDeletedColumnIndicator(), updateDictionaryNames(), updateDictionarySchema(), updateFixlenArrayColumns(), updateFrontendViewAndLinkUsers(), updateFrontendViewSchema(), updateFrontendViewsToDashboards(), updateFsiSchemas(), updateGeoColumns(), updateLinkSchema(), updateLogicalToPhysicalTableLinkSchema(), updatePageSize(), and updateTableDescriptorSchema().

1012  {
1013  cat_write_lock write_lock(this);
1014  cat_sqlite_lock sqlite_lock(getObjForLock());
1017  updateGeoColumns();
1020  updateLinkSchema();
1024  updatePageSize();
1028  if (g_enable_fsi) {
1029  updateFsiSchemas();
1031  }
1034 }
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:470
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:797
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:129
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
void updateCustomExpressionsSchema()
Definition: Catalog.cpp:763
void updateLogicalToPhysicalTableLinkSchema()
Definition: Catalog.cpp:631
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
bool g_enable_fsi
Definition: Catalog.cpp:96

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::CheckAndExecuteMigrationsPostBuildMaps ( )
protected

Definition at line 1036 of file Catalog.cpp.

References checkDateInDaysColumnMigration(), and createDashboardSystemRoles().

1036  {
1039 }
void checkDateInDaysColumnMigration()
Definition: Catalog.cpp:925

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::checkDateInDaysColumnMigration ( )
protected

Definition at line 925 of file Catalog.cpp.

References getCurrentDB(), getObjForLock(), migrations::MigrationMgr::migrateDateInDaysMetadata(), sqliteConnector_, and tableDescriptorMapById_.

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

925  {
926  cat_sqlite_lock sqlite_lock(getObjForLock());
929 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:265
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
static void migrateDateInDaysMetadata(const Catalog_Namespace::TableDescriptorMapById &table_descriptors_by_id, const int database_id, Catalog_Namespace::Catalog *cat, SqliteConnector &sqlite)
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool Catalog_Namespace::Catalog::checkDropRenderGroupColumnsMigration ( )

Definition at line 931 of file Catalog.cpp.

References migrations::MigrationMgr::dropRenderGroupColumns(), and tableDescriptorMapById_.

Referenced by Catalog_Namespace::SysCatalog::checkDropRenderGroupColumnsMigration().

931  {
932  // do not take cat_sqlite_lock here as Catalog functions do that themselves
934 }
static bool dropRenderGroupColumns(const Catalog_Namespace::TableDescriptorMapById &table_descriptors_by_id, Catalog_Namespace::Catalog *cat)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const bool Catalog_Namespace::Catalog::checkMetadataForDeletedRecs ( const TableDescriptor td,
int  column_id 
) const

Definition at line 3903 of file Catalog.cpp.

References CHECK, and TableDescriptor::fragmenter.

3904  {
3905  // check if there are rows deleted by examining the deletedColumn metadata
3906  CHECK(td);
3907  auto fragmenter = td->fragmenter;
3908  if (fragmenter) {
3909  return fragmenter->hasDeletedRows(delete_column_id);
3910  } else {
3911  return false;
3912  }
3913 }
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
#define CHECK(condition)
Definition: Logger.h:291
void Catalog_Namespace::Catalog::checkpoint ( const int  logicalTableId) const

Definition at line 5022 of file Catalog.cpp.

Referenced by UpdelRoll::commitUpdate(), TableOptimizer::vacuumDeletedRows(), and TableOptimizer::vacuumFragmentsAboveMinSelectivity().

5022  {
5023  const auto td = getMetadataForTable(logicalTableId);
5024  const auto shards = getPhysicalTablesDescriptors(td);
5025  for (const auto shard : shards) {
5026  getDataMgr().checkpoint(getCurrentDB().dbId, shard->tableId);
5027  }
5028 }
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:266
void checkpoint(const int db_id, const int tb_id)
Definition: DataMgr.cpp:584
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:265
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4869
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::checkpointWithAutoRollback ( const int  logical_table_id) const

Definition at line 5030 of file Catalog.cpp.

Referenced by AlterTableAlterColumnCommandRecoveryMgr::checkpoint(), Fragmenter_Namespace::LocalInsertConnector::checkpoint(), import_export::Loader::checkpoint(), StorageIOFacility::TransactionParameters::finalizeTransaction(), and TableOptimizer::vacuumFragmentsAboveMinSelectivity().

5030  {
5031  auto table_epochs = getTableEpochs(getDatabaseId(), logical_table_id);
5032  try {
5033  checkpoint(logical_table_id);
5034  } catch (...) {
5035  setTableEpochsLogExceptions(getDatabaseId(), table_epochs);
5036  throw;
5037  }
5038 }
int getDatabaseId() const
Definition: Catalog.h:326
void checkpoint(const int logicalTableId) const
Definition: Catalog.cpp:5022
void setTableEpochsLogExceptions(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3885
std::vector< TableEpochInfo > getTableEpochs(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:3821

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::conditionallyInitializeSystemObjects ( )
private

Definition at line 6382 of file Catalog.cpp.

References g_enable_system_tables.

6382  {
6386  }
6387 }
bool g_enable_system_tables
Definition: SysCatalog.cpp:64
bool isInfoSchemaDb() const
Definition: Catalog.cpp:6389
int32_t Catalog_Namespace::Catalog::createCustomExpression ( std::unique_ptr< CustomExpression custom_expression)

Creates a new custom expression.

Parameters
custom_expression- unique pointer to struct containing custom expression details.
Returns
id of created custom expression

Definition at line 5944 of file Catalog.cpp.

References CHECK, CHECK_EQ, CHECK_GT, and to_string().

5945  {
5946  cat_write_lock write_lock(this);
5947  cat_sqlite_lock sqlite_lock(getObjForLock());
5948  sqliteConnector_.query("BEGIN TRANSACTION");
5949  int32_t custom_expression_id{-1};
5950  try {
5951  auto data_source_type_str =
5952  CustomExpression::dataSourceTypeToString(custom_expression->data_source_type);
5953  auto data_source_id_str = std::to_string(custom_expression->data_source_id);
5954  std::string custom_expr_select_query{
5955  "SELECT id FROM omnisci_custom_expressions WHERE name = ? and data_source_type = "
5956  "? and data_source_id = ? and is_deleted = ?"};
5957  std::vector<std::string> custom_expr_select_params{custom_expression->name,
5958  data_source_type_str,
5959  data_source_id_str,
5960  std::to_string(false)};
5961  sqliteConnector_.query_with_text_params(custom_expr_select_query,
5962  custom_expr_select_params);
5963  if (sqliteConnector_.getNumRows() > 0) {
5964  throw std::runtime_error{
5965  "A custom expression with the given "
5966  "name and data source already exists."};
5967  }
5969  "INSERT INTO omnisci_custom_expressions(name, expression_json, "
5970  "data_source_type, data_source_id, is_deleted) VALUES (?,?,?,?,?)",
5971  std::vector<std::string>{custom_expression->name,
5972  custom_expression->expression_json,
5973  data_source_type_str,
5974  data_source_id_str,
5975  std::to_string(false)});
5976  sqliteConnector_.query_with_text_params(custom_expr_select_query,
5977  custom_expr_select_params);
5978  CHECK_EQ(sqliteConnector_.getNumRows(), static_cast<size_t>(1));
5979  custom_expression->id = sqliteConnector_.getData<int32_t>(0, 0);
5980  custom_expression_id = custom_expression->id;
5981  CHECK(custom_expr_map_by_id_.find(custom_expression->id) ==
5982  custom_expr_map_by_id_.end());
5983  custom_expr_map_by_id_[custom_expression->id] = std::move(custom_expression);
5984  } catch (std::exception& e) {
5985  sqliteConnector_.query("ROLLBACK TRANSACTION");
5986  throw;
5987  }
5988  sqliteConnector_.query("END TRANSACTION");
5989  CHECK_GT(custom_expression_id, 0);
5990  return custom_expression_id;
5991 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
#define CHECK_GT(x, y)
Definition: Logger.h:305
std::string to_string(char const *&&v)
static std::string dataSourceTypeToString(DataSourceType type_enum)
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:717
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

int32_t Catalog_Namespace::Catalog::createDashboard ( DashboardDescriptor vd)

Definition at line 4660 of file Catalog.cpp.

References DashboardDescriptor::dashboardId, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, DashboardDescriptor::dashboardSystemRoleName, generate_dashboard_system_rolename(), DashboardDescriptor::imageHash, to_string(), Catalog_Namespace::sqlite_lock< T >::unlock(), Catalog_Namespace::write_lock< T >::unlock(), DashboardDescriptor::updateTime, and DashboardDescriptor::userId.

4660  {
4661  cat_write_lock write_lock(this);
4662  cat_sqlite_lock sqlite_lock(getObjForLock());
4663  sqliteConnector_.query("BEGIN TRANSACTION");
4664  try {
4665  // TODO(andrew): this should be an upsert
4667  "SELECT id FROM mapd_dashboards WHERE name = ? and userid = ?",
4668  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
4669  if (sqliteConnector_.getNumRows() > 0) {
4671  "UPDATE mapd_dashboards SET state = ?, image_hash = ?, metadata = ?, "
4672  "update_time = "
4673  "datetime('now') where name = ? "
4674  "and userid = ?",
4675  std::vector<std::string>{vd.dashboardState,
4676  vd.imageHash,
4677  vd.dashboardMetadata,
4678  vd.dashboardName,
4679  std::to_string(vd.userId)});
4680  } else {
4682  "INSERT INTO mapd_dashboards (name, state, image_hash, metadata, "
4683  "update_time, "
4684  "userid) "
4685  "VALUES "
4686  "(?,?,?,?, "
4687  "datetime('now'), ?)",
4688  std::vector<std::string>{vd.dashboardName,
4689  vd.dashboardState,
4690  vd.imageHash,
4691  vd.dashboardMetadata,
4692  std::to_string(vd.userId)});
4693  }
4694  } catch (std::exception& e) {
4695  sqliteConnector_.query("ROLLBACK TRANSACTION");
4696  throw;
4697  }
4698  sqliteConnector_.query("END TRANSACTION");
4699 
4700  // now get the auto generated dashboardId
4701  try {
4703  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_dashboards "
4704  "WHERE name = ? and userid = ?",
4705  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
4706  vd.dashboardId = sqliteConnector_.getData<int>(0, 0);
4707  vd.updateTime = sqliteConnector_.getData<std::string>(0, 1);
4708  } catch (std::exception& e) {
4709  throw;
4710  }
4714  sqlite_lock.unlock();
4715  write_lock.unlock();
4716  if (!isInfoSchemaDb()) {
4717  // NOTE(wamsi): Transactionally unsafe
4720  }
4721  return vd.dashboardId;
4722 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
virtual void query(const std::string &queryString)
void addFrontendViewToMap(DashboardDescriptor &vd)
Definition: Catalog.cpp:1683
std::string to_string(char const *&&v)
std::string dashboardSystemRoleName
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1724
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
bool isInfoSchemaDb() const
Definition: Catalog.cpp:6389
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createDashboardSystemRoles ( )
protected

Definition at line 936 of file Catalog.cpp.

References createOrUpdateDashboardSystemRole(), currentDB_, Catalog_Namespace::DBMetadata::dbId, logger::ERROR, generate_dashboard_system_rolename(), SqliteConnector::getData(), Catalog_Namespace::SysCatalog::getGranteesOfSharedDashboards(), SqliteConnector::getNumRows(), getObjForLock(), Catalog_Namespace::SysCatalog::getRoleGrantee(), Catalog_Namespace::SysCatalog::grantRoleBatch(), logger::INFO, Catalog_Namespace::SysCatalog::instance(), LOG, MAPD_VERSION, SqliteConnector::query(), SqliteConnector::query_with_text_params(), run_benchmark_import::result, sqliteConnector_, and to_string().

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

936  {
937  std::unordered_map<std::string, std::pair<int, std::string>> dashboards;
938  std::vector<std::string> dashboard_ids;
939  static const std::string migration_name{"dashboard_roles_migration"};
940  {
941  cat_sqlite_lock sqlite_lock(getObjForLock());
942  sqliteConnector_.query("BEGIN TRANSACTION");
943  try {
944  // migration_history should be present in all catalogs by now
945  // if not then would be created before this migration
947  "select * from mapd_version_history where migration_history = '" +
948  migration_name + "'");
949  if (sqliteConnector_.getNumRows() != 0) {
950  // no need for further execution
951  sqliteConnector_.query("END TRANSACTION");
952  return;
953  }
954  LOG(INFO) << "Performing dashboard internal roles Migration.";
955  sqliteConnector_.query("select id, userid, metadata from mapd_dashboards");
956  for (size_t i = 0; i < sqliteConnector_.getNumRows(); ++i) {
959  sqliteConnector_.getData<string>(i, 0)))) {
960  // Successfully created roles during previous migration/crash
961  // No need to include them
962  continue;
963  }
964  dashboards[sqliteConnector_.getData<string>(i, 0)] = std::make_pair(
965  sqliteConnector_.getData<int>(i, 1), sqliteConnector_.getData<string>(i, 2));
966  dashboard_ids.push_back(sqliteConnector_.getData<string>(i, 0));
967  }
968  } catch (const std::exception& e) {
969  sqliteConnector_.query("ROLLBACK TRANSACTION");
970  throw;
971  }
972  sqliteConnector_.query("END TRANSACTION");
973  }
974  // All current grantees with shared dashboards.
975  const auto active_grantees =
977 
978  try {
979  // NOTE(wamsi): Transactionally unsafe
980  for (auto dash : dashboards) {
981  createOrUpdateDashboardSystemRole(dash.second.second,
982  dash.second.first,
984  std::to_string(currentDB_.dbId), dash.first));
985  auto result = active_grantees.find(dash.first);
986  if (result != active_grantees.end()) {
989  dash.first)},
990  result->second);
991  }
992  }
993  cat_sqlite_lock sqlite_lock(getObjForLock());
994  // check if this has already been completed
996  "select * from mapd_version_history where migration_history = '" +
997  migration_name + "'");
998  if (sqliteConnector_.getNumRows() != 0) {
999  return;
1000  }
1002  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
1003  std::vector<std::string>{std::to_string(MAPD_VERSION), migration_name});
1004  } catch (const std::exception& e) {
1005  LOG(ERROR) << "Failed to create dashboard system roles during migration: "
1006  << e.what();
1007  throw;
1008  }
1009  LOG(INFO) << "Successfully created dashboard system roles during migration.";
1010 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
#define LOG(tag)
Definition: Logger.h:285
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
void grantRoleBatch(const std::vector< std::string > &roles, const std::vector< std::string > &grantees)
static SysCatalog & instance()
Definition: SysCatalog.h:343
static const int32_t MAPD_VERSION
Definition: release.h:32
Role * getRoleGrantee(const std::string &name) const
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1724
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
std::unordered_map< std::string, std::vector< std::string > > getGranteesOfSharedDashboards(const std::vector< std::string > &dashboard_ids)
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createDefaultServersIfNotExists ( )

Creates default local file servers (if they don't already exist).

Definition at line 5331 of file Catalog.cpp.

References CHECK, foreign_storage::DataWrapperType::CSV, g_enable_fsi, shared::kRootUserId, foreign_storage::AbstractFileStorageDataWrapper::LOCAL_FILE_STORAGE_TYPE, foreign_storage::DataWrapperType::PARQUET, foreign_storage::DataWrapperType::REGEX_PARSER, and foreign_storage::AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY.

5331  {
5336 
5337  auto local_csv_server = std::make_unique<foreign_storage::ForeignServer>(
5338  "default_local_delimited",
5340  options,
5342  local_csv_server->validate();
5343  createForeignServerNoLocks(std::move(local_csv_server), true);
5344 
5345 #ifdef ENABLE_IMPORT_PARQUET
5346  auto local_parquet_server = std::make_unique<foreign_storage::ForeignServer>(
5347  "default_local_parquet",
5349  options,
5351  local_parquet_server->validate();
5352  createForeignServerNoLocks(std::move(local_parquet_server), true);
5353 #endif
5354 
5355  auto local_regex_parser_server = std::make_unique<foreign_storage::ForeignServer>(
5356  "default_local_regex_parsed",
5358  options,
5360  local_regex_parser_server->validate();
5361  createForeignServerNoLocks(std::move(local_regex_parser_server), true);
5362 }
const int kRootUserId
static constexpr char const * REGEX_PARSER
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:3291
#define CHECK(condition)
Definition: Logger.h:291
static constexpr char const * CSV
std::map< std::string, std::string, std::less<>> OptionsMap
bool g_enable_fsi
Definition: Catalog.cpp:96
static constexpr char const * PARQUET
void Catalog_Namespace::Catalog::createForeignServer ( std::unique_ptr< foreign_storage::ForeignServer foreign_server,
bool  if_not_exists 
)

Creates a new foreign server DB object.

Parameters
foreign_server- unique pointer to struct containing foreign server details
if_not_exists- flag indicating whether or not an attempt to create a new foreign server should occur if a server with the same name already exists. An exception is thrown if this flag is set to "false" and an attempt is made to create a pre-existing foreign server

Definition at line 3283 of file Catalog.cpp.

3285  {
3286  cat_write_lock write_lock(this);
3287  cat_sqlite_lock sqlite_lock(getObjForLock());
3288  createForeignServerNoLocks(std::move(foreign_server), if_not_exists);
3289 }
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:3291
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
void Catalog_Namespace::Catalog::createForeignServerNoLocks ( std::unique_ptr< foreign_storage::ForeignServer foreign_server,
bool  if_not_exists 
)
private

Same as createForeignServer() but without acquiring locks. This should only be called from within a function/code block that already acquires appropriate locks.

Definition at line 3291 of file Catalog.cpp.

References CHECK, CHECK_EQ, setup::name, and to_string().

3293  {
3294  const auto& name = foreign_server->name;
3295 
3297  "SELECT name from omnisci_foreign_servers where name = ?",
3298  std::vector<std::string>{name});
3299 
3300  if (sqliteConnector_.getNumRows() == 0) {
3301  foreign_server->creation_time = std::time(nullptr);
3303  "INSERT INTO omnisci_foreign_servers (name, data_wrapper_type, owner_user_id, "
3304  "creation_time, "
3305  "options) "
3306  "VALUES (?, ?, ?, ?, ?)",
3307  std::vector<std::string>{name,
3308  foreign_server->data_wrapper_type,
3309  std::to_string(foreign_server->user_id),
3310  std::to_string(foreign_server->creation_time),
3311  foreign_server->getOptionsAsJsonString()});
3313  "SELECT id from omnisci_foreign_servers where name = ?",
3314  std::vector<std::string>{name});
3315  CHECK_EQ(sqliteConnector_.getNumRows(), size_t(1));
3316  foreign_server->id = sqliteConnector_.getData<int32_t>(0, 0);
3317  std::shared_ptr<foreign_storage::ForeignServer> foreign_server_shared =
3318  std::move(foreign_server);
3320  << "Attempting to insert a foreign server into foreign server map that already "
3321  "exists.";
3322  foreignServerMap_[name] = foreign_server_shared;
3323  foreignServerMapById_[foreign_server_shared->id] = foreign_server_shared;
3324  } else if (!if_not_exists) {
3325  throw std::runtime_error{"A foreign server with name \"" + foreign_server->name +
3326  "\" already exists."};
3327  }
3328 
3329  const auto& server_it = foreignServerMap_.find(name);
3330  CHECK(server_it != foreignServerMap_.end());
3331  CHECK(foreignServerMapById_.find(server_it->second->id) != foreignServerMapById_.end());
3332 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
std::string to_string(char const *&&v)
std::string name() const
Definition: Catalog.h:348
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:716
ForeignServerMap foreignServerMap_
Definition: Catalog.h:715
#define CHECK(condition)
Definition: Logger.h:291
virtual size_t getNumRows() const

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::createLink ( LinkDescriptor ld,
size_t  min_length 
)

Definition at line 4813 of file Catalog.cpp.

References LinkDescriptor::link, LinkDescriptor::linkId, to_string(), LinkDescriptor::updateTime, LinkDescriptor::userId, LinkDescriptor::viewMetadata, and LinkDescriptor::viewState.

4813  {
4814  cat_write_lock write_lock(this);
4815  cat_sqlite_lock sqlite_lock(getObjForLock());
4816  sqliteConnector_.query("BEGIN TRANSACTION");
4817  try {
4819  .substr(0, 8);
4821  "SELECT linkid FROM mapd_links WHERE link = ? and userid = ?",
4822  std::vector<std::string>{ld.link, std::to_string(ld.userId)});
4823  if (sqliteConnector_.getNumRows() > 0) {
4825  "UPDATE mapd_links SET update_time = datetime('now') WHERE userid = ? AND "
4826  "link = ?",
4827  std::vector<std::string>{std::to_string(ld.userId), ld.link});
4828  } else {
4830  "INSERT INTO mapd_links (userid, link, view_state, view_metadata, "
4831  "update_time) VALUES (?,?,?,?, datetime('now'))",
4832  std::vector<std::string>{
4833  std::to_string(ld.userId), ld.link, ld.viewState, ld.viewMetadata});
4834  }
4835  // now get the auto generated dashid
4837  "SELECT linkid, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_links "
4838  "WHERE link = ?",
4839  ld.link);
4840  ld.linkId = sqliteConnector_.getData<int>(0, 0);
4841  ld.updateTime = sqliteConnector_.getData<std::string>(0, 1);
4842  } catch (std::exception& e) {
4843  sqliteConnector_.query("ROLLBACK TRANSACTION");
4844  throw;
4845  }
4846  sqliteConnector_.query("END TRANSACTION");
4847  addLinkToMap(ld);
4848  return ld.link;
4849 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
std::string calculateSHA1(const std::string &data)
Definition: Catalog.cpp:4801
void addLinkToMap(LinkDescriptor &ld)
Definition: Catalog.cpp:1770
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createOrUpdateDashboardSystemRole ( const std::string &  view_meta,
const int32_t &  user_id,
const std::string &  dash_role_name 
)
protected

Definition at line 1724 of file Catalog.cpp.

References Grantee::findDbObject(), Grantee::getDbObjects(), TableDBObjectType, and ViewDBObjectType.

Referenced by createDashboardSystemRoles().

1726  {
1727  auto objects = parseDashboardObjects(view_meta, user_id);
1728  Role* rl = SysCatalog::instance().getRoleGrantee(dash_role_name);
1729  if (!rl) {
1730  // Dashboard role does not exist
1731  // create role and grant privileges
1732  // NOTE(wamsi): Transactionally unsafe
1734  dash_role_name, /*user_private_role=*/false, /*is_temporary=*/false);
1735  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1736  } else {
1737  // Dashboard system role already exists
1738  // Add/remove privileges on objects
1739  std::set<DBObjectKey> revoke_keys;
1740  auto ex_objects = rl->getDbObjects(true);
1741  for (auto key : *ex_objects | boost::adaptors::map_keys) {
1742  if (key.permissionType != TableDBObjectType &&
1743  key.permissionType != ViewDBObjectType) {
1744  continue;
1745  }
1746  bool found = false;
1747  for (auto obj : objects) {
1748  found = key == obj.getObjectKey() ? true : false;
1749  if (found) {
1750  break;
1751  }
1752  }
1753  if (!found) {
1754  revoke_keys.insert(key);
1755  }
1756  }
1757  for (auto& key : revoke_keys) {
1758  // revoke privs on object since the object is no
1759  // longer used by the dashboard as source
1760  // NOTE(wamsi): Transactionally unsafe
1762  dash_role_name, *rl->findDbObject(key, true), *this);
1763  }
1764  // Update privileges on remaining objects
1765  // NOTE(wamsi): Transactionally unsafe
1766  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1767  }
1768 }
void revokeDBObjectPrivileges(const std::string &grantee, const DBObject &object, const Catalog_Namespace::Catalog &catalog)
void createRole(const std::string &roleName, const bool user_private_role, const bool is_temporary=false)
Definition: Grantee.h:81
DBObject * findDbObject(const DBObjectKey &objectKey, bool only_direct) const
Definition: Grantee.cpp:85
static SysCatalog & instance()
Definition: SysCatalog.h:343
Role * getRoleGrantee(const std::string &name) const
void grantDBObjectPrivilegesBatch(const std::vector< std::string > &grantees, const std::vector< DBObject > &objects, const Catalog_Namespace::Catalog &catalog)
const DBObjectMap * getDbObjects(bool only_direct) const
Definition: Grantee.h:56
std::vector< DBObject > parseDashboardObjects(const std::string &view_meta, const int &user_id)
Definition: Catalog.cpp:1694

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createShardedTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs 
)

Definition at line 4099 of file Catalog.cpp.

References CHECK, TableDescriptor::nShards, TableDescriptor::shard, table_is_temporary(), TableDescriptor::tableId, and TableDescriptor::tableName.

4102  {
4103  /* create logical table */
4104  TableDescriptor* tdl = &td;
4105  createTable(*tdl, cols, shared_dict_defs, true); // create logical table
4106  int32_t logical_tb_id = tdl->tableId;
4107  std::string logical_table_name = tdl->tableName;
4108 
4109  /* create physical tables and link them to the logical table */
4110  std::vector<int32_t> physicalTables;
4111  for (int32_t i = 1; i <= td.nShards; i++) {
4112  TableDescriptor* tdp = &td;
4113  tdp->tableName = generatePhysicalTableName(logical_table_name, i);
4114  tdp->shard = i - 1;
4115  createTable(*tdp, cols, shared_dict_defs, false); // create physical table
4116  int32_t physical_tb_id = tdp->tableId;
4117 
4118  /* add physical table to the vector of physical tables */
4119  physicalTables.push_back(physical_tb_id);
4120  }
4121 
4122  if (!physicalTables.empty()) {
4123  cat_write_lock write_lock(this);
4124  /* add logical to physical tables correspondence to the map */
4125  const auto it_ok =
4126  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
4127  CHECK(it_ok.second);
4128  /* update sqlite mapd_logical_to_physical in sqlite database */
4129  if (!table_is_temporary(&td)) {
4130  updateLogicalToPhysicalTableMap(logical_tb_id);
4131  }
4132  }
4133 }
std::string tableName
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:2935
bool table_is_temporary(const TableDescriptor *const td)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
std::string generatePhysicalTableName(const std::string &logicalTableName, const size_t shardNumber)
Definition: Catalog.cpp:5085
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:727
void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id)
Definition: Catalog.cpp:645

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createSystemTableServer ( const std::string &  server_name,
const std::string &  data_wrapper_type,
const foreign_storage::OptionsMap options = {} 
)
private

Definition at line 6833 of file Catalog.cpp.

References logger::INFO, shared::kRootUserId, LOG, and run_benchmark_import::tables.

6835  {
6836  auto server = std::make_unique<foreign_storage::ForeignServer>(
6837  server_name, data_wrapper_type, options, shared::kRootUserId);
6838  server->validate();
6839  auto stored_server = getForeignServer(server_name);
6840  if (stored_server && stored_server->options != server->options) {
6841  // Drop all tables for server before dropping server.
6842  auto tables = getAllForeignTablesForForeignServer(stored_server->id);
6843  for (const auto table : tables) {
6844  LOG(INFO) << "Dropping existing \"" << table->tableName << "\" system table for \""
6845  << server_name << "\" foreign server.";
6846  dropTable(table);
6847  }
6848  LOG(INFO) << "Dropping existing \"" << server_name
6849  << "\" system table foreign server.";
6850  dropForeignServer(server_name);
6851  stored_server = nullptr;
6852  }
6853  if (!stored_server) {
6854  LOG(INFO) << "Creating a new \"" << server_name << "\" system table foreign server.";
6855  createForeignServer(std::move(server), true);
6856  }
6857 }
const int kRootUserId
void createForeignServer(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:3283
const foreign_storage::ForeignServer * getForeignServer(const std::string &server_name) const
Definition: Catalog.cpp:3334
#define LOG(tag)
Definition: Logger.h:285
std::vector< const foreign_storage::ForeignTable * > getAllForeignTablesForForeignServer(const int32_t foreign_server_id)
Get all of the foreign tables for associated with a foreign server id.
Definition: Catalog.cpp:3631
void dropForeignServer(const std::string &server_name)
Definition: Catalog.cpp:3450
void dropTable(const TableDescriptor *td)
Definition: Catalog.cpp:4282
void Catalog_Namespace::Catalog::createTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs,
bool  isLogicalTable 
)

Definition at line 2935 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnId, TableDescriptor::columnIdBySpi_, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::default_value, dict_ref_t::dictId, Data_Namespace::DISK_LEVEL, StorageType::FOREIGN_TABLE, TableDescriptor::fragPageSize, TableDescriptor::fragType, g_enable_fsi, g_serialize_temp_tables, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_next_refresh_time(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), TableDescriptor::hasDeletedCol, SQLTypeInfo::is_array(), TableDescriptor::is_system_table, ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, kBIGINT, kBOOLEAN, kENCODING_DICT, TableDescriptor::keyMetainfo, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nColumns, foreign_storage::ForeignTable::next_refresh_time, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, SQLTypeInfo::set_comp_param(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::set_dict_key(), SQLTypeInfo::set_size(), TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::storageType, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), TableDescriptor::userId, TableDescriptor::viewSQL, and ColumnDescriptor::virtualExpr.

2939  {
2940  cat_write_lock write_lock(this);
2941  list<ColumnDescriptor> cds = cols;
2942  list<DictDescriptor> dds;
2943  std::set<std::string> toplevel_column_names;
2944  list<ColumnDescriptor> columns;
2945 
2946  if (!td.storageType.empty() &&
2949  throw std::runtime_error("Only temporary tables can be backed by foreign storage.");
2950  }
2951  dataMgr_->getForeignStorageInterface()->prepareTable(getCurrentDB().dbId, td, cds);
2952  }
2953 
2954  for (auto cd : cds) {
2955  if (cd.columnName == "rowid") {
2956  throw std::runtime_error(
2957  "Cannot create column with name rowid. rowid is a system defined column.");
2958  }
2959  columns.push_back(cd);
2960  toplevel_column_names.insert(cd.columnName);
2961  if (cd.columnType.is_geometry()) {
2962  expandGeoColumn(cd, columns);
2963  }
2964  }
2965  cds.clear();
2966 
2967  ColumnDescriptor cd;
2968  // add row_id column -- Must be last column in the table
2969  cd.columnName = "rowid";
2970  cd.isSystemCol = true;
2971  cd.columnType = SQLTypeInfo(kBIGINT, true);
2972 #ifdef MATERIALIZED_ROWID
2973  cd.isVirtualCol = false;
2974 #else
2975  cd.isVirtualCol = true;
2976  cd.virtualExpr = "MAPD_FRAG_ID * MAPD_ROWS_PER_FRAG + MAPD_FRAG_ROW_ID";
2977 #endif
2978  columns.push_back(cd);
2979  toplevel_column_names.insert(cd.columnName);
2980 
2981  if (td.hasDeletedCol) {
2982  ColumnDescriptor cd_del;
2983  cd_del.columnName = "$deleted$";
2984  cd_del.isSystemCol = true;
2985  cd_del.isVirtualCol = false;
2986  cd_del.columnType = SQLTypeInfo(kBOOLEAN, true);
2987  cd_del.isDeletedCol = true;
2988 
2989  columns.push_back(cd_del);
2990  }
2991 
2992  for (auto& column : columns) {
2993  column.db_id = getDatabaseId();
2994  }
2995 
2996  td.nColumns = columns.size();
2997  // TODO(sy): don't take disk locks or touch sqlite connector for temporary tables
2998  cat_sqlite_lock sqlite_lock(getObjForLock());
2999  sqliteConnector_.query("BEGIN TRANSACTION");
3001  try {
3003  R"(INSERT INTO mapd_tables (name, userid, ncolumns, isview, fragments, frag_type, max_frag_rows, max_chunk_size, frag_page_size, max_rows, partitions, shard_column_id, shard, num_shards, sort_column_id, storage_type, max_rollback_epochs, is_system_table, key_metainfo) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?))",
3004  std::vector<std::string>{td.tableName,
3005  std::to_string(td.userId),
3007  std::to_string(td.isView),
3008  "",
3013  std::to_string(td.maxRows),
3014  td.partitions,
3016  std::to_string(td.shard),
3017  std::to_string(td.nShards),
3019  td.storageType,
3022  td.keyMetainfo});
3023 
3024  // now get the auto generated tableid
3026  "SELECT tableid FROM mapd_tables WHERE name = ?", td.tableName);
3027  td.tableId = sqliteConnector_.getData<int>(0, 0);
3028  int colId = 1;
3029  for (auto cd : columns) {
3031  const bool is_foreign_col =
3032  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
3033  if (!is_foreign_col) {
3034  // Ideally we would like to not persist string dictionaries for system tables,
3035  // since system table content can be highly dynamic and string dictionaries
3036  // are not currently vacuumed. However, in distributed this causes issues
3037  // when the cluster is out of sync (when the agg resets but leaves persist) so
3038  // for the sake of testing we need to leave this as normal dictionaries until
3039  // we solve the distributed issue.
3040  auto use_temp_dictionary = false; // td.is_system_table;
3041  setColumnDictionary(cd, dds, td, isLogicalTable, use_temp_dictionary);
3042  }
3043  }
3044 
3045  if (toplevel_column_names.count(cd.columnName)) {
3046  if (!cd.isGeoPhyCol) {
3047  td.columnIdBySpi_.push_back(colId);
3048  }
3049  }
3050 
3051  using BindType = SqliteConnector::BindType;
3052  std::vector<BindType> types(17, BindType::TEXT);
3053  if (!cd.default_value.has_value()) {
3054  types[16] = BindType::NULL_TYPE;
3055  }
3057  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, "
3058  "coldim, colscale, is_notnull, "
3059  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
3060  "virtual_expr, is_deletedcol, default_value) "
3061  "VALUES (?, ?, ?, ?, ?, "
3062  "?, "
3063  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
3064  std::vector<std::string>{std::to_string(td.tableId),
3065  std::to_string(colId),
3066  cd.columnName,
3075  "",
3078  cd.virtualExpr,
3080  cd.default_value.value_or("NULL")},
3081  types);
3082  cd.tableId = td.tableId;
3083  cd.columnId = colId++;
3084  cds.push_back(cd);
3085  }
3086  if (td.isView) {
3088  "INSERT INTO mapd_views (tableid, sql) VALUES (?,?)",
3089  std::vector<std::string>{std::to_string(td.tableId), td.viewSQL});
3090  }
3092  auto& foreign_table = dynamic_cast<foreign_storage::ForeignTable&>(td);
3093  foreign_table.next_refresh_time = get_next_refresh_time(foreign_table);
3095  "INSERT INTO omnisci_foreign_tables (table_id, server_id, options, "
3096  "last_refresh_time, next_refresh_time) VALUES (?, ?, ?, ?, ?)",
3097  std::vector<std::string>{std::to_string(foreign_table.tableId),
3098  std::to_string(foreign_table.foreign_server->id),
3099  foreign_table.getOptionsAsJsonString(),
3100  std::to_string(foreign_table.last_refresh_time),
3101  std::to_string(foreign_table.next_refresh_time)});
3102  }
3103  } catch (std::exception& e) {
3104  sqliteConnector_.query("ROLLBACK TRANSACTION");
3105  throw;
3106  }
3107  } else { // Temporary table
3108  td.tableId = nextTempTableId_++;
3109  int colId = 1;
3110  for (auto cd : columns) {
3112  const bool is_foreign_col =
3113  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
3114 
3115  if (!is_foreign_col) {
3116  // Create a new temporary dictionary
3117  std::string fileName("");
3118  std::string folderPath("");
3120  nextTempDictId_++;
3121  DictDescriptor dd(dict_ref,
3122  fileName,
3124  false,
3125  1,
3126  folderPath,
3127  true); // Is dictName (2nd argument) used?
3128  dds.push_back(dd);
3129  if (!cd.columnType.is_array()) {
3131  }
3132  cd.columnType.set_comp_param(dict_ref.dictId);
3133  set_dict_key(cd);
3134  }
3135  }
3136  if (toplevel_column_names.count(cd.columnName)) {
3137  if (!cd.isGeoPhyCol) {
3138  td.columnIdBySpi_.push_back(colId);
3139  }
3140  }
3141  cd.tableId = td.tableId;
3142  cd.columnId = colId++;
3143  cds.push_back(cd);
3144  }
3145 
3147  serializeTableJsonUnlocked(&td, cds);
3148  }
3149  }
3150 
3151  try {
3152  auto cache = dataMgr_->getPersistentStorageMgr()->getDiskCache();
3153  if (cache) {
3154  CHECK(!cache->hasCachedMetadataForKeyPrefix({getCurrentDB().dbId, td.tableId}))
3155  << "Disk cache at " + cache->getCacheDirectory()
3156  << " contains preexisting data for new table. Please "
3157  "delete or clear cache before continuing";
3158  }
3159 
3160  addTableToMap(&td, cds, dds);
3161  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
3162  if (!td.storageType.empty() && td.storageType != StorageType::FOREIGN_TABLE) {
3163  dataMgr_->getForeignStorageInterface()->registerTable(this, td, cds);
3164  }
3165  } catch (std::exception& e) {
3166  sqliteConnector_.query("ROLLBACK TRANSACTION");
3167  removeTableFromMap(td.tableName, td.tableId, true);
3168  throw;
3169  }
3170  sqliteConnector_.query("END TRANSACTION");
3171 
3173  write_lock.unlock();
3174  sqlite_lock.unlock();
3176  true); // cause instantiateFragmenter() to be called
3177  }
3178 }
int32_t maxRollbackEpochs
void serializeTableJsonUnlocked(const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
Definition: Catalog.cpp:3180
int64_t get_next_refresh_time(const foreign_storage::ForeignTable &foreign_table)
Definition: Catalog.cpp:2922
std::string virtualExpr
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:392
void set_size(int s)
Definition: sqltypes.h:476
std::string partitions
HOST DEVICE int get_size() const
Definition: sqltypes.h:403
T getData(const int row, const int col)
std::string tableName
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
HOST DEVICE int get_scale() const
Definition: sqltypes.h:396
std::string storageType
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, bool is_logical_table, bool use_temp_dictionary=false)
Definition: Catalog.cpp:4056
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:722
virtual void query(const std::string &queryString)
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:391
std::vector< int > columnIdBySpi_
std::string to_string(char const *&&v)
void addTableToMap(const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
Definition: Catalog.cpp:1542
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:265
int getDatabaseId() const
Definition: Catalog.h:326
specifies the content in-memory of a row in the column metadata table
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:2774
std::string keyMetainfo
bool g_serialize_temp_tables
Definition: Catalog.cpp:106
void set_comp_param(int p)
Definition: sqltypes.h:480
std::optional< std::string > default_value
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:725
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:399
Fragmenter_Namespace::FragmenterType fragType
Data_Namespace::MemoryLevel persistenceLevel
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:393
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1601
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:402
bool setColumnSharedDictionary(ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:3987
#define CHECK(condition)
Definition: Logger.h:291
Descriptor for a dictionary for a string columne.
std::string viewSQL
SQLTypeInfo columnType
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:398
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
static constexpr char const * FOREIGN_TABLE
bool g_enable_fsi
Definition: Catalog.cpp:96
std::string columnName
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
bool is_array() const
Definition: sqltypes.h:583

+ Here is the call graph for this function:

TableDescriptor * Catalog_Namespace::Catalog::createTableFromDiskUnlocked ( int32_t  table_id)
private

Definition at line 5243 of file Catalog.cpp.

References logger::FATAL, StorageType::FOREIGN_TABLE, LOG, and to_string().

5243  {
5244  std::string query(
5245  "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
5246  "max_chunk_size, frag_page_size, max_rows, partitions, shard_column_id, shard, "
5247  "num_shards, key_metainfo, userid, sort_column_id, storage_type, "
5248  "max_rollback_epochs, is_system_table from mapd_tables WHERE tableid = " +
5249  std::to_string(table_id));
5250  sqliteConnector_.query(query);
5251  auto numRows = sqliteConnector_.getNumRows();
5252  if (!numRows) {
5253  throw NoTableFoundException(table_id);
5254  }
5255 
5256  const auto& storage_type = sqliteConnector_.getData<string>(0, 17);
5257  if (!storage_type.empty() && storage_type != StorageType::FOREIGN_TABLE) {
5258  const auto& table_name = sqliteConnector_.getData<string>(0, 1);
5259  LOG(FATAL) << "Unable to read Catalog metadata: storage type is currently not a "
5260  "supported table option (table "
5261  << table_name << " [" << table_id << "] in database " << currentDB_.dbName
5262  << ").";
5263  }
5264 
5265  // TODO(Misiu): Get rid of manual memory allocation and use smart pointers for
5266  // TableDecriptorMap_. Currently we use a smartpointer to cleanup if we catch
5267  // exceptions during initialization and then release ownership into the existing system.
5268  std::unique_ptr<TableDescriptor> td;
5269  td = (storage_type == StorageType::FOREIGN_TABLE)
5270  ? std::make_unique<foreign_storage::ForeignTable>()
5271  : std::make_unique<TableDescriptor>();
5272 
5273  td->tableId = sqliteConnector_.getData<int>(0, 0);
5274  td->tableName = sqliteConnector_.getData<string>(0, 1);
5275  td->nColumns = sqliteConnector_.getData<int>(0, 2);
5276  td->isView = sqliteConnector_.getData<bool>(0, 3);
5277  td->fragments = sqliteConnector_.getData<string>(0, 4);
5278  td->fragType = static_cast<Fragmenter_Namespace::FragmenterType>(
5279  sqliteConnector_.getData<int>(0, 5));
5280  td->maxFragRows = sqliteConnector_.getData<int>(0, 6);
5281  td->maxChunkSize = sqliteConnector_.getData<int64_t>(0, 7);
5282  td->fragPageSize = sqliteConnector_.getData<int>(0, 8);
5283  td->maxRows = sqliteConnector_.getData<int64_t>(0, 9);
5284  td->partitions = sqliteConnector_.getData<string>(0, 10);
5285  td->shardedColumnId = sqliteConnector_.getData<int>(0, 11);
5286  td->shard = sqliteConnector_.getData<int>(0, 12);
5287  td->nShards = sqliteConnector_.getData<int>(0, 13);
5288  td->keyMetainfo = sqliteConnector_.getData<string>(0, 14);
5289  td->userId = sqliteConnector_.getData<int>(0, 15);
5290  td->sortedColumnId =
5291  sqliteConnector_.isNull(0, 16) ? 0 : sqliteConnector_.getData<int>(0, 16);
5292  td->storageType = storage_type;
5293  td->maxRollbackEpochs = sqliteConnector_.getData<int>(0, 18);
5294  td->is_system_table = sqliteConnector_.getData<bool>(0, 19);
5295  td->hasDeletedCol = false;
5296 
5297  if (td->isView) {
5298  updateViewUnlocked(*td);
5299  } else {
5300  td->fragmenter = nullptr;
5301  }
5302 
5303  if (auto ftd = dynamic_cast<foreign_storage::ForeignTable*>(td.get())) {
5305  }
5306 
5307  return td.release();
5308 }
void updateViewUnlocked(TableDescriptor &)
Definition: Catalog.cpp:1395
T getData(const int row, const int col)
#define LOG(tag)
Definition: Logger.h:285
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
void reloadForeignTableUnlocked(foreign_storage::ForeignTable &foreign_table)
Definition: Catalog.cpp:5142
bool isNull(const int row, const int col) const
static constexpr char const * FOREIGN_TABLE
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::delDictionaryNontransactional ( const ColumnDescriptor cd)
private

Definition at line 2280 of file Catalog.cpp.

References CHECK, CHECK_GT, ColumnDescriptor::columnType, g_base_path, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::is_string(), SQLTypeInfo::is_string_array(), shared::kDataDirectoryName, kENCODING_DICT, File_Namespace::renameForDelete(), ColumnDescriptor::tableId, to_string(), and VLOG.

2280  {
2281  cat_write_lock write_lock(this);
2282  cat_sqlite_lock sqlite_lock(getObjForLock());
2283  if (!(cd.columnType.is_string() || cd.columnType.is_string_array())) {
2284  return;
2285  }
2286  if (!(cd.columnType.get_compression() == kENCODING_DICT)) {
2287  return;
2288  }
2289  const auto dictId = cd.columnType.get_comp_param();
2290  CHECK_GT(dictId, 0);
2291  // decrement and zero check dict ref count
2292  const auto td = getMetadataForTable(cd.tableId, false);
2293  CHECK(td);
2295  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
2296  std::to_string(dictId));
2298  "SELECT refcount FROM mapd_dictionaries WHERE dictid = ?", std::to_string(dictId));
2299  const auto refcount = sqliteConnector_.getData<int>(0, 0);
2300  VLOG(3) << "Dictionary " << dictId << "from dropped table has reference count "
2301  << refcount;
2302  if (refcount > 0) {
2303  return;
2304  }
2305  const DictRef dictRef(currentDB_.dbId, dictId);
2306  sqliteConnector_.query_with_text_param("DELETE FROM mapd_dictionaries WHERE dictid = ?",
2307  std::to_string(dictId));
2309  "/DB_" + std::to_string(currentDB_.dbId) + "_DICT_" +
2310  std::to_string(dictId));
2311 
2312  std::unique_ptr<StringDictionaryClient> client;
2313  if (!string_dict_hosts_.empty()) {
2314  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dictRef, true));
2315  }
2316  if (client) {
2317  client->drop(dictRef);
2318  }
2319 
2320  dictDescriptorMapByRef_.erase(dictRef);
2321 }
const std::string kDataDirectoryName
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:724
const DBMetadata currentDB_
Definition: Catalog.h:721
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:711
#define CHECK_GT(x, y)
Definition: Logger.h:305
std::string to_string(char const *&&v)
std::string g_base_path
Definition: SysCatalog.cpp:62
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:399
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:402
#define CHECK(condition)
Definition: Logger.h:291
SQLTypeInfo columnType
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
bool is_string() const
Definition: sqltypes.h:559
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:210
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
bool is_string_array() const
Definition: sqltypes.h:562
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
#define VLOG(n)
Definition: Logger.h:388

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::delDictionaryTransactional ( const ColumnDescriptor cd)

Definition at line 2267 of file Catalog.cpp.

Referenced by AlterTableAlterColumnCommandRecoveryMgr::cleanupDeleteDictionaries(), and AlterTableAlterColumnCommandRecoveryMgr::rollback().

2267  {
2268  cat_write_lock write_lock(this);
2269  cat_sqlite_lock sqlite_lock(this);
2270  sqliteConnector_.query("BEGIN TRANSACTION");
2271  try {
2273  } catch (std::exception& e) {
2274  sqliteConnector_.query("ROLLBACK TRANSACTION");
2275  throw;
2276  }
2277  sqliteConnector_.query("END TRANSACTION");
2278 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
void delDictionaryNontransactional(const ColumnDescriptor &cd)
Definition: Catalog.cpp:2280
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::deleteCustomExpressions ( const std::vector< int32_t > &  custom_expression_ids,
bool  do_soft_delete 
)

Deletes custom expressions with the given ids.

Parameters
custom_expression_ids- ids of custom expressions to delete
do_soft_delete- flag indicating whether or not to do a soft delete

Definition at line 6069 of file Catalog.cpp.

References CHECK, CHECK_EQ, join(), and to_string().

6070  {
6071  cat_write_lock write_lock(this);
6072  cat_sqlite_lock sqlite_lock(getObjForLock());
6073 
6074  std::vector<int32_t> invalid_ids;
6075  for (const auto id : custom_expression_ids) {
6076  if (custom_expr_map_by_id_.find(id) == custom_expr_map_by_id_.end()) {
6077  invalid_ids.emplace_back(id);
6078  }
6079  }
6080  if (!invalid_ids.empty()) {
6081  throw std::runtime_error{"Custom expressions with ids: " + join(invalid_ids, ",") +
6082  " do not exist."};
6083  }
6084  sqliteConnector_.query("BEGIN TRANSACTION");
6085  try {
6086  for (const auto id : custom_expression_ids) {
6088  "SELECT id FROM omnisci_custom_expressions WHERE id = ?",
6089  std::vector<std::string>{std::to_string(id)});
6090  CHECK_EQ(sqliteConnector_.getNumRows(), static_cast<size_t>(1));
6091  if (do_soft_delete) {
6093  "UPDATE omnisci_custom_expressions SET is_deleted = ? WHERE id = ?",
6094  std::vector<std::string>{std::to_string(true), std::to_string(id)});
6095  } else {
6097  "DELETE FROM omnisci_custom_expressions WHERE id = ?",
6098  std::vector<std::string>{std::to_string(id)});
6099  }
6100  }
6101 
6102  for (const auto id : custom_expression_ids) {
6103  if (do_soft_delete) {
6104  auto it = custom_expr_map_by_id_.find(id);
6105  CHECK(it != custom_expr_map_by_id_.end());
6106  it->second->is_deleted = true;
6107  } else {
6108  custom_expr_map_by_id_.erase(id);
6109  }
6110  }
6111  } catch (std::exception& e) {
6112  sqliteConnector_.query("ROLLBACK TRANSACTION");
6113  throw;
6114  }
6115  sqliteConnector_.query("END TRANSACTION");
6116 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
std::string join(T const &container, std::string const &delim)
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:717
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::deleteMetadataForDashboards ( const std::vector< int32_t >  ids,
const UserMetadata user 
)

Definition at line 2011 of file Catalog.cpp.

References DashboardDBObjectType, AccessPrivileges::DELETE_DASHBOARD, and to_string().

2012  {
2013  std::stringstream invalid_ids, restricted_ids;
2014 
2015  for (int32_t dashboard_id : dashboard_ids) {
2016  if (!getMetadataForDashboard(dashboard_id)) {
2017  invalid_ids << (!invalid_ids.str().empty() ? ", " : "") << dashboard_id;
2018  continue;
2019  }
2020  DBObject object(dashboard_id, DashboardDBObjectType);
2021  object.loadKey(*this);
2022  object.setPrivileges(AccessPrivileges::DELETE_DASHBOARD);
2023  std::vector<DBObject> privs = {object};
2024  if (!SysCatalog::instance().checkPrivileges(user, privs)) {
2025  restricted_ids << (!restricted_ids.str().empty() ? ", " : "") << dashboard_id;
2026  }
2027  }
2028 
2029  if (invalid_ids.str().size() > 0 || restricted_ids.str().size() > 0) {
2030  std::stringstream error_message;
2031  error_message << "Delete dashboard(s) failed with error(s):";
2032  if (invalid_ids.str().size() > 0) {
2033  error_message << "\nDashboard id: " << invalid_ids.str()
2034  << " - Dashboard id does not exist";
2035  }
2036  if (restricted_ids.str().size() > 0) {
2037  error_message
2038  << "\nDashboard id: " << restricted_ids.str()
2039  << " - User should be either owner of dashboard or super user to delete it";
2040  }
2041  throw std::runtime_error(error_message.str());
2042  }
2043  std::vector<DBObject> dash_objs;
2044 
2045  for (int32_t dashboard_id : dashboard_ids) {
2046  dash_objs.emplace_back(dashboard_id, DashboardDBObjectType);
2047  }
2048  // BE-5245: Transactionally unsafe (like other combined Catalog/Syscatalog operations)
2050  {
2051  cat_write_lock write_lock(this);
2052  cat_sqlite_lock sqlite_lock(getObjForLock());
2053 
2054  sqliteConnector_.query("BEGIN TRANSACTION");
2055  try {
2056  for (int32_t dashboard_id : dashboard_ids) {
2057  auto dash = getMetadataForDashboard(dashboard_id);
2058  // Dash should still exist if revokeDBObjectPrivileges passed but throw and
2059  // rollback if already deleted
2060  if (!dash) {
2061  throw std::runtime_error(
2062  std::string("Delete dashboard(s) failed with error(s):\nDashboard id: ") +
2063  std::to_string(dashboard_id) + " - Dashboard id does not exist ");
2064  }
2065  std::string user_id = std::to_string(dash->userId);
2066  std::string dash_name = dash->dashboardName;
2067  auto viewDescIt = dashboardDescriptorMap_.find(user_id + ":" + dash_name);
2068  dashboardDescriptorMap_.erase(viewDescIt);
2070  "DELETE FROM mapd_dashboards WHERE name = ? and userid = ?",
2071  std::vector<std::string>{dash_name, user_id});
2072  }
2073  } catch (std::exception& e) {
2074  sqliteConnector_.query("ROLLBACK TRANSACTION");
2075  throw;
2076  }
2077  sqliteConnector_.query("END TRANSACTION");
2078  }
2079 }
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:712
std::string to_string(char const *&&v)
static SysCatalog & instance()
Definition: SysCatalog.h:343
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
static const AccessPrivileges DELETE_DASHBOARD
Definition: DBObject.h:173
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
void revokeDBObjectPrivilegesFromAllBatch(std::vector< DBObject > &objects, Catalog *catalog)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::deleteTableCatalogMetadata ( const TableDescriptor logical_table,
const std::vector< const TableDescriptor * > &  physical_tables 
)
private

Definition at line 4310 of file Catalog.cpp.

References TableDescriptor::tableId, and to_string().

4312  {
4313  cat_write_lock write_lock(this);
4314  cat_sqlite_lock sqlite_lock(getObjForLock());
4315  sqliteConnector_.query("BEGIN TRANSACTION");
4316  try {
4317  // remove corresponding record from the logicalToPhysicalTableMap in sqlite database
4319  "DELETE FROM mapd_logical_to_physical WHERE logical_table_id = ?",
4320  std::to_string(logical_table->tableId));
4321  logicalToPhysicalTableMapById_.erase(logical_table->tableId);
4322  for (auto table : physical_tables) {
4323  eraseTableMetadata(table);
4324  }
4325  } catch (std::exception& e) {
4326  sqliteConnector_.query("ROLLBACK TRANSACTION");
4327  throw;
4328  }
4329  sqliteConnector_.query("END TRANSACTION");
4330 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
void eraseTableMetadata(const TableDescriptor *td)
Definition: Catalog.cpp:4332
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:727

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::doTruncateTable ( const TableDescriptor td)
protected

Definition at line 4143 of file Catalog.cpp.

References CHECK, CHECK_GE, ColumnDescriptor::columnType, Data_Namespace::CPU_LEVEL, dict_ref_t::dictId, DictDescriptor::dictIsTemp, DictDescriptor::dictRef, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), Data_Namespace::GPU_LEVEL, kENCODING_DICT, File_Namespace::renameForDelete(), ColumnDescriptor::tableId, and TableDescriptor::tableId.

4143  {
4144  // must destroy fragmenter before deleteChunks is called.
4146 
4147  const int tableId = td->tableId;
4148  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
4149  // assuming deleteChunksWithPrefix is atomic
4150  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
4151  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
4152 
4153  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
4154 
4155  cat_write_lock write_lock(this);
4156  std::unique_ptr<StringDictionaryClient> client;
4157  if (SysCatalog::instance().isAggregator()) {
4158  CHECK(!string_dict_hosts_.empty());
4159  DictRef dict_ref(currentDB_.dbId, -1);
4160  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
4161  }
4162  // clean up any dictionaries
4163  // delete all column descriptors for the table
4164  for (const auto& columnDescriptor : columnDescriptorMapById_) {
4165  auto cd = columnDescriptor.second;
4166  if (cd->tableId != td->tableId) {
4167  continue;
4168  }
4169  const int dict_id = cd->columnType.get_comp_param();
4170  // Dummy dictionaries created for a shard of a logical table have the id set to zero.
4171  if (cd->columnType.get_compression() == kENCODING_DICT && dict_id) {
4172  const DictRef dict_ref(currentDB_.dbId, dict_id);
4173  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
4174  CHECK(dictIt != dictDescriptorMapByRef_.end());
4175  const auto& dd = dictIt->second;
4176  CHECK_GE(dd->refcount, 1);
4177  // if this is the only table using this dict reset the dict
4178  if (dd->refcount == 1) {
4179  // close the dictionary
4180  dd->stringDict.reset();
4181  File_Namespace::renameForDelete(dd->dictFolderPath);
4182  if (client) {
4183  client->drop(dd->dictRef);
4184  }
4185  if (!dd->dictIsTemp) {
4186  boost::filesystem::create_directory(dd->dictFolderPath);
4187  }
4188  }
4189 
4190  DictDescriptor* new_dd = new DictDescriptor(dd->dictRef,
4191  dd->dictName,
4192  dd->dictNBits,
4193  dd->dictIsShared,
4194  dd->refcount,
4195  dd->dictFolderPath,
4196  dd->dictIsTemp);
4197  dictDescriptorMapByRef_.erase(dictIt);
4198  // now create new Dict -- need to figure out what to do here for temp tables
4199  if (client) {
4200  client->create(new_dd->dictRef, new_dd->dictIsTemp);
4201  }
4202  dictDescriptorMapByRef_[new_dd->dictRef].reset(new_dd);
4204  }
4205  }
4206 }
std::vector< int > ChunkKey
Definition: types.h:36
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:724
const DBMetadata currentDB_
Definition: Catalog.h:721
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:711
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:722
#define CHECK_GE(x, y)
Definition: Logger.h:306
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:710
static SysCatalog & instance()
Definition: SysCatalog.h:343
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1904
int32_t dictId
Definition: DictRef.h:14
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
void removeFragmenterForTable(const int table_id) const
Definition: Catalog.cpp:4260
#define CHECK(condition)
Definition: Logger.h:291
Descriptor for a dictionary for a string columne.
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:210

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropColumn ( const TableDescriptor td,
const ColumnDescriptor cd 
)

Definition at line 2653 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnId, ColumnDescriptor::columnName, TableDescriptor::nShards, TableDescriptor::shard, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), and to_upper().

2653  {
2654  {
2655  cat_write_lock write_lock(this);
2656  cat_sqlite_lock sqlite_lock(getObjForLock());
2657  // caller must handle sqlite/chunk transaction TOGETHER
2659  "DELETE FROM mapd_columns where tableid = ? and columnid = ?",
2660  std::vector<std::string>{std::to_string(td.tableId),
2661  std::to_string(cd.columnId)});
2662 
2664  "UPDATE mapd_tables SET ncolumns = ncolumns - 1 WHERE tableid = ?",
2665  std::vector<std::string>{std::to_string(td.tableId)});
2666 
2667  ColumnDescriptorMap::iterator columnDescIt =
2669  CHECK(columnDescIt != columnDescriptorMap_.end());
2670 
2671  columnDescriptorsForRoll.emplace_back(columnDescIt->second, nullptr);
2672  removeFromColumnMap(columnDescIt->second);
2673  --tableDescriptorMapById_[td.tableId]->nColumns;
2674  }
2675 
2676  // for each shard
2677  if (td.nShards > 0 && td.shard < 0) {
2678  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
2679  const auto shard_cd = getMetadataForColumn(shard->tableId, cd.columnId);
2680  CHECK(shard_cd);
2681  dropColumn(*shard, *shard_cd);
2682  }
2683  }
2684 }
void removeFromColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6992
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:709
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
void dropColumn(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:2653
std::string to_string(char const *&&v)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4869
std::string to_upper(const std::string &str)
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:739
std::string columnName

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropColumnNontransactional ( const TableDescriptor td,
const ColumnDescriptor cd 
)

Definition at line 2607 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnId, ColumnDescriptor::columnName, TableDescriptor::nShards, TableDescriptor::shard, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and to_upper().

2608  {
2609  cat_write_lock write_lock(this);
2610  cat_sqlite_lock sqlite_lock(this);
2611 
2613  "DELETE FROM mapd_columns where tableid = ? and columnid = ?",
2614  std::vector<std::string>{std::to_string(td.tableId), std::to_string(cd.columnId)});
2615 
2617  "UPDATE mapd_tables SET ncolumns = ncolumns - 1 WHERE tableid = ?",
2618  std::vector<std::string>{std::to_string(td.tableId)});
2619 
2620  ColumnDescriptorMap::iterator columnDescIt =
2622  CHECK(columnDescIt != columnDescriptorMap_.end());
2623 
2624  auto ocd = columnDescIt->second;
2625  removeFromColumnMap(ocd);
2626  --tableDescriptorMapById_[td.tableId]->nColumns;
2628  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
2629 
2630  // for each shard
2631  if (td.nShards > 0 && td.shard < 0) {
2632  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
2633  const auto shard_cd = getMetadataForColumn(shard->tableId, cd.columnId);
2634  CHECK(shard_cd);
2635  dropColumnNontransactional(*shard, *shard_cd);
2636  }
2637  }
2638 }
void removeFromColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6992
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
std::string tableName
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:709
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
std::string to_string(char const *&&v)
void removeColumnDescriptor(const ColumnDescriptor *cd)
Definition: Catalog.cpp:2686
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4869
std::string to_upper(const std::string &str)
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:725
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
std::string columnName
void dropColumnNontransactional(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:2607

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropColumnPolicies ( const TableDescriptor td,
const ColumnDescriptor cd 
)

Definition at line 2640 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnId, TableDescriptor::nShards, and TableDescriptor::shard.

2640  {
2641 
2642  // for each shard
2643  if (td.nShards > 0 && td.shard < 0) {
2644  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
2645  const auto shard_cd = getMetadataForColumn(shard->tableId, cd.columnId);
2646  CHECK(shard_cd);
2647  dropColumnPolicies(*shard, *shard_cd);
2648  }
2649  }
2650 }
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4869
void dropColumnPolicies(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:2640
#define CHECK(condition)
Definition: Logger.h:291
void Catalog_Namespace::Catalog::dropColumnTransactional ( const TableDescriptor td,
const ColumnDescriptor cd 
)

Definition at line 2591 of file Catalog.cpp.

Referenced by AlterTableAlterColumnCommandRecoveryMgr::cleanupDropSourceGeoColumns(), and AlterTableAlterColumnCommandRecoveryMgr::rollback().

2592  {
2593  dropColumnPolicies(td, cd);
2594 
2595  cat_write_lock write_lock(this);
2596  cat_sqlite_lock sqlite_lock(this);
2597  sqliteConnector_.query("BEGIN TRANSACTION");
2598  try {
2600  } catch (std::exception& e) {
2601  sqliteConnector_.query("ROLLBACK TRANSACTION");
2602  throw;
2603  }
2604  sqliteConnector_.query("END TRANSACTION");
2605 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
void dropColumnPolicies(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:2640
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
void dropColumnNontransactional(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:2607

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::dropForeignServer ( const std::string &  server_name)

Drops/deletes a foreign server DB object.

Parameters
server_name- Name of foreign server that will be deleted

Definition at line 3450 of file Catalog.cpp.

References CHECK_EQ, and to_string().

3450  {
3451  cat_write_lock write_lock(this);
3452  cat_sqlite_lock sqlite_lock(getObjForLock());
3453 
3455  "SELECT id from omnisci_foreign_servers where name = ?",
3456  std::vector<std::string>{server_name});
3457  auto num_rows = sqliteConnector_.getNumRows();
3458  if (num_rows > 0) {
3459  CHECK_EQ(size_t(1), num_rows);
3460  auto server_id = sqliteConnector_.getData<int32_t>(0, 0);
3462  "SELECT table_id from omnisci_foreign_tables where server_id = ?",
3463  std::to_string(server_id));
3464  if (sqliteConnector_.getNumRows() > 0) {
3465  throw std::runtime_error{"Foreign server \"" + server_name +
3466  "\" is referenced "
3467  "by existing foreign tables and cannot be dropped."};
3468  }
3469  sqliteConnector_.query("BEGIN TRANSACTION");
3470  try {
3472  "DELETE FROM omnisci_foreign_servers WHERE name = ?",
3473  std::vector<std::string>{server_name});
3474  } catch (const std::exception& e) {
3475  sqliteConnector_.query("ROLLBACK TRANSACTION");
3476  throw;
3477  }
3478  sqliteConnector_.query("END TRANSACTION");
3479  foreignServerMap_.erase(server_name);
3480  foreignServerMapById_.erase(server_id);
3481  }
3482 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:716
ForeignServerMap foreignServerMap_
Definition: Catalog.h:715
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropTable ( const TableDescriptor td)

Definition at line 4282 of file Catalog.cpp.

References CHECK, TableDescriptor::isView, TableDBObjectType, TableDescriptor::tableId, TableDescriptor::tableName, and ViewDBObjectType.

Referenced by Catalog_Namespace::anonymous_namespace{Catalog.cpp}::drop_tables().

4282  {
4285  std::vector<const TableDescriptor*> tables_to_drop;
4286  {
4287  cat_read_lock read_lock(this);
4288  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
4289  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
4290  // remove all corresponding physical tables if this is a logical table
4291  const auto physicalTables = physicalTableIt->second;
4292  CHECK(!physicalTables.empty());
4293  for (size_t i = 0; i < physicalTables.size(); i++) {
4294  int32_t physical_tb_id = physicalTables[i];
4295  const TableDescriptor* phys_td =
4296  getMutableMetadataForTableUnlocked(physical_tb_id);
4297  CHECK(phys_td);
4298  tables_to_drop.emplace_back(phys_td);
4299  }
4300  }
4301  tables_to_drop.emplace_back(td);
4302  }
4303 
4304  for (auto table : tables_to_drop) {
4305  eraseTablePhysicalData(table);
4306  }
4307  deleteTableCatalogMetadata(td, tables_to_drop);
4308 }
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:5067
std::string tableName
void revokeDBObjectPrivilegesFromAll(DBObject object, Catalog *catalog)
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
void deleteTableCatalogMetadata(const TableDescriptor *logical_table, const std::vector< const TableDescriptor * > &physical_tables)
Definition: Catalog.cpp:4310
TableDescriptor * getMutableMetadataForTableUnlocked(int table_id) const
Definition: Catalog.cpp:1896
static SysCatalog & instance()
Definition: SysCatalog.h:343
#define CHECK(condition)
Definition: Logger.h:291
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:727

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::dropTableFromJsonUnlocked ( const std::string &  table_name) const
protected

Definition at line 3251 of file Catalog.cpp.

References CHECK, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and VLOG.

3251  {
3252  // relies on the catalog write lock
3253  using namespace rapidjson;
3254 
3255  VLOG(1) << "Dropping temporary table " << table_name << " to JSON for Calcite.";
3256 
3257  const auto db_name = currentDB_.dbName;
3258  const auto file_path = table_json_filepath(basePath_, db_name);
3259 
3260  CHECK(boost::filesystem::exists(file_path));
3261  Document d;
3262 
3263  std::ifstream reader(file_path.string());
3264  CHECK(reader.is_open());
3265  IStreamWrapper json_read_wrapper(reader);
3266  d.ParseStream(json_read_wrapper);
3267 
3268  CHECK(d.IsObject());
3269  auto table_name_ref = StringRef(table_name.c_str());
3270  CHECK(d.HasMember(table_name_ref));
3271  CHECK(d.RemoveMember(table_name_ref));
3272 
3273  // Overwrite the existing file
3274  std::ofstream writer(file_path.string(), std::ios::trunc | std::ios::out);
3275  CHECK(writer.is_open());
3276  OStreamWrapper json_wrapper(writer);
3277 
3278  Writer<OStreamWrapper> json_writer(json_wrapper);
3279  d.Accept(json_writer);
3280  writer.close();
3281 }
const DBMetadata currentDB_
Definition: Catalog.h:721
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:162
#define CHECK(condition)
Definition: Logger.h:291
#define VLOG(n)
Definition: Logger.h:388

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::dumpCreateServer ( const std::string &  name,
bool  multiline_formatting = true 
) const

Definition at line 5746 of file Catalog.cpp.

References join().

5747  {
5748  cat_read_lock read_lock(this);
5749  auto server_it = foreignServerMap_.find(name);
5750  if (server_it == foreignServerMap_.end()) {
5751  throw std::runtime_error("Foreign server " + name + " does not exist.");
5752  }
5753  auto server = server_it->second.get();
5754  std::ostringstream os;
5755  os << "CREATE SERVER " << name << " FOREIGN DATA WRAPPER " << server->data_wrapper_type;
5756  std::vector<std::string> with_options;
5757  for (const auto& [option, value] : server->options) {
5758  with_options.emplace_back(option + "='" + value + "'");
5759  }
5760  if (!with_options.empty()) {
5761  if (!multiline_formatting) {
5762  os << " ";
5763  } else {
5764  os << "\n";
5765  }
5766  os << "WITH (" + boost::algorithm::join(with_options, ", ") + ")";
5767  }
5768  os << ";";
5769  return os.str();
5770 }
std::string join(T const &container, std::string const &delim)
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
std::string name() const
Definition: Catalog.h:348
ForeignServerMap foreignServerMap_
Definition: Catalog.h:715

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::dumpCreateTable ( const TableDescriptor td,
bool  multiline_formatting = true,
bool  dump_defaults = false 
) const

Definition at line 5567 of file Catalog.cpp.

5569  {
5570  cat_read_lock read_lock(this);
5571  return dumpCreateTableUnlocked(td, multiline_formatting, dump_defaults);
5572 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
std::string dumpCreateTableUnlocked(const TableDescriptor *td, bool multiline_formatting, bool dump_defaults) const
Definition: Catalog.cpp:5585
std::optional< std::string > Catalog_Namespace::Catalog::dumpCreateTable ( int32_t  table_id,
bool  multiline_formatting = true,
bool  dump_defaults = false 
) const

Definition at line 5574 of file Catalog.cpp.

5576  {
5577  cat_read_lock read_lock(this);
5578  const auto td = getMutableMetadataForTableUnlocked(table_id);
5579  if (!td) {
5580  return {};
5581  }
5582  return dumpCreateTableUnlocked(td, multiline_formatting, dump_defaults);
5583 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
std::string dumpCreateTableUnlocked(const TableDescriptor *td, bool multiline_formatting, bool dump_defaults) const
Definition: Catalog.cpp:5585
TableDescriptor * getMutableMetadataForTableUnlocked(int table_id) const
Definition: Catalog.cpp:1896
std::string Catalog_Namespace::Catalog::dumpCreateTableUnlocked ( const TableDescriptor td,
bool  multiline_formatting,
bool  dump_defaults 
) const
private

Definition at line 5585 of file Catalog.cpp.

References CHECK, Data_Namespace::CPU_LEVEL, DEFAULT_FRAGMENT_ROWS, DEFAULT_MAX_CHUNK_SIZE, DEFAULT_MAX_ROLLBACK_EPOCHS, DEFAULT_MAX_ROWS, DEFAULT_PAGE_SIZE, TableDescriptor::fragPageSize, g_leaf_count, TableDescriptor::hasDeletedCol, TableDescriptor::is_system_table, TableDescriptor::isView, join(), kCHAR, kENCODING_DICT, kENCODING_GEOINT, kTEXT, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and TableDescriptor::viewSQL.

5587  {
5588  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
5589  std::ostringstream os;
5590 
5591  if (foreign_table && !td->is_system_table) {
5592  os << "CREATE FOREIGN TABLE " << td->tableName << " (";
5593  } else if (!td->isView) {
5594  os << "CREATE ";
5596  os << "TEMPORARY ";
5597  }
5598  os << "TABLE " + td->tableName + " (";
5599  } else {
5600  os << "CREATE VIEW " + td->tableName + " AS " << td->viewSQL;
5601  return os.str();
5602  }
5603  // scan column defines
5604  std::vector<std::string> additional_info;
5605  std::set<std::string> shared_dict_column_names;
5606 
5607  gatherAdditionalInfo(additional_info, shared_dict_column_names, td);
5608 
5609  // gather column defines
5610  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
5611  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
5612  bool first = true;
5613  for (const auto cd : cds) {
5614  if (!(cd->isSystemCol || cd->isVirtualCol)) {
5615  const auto& ti = cd->columnType;
5616  if (!first) {
5617  os << ",";
5618  if (!multiline_formatting) {
5619  os << " ";
5620  }
5621  } else {
5622  first = false;
5623  }
5624  if (multiline_formatting) {
5625  os << "\n ";
5626  }
5627  // column name
5628  os << quoteIfRequired(cd->columnName);
5629  // CHAR is perculiar... better dump it as TEXT(32) like \d does
5630  if (ti.get_type() == SQLTypes::kCHAR) {
5631  os << " "
5632  << "TEXT";
5633  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
5634  os << " "
5635  << "TEXT[]";
5636  } else {
5637  os << " " << ti.get_type_name();
5638  }
5639  os << (ti.get_notnull() ? " NOT NULL" : "");
5640  if (cd->default_value.has_value()) {
5641  os << " DEFAULT " << cd->getDefaultValueLiteral();
5642  }
5643  if (shared_dict_column_names.find(cd->columnName) ==
5644  shared_dict_column_names.end()) {
5645  // avoids "Column ... shouldn't specify an encoding, it borrows it
5646  // from the referenced column"
5647  if (ti.is_string() || (ti.is_array() && ti.get_subtype() == kTEXT)) {
5648  auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
5649  if (ti.get_compression() == kENCODING_DICT) {
5650  os << " ENCODING " << ti.get_compression_name() << "(" << (size * 8) << ")";
5651  } else {
5652  os << " ENCODING NONE";
5653  }
5654  } else if (ti.is_date_in_days() ||
5655  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
5656  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
5657  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
5658  } else if (ti.is_geometry()) {
5659  if (ti.get_compression() == kENCODING_GEOINT) {
5660  os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
5661  << ")";
5662  } else {
5663  os << " ENCODING NONE";
5664  }
5665  }
5666  }
5667  }
5668  }
5669  // gather SHARED DICTIONARYs
5670  if (additional_info.size()) {
5671  std::string comma;
5672  if (!multiline_formatting) {
5673  comma = ", ";
5674  } else {
5675  comma = ",\n ";
5676  }
5677  os << comma;
5678  os << boost::algorithm::join(additional_info, comma);
5679  }
5680  os << ")";
5681 
5682  std::vector<std::string> with_options;
5683  if (foreign_table && !td->is_system_table) {
5684  if (multiline_formatting) {
5685  os << "\n";
5686  } else {
5687  os << " ";
5688  }
5689  os << "SERVER " << foreign_table->foreign_server->name;
5690 
5691  // gather WITH options ...
5692  for (const auto& [option, value] : foreign_table->options) {
5693  with_options.emplace_back(option + "='" + value + "'");
5694  }
5695  }
5696 
5697  if (dump_defaults || td->maxFragRows != DEFAULT_FRAGMENT_ROWS) {
5698  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
5699  }
5700  if (dump_defaults || td->maxChunkSize != DEFAULT_MAX_CHUNK_SIZE) {
5701  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
5702  }
5703  if (!foreign_table && (dump_defaults || td->fragPageSize != DEFAULT_PAGE_SIZE)) {
5704  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
5705  }
5706  if (!foreign_table && (dump_defaults || td->maxRows != DEFAULT_MAX_ROWS)) {
5707  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
5708  }
5709  if ((dump_defaults || td->maxRollbackEpochs != DEFAULT_MAX_ROLLBACK_EPOCHS) &&
5710  td->maxRollbackEpochs != -1) {
5711  with_options.push_back("MAX_ROLLBACK_EPOCHS=" +
5713  }
5714  if (!foreign_table && (dump_defaults || !td->hasDeletedCol)) {
5715  with_options.emplace_back(td->hasDeletedCol ? "VACUUM='DELAYED'"
5716  : "VACUUM='IMMEDIATE'");
5717  }
5718  if (!foreign_table && !td->partitions.empty()) {
5719  with_options.push_back("PARTITIONS='" + td->partitions + "'");
5720  }
5721  if (!foreign_table && td->nShards > 0) {
5722  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
5723  CHECK(shard_cd);
5724  with_options.push_back(
5725  "SHARD_COUNT=" +
5726  std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
5727  }
5728  if (!foreign_table && td->sortedColumnId > 0) {
5729  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
5730  CHECK(sort_cd);
5731  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
5732  }
5733 
5734  if (!with_options.empty()) {
5735  if (!multiline_formatting) {
5736  os << " ";
5737  } else {
5738  os << "\n";
5739  }
5740  os << "WITH (" + boost::algorithm::join(with_options, ", ") + ")";
5741  }
5742  os << ";";
5743  return os.str();
5744 }
int32_t maxRollbackEpochs
std::string partitions
std::string tableName
std::string join(T const &container, std::string const &delim)
#define DEFAULT_MAX_CHUNK_SIZE
std::string to_string(char const *&&v)
std::string quoteIfRequired(const std::string &column_name) const
Definition: Catalog.cpp:5867
#define DEFAULT_MAX_ROWS
void gatherAdditionalInfo(std::vector< std::string > &additional_info, std::set< std::string > &shared_dict_column_names, const TableDescriptor *td) const
Definition: Catalog.cpp:5878
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
#define DEFAULT_MAX_ROLLBACK_EPOCHS
specifies the content in-memory of a row in the table metadata table
#define DEFAULT_PAGE_SIZE
Definition: sqltypes.h:79
#define DEFAULT_FRAGMENT_ROWS
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:2172
Data_Namespace::MemoryLevel persistenceLevel
Definition: sqltypes.h:68
#define CHECK(condition)
Definition: Logger.h:291
std::string viewSQL
size_t g_leaf_count
Definition: ParserNode.cpp:78

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::dumpSchema ( const TableDescriptor td) const

Definition at line 5440 of file Catalog.cpp.

References CHECK, DEFAULT_MAX_ROLLBACK_EPOCHS, TableDescriptor::fragPageSize, g_leaf_count, TableDescriptor::hasDeletedCol, TableDescriptor::is_system_table, join(), kCHAR, kENCODING_DICT, kENCODING_GEOINT, kTEXT, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::tableId, and to_string().

Referenced by TableArchiver::dumpTable().

5440  {
5441  CHECK(!td->is_system_table);
5442  cat_read_lock read_lock(this);
5443 
5444  std::ostringstream os;
5445  os << "CREATE TABLE @T (";
5446  // gather column defines
5447  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
5448  std::string comma;
5449  std::vector<std::string> shared_dicts;
5450  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
5451  for (const auto cd : cds) {
5452  if (!(cd->isSystemCol || cd->isVirtualCol)) {
5453  const auto& ti = cd->columnType;
5454  os << comma << quoteIfRequired(cd->columnName);
5455  // CHAR is perculiar... better dump it as TEXT(32) like \d does
5456  if (ti.get_type() == SQLTypes::kCHAR) {
5457  os << " "
5458  << "TEXT";
5459  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
5460  os << " "
5461  << "TEXT[]";
5462  } else {
5463  os << " " << ti.get_type_name();
5464  }
5465  os << (ti.get_notnull() ? " NOT NULL" : "");
5466  if (cd->default_value.has_value()) {
5467  os << " DEFAULT " << cd->getDefaultValueLiteral();
5468  }
5469  if (ti.is_string() || (ti.is_array() && ti.get_subtype() == kTEXT)) {
5470  auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
5471  if (ti.get_compression() == kENCODING_DICT) {
5472  // if foreign reference, get referenced tab.col
5473  const auto dict_id = ti.get_comp_param();
5474  const DictRef dict_ref(currentDB_.dbId, dict_id);
5475  const auto dict_it = dictDescriptorMapByRef_.find(dict_ref);
5476  CHECK(dict_it != dictDescriptorMapByRef_.end());
5477  const auto dict_name = dict_it->second->dictName;
5478  // when migrating a table, any foreign dict ref will be dropped
5479  // and the first cd of a dict will become root of the dict
5480  if (dict_root_cds.end() == dict_root_cds.find(dict_name)) {
5481  dict_root_cds[dict_name] = cd;
5482  os << " ENCODING " << ti.get_compression_name() << "(" << (size * 8) << ")";
5483  } else {
5484  const auto dict_root_cd = dict_root_cds[dict_name];
5485  shared_dicts.push_back("SHARED DICTIONARY (" + cd->columnName +
5486  ") REFERENCES @T(" + dict_root_cd->columnName + ")");
5487  // "... shouldn't specify an encoding, it borrows from the referenced
5488  // column"
5489  }
5490  } else {
5491  os << " ENCODING NONE";
5492  }
5493  } else if (ti.is_date_in_days() ||
5494  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
5495  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
5496  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
5497  } else if (ti.is_geometry()) {
5498  if (ti.get_compression() == kENCODING_GEOINT) {
5499  os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
5500  << ")";
5501  } else {
5502  os << " ENCODING NONE";
5503  }
5504  }
5505  comma = ", ";
5506  }
5507  }
5508  // gather SHARED DICTIONARYs
5509  if (shared_dicts.size()) {
5510  os << ", " << boost::algorithm::join(shared_dicts, ", ");
5511  }
5512  // gather WITH options ...
5513  std::vector<std::string> with_options;
5514  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
5515  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
5516  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
5517  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
5518  with_options.emplace_back(td->hasDeletedCol ? "VACUUM='DELAYED'"
5519  : "VACUUM='IMMEDIATE'");
5520  if (!td->partitions.empty()) {
5521  with_options.push_back("PARTITIONS='" + td->partitions + "'");
5522  }
5523  if (td->nShards > 0) {
5524  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
5525  CHECK(shard_cd);
5526  os << ", SHARD KEY(" << shard_cd->columnName << ")";
5527  with_options.push_back(
5528  "SHARD_COUNT=" +
5529  std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
5530  }
5531  if (td->sortedColumnId > 0) {
5532  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
5533  CHECK(sort_cd);
5534  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
5535  }
5537  td->maxRollbackEpochs != -1) {
5538  with_options.push_back("MAX_ROLLBACK_EPOCHS=" +
5540  }
5541  os << ") WITH (" + boost::algorithm::join(with_options, ", ") + ");";
5542  return os.str();
5543 }
int32_t maxRollbackEpochs
std::string partitions
const DBMetadata currentDB_
Definition: Catalog.h:721
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:711
std::string join(T const &container, std::string const &delim)
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
std::string to_string(char const *&&v)
std::string quoteIfRequired(const std::string &column_name) const
Definition: Catalog.cpp:5867
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
#define DEFAULT_MAX_ROLLBACK_EPOCHS
specifies the content in-memory of a row in the table metadata table
Definition: sqltypes.h:79
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:2172
Definition: sqltypes.h:68
#define CHECK(condition)
Definition: Logger.h:291
size_t g_leaf_count
Definition: ParserNode.cpp:78

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::eraseDbMetadata ( )

Definition at line 5049 of file Catalog.cpp.

References shared::kCatalogDirectoryName, and run_benchmark_import::tables.

5049  {
5050  const auto tables = getAllTableMetadata();
5051  for (const auto table : tables) {
5052  eraseTableMetadata(table);
5053  }
5054  // Physically erase database metadata
5055  boost::filesystem::remove(basePath_ + "/" + shared::kCatalogDirectoryName + "/" +
5056  currentDB_.dbName);
5057  calciteMgr_->updateMetadata(currentDB_.dbName, "");
5058 }
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:2188
const DBMetadata currentDB_
Definition: Catalog.h:721
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:725
const std::string kCatalogDirectoryName
void eraseTableMetadata(const TableDescriptor *td)
Definition: Catalog.cpp:4332
void Catalog_Namespace::Catalog::eraseDbPhysicalData ( )

Definition at line 5060 of file Catalog.cpp.

References run_benchmark_import::tables.

5060  {
5061  const auto tables = getAllTableMetadata();
5062  for (const auto table : tables) {
5063  eraseTablePhysicalData(table);
5064  }
5065 }
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:5067
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:2188
void Catalog_Namespace::Catalog::eraseTableMetadata ( const TableDescriptor td)
protected

Definition at line 4332 of file Catalog.cpp.

References g_serialize_temp_tables, INJECT_TIMER, table_is_temporary(), TableDescriptor::tableId, and TableDescriptor::tableName.

4332  {
4336  }
4337  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
4338  {
4339  INJECT_TIMER(removeTableFromMap_);
4341  }
4342 }
void executeDropTableSqliteQueries(const TableDescriptor *td)
Definition: Catalog.cpp:4344
std::string tableName
const DBMetadata currentDB_
Definition: Catalog.h:721
void dropTableFromJsonUnlocked(const std::string &table_name) const
Definition: Catalog.cpp:3251
#define INJECT_TIMER(DESC)
Definition: measure.h:96
bool g_serialize_temp_tables
Definition: Catalog.cpp:106
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:725
bool table_is_temporary(const TableDescriptor *const td)
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1601

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::eraseTablePhysicalData ( const TableDescriptor td)

Definition at line 5067 of file Catalog.cpp.

References Data_Namespace::CPU_LEVEL, Data_Namespace::GPU_LEVEL, INJECT_TIMER, TableDescriptor::isView, and TableDescriptor::tableId.

5067  {
5068  const int tableId = td->tableId;
5069  // must destroy fragmenter before deleteChunks is called.
5070  removeFragmenterForTable(tableId);
5071 
5072  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
5073  {
5074  INJECT_TIMER(deleteChunksWithPrefix);
5075  // assuming deleteChunksWithPrefix is atomic
5076  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
5077  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
5078  }
5079  if (!td->isView) {
5080  INJECT_TIMER(Remove_Table);
5081  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
5082  }
5083 }
std::vector< int > ChunkKey
Definition: types.h:36
const DBMetadata currentDB_
Definition: Catalog.h:721
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:722
#define INJECT_TIMER(DESC)
Definition: measure.h:96
void removeFragmenterForTable(const int table_id) const
Definition: Catalog.cpp:4260
template<typename F , typename... Args>
void Catalog_Namespace::Catalog::execInTransaction ( F &&  f,
Args &&...  args 
)
private

Definition at line 7002 of file Catalog.cpp.

References run_benchmark_import::args, and f().

7002  {
7003  cat_write_lock write_lock(this);
7004  cat_sqlite_lock sqlite_lock(this);
7005  sqliteConnector_.query("BEGIN TRANSACTION");
7006  try {
7007  (this->*f)(std::forward<Args>(args)...);
7008  } catch (std::exception&) {
7009  sqliteConnector_.query("ROLLBACK TRANSACTION");
7010  throw;
7011  }
7012  sqliteConnector_.query("END TRANSACTION");
7013 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
torch::Tensor f(torch::Tensor x, torch::Tensor W_target, torch::Tensor b_target)
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::executeDropTableSqliteQueries ( const TableDescriptor td)
protected

Definition at line 4344 of file Catalog.cpp.

References StorageType::FOREIGN_TABLE, TableDescriptor::isView, kENCODING_DICT, TableDescriptor::storageType, TableDescriptor::tableId, and to_string().

4344  {
4345  const int tableId = td->tableId;
4346  sqliteConnector_.query_with_text_param("DELETE FROM mapd_tables WHERE tableid = ?",
4347  std::to_string(tableId));
4349  "select comp_param from mapd_columns where compression = ? and tableid = ?",
4350  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
4351  int numRows = sqliteConnector_.getNumRows();
4352  std::vector<int> dict_id_list;
4353  for (int r = 0; r < numRows; ++r) {
4354  dict_id_list.push_back(sqliteConnector_.getData<int>(r, 0));
4355  }
4356  for (auto dict_id : dict_id_list) {
4358  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
4359  std::vector<std::string>{std::to_string(dict_id)});
4360  }
4362  "DELETE FROM mapd_dictionaries WHERE dictid in (select comp_param from "
4363  "mapd_columns where compression = ? "
4364  "and tableid = ?) and refcount = 0",
4365  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
4366  sqliteConnector_.query_with_text_param("DELETE FROM mapd_columns WHERE tableid = ?",
4367  std::to_string(tableId));
4368  if (td->isView) {
4369  sqliteConnector_.query_with_text_param("DELETE FROM mapd_views WHERE tableid = ?",
4370  std::to_string(tableId));
4371  }
4374  "DELETE FROM omnisci_foreign_tables WHERE table_id = ?", std::to_string(tableId));
4375  }
4376 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
std::string storageType
std::string to_string(char const *&&v)
static constexpr char const * FOREIGN_TABLE
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::expandGeoColumn ( const ColumnDescriptor cd,
std::list< ColumnDescriptor > &  columns 
)
static

Definition at line 2774 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, IS_GEO, kARRAY, kDOUBLE, kENCODING_GEOINT, kENCODING_NONE, kINT, kLINESTRING, kMULTILINESTRING, kMULTIPOINT, kMULTIPOLYGON, kPOINT, kPOLYGON, kTINYINT, SQLTypeInfo::set_size(), and SQLTypeInfo::set_subtype().

2775  {
2776  const auto& col_ti = cd.columnType;
2777  if (IS_GEO(col_ti.get_type())) {
2778  switch (col_ti.get_type()) {
2779  case kPOINT: {
2780  ColumnDescriptor physical_cd_coords(true);
2781  physical_cd_coords.columnName = cd.columnName + "_coords";
2782  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2783  // Raw data: compressed/uncompressed coords
2784  coords_ti.set_subtype(kTINYINT);
2785  size_t unit_size;
2786  if (col_ti.get_compression() == kENCODING_GEOINT &&
2787  col_ti.get_comp_param() == 32) {
2788  unit_size = 4 * sizeof(int8_t);
2789  } else {
2790  CHECK(col_ti.get_compression() == kENCODING_NONE);
2791  unit_size = 8 * sizeof(int8_t);
2792  }
2793  coords_ti.set_size(2 * unit_size);
2794  physical_cd_coords.columnType = coords_ti;
2795  columns.push_back(physical_cd_coords);
2796 
2797  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2798 
2799  break;
2800  }
2801  case kMULTIPOINT:
2802  case kLINESTRING: {
2803  ColumnDescriptor physical_cd_coords(true);
2804  physical_cd_coords.columnName = cd.columnName + "_coords";
2805  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2806  // Raw data: compressed/uncompressed coords
2807  coords_ti.set_subtype(kTINYINT);
2808  physical_cd_coords.columnType = coords_ti;
2809  columns.push_back(physical_cd_coords);
2810 
2811  ColumnDescriptor physical_cd_bounds(true);
2812  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2813  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2814  bounds_ti.set_subtype(kDOUBLE);
2815  bounds_ti.set_size(4 * sizeof(double));
2816  physical_cd_bounds.columnType = bounds_ti;
2817  columns.push_back(physical_cd_bounds);
2818 
2819  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2820 
2821  break;
2822  }
2823  case kMULTILINESTRING: {
2824  ColumnDescriptor physical_cd_coords(true);
2825  physical_cd_coords.columnName = cd.columnName + "_coords";
2826  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2827  // Raw data: compressed/uncompressed coords
2828  coords_ti.set_subtype(kTINYINT);
2829  physical_cd_coords.columnType = coords_ti;
2830  columns.push_back(physical_cd_coords);
2831 
2832  ColumnDescriptor physical_cd_linestring_sizes(true);
2833  physical_cd_linestring_sizes.columnName = cd.columnName + "_linestring_sizes";
2834  SQLTypeInfo linestring_sizes_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2835  linestring_sizes_ti.set_subtype(kINT);
2836  physical_cd_linestring_sizes.columnType = linestring_sizes_ti;
2837  columns.push_back(physical_cd_linestring_sizes);
2838 
2839  ColumnDescriptor physical_cd_bounds(true);
2840  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2841  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2842  bounds_ti.set_subtype(kDOUBLE);
2843  bounds_ti.set_size(4 * sizeof(double));
2844  physical_cd_bounds.columnType = bounds_ti;
2845  columns.push_back(physical_cd_bounds);
2846 
2847  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2848 
2849  break;
2850  }
2851  case kPOLYGON: {
2852  ColumnDescriptor physical_cd_coords(true);
2853  physical_cd_coords.columnName = cd.columnName + "_coords";
2854  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2855  // Raw data: compressed/uncompressed coords
2856  coords_ti.set_subtype(kTINYINT);
2857  physical_cd_coords.columnType = coords_ti;
2858  columns.push_back(physical_cd_coords);
2859 
2860  ColumnDescriptor physical_cd_ring_sizes(true);
2861  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
2862  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2863  ring_sizes_ti.set_subtype(kINT);
2864  physical_cd_ring_sizes.columnType = ring_sizes_ti;
2865  columns.push_back(physical_cd_ring_sizes);
2866 
2867  ColumnDescriptor physical_cd_bounds(true);
2868  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2869  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2870  bounds_ti.set_subtype(kDOUBLE);
2871  bounds_ti.set_size(4 * sizeof(double));
2872  physical_cd_bounds.columnType = bounds_ti;
2873  columns.push_back(physical_cd_bounds);
2874 
2875  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2876 
2877  break;
2878  }
2879  case kMULTIPOLYGON: {
2880  ColumnDescriptor physical_cd_coords(true);
2881  physical_cd_coords.columnName = cd.columnName + "_coords";
2882  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2883  // Raw data: compressed/uncompressed coords
2884  coords_ti.set_subtype(kTINYINT);
2885  physical_cd_coords.columnType = coords_ti;
2886  columns.push_back(physical_cd_coords);
2887 
2888  ColumnDescriptor physical_cd_ring_sizes(true);
2889  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
2890  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2891  ring_sizes_ti.set_subtype(kINT);
2892  physical_cd_ring_sizes.columnType = ring_sizes_ti;
2893  columns.push_back(physical_cd_ring_sizes);
2894 
2895  ColumnDescriptor physical_cd_poly_rings(true);
2896  physical_cd_poly_rings.columnName = cd.columnName + "_poly_rings";
2897  SQLTypeInfo poly_rings_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2898  poly_rings_ti.set_subtype(kINT);
2899  physical_cd_poly_rings.columnType = poly_rings_ti;
2900  columns.push_back(physical_cd_poly_rings);
2901 
2902  ColumnDescriptor physical_cd_bounds(true);
2903  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2904  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2905  bounds_ti.set_subtype(kDOUBLE);
2906  bounds_ti.set_size(4 * sizeof(double));
2907  physical_cd_bounds.columnType = bounds_ti;
2908  columns.push_back(physical_cd_bounds);
2909 
2910  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2911 
2912  break;
2913  }
2914  default:
2915  throw runtime_error("Unrecognized geometry type.");
2916  break;
2917  }
2918  }
2919 }
void set_size(int s)
Definition: sqltypes.h:476
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:469
specifies the content in-memory of a row in the column metadata table
#define CHECK(condition)
Definition: Logger.h:291
Definition: sqltypes.h:72
SQLTypeInfo columnType
std::string columnName
#define IS_GEO(T)
Definition: sqltypes.h:310

+ Here is the call graph for this function:

bool Catalog_Namespace::Catalog::filterTableByTypeAndUser ( const TableDescriptor td,
const UserMetadata user_metadata,
const GetTablesType  get_tables_type 
) const
private

Definition at line 4936 of file Catalog.cpp.

References GET_PHYSICAL_TABLES, GET_VIEWS, TableDescriptor::isView, DBObject::loadKey(), TableDescriptor::shard, TableDBObjectType, TableDescriptor::tableName, and ViewDBObjectType.

4938  {
4939  if (td->shard >= 0) {
4940  // skip shards, they're not standalone tables
4941  return false;
4942  }
4943  switch (get_tables_type) {
4944  case GET_PHYSICAL_TABLES: {
4945  if (td->isView) {
4946  return false;
4947  }
4948  break;
4949  }
4950  case GET_VIEWS: {
4951  if (!td->isView) {
4952  return false;
4953  }
4954  break;
4955  }
4956  default:
4957  break;
4958  }
4960  dbObject.loadKey(*this);
4961  std::vector<DBObject> privObjects = {dbObject};
4962  if (!SysCatalog::instance().hasAnyPrivileges(user_metadata, privObjects)) {
4963  // skip table, as there are no privileges to access it
4964  return false;
4965  }
4966  return true;
4967 }
std::string tableName
static SysCatalog & instance()
Definition: SysCatalog.h:343

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::gatherAdditionalInfo ( std::vector< std::string > &  additional_info,
std::set< std::string > &  shared_dict_column_names,
const TableDescriptor td 
) const
private

Definition at line 5878 of file Catalog.cpp.

References CHECK, logger::ERROR, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), kENCODING_DICT, LOG, TableDescriptor::nShards, TableDescriptor::shardedColumnId, TableDescriptor::tableId, and TableDescriptor::tableName.

5880  {
5881  if (td->nShards > 0) {
5882  ColumnIdKey columnIdKey(td->tableId, td->shardedColumnId);
5883  auto scd = columnDescriptorMapById_.find(columnIdKey)->second;
5884  CHECK(scd);
5885  std::string txt = "SHARD KEY (" + quoteIfRequired(scd->columnName) + ")";
5886  additional_info.emplace_back(txt);
5887  }
5888  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
5889  for (const auto cd : cds) {
5890  if (!(cd->isSystemCol || cd->isVirtualCol)) {
5891  const SQLTypeInfo& ti = cd->columnType;
5892  if (ti.get_compression() != kENCODING_DICT) {
5893  continue;
5894  }
5895  auto dictId = ti.get_comp_param();
5896 
5897  // now we need to check how many other users have this dictionary
5898 
5899  DictRef dict_ref(currentDB_.dbId, dictId);
5900  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
5901  if (dictIt == dictDescriptorMapByRef_.end()) {
5902  LOG(ERROR) << "missing dictionary " << dictId << " for table " << td->tableName;
5903  continue;
5904  }
5905 
5906  const auto& dd = dictIt->second;
5907  if (dd->refcount > 1) {
5908  auto lowest_table = td->tableId;
5909  auto lowest_column = cd->columnId;
5910  std::string lowest_column_name;
5911  // we have multiple tables using this dictionary
5912  // find the other occurances and keep the "lowest"
5913  for (auto const& [key, val] : columnDescriptorMap_) {
5914  if (val->columnType.get_compression() == kENCODING_DICT &&
5915  val->columnType.get_comp_param() == dictId &&
5916  !(val->tableId == td->tableId && val->columnId == cd->columnId)) {
5917  if (val->tableId < lowest_table) {
5918  lowest_table = val->tableId;
5919  lowest_column = val->columnId;
5920  lowest_column_name = val->columnName;
5921  }
5922  if (val->columnId < lowest_column) {
5923  lowest_column = val->columnId;
5924  lowest_column_name = val->columnName;
5925  }
5926  }
5927  }
5928  if (lowest_table != td->tableId || lowest_column != cd->columnId) {
5929  // we are referencing a different tables dictionary
5930  auto lowest_td = tableDescriptorMapById_.find(lowest_table)->second;
5931  CHECK(lowest_td);
5932  std::string txt = "SHARED DICTIONARY (" + quoteIfRequired(cd->columnName) +
5933  ") REFERENCES " + lowest_td->tableName + "(" +
5934  quoteIfRequired(lowest_column_name) + ")";
5935 
5936  additional_info.emplace_back(txt);
5937  shared_dict_column_names.insert(cd->columnName);
5938  }
5939  }
5940  }
5941  }
5942 }
std::string tableName
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:709
#define LOG(tag)
Definition: Logger.h:285
const DBMetadata currentDB_
Definition: Catalog.h:721
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:711
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:710
std::string quoteIfRequired(const std::string &column_name) const
Definition: Catalog.cpp:5867
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:399
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:2172
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:402
#define CHECK(condition)
Definition: Logger.h:291
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::generatePhysicalTableName ( const std::string &  logicalTableName,
const size_t  shardNumber 
)
protected

Definition at line 5085 of file Catalog.cpp.

References to_string().

5086  {
5087  std::string physicalTableName =
5088  logicalTableName + physicalTableNameTag_ + std::to_string(shardNumber);
5089  return (physicalTableName);
5090 }
std::string to_string(char const *&&v)
static const std::string physicalTableNameTag_
Definition: Catalog.h:729

+ Here is the call graph for this function:

std::list< const ColumnDescriptor * > Catalog_Namespace::Catalog::getAllColumnMetadataForTable ( const int  tableId,
const bool  fetchSystemColumns,
const bool  fetchVirtualColumns,
const bool  fetchPhysicalColumns 
) const

Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular table specified by table id.

Parameters
tableIdtable id we want the column metadata for
Returns
list of pointers to const ColumnDescriptor structs - one for each and every column in the table

Definition at line 2172 of file Catalog.cpp.

Referenced by Analyzer::RangeTableEntry::add_all_column_descs(), Parser::InsertStmt::analyze(), Parser::InsertValuesStmt::analyze(), Fragmenter_Namespace::copyDataOfShard(), anonymous_namespace{ForeignDataImporter.cpp}::create_fragment_buffers(), TableArchiver::dumpTable(), Analyzer::RangeTableEntry::expand_star_in_targetlist(), import_export::fill_missing_columns(), import_export::ForeignDataImporter::finalize(), import_export::anonymous_namespace{ForeignDataImporter.cpp}::get_proxy_foreign_table_fragment_size(), anonymous_namespace{TableOptimizer.cpp}::get_uncached_cpu_chunk_keys(), Parser::InsertValuesStmt::InsertValuesStmt(), TableOptimizer::recomputeMetadata(), TableArchiver::restoreTable(), AlterTableAlterColumnCommandRecoveryMgr::rollback(), and validate_shared_dictionary().

2176  {
2177  cat_read_lock read_lock(this);
2178  std::list<const ColumnDescriptor*> columnDescriptors;
2181  columnDescriptors,
2182  fetchSystemColumns,
2183  fetchVirtualColumns,
2184  fetchPhysicalColumns);
2185  return columnDescriptors;
2186 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
TableDescriptor * getMutableMetadataForTableUnlocked(int table_id) const
Definition: Catalog.cpp:1896
void getAllColumnMetadataForTableImpl(const TableDescriptor *td, std::list< const ColumnDescriptor * > &colDescs, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Definition: Catalog.cpp:2142

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::getAllColumnMetadataForTableImpl ( const TableDescriptor td,
std::list< const ColumnDescriptor * > &  colDescs,
const bool  fetchSystemColumns,
const bool  fetchVirtualColumns,
const bool  fetchPhysicalColumns 
) const
protected

Definition at line 2142 of file Catalog.cpp.

References TableDescriptor::tableId.

2147  {
2148  int32_t skip_physical_cols = 0;
2149  for (const auto& columnDescriptor : columnDescriptorMapById_) {
2150  if (!fetchPhysicalColumns && skip_physical_cols > 0) {
2151  --skip_physical_cols;
2152  continue;
2153  }
2154  auto cd = columnDescriptor.second;
2155  if (cd->tableId != td->tableId) {
2156  continue;
2157  }
2158  if (!fetchSystemColumns && cd->isSystemCol) {
2159  continue;
2160  }
2161  if (!fetchVirtualColumns && cd->isVirtualCol) {
2162  continue;
2163  }
2164  if (!fetchPhysicalColumns) {
2165  const auto& col_ti = cd->columnType;
2166  skip_physical_cols = col_ti.get_physical_cols();
2167  }
2168  columnDescriptors.push_back(cd);
2169  }
2170 }
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:710
list< const DashboardDescriptor * > Catalog_Namespace::Catalog::getAllDashboardsMetadata ( ) const

Definition at line 2208 of file Catalog.cpp.

2208  {
2209  cat_read_lock read_lock(this);
2210  list<const DashboardDescriptor*> dashboards;
2211  for (auto dashboard_entry : dashboardDescriptorMap_) {
2212  dashboards.push_back(dashboard_entry.second.get());
2213  }
2214  return dashboards;
2215 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:712
std::vector< DashboardDescriptor > Catalog_Namespace::Catalog::getAllDashboardsMetadataCopy ( ) const

Definition at line 2217 of file Catalog.cpp.

2217  {
2218  cat_read_lock read_lock(this);
2219  std::vector<DashboardDescriptor> dashboards;
2220  dashboards.reserve(dashboardDescriptorMap_.size());
2221  for (auto dashboard_entry : dashboardDescriptorMap_) {
2222  dashboards.emplace_back(*dashboard_entry.second);
2223  }
2224  return dashboards;
2225 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:712
std::list< const DictDescriptor * > Catalog_Namespace::Catalog::getAllDictionariesWithColumnInName ( const ColumnDescriptor cd)

Definition at line 2323 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, and ColumnDescriptor::tableId.

Referenced by AlterTableAlterColumnCommandRecoveryMgr::rollback().

2324  {
2325  cat_read_lock read_lock(this);
2326  std::list<const DictDescriptor*> dds;
2327 
2328  auto table_name_opt = getTableName(cd->tableId);
2329  CHECK(table_name_opt.has_value());
2330  auto table_name = table_name_opt.value();
2331 
2332  for (const auto& [dkey, dd] : dictDescriptorMapByRef_) {
2333  if (dd->dictName.find(table_name + "_" + cd->columnName + "_dict") !=
2334  std::string::npos) {
2335  dds.push_back(dd.get());
2336  }
2337  }
2338 
2339  return dds;
2340 }
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:711
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
#define CHECK(condition)
Definition: Logger.h:291
std::optional< std::string > getTableName(int32_t table_id) const
Definition: Catalog.cpp:1869
std::string columnName

+ Here is the caller graph for this function:

std::vector< const foreign_storage::ForeignTable * > Catalog_Namespace::Catalog::getAllForeignTablesForForeignServer ( const int32_t  foreign_server_id)

Get all of the foreign tables for associated with a foreign server id.

Parameters
foreign_server_id- id of foreign server
Returns
std::vector<const ForeignTable*> a vector containing foreign tables which use the foreign server

Definition at line 3631 of file Catalog.cpp.

References CHECK, and StorageType::FOREIGN_TABLE.

3631  {
3632  cat_read_lock read_lock(this);
3633  std::vector<const foreign_storage::ForeignTable*> foreign_tables;
3634  for (auto entry : tableDescriptorMapById_) {
3635  auto table_descriptor = entry.second;
3636  if (table_descriptor->storageType == StorageType::FOREIGN_TABLE) {
3637  auto foreign_table = dynamic_cast<foreign_storage::ForeignTable*>(table_descriptor);
3638  CHECK(foreign_table);
3639  if (foreign_table->foreign_server->id == foreign_server_id) {
3640  foreign_tables.emplace_back(foreign_table);
3641  }
3642  }
3643  }
3644  return foreign_tables;
3645 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
#define CHECK(condition)
Definition: Logger.h:291
static constexpr char const * FOREIGN_TABLE
std::vector< const TableDescriptor * > Catalog_Namespace::Catalog::getAllForeignTablesForRefresh ( ) const

Gets all the foreign tables that are pending refreshes. The list of tables includes tables that are configured for scheduled refreshes with next refresh timestamps that are in the past.

Returns
foreign tables pending refreshes

Definition at line 5783 of file Catalog.cpp.

References CHECK, StorageType::FOREIGN_TABLE, foreign_storage::RefreshTimeCalculator::getCurrentTime(), foreign_storage::ForeignTable::REFRESH_TIMING_TYPE_KEY, foreign_storage::ForeignTable::SCHEDULE_REFRESH_TIMING_TYPE, and run_benchmark_import::tables.

5783  {
5784  cat_read_lock read_lock(this);
5785  std::vector<const TableDescriptor*> tables;
5786  for (auto entry : tableDescriptorMapById_) {
5787  auto table_descriptor = entry.second;
5788  if (table_descriptor->storageType == StorageType::FOREIGN_TABLE) {
5789  auto foreign_table = dynamic_cast<foreign_storage::ForeignTable*>(table_descriptor);
5790  CHECK(foreign_table);
5791  auto timing_type_entry = foreign_table->options.find(
5793  CHECK(timing_type_entry != foreign_table->options.end());
5795  if (timing_type_entry->second ==
5797  foreign_table->next_refresh_time <= current_time) {
5798  tables.emplace_back(foreign_table);
5799  }
5800  }
5801  }
5802  return tables;
5803 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
static constexpr const char * REFRESH_TIMING_TYPE_KEY
Definition: ForeignTable.h:43
#define CHECK(condition)
Definition: Logger.h:291
static constexpr char const * FOREIGN_TABLE
static constexpr const char * SCHEDULE_REFRESH_TIMING_TYPE
Definition: ForeignTable.h:53

+ Here is the call graph for this function:

std::vector< std::pair< int32_t, int32_t > > Catalog_Namespace::Catalog::getAllPersistedTableAndShardIds ( ) const

Definition at line 4889 of file Catalog.cpp.

4890  {
4891  cat_read_lock read_lock(this);
4892  std::vector<std::pair<int32_t, int32_t>> table_and_shard_ids;
4893  table_and_shard_ids.reserve(tableDescriptorMapById_.size());
4894  for (const auto [table_id, td] : tableDescriptorMapById_) {
4895  // Only include ids for physical persisted tables
4896  if (!td->isView && !td->isTemporaryTable() && !td->isForeignTable() &&
4897  logicalToPhysicalTableMapById_.find(table_id) ==
4899  table_and_shard_ids.emplace_back(table_id, td->shard);
4900  }
4901  }
4902  return table_and_shard_ids;
4903 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:727
list< const TableDescriptor * > Catalog_Namespace::Catalog::getAllTableMetadata ( ) const

Definition at line 2188 of file Catalog.cpp.

Referenced by system_validator::SingleNodeValidator::validate().

2188  {
2189  cat_read_lock read_lock(this);
2190  list<const TableDescriptor*> table_list;
2191  for (auto p : tableDescriptorMapById_) {
2192  table_list.push_back(p.second);
2193  }
2194  return table_list;
2195 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708

+ Here is the caller graph for this function:

std::vector< TableDescriptor > Catalog_Namespace::Catalog::getAllTableMetadataCopy ( ) const

Definition at line 2197 of file Catalog.cpp.

References run_benchmark_import::tables.

2197  {
2198  cat_read_lock read_lock(this);
2199  std::vector<TableDescriptor> tables;
2200  tables.reserve(tableDescriptorMapById_.size());
2201  for (auto table_entry : tableDescriptorMapById_) {
2202  tables.emplace_back(*table_entry.second);
2203  tables.back().fragmenter = nullptr;
2204  }
2205  return tables;
2206 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
const TableDescriptor * Catalog_Namespace::Catalog::getCachedTableDescriptor ( const std::map< std::string, int > &  cached_table_map,
const std::string &  cur_table_name 
)
private

Definition at line 4480 of file Catalog.cpp.

4482  {
4483  if (auto it = cached_table_map.find(cur_table_name); it != cached_table_map.end()) {
4484  auto table_id = it->second;
4485  return (table_id == -1) ? NULL : getMetadataForTable(table_id);
4486  }
4487  return getMetadataForTable(cur_table_name);
4488 }
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
std::shared_ptr<Calcite> Catalog_Namespace::Catalog::getCalciteMgr ( ) const
inline

Definition at line 267 of file Catalog.h.

References calciteMgr_.

267 { return calciteMgr_; }
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:725
const std::string& Catalog_Namespace::Catalog::getCatalogBasePath ( ) const
inline

Definition at line 273 of file Catalog.h.

References basePath_.

Referenced by AlterTableAlterColumnCommandRecoveryMgr::getRecoveryFilepathInfo().

273 { return basePath_; }

+ Here is the caller graph for this function:

std::string Catalog_Namespace::Catalog::getColumnDictDirectory ( const ColumnDescriptor cd,
bool  file_name_only = true 
) const

Definition at line 5388 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnType, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::is_string(), SQLTypeInfo::is_string_array(), and kENCODING_DICT.

Referenced by TableArchiver::dumpTable(), and TableArchiver::restoreTable().

5389  {
5390  if ((cd->columnType.is_string() || cd->columnType.is_string_array()) &&
5392  cd->columnType.get_comp_param() > 0) {
5393  const auto dictId = cd->columnType.get_comp_param();
5394  const DictRef dictRef(currentDB_.dbId, dictId);
5395  const auto dit = dictDescriptorMapByRef_.find(dictRef);
5396  CHECK(dit != dictDescriptorMapByRef_.end());
5397  CHECK(dit->second);
5398  if (file_name_only) {
5399  boost::filesystem::path file_path(dit->second->dictFolderPath);
5400  return file_path.filename().string();
5401  } else {
5402  return dit->second->dictFolderPath;
5403  }
5404  }
5405  return std::string();
5406 }
const DBMetadata currentDB_
Definition: Catalog.h:721
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:711
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:399
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:402
#define CHECK(condition)
Definition: Logger.h:291
SQLTypeInfo columnType
bool is_string() const
Definition: sqltypes.h:559
bool is_string_array() const
Definition: sqltypes.h:562

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const int Catalog_Namespace::Catalog::getColumnIdBySpi ( const int  tableId,
const size_t  spi 
) const

Definition at line 1996 of file Catalog.cpp.

1996  {
1997  cat_read_lock read_lock(this);
1998  return getColumnIdBySpiUnlocked(table_id, spi);
1999 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
const int getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const
Definition: Catalog.cpp:1978
const int Catalog_Namespace::Catalog::getColumnIdBySpiUnlocked ( const int  table_id,
const size_t  spi 
) const
protected

Definition at line 1978 of file Catalog.cpp.

References CHECK, SPIMAP_MAGIC1, and SPIMAP_MAGIC2.

1978  {
1979  const auto tabDescIt = tableDescriptorMapById_.find(table_id);
1980  CHECK(tableDescriptorMapById_.end() != tabDescIt);
1981  const auto& columnIdBySpi = tabDescIt->second->columnIdBySpi_;
1982 
1983  auto spx = spi;
1984  int phi = 0;
1985  if (spx >= SPIMAP_MAGIC1) // see Catalog.h
1986  {
1987  phi = (spx - SPIMAP_MAGIC1) % SPIMAP_MAGIC2;
1988  spx = (spx - SPIMAP_MAGIC1) / SPIMAP_MAGIC2;
1989  }
1990 
1991  CHECK(0 < spx && spx <= columnIdBySpi.size())
1992  << "spx = " << spx << ", size = " << columnIdBySpi.size();
1993  return columnIdBySpi[spx - 1] + phi;
1994 }
#define SPIMAP_MAGIC1
Definition: Catalog.h:82
#define SPIMAP_MAGIC2
Definition: Catalog.h:83
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
#define CHECK(condition)
Definition: Logger.h:291
const std::optional< std::string > Catalog_Namespace::Catalog::getColumnName ( int  table_id,
int  column_id 
) const

Definition at line 1968 of file Catalog.cpp.

1969  {
1970  cat_read_lock read_lock(this);
1971  auto it = columnDescriptorMapById_.find(ColumnIdKey{table_id, column_id});
1972  if (it == columnDescriptorMapById_.end()) {
1973  return {};
1974  }
1975  return it->second->columnName;
1976 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:710
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39
const DBMetadata& Catalog_Namespace::Catalog::getCurrentDB ( ) const
inline

Definition at line 265 of file Catalog.h.

References currentDB_.

Referenced by lockmgr::TableSchemaLockContainer< ReadLock >::acquireTableDescriptor(), lockmgr::TableSchemaLockContainer< WriteLock >::acquireTableDescriptor(), checkDateInDaysColumnMigration(), lockmgr::helpers::chunk_key_for_table(), AlterTableAlterColumnCommandRecoveryMgr::cleanupDropSourceGeoColumns(), Catalog_Namespace::SysCatalog::deleteObjectDescriptorMap(), TableArchiver::dumpTable(), Fragmenter_Namespace::get_chunks(), getChunkAccessorTable(), Fragmenter_Namespace::InsertOrderFragmenter::getChunksForAllColumns(), AlterTableAlterColumnCommandRecoveryMgr::getRecoveryFilepathInfo(), Parser::getTableChunkKey(), import_export::Loader::init(), DBHandler::interruptQuery(), DBObject::loadKey(), name(), query_state::operator<<(), TableOptimizer::recomputeMetadata(), AlterTableAlterColumnCommandRecoveryMgr::recoverAlterTableAlterColumnFromFile(), foreign_storage::refresh_foreign_table_unlocked(), ForeignStorageInterface::registerTable(), Catalog_Namespace::SysCatalog::renameDBObject(), Catalog_Namespace::SysCatalog::renameObjectsInDescriptorMap(), TableArchiver::restoreTable(), AlterTableAlterColumnCommandRecoveryMgr::rollback(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumns(), Catalog_Namespace::SysCatalog::updateObjectDescriptorMap(), lockmgr::validate_table_descriptor_after_lock(), and StorageIOFacility::yieldUpdateCallback().

265 { return currentDB_; }
const DBMetadata currentDB_
Definition: Catalog.h:721

+ Here is the caller graph for this function:

const CustomExpression * Catalog_Namespace::Catalog::getCustomExpression ( int32_t  custom_expression_id) const

Gets a pointer to the custom expression object with the given id.

Parameters
custom_expression_id- id of custom expression to get
Returns
pointer to custom expression object. nullptr is returned if no custom expression is found for the given id.

Definition at line 5993 of file Catalog.cpp.

5993  {
5994  cat_read_lock read_lock(this);
5995  auto it = custom_expr_map_by_id_.find(custom_expression_id);
5996  if (it != custom_expr_map_by_id_.end()) {
5997  return it->second.get();
5998  }
5999  return nullptr;
6000 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:717
std::unique_ptr< CustomExpression > Catalog_Namespace::Catalog::getCustomExpressionFromConnector ( size_t  row)
private

Definition at line 1526 of file Catalog.cpp.

References setup::name.

1526  {
1527  auto id = sqliteConnector_.getData<int>(row, 0);
1528  auto name = sqliteConnector_.getData<string>(row, 1);
1529  auto expression_json = sqliteConnector_.getData<string>(row, 2);
1530  auto data_source_type_str = sqliteConnector_.getData<string>(row, 3);
1531  auto data_source_id = sqliteConnector_.getData<int>(row, 4);
1532  auto is_deleted = sqliteConnector_.getData<bool>(row, 5);
1533  return std::make_unique<CustomExpression>(
1534  id,
1535  name,
1536  expression_json,
1537  CustomExpression::dataSourceTypeFromString(data_source_type_str),
1538  data_source_id,
1539  is_deleted);
1540 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
static DataSourceType dataSourceTypeFromString(const std::string &type_str)
std::string name() const
Definition: Catalog.h:348
const std::unique_ptr< const CustomExpression > Catalog_Namespace::Catalog::getCustomExpressionFromStorage ( int32_t  custom_expression_id)

Gets a pointer to a struct containing custom expression details fetched from storage. This is intended for use in tests, when asserting that expected custom expression data is persisted.

Parameters
custom_expression_id- id of custom expression to get
Returns
pointer to custom expression object. nullptr is returned if no custom expression is found for the given id.

Definition at line 6002 of file Catalog.cpp.

References CHECK_EQ, and to_string().

6003  {
6004  cat_sqlite_lock sqlite_lock(getObjForLock());
6006  "SELECT id, name, expression_json, data_source_type, data_source_id, "
6007  "is_deleted FROM omnisci_custom_expressions WHERE id = ?",
6008  std::vector<std::string>{to_string(custom_expression_id)});
6009  if (sqliteConnector_.getNumRows() > 0) {
6010  CHECK_EQ(sqliteConnector_.getNumRows(), static_cast<size_t>(1));
6012  }
6013  return nullptr;
6014 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
std::unique_ptr< CustomExpression > getCustomExpressionFromConnector(size_t row)
Definition: Catalog.cpp:1526
std::string to_string(char const *&&v)
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

std::vector< const CustomExpression * > Catalog_Namespace::Catalog::getCustomExpressionsForUser ( const UserMetadata user) const

Gets pointers to all the custom expression objects that the given user has access to. For custom expressions that are associated with a table data source, custom expressions for tables that the given user has SELECT access to are returned.

Parameters
user- user for which to get accessible custom expressions
Returns
pointer to custom expression objects that the given user has access to

Definition at line 6016 of file Catalog.cpp.

References CHECK, AccessPrivileges::SELECT_FROM_TABLE, and TableDBObjectType.

6017  {
6018  std::vector<const CustomExpression*> all_custom_expressions;
6019  {
6020  // Get custom expression pointers separately in order to avoid holding the catalog
6021  // read lock while checking privileges (which may cause a deadlock).
6022  cat_read_lock read_lock(this);
6023  for (const auto& [id, custom_expression] : custom_expr_map_by_id_) {
6024  all_custom_expressions.emplace_back(custom_expression.get());
6025  }
6026  }
6027 
6028  std::vector<const CustomExpression*> filtered_custom_expressions;
6029  for (const auto custom_expression : all_custom_expressions) {
6030  CHECK(custom_expression->data_source_type == DataSourceType::TABLE);
6031  DBObject db_object{custom_expression->data_source_id, TableDBObjectType};
6032  db_object.loadKey(*this);
6033  db_object.setPrivileges(AccessPrivileges::SELECT_FROM_TABLE);
6034  if (SysCatalog::instance().checkPrivileges(user, {db_object})) {
6035  filtered_custom_expressions.emplace_back(custom_expression);
6036  }
6037  }
6038  return filtered_custom_expressions;
6039 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
static const AccessPrivileges SELECT_FROM_TABLE
Definition: DBObject.h:160
static SysCatalog & instance()
Definition: SysCatalog.h:343
#define CHECK(condition)
Definition: Logger.h:291
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:717
const std::string Catalog_Namespace::Catalog::getCustomExpressionsSchema ( bool  if_not_exists = false)
static

Gets the DDL statement used to create the custom expressions table.

Parameters
if_not_exists- flag the indicates whether or not to include the "IF NOT EXISTS" phrase in the DDL statement.
Returns
string containing DDL statement

Definition at line 790 of file Catalog.cpp.

Referenced by Catalog_Namespace::SysCatalog::createDatabase(), and updateCustomExpressionsSchema().

790  {
791  return "CREATE TABLE " + (if_not_exists ? std::string{"IF NOT EXISTS "} : "") +
792  "omnisci_custom_expressions(id integer primary key, name text, " +
793  "expression_json text, data_source_type text, " +
794  "data_source_id integer, is_deleted boolean)";
795 }

+ Here is the caller graph for this function:

int Catalog_Namespace::Catalog::getDatabaseId ( ) const
inline

Definition at line 326 of file Catalog.h.

References currentDB_, and Catalog_Namespace::DBMetadata::dbId.

Referenced by UpdelRoll::addDirtyChunk(), Parser::ColumnRef::analyze(), UpdelRoll::cancelUpdate(), AlterTableAlterColumnCommandRecoveryMgr::cleanupClearRemainingChunks(), UpdelRoll::commitUpdate(), anonymous_namespace{ForeignDataImporter.cpp}::create_fragment_buffers(), DictionaryValueConverter< TARGET_TYPE >::DictionaryValueConverter(), Analyzer::RangeTableEntry::expand_star_in_targetlist(), anonymous_namespace{DdlCommandExecutor.cpp}::get_agg_storage_stats(), anonymous_namespace{TableOptimizer.cpp}::get_uncached_cpu_chunk_keys(), TableOptimizer::getDeletedColumnStats(), lockmgr::TableLockMgrImpl< TableDataLockMgr >::getMutexTracker(), getSyntheticColumnVar(), anonymous_namespace{ForeignDataImporter.cpp}::load_foreign_data_buffers(), Catalog_Namespace::SysCatalog::reassignObjectOwners(), TableOptimizer::recomputeColumnMetadata(), AlterTableAlterColumnCommandRecoveryMgr::recoverAlterTableAlterColumnFromFile(), foreign_storage::refresh_foreign_table_unlocked(), AlterTableAlterColumnCommandRecoveryMgr::rollback(), Fragmenter_Namespace::LocalInsertConnector::rollback(), UpdelRoll::stageUpdate(), TableOptimizer::vacuumDeletedRows(), TableOptimizer::vacuumFragments(), TableOptimizer::vacuumFragmentsAboveMinSelectivity(), system_validator::SingleNodeValidator::validate(), anonymous_namespace{DdlCommandExecutor.cpp}::validate_alter_type_metadata(), and anonymous_namespace{QueryPhysicalInputsCollector.cpp}::RelAlgPhysicalTableInputsVisitor::visit().

326 { return currentDB_.dbId; }
const DBMetadata currentDB_
Definition: Catalog.h:721

+ Here is the caller graph for this function:

Data_Namespace::DataMgr& Catalog_Namespace::Catalog::getDataMgr ( ) const
inline

Definition at line 266 of file Catalog.h.

References dataMgr_.

Referenced by UpdelRoll::cancelUpdate(), AlterTableAlterColumnCommandRecoveryMgr::cleanupClearChunk(), AlterTableAlterColumnCommandRecoveryMgr::cleanupClearRemainingChunks(), AlterTableAlterColumnCommandRecoveryMgr::cleanupDropSourceGeoColumns(), anonymous_namespace{TableOptimizer.cpp}::delete_cpu_chunks(), TableArchiver::dumpTable(), anonymous_namespace{DdlCommandExecutor.cpp}::get_agg_storage_stats(), Fragmenter_Namespace::get_chunks(), anonymous_namespace{TableOptimizer.cpp}::get_uncached_cpu_chunk_keys(), getChunkAccessorTable(), Fragmenter_Namespace::InsertOrderFragmenter::getChunksForAllColumns(), anonymous_namespace{DBHandler.cpp}::log_system_cpu_memory_status(), TableOptimizer::recomputeMetadata(), AlterTableAlterColumnCommandRecoveryMgr::recoverAlterTableAlterColumnFromFile(), foreign_storage::refresh_foreign_table_unlocked(), TableArchiver::restoreTable(), AlterTableAlterColumnCommandRecoveryMgr::rollback(), UpdelRoll::stageUpdate(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), UpdelRoll::updateFragmenterAndCleanupChunks(), TableOptimizer::vacuumDeletedRows(), TableOptimizer::vacuumFragments(), and anonymous_namespace{DdlCommandExecutor.cpp}::validate_alter_type_metadata().

266 { return *dataMgr_; }
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:722

+ Here is the caller graph for this function:

const ColumnDescriptor * Catalog_Namespace::Catalog::getDeletedColumn ( const TableDescriptor td) const

Definition at line 3897 of file Catalog.cpp.

Referenced by TableOptimizer::getDeletedColumnStats(), TableOptimizer::recomputeDeletedColumnMetadata(), and TableOptimizer::vacuumFragments().

3897  {
3898  cat_read_lock read_lock(this);
3899  const auto it = deletedColumnPerTable_.find(td);
3900  return it != deletedColumnPerTable_.end() ? it->second : nullptr;
3901 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
DeletedColumnPerTableMap deletedColumnPerTable_
Definition: Catalog.h:774

+ Here is the caller graph for this function:

const ColumnDescriptor * Catalog_Namespace::Catalog::getDeletedColumnIfRowsDeleted ( const TableDescriptor td) const

Definition at line 3915 of file Catalog.cpp.

References ColumnDescriptor::columnId.

3916  {
3917  std::vector<const TableDescriptor*> tds;
3918  const ColumnDescriptor* cd;
3919  {
3920  cat_read_lock read_lock(this);
3921 
3922  const auto it = deletedColumnPerTable_.find(td);
3923  // if not a table that supports delete return nullptr, nothing more to do
3924  if (it == deletedColumnPerTable_.end()) {
3925  return nullptr;
3926  }
3927  cd = it->second;
3928  tds = getPhysicalTablesDescriptors(td, false);
3929  }
3930  // individual tables are still protected by higher level locks
3931  for (auto tdd : tds) {
3932  if (checkMetadataForDeletedRecs(tdd, cd->columnId)) {
3933  return cd;
3934  }
3935  }
3936  // no deletes so far recorded in metadata
3937  return nullptr;
3938 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
DeletedColumnPerTableMap deletedColumnPerTable_
Definition: Catalog.h:774
const bool checkMetadataForDeletedRecs(const TableDescriptor *td, int column_id) const
Definition: Catalog.cpp:3903
specifies the content in-memory of a row in the column metadata table
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4869
void Catalog_Namespace::Catalog::getDictionary ( const ColumnDescriptor cd,
std::map< int, StringDictionary * > &  stringDicts 
)

Definition at line 2342 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, kENCODING_DICT, ColumnDescriptor::tableId, and to_upper().

2343  {
2344  // learn 'committed' ColumnDescriptor of this column
2345  auto cit = columnDescriptorMap_.find(ColumnKey(cd.tableId, to_upper(cd.columnName)));
2346  CHECK(cit != columnDescriptorMap_.end());
2347  auto& ccd = *cit->second;
2348 
2349  if (!(ccd.columnType.is_string() || ccd.columnType.is_string_array())) {
2350  return;
2351  }
2352  if (!(ccd.columnType.get_compression() == kENCODING_DICT)) {
2353  return;
2354  }
2355  if (!(ccd.columnType.get_comp_param() > 0)) {
2356  return;
2357  }
2358 
2359  auto dictId = ccd.columnType.get_comp_param();
2360  getMetadataForDict(dictId);
2361 
2362  const DictRef dictRef(currentDB_.dbId, dictId);
2363  auto dit = dictDescriptorMapByRef_.find(dictRef);
2364  CHECK(dit != dictDescriptorMapByRef_.end());
2365  CHECK(dit->second);
2366  CHECK(dit->second.get()->stringDict);
2367  stringDicts[ccd.columnId] = dit->second.get()->stringDict.get();
2368 }
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:709
const DBMetadata currentDB_
Definition: Catalog.h:721
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:711
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1904
std::string to_upper(const std::string &str)
#define CHECK(condition)
Definition: Logger.h:291
std::string columnName

+ Here is the call graph for this function:

const std::map< int, const ColumnDescriptor * > Catalog_Namespace::Catalog::getDictionaryToColumnMapping ( )

Definition at line 4905 of file Catalog.cpp.

References kENCODING_DICT, and run_benchmark_import::tables.

4905  {
4906  cat_read_lock read_lock(this);
4907 
4908  std::map<int, const ColumnDescriptor*> mapping;
4909 
4910  const auto tables = getAllTableMetadata();
4911  for (const auto td : tables) {
4912  if (td->shard >= 0) {
4913  // skip shards, they're not standalone tables
4914  continue;
4915  }
4916 
4917  for (auto& cd : getAllColumnMetadataForTable(td->tableId, false, false, true)) {
4918  const auto& ti = cd->columnType;
4919  if (ti.is_string()) {
4920  if (ti.get_compression() == kENCODING_DICT) {
4921  // if foreign reference, get referenced tab.col
4922  const auto dict_id = ti.get_comp_param();
4923 
4924  // ignore temp (negative) dictionaries
4925  if (dict_id > 0 && mapping.end() == mapping.find(dict_id)) {
4926  mapping[dict_id] = cd;
4927  }
4928  }
4929  }
4930  }
4931  }
4932 
4933  return mapping;
4934 }
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:2188
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:2172
const foreign_storage::ForeignServer * Catalog_Namespace::Catalog::getForeignServer ( const std::string &  server_name) const

Gets a pointer to a struct containing foreign server details.

Parameters
server_name- Name of foreign server whose details will be fetched
Returns
pointer to a struct containing foreign server details. nullptr is returned if no foreign server exists with the given name

Definition at line 3334 of file Catalog.cpp.

Referenced by DBObject::loadKey().

3335  {
3336  foreign_storage::ForeignServer* foreign_server = nullptr;
3337  cat_read_lock read_lock(this);
3338 
3339  if (foreignServerMap_.find(server_name) != foreignServerMap_.end()) {
3340  foreign_server = foreignServerMap_.find(server_name)->second.get();
3341  }
3342  return foreign_server;
3343 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
ForeignServerMap foreignServerMap_
Definition: Catalog.h:715

+ Here is the caller graph for this function:

const std::unique_ptr< const foreign_storage::ForeignServer > Catalog_Namespace::Catalog::getForeignServerFromStorage ( const std::string &  server_name)

Gets a pointer to a struct containing foreign server details fetched from storage. This is mainly used for testing when asserting that expected catalog data is persisted.

Parameters
server_name- Name of foreign server whose details will be fetched
Returns
pointer to a struct containing foreign server details. nullptr is returned if no foreign server exists with the given name

Definition at line 3346 of file Catalog.cpp.

3346  {
3347  std::unique_ptr<foreign_storage::ForeignServer> foreign_server = nullptr;
3348  cat_sqlite_lock sqlite_lock(getObjForLock());
3350  "SELECT id, name, data_wrapper_type, options, owner_user_id, creation_time "
3351  "FROM omnisci_foreign_servers WHERE name = ?",
3352  std::vector<std::string>{server_name});
3353  if (sqliteConnector_.getNumRows() > 0) {
3354  foreign_server = std::make_unique<foreign_storage::ForeignServer>(
3355  sqliteConnector_.getData<int>(0, 0),
3356  sqliteConnector_.getData<std::string>(0, 1),
3357  sqliteConnector_.getData<std::string>(0, 2),
3358  sqliteConnector_.getData<std::string>(0, 3),
3359  sqliteConnector_.getData<std::int32_t>(0, 4),
3360  sqliteConnector_.getData<std::int32_t>(0, 5));
3361  }
3362  return foreign_server;
3363 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const
const std::string Catalog_Namespace::Catalog::getForeignServerSchema ( bool  if_not_exists = false)
static

Gets the DDL statement used to create a foreign server schema.

Parameters
if_not_exists- flag that indicates whether or not to include the "IF NOT EXISTS" phrase in the DDL statement
Returns
string containing DDL statement

Definition at line 775 of file Catalog.cpp.

Referenced by Catalog_Namespace::SysCatalog::createDatabase(), and updateFsiSchemas().

775  {
776  return "CREATE TABLE " + (if_not_exists ? std::string{"IF NOT EXISTS "} : "") +
777  "omnisci_foreign_servers(id integer primary key, name text unique, " +
778  "data_wrapper_type text, owner_user_id integer, creation_time integer, " +
779  "options text)";
780 }

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::getForeignServersForUser ( const rapidjson::Value *  filters,
const UserMetadata user,
std::vector< const foreign_storage::ForeignServer * > &  results 
)

Performs a query on all foreign servers accessible to user with optional filter, and returns pointers toresulting server objects

Parameters
filters- Json Value representing SQL WHERE clause to filter results, eg.: "WHERE attribute1 = value1 AND attribute2 LIKE value2", or Null Value Array of Values with attribute, value, operator, and chain specifier after first entry
user- user to retrieve server names
results- results returned as a vector of pointers to const foreign_storage::ForeignServer

Definition at line 3484 of file Catalog.cpp.

References CHECK, shared::contains(), DBObject::loadKey(), foreign_storage::ForeignServer::name, ServerDBObjectType, and to_string().

3487  {
3488  sys_read_lock syscat_read_lock(&SysCatalog::instance());
3489  cat_read_lock read_lock(this);
3490  cat_sqlite_lock sqlite_lock(getObjForLock());
3491  // Customer facing and internal SQlite names
3492  std::map<std::string, std::string> col_names{{"server_name", "name"},
3493  {"data_wrapper", "data_wrapper_type"},
3494  {"created_at", "creation_time"},
3495  {"options", "options"}};
3496 
3497  // TODO add "owner" when FSI privilege is implemented
3498  std::stringstream filter_string;
3499  std::vector<std::string> arguments;
3500 
3501  if (filters != nullptr) {
3502  // Create SQL WHERE clause for SQLite query
3503  int num_filters = 0;
3504  filter_string << " WHERE";
3505  for (auto& filter_def : filters->GetArray()) {
3506  if (num_filters > 0) {
3507  filter_string << " " << std::string(filter_def["chain"].GetString());
3508  ;
3509  }
3510 
3511  if (col_names.find(std::string(filter_def["attribute"].GetString())) ==
3512  col_names.end()) {
3513  throw std::runtime_error{"Attribute with name \"" +
3514  std::string(filter_def["attribute"].GetString()) +
3515  "\" does not exist."};
3516  }
3517 
3518  filter_string << " " << col_names[std::string(filter_def["attribute"].GetString())];
3519 
3520  bool equals_operator = false;
3521  if (std::strcmp(filter_def["operation"].GetString(), "EQUALS") == 0) {
3522  filter_string << " = ? ";
3523  equals_operator = true;
3524  } else {
3525  filter_string << " LIKE ? ";
3526  }
3527 
3528  bool timestamp_column =
3529  (std::strcmp(filter_def["attribute"].GetString(), "created_at") == 0);
3530 
3531  if (timestamp_column && !equals_operator) {
3532  throw std::runtime_error{"LIKE operator is incompatible with TIMESTAMP data"};
3533  }
3534 
3535  if (timestamp_column && equals_operator) {
3536  arguments.push_back(std::to_string(
3537  dateTimeParse<kTIMESTAMP>(filter_def["value"].GetString(), 0)));
3538  } else {
3539  arguments.emplace_back(filter_def["value"].GetString());
3540  }
3541 
3542  num_filters++;
3543  }
3544  }
3545  // Create select query for the omnisci_foreign_servers table
3546  std::string query = std::string("SELECT name from omnisci_foreign_servers ");
3547  query += filter_string.str();
3548 
3549  sqliteConnector_.query_with_text_params(query, arguments);
3550  auto num_rows = sqliteConnector_.getNumRows();
3551 
3552  if (sqliteConnector_.getNumRows() == 0) {
3553  return;
3554  }
3555 
3557  // Return pointers to objects
3558  results.reserve(num_rows);
3559  for (size_t row = 0; row < num_rows; ++row) {
3560  const auto& server_name = sqliteConnector_.getData<std::string>(row, 0);
3561  if (shared::contains(INTERNAL_SERVERS, server_name)) {
3562  continue;
3563  }
3564  const foreign_storage::ForeignServer* foreign_server = getForeignServer(server_name);
3565  CHECK(foreign_server != nullptr);
3566 
3567  DBObject dbObject(foreign_server->name, ServerDBObjectType);
3568  dbObject.loadKey(*this);
3569  std::vector<DBObject> privObjects = {dbObject};
3570  if (!SysCatalog::instance().hasAnyPrivileges(user, privObjects)) {
3571  // skip server, as there are no privileges to access it
3572  continue;
3573  }
3574  results.push_back(foreign_server);
3575  }
3576 }
bool contains(const T &container, const U &element)
Definition: misc.h:195
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
const foreign_storage::ForeignServer * getForeignServer(const std::string &server_name) const
Definition: Catalog.cpp:3334
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
static constexpr std::array< const char *, 6 > INTERNAL_SERVERS
Definition: Catalog.h:898
std::string to_string(char const *&&v)
static SysCatalog & instance()
Definition: SysCatalog.h:343
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
virtual size_t getNumCols() const
#define CHECK(condition)
Definition: Logger.h:291
read_lock< SysCatalog > sys_read_lock
Definition: Catalog.cpp:120
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

const foreign_storage::ForeignTable * Catalog_Namespace::Catalog::getForeignTable ( const std::string &  tableName) const

Definition at line 1827 of file Catalog.cpp.

1828  {
1829  cat_read_lock read_lock(this);
1830  return getForeignTableUnlocked(tableName);
1831 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
foreign_storage::ForeignTable * getForeignTableUnlocked(const std::string &tableName) const
Definition: Catalog.cpp:1818
const foreign_storage::ForeignTable * Catalog_Namespace::Catalog::getForeignTable ( int  table_id) const

Definition at line 2133 of file Catalog.cpp.

References CHECK.

2133  {
2134  cat_read_lock read_lock(this);
2135  const auto table = getMutableMetadataForTableUnlocked(table_id);
2136  CHECK(table);
2137  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(table);
2138  CHECK(foreign_table);
2139  return foreign_table;
2140 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
TableDescriptor * getMutableMetadataForTableUnlocked(int table_id) const
Definition: Catalog.cpp:1896
#define CHECK(condition)
Definition: Logger.h:291
const std::unique_ptr< const foreign_storage::ForeignTable > Catalog_Namespace::Catalog::getForeignTableFromStorage ( int  table_id)

Gets a pointer to a struct containing foreign table details fetched from storage. This is mainly used for testing when asserting that expected catalog data is persisted.

Parameters
table_name- Name of foreign table whose details will be fetched
Returns
pointer to a struct containing foreign table details. nullptr is returned if no foreign table exists with the given name

Definition at line 3366 of file Catalog.cpp.

References CHECK_EQ, and to_string().

3366  {
3367  std::unique_ptr<foreign_storage::ForeignTable> foreign_table = nullptr;
3368  cat_sqlite_lock sqlite_lock(getObjForLock());
3370  "SELECT table_id, server_id, options, last_refresh_time, next_refresh_time from "
3371  "omnisci_foreign_tables WHERE table_id = ?",
3372  std::vector<std::string>{to_string(table_id)});
3373  auto num_rows = sqliteConnector_.getNumRows();
3374  if (num_rows > 0) {
3375  CHECK_EQ(size_t(1), num_rows);
3376  foreign_table = std::make_unique<foreign_storage::ForeignTable>(
3377  sqliteConnector_.getData<int>(0, 0),
3378  foreignServerMapById_[sqliteConnector_.getData<int32_t>(0, 1)].get(),
3379  sqliteConnector_.getData<std::string>(0, 2),
3380  sqliteConnector_.getData<int64_t>(0, 3),
3381  sqliteConnector_.getData<int64_t>(0, 4));
3382  }
3383  return foreign_table;
3384 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
std::string to_string(char const *&&v)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:716
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

const std::string Catalog_Namespace::Catalog::getForeignTableSchema ( bool  if_not_exists = false)
static

Gets the DDL statement used to create a foreign table schema.

Parameters
if_not_exists- flag that indicates whether or not to include the "IF NOT EXISTS" phrase in the DDL statement
Returns
string containing DDL statement

Definition at line 782 of file Catalog.cpp.

Referenced by Catalog_Namespace::SysCatalog::createDatabase(), and updateFsiSchemas().

782  {
783  return "CREATE TABLE " + (if_not_exists ? std::string{"IF NOT EXISTS "} : "") +
784  "omnisci_foreign_tables(table_id integer unique, server_id integer, " +
785  "options text, last_refresh_time integer, next_refresh_time integer, " +
786  "FOREIGN KEY(table_id) REFERENCES mapd_tables(tableid), " +
787  "FOREIGN KEY(server_id) REFERENCES omnisci_foreign_servers(id))";
788 }

+ Here is the caller graph for this function:

foreign_storage::ForeignTable * Catalog_Namespace::Catalog::getForeignTableUnlocked ( const std::string &  tableName) const
private

Definition at line 1818 of file Catalog.cpp.

References to_upper().

1819  {
1820  auto tableDescIt = tableDescriptorMap_.find(to_upper(tableName));
1821  if (tableDescIt == tableDescriptorMap_.end()) { // check to make sure table exists
1822  return nullptr;
1823  }
1824  return dynamic_cast<foreign_storage::ForeignTable*>(tableDescIt->second);
1825 }
std::string to_upper(const std::string &str)
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:707

+ Here is the call graph for this function:

int Catalog_Namespace::Catalog::getLogicalTableId ( const int  physicalTableId) const

Definition at line 5008 of file Catalog.cpp.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumns(), and TableOptimizer::vacuumFragments().

5008  {
5009  cat_read_lock read_lock(this);
5010  for (const auto& l : logicalToPhysicalTableMapById_) {
5011  if (l.second.end() != std::find_if(l.second.begin(),
5012  l.second.end(),
5013  [&](decltype(*l.second.begin()) tid) -> bool {
5014  return physicalTableId == tid;
5015  })) {
5016  return l.first;
5017  }
5018  }
5019  return physicalTableId;
5020 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:727

+ Here is the caller graph for this function:

const ColumnDescriptor* Catalog_Namespace::Catalog::getMetadataForColumn ( int  tableId,
const std::string &  colName 
) const

Referenced by Parser::InsertStmt::analyze(), Parser::InsertValuesStmt::analyze(), Parser::QuerySpec::analyze_select_clause(), RelModify::applyUpdateModificationsToInputNode(), AlterTableAlterColumnCommandRecoveryMgr::cleanupDropSourceGeoColumns(), GeoLinestringValueConverter::GeoLinestringValueConverter(), GeoMultiLinestringValueConverter::GeoMultiLinestringValueConverter(), GeoMultiPointValueConverter::GeoMultiPointValueConverter(), GeoMultiPolygonValueConverter::GeoMultiPolygonValueConverter(), GeoPointValueConverter::GeoPointValueConverter(), GeoPolygonValueConverter::GeoPolygonValueConverter(), anonymous_namespace{DdlCommandExecutor.cpp}::get_alter_column_src_dst_cds(), Fragmenter_Namespace::get_chunks(), Analyzer::RangeTableEntry::get_column_desc(), foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::get_columns(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_foreign_col(), getChunkAccessorTable(), Fragmenter_Namespace::InsertOrderFragmenter::getChunksForAllColumns(), AlterTableAlterColumnCommandRecoveryMgr::getSrcDstCds(), getSyntheticColumnVar(), foreign_storage::anonymous_namespace{InternalSystemDataWrapper.cpp}::initialize_chunks(), anonymous_namespace{ForeignDataImporter.cpp}::load_foreign_data_buffers(), serialize_column_ref(), import_export::Importer::set_geo_physical_import_buffer(), import_export::Importer::set_geo_physical_import_buffer_columnar(), Fragmenter_Namespace::SortedOrderFragmenter::sortData(), and Fragmenter_Namespace::InsertOrderFragmenter::updateColumn().

+ Here is the caller graph for this function:

const ColumnDescriptor * Catalog_Namespace::Catalog::getMetadataForColumn ( int  tableId,
int  columnId 
) const

Definition at line 1957 of file Catalog.cpp.

1957  {
1958  cat_read_lock read_lock(this);
1959  ColumnIdKey columnIdKey(table_id, column_id);
1960  auto colDescIt = columnDescriptorMapById_.find(columnIdKey);
1961  if (colDescIt == columnDescriptorMapById_
1962  .end()) { // need to check to make sure column exists for table
1963  return nullptr;
1964  }
1965  return colDescIt->second;
1966 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:710
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39
const ColumnDescriptor * Catalog_Namespace::Catalog::getMetadataForColumnBySpi ( const int  tableId,
const size_t  spi 
) const

Definition at line 2001 of file Catalog.cpp.

2002  {
2003  cat_read_lock read_lock(this);
2004 
2005  const auto columnId = getColumnIdBySpiUnlocked(tableId, spi);
2006  ColumnIdKey columnIdKey(tableId, columnId);
2007  const auto colDescIt = columnDescriptorMapById_.find(columnIdKey);
2008  return columnDescriptorMapById_.end() == colDescIt ? nullptr : colDescIt->second;
2009 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:710
const int getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const
Definition: Catalog.cpp:1978
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39
const DashboardDescriptor* Catalog_Namespace::Catalog::getMetadataForDashboard ( const std::string &  userId,
const std::string &  dashName 
) const

Referenced by anonymous_namespace{DBHandler.cpp}::dashboard_exists(), and DBObject::loadKey().

+ Here is the caller graph for this function:

const DashboardDescriptor * Catalog_Namespace::Catalog::getMetadataForDashboard ( const int32_t  dashboard_id) const

Definition at line 2093 of file Catalog.cpp.

References setup::name, and to_string().

2093  {
2094  cat_read_lock read_lock(this);
2095  std::string userId;
2096  std::string name;
2097  bool found{false};
2098  {
2099  for (auto descp : dashboardDescriptorMap_) {
2100  auto dash = descp.second.get();
2101  if (dash->dashboardId == id) {
2102  userId = std::to_string(dash->userId);
2103  name = dash->dashboardName;
2104  found = true;
2105  break;
2106  }
2107  }
2108  }
2109  if (found) {
2110  return getMetadataForDashboard(userId, name);
2111  }
2112  return nullptr;
2113 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:712
std::string to_string(char const *&&v)
std::string name() const
Definition: Catalog.h:348
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const

+ Here is the call graph for this function:

const DictDescriptor * Catalog_Namespace::Catalog::getMetadataForDict ( int  dict_ref,
bool  loadDict = true 
) const

Definition at line 1904 of file Catalog.cpp.

References measure< TimeT >::execution(), g_cache_string_hash, logger::INFO, and LOG.

Referenced by AlterTableAlterColumnCommandRecoveryMgr::checkpoint(), DictionaryValueConverter< TARGET_TYPE >::DictionaryValueConverter(), import_export::fill_missing_columns(), import_export::Loader::init(), foreign_storage::anonymous_namespace{InternalSystemDataWrapper.cpp}::initialize_import_buffers(), anonymous_namespace{RelAlgExecutor.cpp}::insert_one_dict_str(), ArrowForeignStorageBase::parseArrowTable(), DBHandler::populateThriftColumnType(), and Fragmenter_Namespace::InsertOrderFragmenter::updateColumn().

1905  {
1906  cat_read_lock read_lock(this);
1907  const DictRef dictRef(currentDB_.dbId, dict_id);
1908  auto dictDescIt = dictDescriptorMapByRef_.find(dictRef);
1909  if (dictDescIt ==
1910  dictDescriptorMapByRef_.end()) { // check to make sure dictionary exists
1911  return nullptr;
1912  }
1913  auto& dd = dictDescIt->second;
1914 
1915  if (load_dict) {
1916  std::lock_guard string_dict_lock(*dd->string_dict_mutex);
1917  if (!dd->stringDict) {
1918  auto time_ms = measure<>::execution([&]() {
1919  if (string_dict_hosts_.empty()) {
1920  if (dd->dictIsTemp) {
1921  dd->stringDict = std::make_shared<StringDictionary>(
1922  dd->dictRef, dd->dictFolderPath, true, true, g_cache_string_hash);
1923  } else {
1924  dd->stringDict = std::make_shared<StringDictionary>(
1925  dd->dictRef, dd->dictFolderPath, false, true, g_cache_string_hash);
1926  }
1927  } else {
1928  dd->stringDict =
1929  std::make_shared<StringDictionary>(string_dict_hosts_.front(), dd->dictRef);
1930  }
1931  });
1932  LOG(INFO) << "Time to load Dictionary " << dd->dictRef.dbId << "_"
1933  << dd->dictRef.dictId << " was " << time_ms << "ms";
1934  }
1935  }
1936 
1937  return dd.get();
1938 }
std::lock_guard< T > lock_guard
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:285
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:724
const DBMetadata currentDB_
Definition: Catalog.h:721
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:711
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
bool g_cache_string_hash

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const LinkDescriptor* Catalog_Namespace::Catalog::getMetadataForLink ( const std::string &  link) const
const LinkDescriptor * Catalog_Namespace::Catalog::getMetadataForLink ( int  linkId) const

Definition at line 2124 of file Catalog.cpp.

2124  {
2125  cat_read_lock read_lock(this);
2126  auto linkDescIt = linkDescriptorMapById_.find(linkId);
2127  if (linkDescIt == linkDescriptorMapById_.end()) { // check to make sure view exists
2128  return nullptr;
2129  }
2130  return linkDescIt->second;
2131 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:714
const TableDescriptor* Catalog_Namespace::Catalog::getMetadataForTable ( const std::string &  tableName,
const bool  populateFragmenter = true 
) const

Returns a pointer to a const TableDescriptor struct matching the provided tableName.

Parameters
tableNametable specified column belongs to
Returns
pointer to const TableDescriptor object queried for or nullptr if it does not exist.

Referenced by lockmgr::TableSchemaLockContainer< ReadLock >::acquireTableDescriptor(), lockmgr::TableSchemaLockContainer< WriteLock >::acquireTableDescriptor(), Parser::InsertStmt::analyze(), Parser::QuerySpec::analyze_from_clause(), UpdelRoll::cancelUpdate(), anonymous_namespace{DBHandler.cpp}::check_and_reset_in_memory_system_table(), UpdelRoll::commitUpdate(), Fragmenter_Namespace::compute_row_indices_of_shards(), Fragmenter_Namespace::computeRowIndicesOfShards(), Fragmenter_Namespace::copy_data_of_shard(), Fragmenter_Namespace::copyDataOfShard(), anonymous_namespace{DBHandler.cpp}::create_custom_expr_from_thrift_obj(), anonymous_namespace{DBHandler.cpp}::create_thrift_obj_from_custom_expr(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::drop_tables(), RelAlgExecutor::executeSimpleInsert(), Executor::executeUpdate(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_foreign_col(), getSyntheticColumnVar(), Fragmenter_Namespace::LocalInsertConnector::insertChunksToLeaf(), Fragmenter_Namespace::LocalInsertConnector::insertDataToLeaf(), Parser::InsertValuesStmt::InsertValuesStmt(), DBObject::loadKey(), Parser::anonymous_namespace{ParserNode.cpp}::loadTable(), TableOptimizer::recomputeMetadataUnlocked(), TableArchiver::restoreTable(), serialize_table_ref(), Fragmenter_Namespace::set_chunk_metadata(), Fragmenter_Namespace::SortedOrderFragmenter::sortData(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumnMetadata(), anonymous_namespace{DdlCommandExecutor.cpp}::vacuum_table_if_required(), TableOptimizer::vacuumFragmentsAboveMinSelectivity(), validate_shared_dictionary(), lockmgr::validate_table_descriptor_after_lock(), Parser::verifyObject(), StorageIOFacility::yieldDeleteCallback(), and StorageIOFacility::yieldUpdateCallback().

+ Here is the caller graph for this function:

const TableDescriptor * Catalog_Namespace::Catalog::getMetadataForTable ( int  tableId,
bool  populateFragmenter = true 
) const

Definition at line 1852 of file Catalog.cpp.

References Catalog_Namespace::read_lock< T >::unlock().

1853  {
1854  cat_read_lock read_lock(this);
1855  auto td = getMutableMetadataForTableUnlocked(table_id);
1856  if (!td) {
1857  return nullptr;
1858  }
1859  read_lock.unlock();
1860  if (populateFragmenter) {
1861  std::unique_lock<std::mutex> td_lock(*td->mutex_.get());
1862  if (td->fragmenter == nullptr && !td->isView) {
1864  }
1865  }
1866  return td;
1867 }
void instantiateFragmenter(TableDescriptor *td) const
Definition: Catalog.cpp:1778
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
TableDescriptor * getMutableMetadataForTableUnlocked(int table_id) const
Definition: Catalog.cpp:1896

+ Here is the call graph for this function:

TableDescriptor * Catalog_Namespace::Catalog::getMutableMetadataForTableUnlocked ( int  table_id) const
private

Definition at line 1896 of file Catalog.cpp.

1896  {
1897  auto tableDescIt = tableDescriptorMapById_.find(table_id);
1898  if (tableDescIt == tableDescriptorMapById_.end()) { // check to make sure table exists
1899  return nullptr;
1900  }
1901  return tableDescIt->second;
1902 }
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
TableDescriptor * Catalog_Namespace::Catalog::getMutableMetadataForTableUnlocked ( const std::string &  table_name) const
private

Definition at line 1887 of file Catalog.cpp.

References to_upper().

1888  {
1889  auto it = tableDescriptorMap_.find(to_upper(table_name));
1890  if (it == tableDescriptorMap_.end()) {
1891  return nullptr;
1892  }
1893  return it->second;
1894 }
std::string to_upper(const std::string &str)
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:707

+ Here is the call graph for this function:

int Catalog_Namespace::Catalog::getNextAddedColumnId ( const TableDescriptor td)

Definition at line 2441 of file Catalog.cpp.

References TableDescriptor::tableId, and to_string().

2441  {
2442  cat_read_lock read_lock(this);
2444  "SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?",
2445  std::vector<std::string>{std::to_string(td.tableId)});
2446  return sqliteConnector_.getData<int>(0, 0);
2447 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
std::string to_string(char const *&&v)

+ Here is the call graph for this function:

const Catalog * Catalog_Namespace::Catalog::getObjForLock ( )
private

Definition at line 257 of file Catalog.cpp.

References Catalog_Namespace::SysCatalog::getDummyCatalog(), initialized_, and Catalog_Namespace::SysCatalog::instance().

Referenced by CheckAndExecuteMigrations(), checkDateInDaysColumnMigration(), createDashboardSystemRoles(), recordOwnershipOfObjectsInObjectPermissions(), reloadTableMetadata(), renameLegacyDataWrappers(), updateCustomExpressionsSchema(), updateDefaultColumnValues(), updateDeletedColumnIndicator(), updateDictionaryNames(), updateDictionarySchema(), updateFixlenArrayColumns(), updateFrontendViewAndLinkUsers(), updateFrontendViewSchema(), updateFrontendViewsToDashboards(), updateFsiSchemas(), updateGeoColumns(), updateLinkSchema(), updateLogicalToPhysicalTableLinkSchema(), updateLogicalToPhysicalTableMap(), updatePageSize(), and updateTableDescriptorSchema().

257  {
258  if (initialized_) {
259  return this;
260  } else {
261  return SysCatalog::instance().getDummyCatalog().get();
262  }
263 }
std::shared_ptr< Catalog > getDummyCatalog()
Definition: SysCatalog.h:369
static SysCatalog & instance()
Definition: SysCatalog.h:343

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::vector< const TableDescriptor * > Catalog_Namespace::Catalog::getPhysicalTablesDescriptors ( const TableDescriptor logical_table_desc,
bool  populate_fragmenter = true 
) const

Definition at line 4869 of file Catalog.cpp.

References CHECK, TableDescriptor::tableId, and Catalog_Namespace::read_lock< T >::unlock().

Referenced by Fragmenter_Namespace::copy_data_of_shard(), Fragmenter_Namespace::copyDataOfShard(), import_export::Loader::distributeToShardsNewColumns(), import_export::Loader::dropColumns(), anonymous_namespace{DdlCommandExecutor.cpp}::get_agg_storage_stats(), import_export::Loader::loadImpl(), TableOptimizer::recomputeMetadata(), and TableOptimizer::vacuumDeletedRows().

4871  {
4872  cat_read_lock read_lock(this);
4873  const auto physicalTableIt =
4874  logicalToPhysicalTableMapById_.find(logical_table_desc->tableId);
4875  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
4876  return {logical_table_desc};
4877  }
4878  const auto physicalTablesIds = physicalTableIt->second;
4879  CHECK(!physicalTablesIds.empty());
4880  read_lock.unlock();
4881  std::vector<const TableDescriptor*> physicalTables;
4882  for (size_t i = 0; i < physicalTablesIds.size(); i++) {
4883  physicalTables.push_back(
4884  getMetadataForTable(physicalTablesIds[i], populate_fragmenter));
4885  }
4886  return physicalTables;
4887 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
#define CHECK(condition)
Definition: Logger.h:291
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:727

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const ColumnDescriptor * Catalog_Namespace::Catalog::getShardColumnMetadataForTable ( const TableDescriptor td) const

Definition at line 4851 of file Catalog.cpp.

References TableDescriptor::shardedColumnId, and TableDescriptor::tableId.

Referenced by RelModify::applyUpdateModificationsToInputNode(), Fragmenter_Namespace::compute_row_indices_of_shards(), and Fragmenter_Namespace::computeRowIndicesOfShards().

4852  {
4853  cat_read_lock read_lock(this);
4854 
4855  const auto column_descriptors =
4856  getAllColumnMetadataForTable(td->tableId, false, true, true);
4857 
4858  const ColumnDescriptor* shard_cd{nullptr};
4859  int i = 1;
4860  for (auto cd_itr = column_descriptors.begin(); cd_itr != column_descriptors.end();
4861  ++cd_itr, ++i) {
4862  if (i == td->shardedColumnId) {
4863  shard_cd = *cd_itr;
4864  }
4865  }
4866  return shard_cd;
4867 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
specifies the content in-memory of a row in the column metadata table
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:2172

+ Here is the caller graph for this function:

SqliteConnector& Catalog_Namespace::Catalog::getSqliteConnector ( )
inline

Definition at line 327 of file Catalog.h.

References sqliteConnector_.

327 { return sqliteConnector_; }
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const std::vector< LeafHostInfo > & Catalog_Namespace::Catalog::getStringDictionaryHosts ( ) const

Definition at line 1940 of file Catalog.cpp.

1940  {
1941  return string_dict_hosts_;
1942 }
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:724
std::pair< foreign_storage::ForeignTable, std::list< ColumnDescriptor > > Catalog_Namespace::Catalog::getSystemTableSchema ( const std::string &  table_name,
const std::string &  server_name,
const std::vector< std::pair< std::string, SQLTypeInfo >> &  column_type_by_name,
bool  is_in_memory_system_table 
)
private

Definition at line 6860 of file Catalog.cpp.

References CHECK, DEFAULT_FRAGMENT_ROWS, DEFAULT_MAX_CHUNK_SIZE, DEFAULT_MAX_ROWS, DEFAULT_PAGE_SIZE, foreign_storage::ForeignTable::foreign_server, StorageType::FOREIGN_TABLE, TableDescriptor::fragmenter, TableDescriptor::fragments, TableDescriptor::fragPageSize, TableDescriptor::fragType, TableDescriptor::hasDeletedCol, Fragmenter_Namespace::INSERT_ORDER, TableDescriptor::is_in_memory_system_table, TableDescriptor::is_system_table, TableDescriptor::isView, TableDescriptor::keyMetainfo, shared::kRootUserId, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRows, TableDescriptor::nColumns, TableDescriptor::partitions, TableDescriptor::storageType, TableDescriptor::tableName, and TableDescriptor::userId.

6864  {
6865  foreign_storage::ForeignTable foreign_table;
6866  foreign_table.tableName = table_name;
6867  foreign_table.nColumns = column_type_by_name.size();
6868  foreign_table.isView = false;
6869  foreign_table.is_system_table = true;
6870  foreign_table.is_in_memory_system_table = is_in_memory_system_table;
6871  foreign_table.fragmenter = nullptr;
6873  foreign_table.maxFragRows = DEFAULT_FRAGMENT_ROWS;
6874  foreign_table.maxChunkSize = DEFAULT_MAX_CHUNK_SIZE;
6875  foreign_table.fragPageSize = DEFAULT_PAGE_SIZE;
6876  foreign_table.maxRows = DEFAULT_MAX_ROWS;
6877  foreign_table.userId = shared::kRootUserId;
6878  foreign_table.storageType = StorageType::FOREIGN_TABLE;
6879  foreign_table.hasDeletedCol = false;
6880  foreign_table.keyMetainfo = "[]";
6881  foreign_table.fragments = "";
6882  foreign_table.partitions = "";
6883  foreign_table.foreign_server = getForeignServer(server_name);
6884  CHECK(foreign_table.foreign_server);
6885 
6886  list<ColumnDescriptor> columns;
6887  for (const auto& [column_name, column_type] : column_type_by_name) {
6888  columns.emplace_back();
6889  auto& cd = columns.back();
6890  cd.columnName = column_name;
6891  cd.columnType = column_type;
6892  cd.isSystemCol = false;
6893  cd.isVirtualCol = false;
6894  }
6895  return {foreign_table, columns};
6896 }
std::string partitions
const int kRootUserId
std::string tableName
const foreign_storage::ForeignServer * getForeignServer(const std::string &server_name) const
Definition: Catalog.cpp:3334
std::string storageType
#define DEFAULT_MAX_CHUNK_SIZE
std::string fragments
bool is_in_memory_system_table
#define DEFAULT_MAX_ROWS
std::string keyMetainfo
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
#define DEFAULT_PAGE_SIZE
#define DEFAULT_FRAGMENT_ROWS
Fragmenter_Namespace::FragmenterType fragType
const ForeignServer * foreign_server
Definition: ForeignTable.h:57
#define CHECK(condition)
Definition: Logger.h:291
static constexpr char const * FOREIGN_TABLE
std::vector< std::string > Catalog_Namespace::Catalog::getTableDataDirectories ( const TableDescriptor td) const

Definition at line 5374 of file Catalog.cpp.

Referenced by TableArchiver::dumpTable(), and TableArchiver::restoreTable().

5375  {
5376  const auto global_file_mgr = getDataMgr().getGlobalFileMgr();
5377  std::vector<std::string> file_paths;
5378  for (auto shard : getPhysicalTablesDescriptors(td)) {
5379  const auto file_mgr = dynamic_cast<File_Namespace::FileMgr*>(
5380  global_file_mgr->getFileMgr(currentDB_.dbId, shard->tableId));
5381  boost::filesystem::path file_path(file_mgr->getFileMgrBasePath());
5382  file_paths.push_back(file_path.filename().string());
5383  }
5384  return file_paths;
5385 }
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:266
const DBMetadata currentDB_
Definition: Catalog.h:721
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4869
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
Definition: DataMgr.cpp:649

+ Here is the caller graph for this function:

std::vector< std::string > Catalog_Namespace::Catalog::getTableDictDirectories ( const TableDescriptor td) const

Definition at line 5409 of file Catalog.cpp.

References TableDescriptor::tableId.

Referenced by TableArchiver::dumpTable(), and TableArchiver::restoreTable().

5410  {
5411  std::vector<std::string> file_paths;
5412  for (auto cd : getAllColumnMetadataForTable(td->tableId, false, false, true)) {
5413  auto file_base = getColumnDictDirectory(cd);
5414  if (!file_base.empty() &&
5415  file_paths.end() == std::find(file_paths.begin(), file_paths.end(), file_base)) {
5416  file_paths.push_back(file_base);
5417  }
5418  }
5419  return file_paths;
5420 }
std::string getColumnDictDirectory(const ColumnDescriptor *cd, bool file_name_only=true) const
Definition: Catalog.cpp:5388
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:2172

+ Here is the caller graph for this function:

std::set< std::string > Catalog_Namespace::Catalog::getTableDictDirectoryPaths ( int32_t  table_id) const

Definition at line 5422 of file Catalog.cpp.

5422  {
5423  cat_read_lock read_lock(this);
5424  std::set<std::string> directory_paths;
5425  auto it = dict_columns_by_table_id_.find(table_id);
5426  if (it != dict_columns_by_table_id_.end()) {
5427  for (auto cd : it->second) {
5428  auto directory_path = getColumnDictDirectory(cd, false);
5429  if (!directory_path.empty()) {
5430  directory_paths.emplace(directory_path);
5431  }
5432  }
5433  }
5434  return directory_paths;
5435 }
TableDictColumnsMap dict_columns_by_table_id_
Definition: Catalog.h:718
std::string getColumnDictDirectory(const ColumnDescriptor *cd, bool file_name_only=true) const
Definition: Catalog.cpp:5388
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
int32_t Catalog_Namespace::Catalog::getTableEpoch ( const int32_t  db_id,
const int32_t  table_id 
) const

Definition at line 3579 of file Catalog.cpp.

References CHECK, logger::ERROR, logger::INFO, and LOG.

Referenced by TableArchiver::dumpTable().

3579  {
3580  cat_read_lock read_lock(this);
3581  const auto td = getMetadataForTable(table_id, false);
3582  if (!td) {
3583  std::stringstream table_not_found_error_message;
3584  table_not_found_error_message << "Table (" << db_id << "," << table_id
3585  << ") not found";
3586  throw std::runtime_error(table_not_found_error_message.str());
3587  }
3588  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(table_id);
3589  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
3590  // check all shards have same checkpoint
3591  const auto physicalTables = physicalTableIt->second;
3592  CHECK(!physicalTables.empty());
3593  size_t curr_epoch{0}, first_epoch{0};
3594  int32_t first_table_id{0};
3595  bool are_epochs_inconsistent{false};
3596  for (size_t i = 0; i < physicalTables.size(); i++) {
3597  int32_t physical_tb_id = physicalTables[i];
3598  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id, false);
3599  CHECK(phys_td);
3600 
3601  curr_epoch = dataMgr_->getTableEpoch(db_id, physical_tb_id);
3602  LOG(INFO) << "Got sharded table epoch for db id: " << db_id
3603  << ", table id: " << physical_tb_id << ", epoch: " << curr_epoch;
3604  if (i == 0) {
3605  first_epoch = curr_epoch;
3606  first_table_id = physical_tb_id;
3607  } else if (first_epoch != curr_epoch) {
3608  are_epochs_inconsistent = true;
3609  LOG(ERROR) << "Epochs on shards do not all agree on table id: " << table_id
3610  << ", db id: " << db_id
3611  << ". First table (table id: " << first_table_id
3612  << ") has epoch: " << first_epoch << ". Table id: " << physical_tb_id
3613  << ", has inconsistent epoch: " << curr_epoch
3614  << ". See previous INFO logs for all epochs and their table ids.";
3615  }
3616  }
3617  if (are_epochs_inconsistent) {
3618  // oh dear the shards do not agree on the epoch for this table
3619  return -1;
3620  }
3621  return curr_epoch;
3622  } else {
3623  auto epoch = dataMgr_->getTableEpoch(db_id, table_id);
3624  LOG(INFO) << "Got table epoch for db id: " << db_id << ", table id: " << table_id
3625  << ", epoch: " << epoch;
3626  return epoch;
3627  }
3628 }
#define LOG(tag)
Definition: Logger.h:285
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:722
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
#define CHECK(condition)
Definition: Logger.h:291
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:727

+ Here is the caller graph for this function:

std::vector< TableEpochInfo > Catalog_Namespace::Catalog::getTableEpochs ( const int32_t  db_id,
const int32_t  table_id 
) const

Definition at line 3821 of file Catalog.cpp.

References CHECK, logger::INFO, and LOG.

Referenced by UpdelRoll::cancelUpdate(), UpdelRoll::commitUpdate(), import_export::Loader::getTableEpochs(), AlterTableAlterColumnCommandRecoveryMgr::recoverAlterTableAlterColumnFromFile(), TableOptimizer::vacuumDeletedRows(), TableOptimizer::vacuumFragmentsAboveMinSelectivity(), and system_validator::SingleNodeValidator::validate().

3822  {
3823  cat_read_lock read_lock(this);
3824  std::vector<TableEpochInfo> table_epochs;
3825  const auto physical_table_it = logicalToPhysicalTableMapById_.find(table_id);
3826  if (physical_table_it != logicalToPhysicalTableMapById_.end()) {
3827  const auto physical_tables = physical_table_it->second;
3828  CHECK(!physical_tables.empty());
3829 
3830  for (const auto physical_tb_id : physical_tables) {
3831  const auto phys_td = getMutableMetadataForTableUnlocked(physical_tb_id);
3832  CHECK(phys_td);
3833 
3834  auto table_id = phys_td->tableId;
3835  auto epoch = dataMgr_->getTableEpoch(db_id, phys_td->tableId);
3836  table_epochs.emplace_back(table_id, epoch);
3837  LOG(INFO) << "Got sharded table epoch for db id: " << db_id
3838  << ", table id: " << table_id << ", epoch: " << epoch;
3839  }
3840  } else {
3841  auto epoch = dataMgr_->getTableEpoch(db_id, table_id);
3842  LOG(INFO) << "Got table epoch for db id: " << db_id << ", table id: " << table_id
3843  << ", epoch: " << epoch;
3844  table_epochs.emplace_back(table_id, epoch);
3845  }
3846  return table_epochs;
3847 }
#define LOG(tag)
Definition: Logger.h:285
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:722
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
TableDescriptor * getMutableMetadataForTableUnlocked(int table_id) const
Definition: Catalog.cpp:1896
#define CHECK(condition)
Definition: Logger.h:291
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:727

+ Here is the caller graph for this function:

std::optional< int32_t > Catalog_Namespace::Catalog::getTableId ( const std::string &  table_name) const

Definition at line 1878 of file Catalog.cpp.

Referenced by lockmgr::helpers::chunk_key_for_table(), and lockmgr::TableLockMgrImpl< TableDataLockMgr >::validateAndGetExistingTableId().

1878  {
1879  cat_read_lock read_lock(this);
1880  auto td = getMutableMetadataForTableUnlocked(table_name);
1881  if (!td) {
1882  return {};
1883  }
1884  return td->tableId;
1885 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
TableDescriptor * getMutableMetadataForTableUnlocked(int table_id) const
Definition: Catalog.cpp:1896

+ Here is the caller graph for this function:

std::optional< std::string > Catalog_Namespace::Catalog::getTableName ( int32_t  table_id) const

Definition at line 1869 of file Catalog.cpp.

Referenced by lockmgr::TableSchemaLockContainer< ReadLock >::acquireTableDescriptor(), lockmgr::TableSchemaLockContainer< WriteLock >::acquireTableDescriptor(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_checked_table_name(), AlterTableAlterColumnCommandRecoveryMgr::getRecoveryFilepathInfo(), and AlterTableAlterColumnCommandRecoveryMgr::recoverAlterTableAlterColumnFromFile().

1869  {
1870  cat_read_lock read_lock(this);
1871  auto td = getMutableMetadataForTableUnlocked(table_id);
1872  if (!td) {
1873  return {};
1874  }
1875  return td->tableName;
1876 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
TableDescriptor * getMutableMetadataForTableUnlocked(int table_id) const
Definition: Catalog.cpp:1896

+ Here is the caller graph for this function:

std::vector< std::string > Catalog_Namespace::Catalog::getTableNamesForUser ( const UserMetadata user,
const GetTablesType  get_tables_type 
) const

Get names of all tables accessible to user.

Parameters
user- user to retrieve table names for
get_tables_type- enum indicating if tables, views or tables & views should be returned
Returns
table_names - vector of table names accessible by user

Definition at line 4969 of file Catalog.cpp.

References run_benchmark_import::tables.

Referenced by DBHandler::get_tables_impl().

4971  {
4972  sys_read_lock syscat_read_lock(&SysCatalog::instance());
4973  cat_read_lock read_lock(this);
4974  std::vector<std::string> table_names;
4975  const auto tables = getAllTableMetadata();
4976  for (const auto td : tables) {
4977  if (filterTableByTypeAndUser(td, user_metadata, get_tables_type)) {
4978  table_names.push_back(td->tableName);
4979  }
4980  }
4981  return table_names;
4982 }
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:2188
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
bool filterTableByTypeAndUser(const TableDescriptor *td, const UserMetadata &user_metadata, const GetTablesType get_tables_type) const
Definition: Catalog.cpp:4936
static SysCatalog & instance()
Definition: SysCatalog.h:343
read_lock< SysCatalog > sys_read_lock
Definition: Catalog.cpp:120

+ Here is the caller graph for this function:

std::vector< TableMetadata > Catalog_Namespace::Catalog::getTablesMetadataForUser ( const UserMetadata user_metadata,
const GetTablesType  get_tables_type,
const std::string &  filter_table_name 
) const

Get table descriptors of all tables accessible to user.

Parameters
user- user to retrieve table descriptors for
get_tables_type- enum indicating if tables, views or tables & views should be returned
Returns
table_descriptors - vector of table descriptors accessible by user

Definition at line 4984 of file Catalog.cpp.

References run_benchmark_import::tables.

4987  {
4988  sys_read_lock syscat_read_lock(&SysCatalog::instance());
4989  cat_read_lock read_lock(this);
4990 
4991  std::vector<TableMetadata> tables_metadata;
4992  const auto tables = getAllTableMetadata();
4993  for (const auto td : tables) {
4994  if (filterTableByTypeAndUser(td, user_metadata, get_tables_type)) {
4995  if (!filter_table_name.empty()) {
4996  if (td->tableName != filter_table_name) {
4997  continue;
4998  }
4999  }
5000  TableMetadata table_metadata(td); // Makes a copy, not safe to access raw table
5001  // descriptor outside catalog lock
5002  tables_metadata.emplace_back(table_metadata);
5003  }
5004  }
5005  return tables_metadata;
5006 }
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:2188
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
bool filterTableByTypeAndUser(const TableDescriptor *td, const UserMetadata &user_metadata, const GetTablesType get_tables_type) const
Definition: Catalog.cpp:4936
static SysCatalog & instance()
Definition: SysCatalog.h:343
read_lock< SysCatalog > sys_read_lock
Definition: Catalog.cpp:120
size_t Catalog_Namespace::Catalog::getTotalMemorySizeForDictionariesForDatabase ( ) const

Definition at line 2370 of file Catalog.cpp.

Referenced by anonymous_namespace{DBHandler.cpp}::log_cache_size().

2370  {
2371  size_t ret{0};
2372  for (auto const& kv : dictDescriptorMapByRef_) {
2373  if (kv.first.dbId == currentDB_.dbId) {
2374  auto dictionary = kv.second.get()->stringDict.get();
2375  if (dictionary) {
2376  ret += dictionary->computeCacheSize();
2377  }
2378  }
2379  }
2380  return ret;
2381 }
const DBMetadata currentDB_
Definition: Catalog.h:721
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:711

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::initializeDashboardsSystemTable ( )
private

Definition at line 6582 of file Catalog.cpp.

References Catalog_Namespace::DASHBOARDS_SYS_TABLE_NAME, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_encoded_text_type(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_var_encoded_text_array_type(), kINT, and kTIMESTAMP.

6582  {
6583  auto [foreign_table, columns] =
6586  {{"database_id", {kINT}},
6587  {"database_name", get_encoded_text_type()},
6588  {"dashboard_id", {kINT}},
6589  {"dashboard_name", get_encoded_text_type()},
6590  {"owner_id", {kINT}},
6591  {"owner_user_name", get_encoded_text_type()},
6592  {"last_updated_at", {kTIMESTAMP}},
6593  {"data_sources", get_var_encoded_text_array_type()}},
6594  true);
6595  recreateSystemTableIfUpdated(foreign_table, columns);
6596 }
std::pair< foreign_storage::ForeignTable, std::list< ColumnDescriptor > > getSystemTableSchema(const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name, bool is_in_memory_system_table)
Definition: Catalog.cpp:6860
static constexpr const char * DASHBOARDS_SYS_TABLE_NAME
Definition: Catalog.h:114
static constexpr const char * CATALOG_SERVER_NAME
Definition: Catalog.h:892
Definition: sqltypes.h:72
bool recreateSystemTableIfUpdated(foreign_storage::ForeignTable &foreign_table, const std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:6898

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::initializeDatabasesSystemTable ( )
private

Definition at line 6519 of file Catalog.cpp.

References Catalog_Namespace::DATABASES_SYS_TABLE_NAME, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_encoded_text_type(), and kINT.

6519  {
6520  auto [foreign_table, columns] =
6523  {{"database_id", {kINT}},
6524  {"database_name", get_encoded_text_type()},
6525  {"owner_id", {kINT}},
6526  {"owner_user_name", get_encoded_text_type()}},
6527  true);
6528  recreateSystemTableIfUpdated(foreign_table, columns);
6529 }
std::pair< foreign_storage::ForeignTable, std::list< ColumnDescriptor > > getSystemTableSchema(const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name, bool is_in_memory_system_table)
Definition: Catalog.cpp:6860
static constexpr const char * DATABASES_SYS_TABLE_NAME
Definition: Catalog.h:115
static constexpr const char * CATALOG_SERVER_NAME
Definition: Catalog.h:892
Definition: sqltypes.h:72
bool recreateSystemTableIfUpdated(foreign_storage::ForeignTable &foreign_table, const std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:6898

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::initializeExecutorResourcePoolSummarySystemTable ( )
private

Definition at line 6673 of file Catalog.cpp.

References Catalog_Namespace::EXECUTOR_RESOURCE_POOL_SUMMARY_SYS_TABLE_NAME, kBIGINT, and kINT.

6673  {
6674  auto [foreign_table, columns] =
6677  {{"total_cpu_slots", {kINT}},
6678  {"total_gpu_slots", {kINT}},
6679  {"total_cpu_result_mem", {kBIGINT}},
6680  {"total_cpu_buffer_pool_mem", {kBIGINT}},
6681  {"total_gpu_buffer_pool_mem", {kBIGINT}},
6682  {"allocated_cpu_slots", {kINT}},
6683  {"allocated_gpu_slots", {kINT}},
6684  {"allocated_cpu_result_mem", {kBIGINT}},
6685  {"allocated_cpu_buffer_pool_mem", {kBIGINT}},
6686  {"allocated_gpu_buffer_pool_mem", {kBIGINT}},
6687  {"allocated_cpu_buffers", {kINT}},
6688  {"allocated_gpu_buffers", {kINT}},
6689  {"allocated_temp_cpu_buffer_pool_mem", {kBIGINT}},
6690  {"allocated_temp_gpu_buffer_pool_mem", {kBIGINT}},
6691  {"total_requests", {kBIGINT}},
6692  {"outstanding_requests", {kINT}},
6693  {"outstanding_cpu_slots_requests", {kINT}},
6694  {"outstanding_gpu_slots_requests", {kINT}},
6695  {"outstanding_cpu_result_mem_requests", {kINT}},
6696  {"outstanding_cpu_buffer_pool_mem_requests", {kINT}},
6697  {"outstanding_gpu_buffer_pool_mem_requests", {kINT}}},
6698  true);
6699  recreateSystemTableIfUpdated(foreign_table, columns);
6700 }
std::pair< foreign_storage::ForeignTable, std::list< ColumnDescriptor > > getSystemTableSchema(const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name, bool is_in_memory_system_table)
Definition: Catalog.cpp:6860
static constexpr const char * EXECUTOR_RESOURCE_POOL_SUMMARY_SYS_TABLE_NAME
Definition: Catalog.h:122
static constexpr const char * EXECUTOR_STATS_SERVER_NAME
Definition: Catalog.h:895
Definition: sqltypes.h:72
bool recreateSystemTableIfUpdated(foreign_storage::ForeignTable &foreign_table, const std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:6898
void Catalog_Namespace::Catalog::initializeMemoryDetailsSystemTable ( )
private

Definition at line 6623 of file Catalog.cpp.

References Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_encoded_text_type(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_var_array_type(), kBIGINT, kINT, and Catalog_Namespace::MEMORY_DETAILS_SYS_TABLE_NAME.

6623  {
6624  auto [foreign_table, columns] =
6627  {{"node", get_encoded_text_type()},
6628  {"database_id", {kINT}},
6629  {"database_name", get_encoded_text_type()},
6630  {"table_id", {kINT}},
6631  {"table_name", get_encoded_text_type()},
6632  {"column_id", {kINT}},
6633  {"column_name", get_encoded_text_type()},
6634  {"chunk_key", get_var_array_type(kINT)},
6635  {"device_id", {kINT}},
6636  {"device_type", get_encoded_text_type()},
6637  {"memory_status", get_encoded_text_type()},
6638  {"page_count", {kBIGINT}},
6639  {"page_size", {kBIGINT}},
6640  {"slab_id", {kINT}},
6641  {"start_page", {kBIGINT}},
6642  {"last_touch_epoch", {kBIGINT}}},
6643  true);
6644  recreateSystemTableIfUpdated(foreign_table, columns);
6645 }
static constexpr const char * MEMORY_DETAILS_SYS_TABLE_NAME
Definition: Catalog.h:120
std::pair< foreign_storage::ForeignTable, std::list< ColumnDescriptor > > getSystemTableSchema(const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name, bool is_in_memory_system_table)
Definition: Catalog.cpp:6860
Definition: sqltypes.h:72
static constexpr const char * MEMORY_STATS_SERVER_NAME
Definition: Catalog.h:893
bool recreateSystemTableIfUpdated(foreign_storage::ForeignTable &foreign_table, const std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:6898

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::initializeMemorySummarySystemTable ( )
private

Definition at line 6607 of file Catalog.cpp.

References Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_encoded_text_type(), kBIGINT, kINT, and Catalog_Namespace::MEMORY_SUMMARY_SYS_TABLE_NAME.

6607  {
6608  auto [foreign_table, columns] =
6611  {{"node", get_encoded_text_type()},
6612  {"device_id", {kINT}},
6613  {"device_type", get_encoded_text_type()},
6614  {"max_page_count", {kBIGINT}},
6615  {"page_size", {kBIGINT}},
6616  {"allocated_page_count", {kBIGINT}},
6617  {"used_page_count", {kBIGINT}},
6618  {"free_page_count", {kBIGINT}}},
6619  true);
6620  recreateSystemTableIfUpdated(foreign_table, columns);
6621 }
std::pair< foreign_storage::ForeignTable, std::list< ColumnDescriptor > > getSystemTableSchema(const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name, bool is_in_memory_system_table)
Definition: Catalog.cpp:6860
static constexpr const char * MEMORY_SUMMARY_SYS_TABLE_NAME
Definition: Catalog.h:119
Definition: sqltypes.h:72
static constexpr const char * MEMORY_STATS_SERVER_NAME
Definition: Catalog.h:893
bool recreateSystemTableIfUpdated(foreign_storage::ForeignTable &foreign_table, const std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:6898

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::initializeMLModelMetadataSystemTable ( )
private

Definition at line 6702 of file Catalog.cpp.

References Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_encoded_text_type(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_var_encoded_text_array_type(), kBIGINT, kDOUBLE, and Catalog_Namespace::ML_MODEL_METADATA_SYS_TABLE_NAME.

6702  {
6703  auto [foreign_table, columns] =
6706  {{"model_name", get_encoded_text_type()},
6707  {"model_type", get_encoded_text_type()},
6708  {"predicted", get_encoded_text_type()},
6709  {"features", get_var_encoded_text_array_type()},
6710  {"training_query", get_encoded_text_type()},
6711  {"num_logical_features", {kBIGINT}},
6712  {"num_physical_features", {kBIGINT}},
6713  {"num_categorical_features", {kBIGINT}},
6714  {"num_numeric_features", {kBIGINT}},
6715  {"train_fraction", {kDOUBLE}},
6716  {"eval_fraction", {kDOUBLE}}},
6717  true);
6718  recreateSystemTableIfUpdated(foreign_table, columns);
6719 }
std::pair< foreign_storage::ForeignTable, std::list< ColumnDescriptor > > getSystemTableSchema(const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name, bool is_in_memory_system_table)
Definition: Catalog.cpp:6860
static constexpr const char * ML_METADATA_SERVER_NAME
Definition: Catalog.h:896
static constexpr const char * ML_MODEL_METADATA_SYS_TABLE_NAME
Definition: Catalog.h:124
bool recreateSystemTableIfUpdated(foreign_storage::ForeignTable &foreign_table, const std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:6898

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::initializePermissionsSystemTable ( )
private

Definition at line 6531 of file Catalog.cpp.

References Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_encoded_text_type(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_var_encoded_text_array_type(), kBOOLEAN, kINT, and Catalog_Namespace::PERMISSIONS_SYS_TABLE_NAME.

6531  {
6532  auto [foreign_table, columns] =
6535  {{"role_name", get_encoded_text_type()},
6536  {"is_user_role", {kBOOLEAN}},
6537  {"database_id", {kINT}},
6538  {"database_name", get_encoded_text_type()},
6539  {"object_name", get_encoded_text_type()},
6540  {"object_id", {kINT}},
6541  {"object_owner_id", {kINT}},
6542  {"object_owner_user_name", get_encoded_text_type()},
6543  {"object_permission_type", get_encoded_text_type()},
6544  {"object_permissions", get_var_encoded_text_array_type()}},
6545  true);
6546  recreateSystemTableIfUpdated(foreign_table, columns);
6547 }
std::pair< foreign_storage::ForeignTable, std::list< ColumnDescriptor > > getSystemTableSchema(const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name, bool is_in_memory_system_table)
Definition: Catalog.cpp:6860
static constexpr const char * PERMISSIONS_SYS_TABLE_NAME
Definition: Catalog.h:116
static constexpr const char * CATALOG_SERVER_NAME
Definition: Catalog.h:892
Definition: sqltypes.h:72
bool recreateSystemTableIfUpdated(foreign_storage::ForeignTable &foreign_table, const std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:6898

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::initializeRequestLogsSystemTables ( )
private

Definition at line 6746 of file Catalog.cpp.

References Catalog_Namespace::anonymous_namespace{Catalog.cpp}::clear_cached_table_data(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_encoded_text_type(), kBIGINT, kINT, kTIMESTAMP, Catalog_Namespace::REQUEST_LOGS_SYS_TABLE_NAME, and Catalog_Namespace::anonymous_namespace{Catalog.cpp}::set_common_db_log_system_table_options().

6746  {
6747  auto [foreign_table, columns] =
6750  {{"log_timestamp", {kTIMESTAMP}},
6751  {"severity", get_encoded_text_type()},
6752  {"process_id", {kINT}},
6753  {"query_id", {kINT}},
6754  {"thread_id", {kINT}},
6755  {"file_location", get_encoded_text_type()},
6756  {"api_name", get_encoded_text_type()},
6757  {"request_duration_ms", {kBIGINT}},
6758  {"database_name", get_encoded_text_type()},
6759  {"user_name", get_encoded_text_type()},
6760  {"public_session_id", get_encoded_text_type()},
6761  {"query_string", get_encoded_text_type()},
6762  {"client", get_encoded_text_type()},
6763  {"dashboard_id", {kINT}},
6764  {"dashboard_name", get_encoded_text_type()},
6765  {"chart_id", {kINT}},
6766  {"execution_time_ms", {kBIGINT}},
6767  {"total_time_ms", {kBIGINT}}},
6768  false);
6771  // Matches request logs like those seen in the "heavydb.INFO.20220518-210103.log" test
6772  // file (specifically, lines containing " stdlog ").
6773  foreign_table.options[RegexFileBufferParser::LINE_REGEX_KEY] =
6774  "^([^\\s]+)\\s(\\w)\\s(\\d+)\\s(\\d+)\\s(\\d+)\\s([^\\s]+)\\s(?:stdlog)\\s(\\w+)"
6775  "\\s(?:\\d+)\\s(\\d+)\\s(\\w+)\\s([^\\s]+)\\s([^\\s]+)\\s(\\{[^\\}]+\\})\\s(\\{[^"
6776  "\\}]+\\})$";
6777  if (recreateSystemTableIfUpdated(foreign_table, columns)) {
6778  // Clear table cache if the table schema is updated
6779  clear_cached_table_data(dataMgr_.get(), currentDB_.dbId, foreign_table.tableId);
6780  }
6781 }
std::pair< foreign_storage::ForeignTable, std::list< ColumnDescriptor > > getSystemTableSchema(const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name, bool is_in_memory_system_table)
Definition: Catalog.cpp:6860
const DBMetadata currentDB_
Definition: Catalog.h:721
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:722
void clear_cached_table_data(const Data_Namespace::DataMgr *data_mgr, int32_t db_id, int32_t table_id)
Definition: Catalog.cpp:6458
static constexpr const char * LOGS_SERVER_NAME
Definition: Catalog.h:897
static constexpr const char * REQUEST_LOGS_SYS_TABLE_NAME
Definition: Catalog.h:126
void set_common_db_log_system_table_options(foreign_storage::ForeignTable &foreign_table)
Definition: Catalog.cpp:6446
Definition: sqltypes.h:72
bool recreateSystemTableIfUpdated(foreign_storage::ForeignTable &foreign_table, const std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:6898

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::initializeRoleAssignmentsSystemTable ( )
private

Definition at line 6598 of file Catalog.cpp.

References Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_encoded_text_type(), and Catalog_Namespace::ROLE_ASSIGNMENTS_SYS_TABLE_NAME.

6598  {
6599  auto [foreign_table, columns] = getSystemTableSchema(
6602  {{"role_name", get_encoded_text_type()}, {"user_name", get_encoded_text_type()}},
6603  true);
6604  recreateSystemTableIfUpdated(foreign_table, columns);
6605 }
std::pair< foreign_storage::ForeignTable, std::list< ColumnDescriptor > > getSystemTableSchema(const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name, bool is_in_memory_system_table)
Definition: Catalog.cpp:6860
static constexpr const char * ROLE_ASSIGNMENTS_SYS_TABLE_NAME
Definition: Catalog.h:118
static constexpr const char * CATALOG_SERVER_NAME
Definition: Catalog.h:892
bool recreateSystemTableIfUpdated(foreign_storage::ForeignTable &foreign_table, const std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:6898

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::initializeRolesSystemTable ( )
private

Definition at line 6549 of file Catalog.cpp.

References Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_encoded_text_type(), and Catalog_Namespace::ROLES_SYS_TABLE_NAME.

6549  {
6550  auto [foreign_table, columns] =
6553  {{"role_name", get_encoded_text_type()}},
6554  true);
6555  recreateSystemTableIfUpdated(foreign_table, columns);
6556 }
std::pair< foreign_storage::ForeignTable, std::list< ColumnDescriptor > > getSystemTableSchema(const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name, bool is_in_memory_system_table)
Definition: Catalog.cpp:6860
static constexpr const char * ROLES_SYS_TABLE_NAME
Definition: Catalog.h:117
static constexpr const char * CATALOG_SERVER_NAME
Definition: Catalog.h:892
bool recreateSystemTableIfUpdated(foreign_storage::ForeignTable &foreign_table, const std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:6898

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::initializeServerLogsSystemTables ( )
private

Definition at line 6721 of file Catalog.cpp.

References Catalog_Namespace::anonymous_namespace{Catalog.cpp}::clear_cached_table_data(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_encoded_text_type(), kINT, kTIMESTAMP, Catalog_Namespace::SERVER_LOGS_SYS_TABLE_NAME, and Catalog_Namespace::anonymous_namespace{Catalog.cpp}::set_common_db_log_system_table_options().

6721  {
6722  auto [foreign_table, columns] =
6725  {{"node", get_encoded_text_type()},
6726  {"log_timestamp", {kTIMESTAMP}},
6727  {"severity", get_encoded_text_type()},
6728  {"process_id", {kINT}},
6729  {"query_id", {kINT}},
6730  {"thread_id", {kINT}},
6731  {"file_location", get_encoded_text_type()},
6732  {"message", get_encoded_text_type()}},
6733  false);
6736  // Matches server logs like those seen in the "heavydb.INFO.20220518-210103.log" test
6737  // file.
6738  foreign_table.options[RegexFileBufferParser::LINE_REGEX_KEY] =
6739  "^([^\\s]+)\\s(\\w)\\s(\\d+)\\s(\\d+)\\s(\\d+)\\s([^\\s]+)\\s(.+)$";
6740  if (recreateSystemTableIfUpdated(foreign_table, columns)) {
6741  // Clear table cache if the table schema is updated
6742  clear_cached_table_data(dataMgr_.get(), currentDB_.dbId, foreign_table.tableId);
6743  }
6744 }
std::pair< foreign_storage::ForeignTable, std::list< ColumnDescriptor > > getSystemTableSchema(const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name, bool is_in_memory_system_table)
Definition: Catalog.cpp:6860
static constexpr const char * SERVER_LOGS_SYS_TABLE_NAME
Definition: Catalog.h:125
const DBMetadata currentDB_
Definition: Catalog.h:721
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:722
void clear_cached_table_data(const Data_Namespace::DataMgr *data_mgr, int32_t db_id, int32_t table_id)
Definition: Catalog.cpp:6458
static constexpr const char * LOGS_SERVER_NAME
Definition: Catalog.h:897
void set_common_db_log_system_table_options(foreign_storage::ForeignTable &foreign_table)
Definition: Catalog.cpp:6446
Definition: sqltypes.h:72
bool recreateSystemTableIfUpdated(foreign_storage::ForeignTable &foreign_table, const std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:6898

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::initializeStorageDetailsSystemTable ( )
private

Definition at line 6647 of file Catalog.cpp.

References Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_encoded_text_type(), kBIGINT, kINT, and Catalog_Namespace::STORAGE_DETAILS_SYS_TABLE_NAME.

6647  {
6648  auto [foreign_table, columns] =
6651  {{"node", get_encoded_text_type()},
6652  {"database_id", {kINT}},
6653  {"database_name", get_encoded_text_type()},
6654  {"table_id", {kINT}},
6655  {"table_name", get_encoded_text_type()},
6656  {"epoch", {kINT}},
6657  {"epoch_floor", {kINT}},
6658  {"fragment_count", {kINT}},
6659  {"shard_id", {kINT}},
6660  {"data_file_count", {kINT}},
6661  {"metadata_file_count", {kINT}},
6662  {"total_data_file_size", {kBIGINT}},
6663  {"total_data_page_count", {kBIGINT}},
6664  {"total_free_data_page_count", {kBIGINT}},
6665  {"total_metadata_file_size", {kBIGINT}},
6666  {"total_metadata_page_count", {kBIGINT}},
6667  {"total_free_metadata_page_count", {kBIGINT}},
6668  {"total_dictionary_data_file_size", {kBIGINT}}},
6669  true);
6670  recreateSystemTableIfUpdated(foreign_table, columns);
6671 }
std::pair< foreign_storage::ForeignTable, std::list< ColumnDescriptor > > getSystemTableSchema(const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name, bool is_in_memory_system_table)
Definition: Catalog.cpp:6860
static constexpr const char * STORAGE_DETAILS_SYS_TABLE_NAME
Definition: Catalog.h:121
Definition: sqltypes.h:72
bool recreateSystemTableIfUpdated(foreign_storage::ForeignTable &foreign_table, const std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:6898
static constexpr const char * STORAGE_STATS_SERVER_NAME
Definition: Catalog.h:894

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::initializeSystemServers ( )
private

Definition at line 6393 of file Catalog.cpp.

References foreign_storage::AbstractFileStorageDataWrapper::BASE_PATH_KEY, g_enable_logs_system_tables, logger::get_log_dir_path(), foreign_storage::DataWrapperType::INTERNAL_CATALOG, foreign_storage::DataWrapperType::INTERNAL_EXECUTOR_STATS, foreign_storage::DataWrapperType::INTERNAL_LOGS, foreign_storage::DataWrapperType::INTERNAL_MEMORY_STATS, foreign_storage::DataWrapperType::INTERNAL_ML_MODEL_METADATA, foreign_storage::DataWrapperType::INTERNAL_STORAGE_STATS, foreign_storage::AbstractFileStorageDataWrapper::LOCAL_FILE_STORAGE_TYPE, and foreign_storage::AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY.

6393  {
6404 
6406  foreign_storage::OptionsMap log_server_options;
6407  log_server_options
6411  logger::get_log_dir_path().string();
6414  log_server_options);
6415  }
6416 }
bool g_enable_logs_system_tables
Definition: Catalog.cpp:100
static constexpr char const * INTERNAL_STORAGE_STATS
static constexpr const char * ML_METADATA_SERVER_NAME
Definition: Catalog.h:896
fs::path get_log_dir_path()
Definition: Logger.cpp:904
static constexpr const char * LOGS_SERVER_NAME
Definition: Catalog.h:897
static constexpr char const * INTERNAL_CATALOG
static constexpr char const * INTERNAL_ML_MODEL_METADATA
static constexpr const char * EXECUTOR_STATS_SERVER_NAME
Definition: Catalog.h:895
static constexpr char const * INTERNAL_EXECUTOR_STATS
static constexpr char const * INTERNAL_MEMORY_STATS
static constexpr const char * CATALOG_SERVER_NAME
Definition: Catalog.h:892
std::map< std::string, std::string, std::less<>> OptionsMap
static constexpr const char * MEMORY_STATS_SERVER_NAME
Definition: Catalog.h:893
void createSystemTableServer(const std::string &server_name, const std::string &data_wrapper_type, const foreign_storage::OptionsMap &options={})
Definition: Catalog.cpp:6833
static constexpr char const * INTERNAL_LOGS
static constexpr const char * STORAGE_STATS_SERVER_NAME
Definition: Catalog.h:894

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::initializeSystemTables ( )
private

Definition at line 6476 of file Catalog.cpp.

References Catalog_Namespace::anonymous_namespace{Catalog.cpp}::drop_tables(), g_enable_logs_system_tables, Catalog_Namespace::REQUEST_LOGS_SYS_TABLE_NAME, Catalog_Namespace::SERVER_LOGS_SYS_TABLE_NAME, Catalog_Namespace::WS_SERVER_ACCESS_LOGS_SYS_TABLE_NAME, and Catalog_Namespace::WS_SERVER_LOGS_SYS_TABLE_NAME.

6476  {
6489 
6493  } else {
6494  drop_tables(*this,
6501  }
6502  }
6503 }
static constexpr const char * WS_SERVER_ACCESS_LOGS_SYS_TABLE_NAME
Definition: Catalog.h:128
void initializePermissionsSystemTable()
Definition: Catalog.cpp:6531
static constexpr const char * SERVER_LOGS_SYS_TABLE_NAME
Definition: Catalog.h:125
bool g_enable_logs_system_tables
Definition: Catalog.cpp:100
const foreign_storage::ForeignServer * getForeignServer(const std::string &server_name) const
Definition: Catalog.cpp:3334
void initializeServerLogsSystemTables()
Definition: Catalog.cpp:6721
static constexpr const char * LOGS_SERVER_NAME
Definition: Catalog.h:897
void initializeMLModelMetadataSystemTable()
Definition: Catalog.cpp:6702
static constexpr const char * REQUEST_LOGS_SYS_TABLE_NAME
Definition: Catalog.h:126
void initializeMemorySummarySystemTable()
Definition: Catalog.cpp:6607
void initializeStorageDetailsSystemTable()
Definition: Catalog.cpp:6647
void dropForeignServer(const std::string &server_name)
Definition: Catalog.cpp:3450
void initializeRoleAssignmentsSystemTable()
Definition: Catalog.cpp:6598
void initializeMemoryDetailsSystemTable()
Definition: Catalog.cpp:6623
void initializeRequestLogsSystemTables()
Definition: Catalog.cpp:6746
void initializeExecutorResourcePoolSummarySystemTable()
Definition: Catalog.cpp:6673
void drop_tables(Catalog &catalog, const std::vector< std::string > &table_names)
Definition: Catalog.cpp:6467
static constexpr const char * WS_SERVER_LOGS_SYS_TABLE_NAME
Definition: Catalog.h:127
void initializeDashboardsSystemTable()
Definition: Catalog.cpp:6582

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::initializeTablesSystemTable ( )
private

Definition at line 6558 of file Catalog.cpp.

References Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_encoded_text_type(), kBIGINT, kINT, and Catalog_Namespace::TABLES_SYS_TABLE_NAME.

6558  {
6559  auto [foreign_table, columns] =
6562  {{"database_id", {kINT}},
6563  {"database_name", get_encoded_text_type()},
6564  {"table_id", {kINT}},
6565  {"table_name", get_encoded_text_type()},
6566  {"owner_id", {kINT}},
6567  {"owner_user_name", get_encoded_text_type()},
6568  {"column_count", {kINT}},
6569  {"table_type", get_encoded_text_type()},
6570  {"view_sql", get_encoded_text_type()},
6571  {"max_fragment_size", {kINT}},
6572  {"max_chunk_size", {kBIGINT}},
6573  {"fragment_page_size", {kINT}},
6574  {"max_rows", {kBIGINT}},
6575  {"max_rollback_epochs", {kINT}},
6576  {"shard_count", {kINT}},
6577  {"ddl_statement", get_encoded_text_type()}},
6578  true);
6579  recreateSystemTableIfUpdated(foreign_table, columns);
6580 }
std::pair< foreign_storage::ForeignTable, std::list< ColumnDescriptor > > getSystemTableSchema(const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name, bool is_in_memory_system_table)
Definition: Catalog.cpp:6860
static constexpr const char * TABLES_SYS_TABLE_NAME
Definition: Catalog.h:113
static constexpr const char * CATALOG_SERVER_NAME
Definition: Catalog.h:892
Definition: sqltypes.h:72
bool recreateSystemTableIfUpdated(foreign_storage::ForeignTable &foreign_table, const std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:6898

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::initializeUsersSystemTable ( )
private

Definition at line 6505 of file Catalog.cpp.

References Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_encoded_text_type(), kBOOLEAN, kINT, and Catalog_Namespace::USERS_SYS_TABLE_NAME.

6505  {
6506  auto [foreign_table, columns] =
6509  {{"user_id", {kINT}},
6510  {"user_name", get_encoded_text_type()},
6511  {"is_super_user", {kBOOLEAN}},
6512  {"default_db_id", {kINT}},
6513  {"default_db_name", get_encoded_text_type()},
6514  {"can_login", {kBOOLEAN}}},
6515  true);
6516  recreateSystemTableIfUpdated(foreign_table, columns);
6517 }
std::pair< foreign_storage::ForeignTable, std::list< ColumnDescriptor > > getSystemTableSchema(const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name, bool is_in_memory_system_table)
Definition: Catalog.cpp:6860
static constexpr const char * USERS_SYS_TABLE_NAME
Definition: Catalog.h:112
static constexpr const char * CATALOG_SERVER_NAME
Definition: Catalog.h:892
Definition: sqltypes.h:72
bool recreateSystemTableIfUpdated(foreign_storage::ForeignTable &foreign_table, const std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:6898

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::initializeWebServerAccessLogsSystemTables ( )
private

Definition at line 6806 of file Catalog.cpp.

References Catalog_Namespace::anonymous_namespace{Catalog.cpp}::clear_cached_table_data(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_encoded_text_type(), kBIGINT, kSMALLINT, kTIMESTAMP, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::set_common_log_system_table_options(), and Catalog_Namespace::WS_SERVER_ACCESS_LOGS_SYS_TABLE_NAME.

6806  {
6807  auto [foreign_table, columns] =
6810  {{"ip_address", get_encoded_text_type()},
6811  {"log_timestamp", {kTIMESTAMP}},
6812  {"http_method", get_encoded_text_type()},
6813  {"endpoint", get_encoded_text_type()},
6814  {"http_status", {kSMALLINT}},
6815  {"response_size", {kBIGINT}}},
6816  false);
6817  set_common_log_system_table_options(foreign_table);
6819  foreign_table.options[AbstractFileStorageDataWrapper::REGEX_PATH_FILTER_KEY] =
6820  ".*heavy_web_server.*ACCESS\\..*";
6822  // Matches web server access logs like those seen in the
6823  // "heavy_web_server.test.log.ACCESS.20220518-210103.307016" test file.
6824  foreign_table.options[RegexFileBufferParser::LINE_REGEX_KEY] =
6825  "^(\\d+\\.\\d+\\.\\d+\\.\\d+)\\s+\\-\\s+\\-\\s+\\[([^\\]]+)\\]\\s+\"(\\w+)\\s+([^"
6826  "\\s]+)\\s+HTTP\\/1\\.1\"\\s+(\\d+)\\s+(\\d+)$";
6827  if (recreateSystemTableIfUpdated(foreign_table, columns)) {
6828  // Clear table cache if the table schema is updated
6829  clear_cached_table_data(dataMgr_.get(), currentDB_.dbId, foreign_table.tableId);
6830  }
6831 }
static constexpr const char * WS_SERVER_ACCESS_LOGS_SYS_TABLE_NAME
Definition: Catalog.h:128
std::pair< foreign_storage::ForeignTable, std::list< ColumnDescriptor > > getSystemTableSchema(const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name, bool is_in_memory_system_table)
Definition: Catalog.cpp:6860
void set_common_log_system_table_options(foreign_storage::ForeignTable &foreign_table)
Definition: Catalog.cpp:6436
const DBMetadata currentDB_
Definition: Catalog.h:721
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:722
void clear_cached_table_data(const Data_Namespace::DataMgr *data_mgr, int32_t db_id, int32_t table_id)
Definition: Catalog.cpp:6458
static constexpr const char * LOGS_SERVER_NAME
Definition: Catalog.h:897
bool recreateSystemTableIfUpdated(foreign_storage::ForeignTable &foreign_table, const std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:6898

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::initializeWebServerLogsSystemTables ( )
private

Definition at line 6783 of file Catalog.cpp.

References Catalog_Namespace::anonymous_namespace{Catalog.cpp}::clear_cached_table_data(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_encoded_text_type(), kTIMESTAMP, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::set_common_log_system_table_options(), and Catalog_Namespace::WS_SERVER_LOGS_SYS_TABLE_NAME.

6783  {
6784  auto [foreign_table, columns] =
6787  {{"log_timestamp", {kTIMESTAMP}},
6788  {"severity", get_encoded_text_type()},
6789  {"message", get_encoded_text_type()}},
6790  false);
6791  set_common_log_system_table_options(foreign_table);
6793  foreign_table.options[AbstractFileStorageDataWrapper::REGEX_PATH_FILTER_KEY] =
6794  ".*heavy_web_server.*ALL\\..*";
6796  // Matches web server logs like those seen in the
6797  // "heavy_web_server.test.log.ALL.20220518-210103.307016" test file.
6798  foreign_table.options[RegexFileBufferParser::LINE_REGEX_KEY] =
6799  "^time=\"([^\"]+)\"\\slevel=([^\\s]+)\\smsg=\"([^\"]+)\"$";
6800  if (recreateSystemTableIfUpdated(foreign_table, columns)) {
6801  // Clear table cache if the table schema is updated
6802  clear_cached_table_data(dataMgr_.get(), currentDB_.dbId, foreign_table.tableId);
6803  }
6804 }
std::pair< foreign_storage::ForeignTable, std::list< ColumnDescriptor > > getSystemTableSchema(const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name, bool is_in_memory_system_table)
Definition: Catalog.cpp:6860
void set_common_log_system_table_options(foreign_storage::ForeignTable &foreign_table)
Definition: Catalog.cpp:6436
const DBMetadata currentDB_
Definition: Catalog.h:721
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:722
void clear_cached_table_data(const Data_Namespace::DataMgr *data_mgr, int32_t db_id, int32_t table_id)
Definition: Catalog.cpp:6458
static constexpr const char * LOGS_SERVER_NAME
Definition: Catalog.h:897
bool recreateSystemTableIfUpdated(foreign_storage::ForeignTable &foreign_table, const std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:6898
static constexpr const char * WS_SERVER_LOGS_SYS_TABLE_NAME
Definition: Catalog.h:127

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::instantiateFragmenter ( TableDescriptor td) const
protected

Definition at line 1778 of file Catalog.cpp.

References CHECK_EQ, measure< TimeT >::execution(), TableDescriptor::fragmenter, TableDescriptor::fragPageSize, TableDescriptor::fragType, logger::INFO, Fragmenter_Namespace::INSERT_ORDER, LOG, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRows, TableDescriptor::persistenceLevel, TableDescriptor::shard, TableDescriptor::sortedColumnId, TableDescriptor::storageType, TableDescriptor::tableId, and TableDescriptor::tableName.

1778  {
1779  auto time_ms = measure<>::execution([&]() {
1780  // instanciate table fragmenter upon first use
1781  // assume only insert order fragmenter is supported
1783  vector<Chunk> chunkVec;
1784  auto columnDescs = getAllColumnMetadataForTable(td->tableId, true, false, true);
1785  Chunk::translateColumnDescriptorsToChunkVec(columnDescs, chunkVec);
1786  ChunkKey chunkKeyPrefix = {currentDB_.dbId, td->tableId};
1787  if (td->sortedColumnId > 0) {
1788  td->fragmenter = std::make_shared<SortedOrderFragmenter>(chunkKeyPrefix,
1789  chunkVec,
1790  dataMgr_.get(),
1791  const_cast<Catalog*>(this),
1792  td->tableId,
1793  td->shard,
1794  td->maxFragRows,
1795  td->maxChunkSize,
1796  td->fragPageSize,
1797  td->maxRows,
1798  td->persistenceLevel);
1799  } else {
1800  td->fragmenter = std::make_shared<InsertOrderFragmenter>(chunkKeyPrefix,
1801  chunkVec,
1802  dataMgr_.get(),
1803  const_cast<Catalog*>(this),
1804  td->tableId,
1805  td->shard,
1806  td->maxFragRows,
1807  td->maxChunkSize,
1808  td->fragPageSize,
1809  td->maxRows,
1810  td->persistenceLevel,
1811  !td->storageType.empty());
1812  }
1813  });
1814  LOG(INFO) << "Instantiating Fragmenter for table " << td->tableName << " took "
1815  << time_ms << "ms";
1816 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
std::vector< int > ChunkKey
Definition: types.h:36
std::string tableName
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:285
const DBMetadata currentDB_
Definition: Catalog.h:721
std::string storageType
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:722
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
Fragmenter_Namespace::FragmenterType fragType
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:2172
Data_Namespace::MemoryLevel persistenceLevel
Catalog()
Constructor builds a hollow catalog used during constructor of other catalogs.
Definition: Catalog.cpp:171

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::invalidateCachesForTable ( const int  table_id)

Definition at line 4229 of file Catalog.cpp.

References CHECK, CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, Executor::clearExternalCaches(), Data_Namespace::CPU_LEVEL, and Data_Namespace::GPU_LEVEL.

4229  {
4230  // When called, exactly one thread has a LockMgr data or insert lock for the table.
4231  cat_read_lock read_lock(this);
4232  ChunkKey const table_key{getDatabaseId(), table_id};
4233  auto td = getMutableMetadataForTableUnlocked(table_id);
4234  CHECK(td);
4238 
4240 
4241  // TODO(sy): doTruncateTable() says "destroy fragmenter before deleteChunks is called"
4242  // removeFragmenterForTable(table_key[CHUNK_KEY_TABLE_IDX]);
4243  if (td->fragmenter != nullptr) {
4244  auto tableDescIt = tableDescriptorMapById_.find(table_id);
4245  CHECK(tableDescIt != tableDescriptorMapById_.end());
4246  tableDescIt->second->fragmenter = nullptr;
4247  CHECK(td->fragmenter == nullptr);
4248  }
4250  table_key[CHUNK_KEY_TABLE_IDX]);
4251  if (dynamic_cast<foreign_storage::ForeignTable*>(td)) {
4252  dataMgr_->getPersistentStorageMgr()->getForeignStorageMgr()->refreshTable(table_key,
4253  true);
4254  } else {
4255  dataMgr_->removeMutableTableDiskCacheData(currentDB_.dbId, table_id);
4256  }
4258 }
void instantiateFragmenter(TableDescriptor *td) const
Definition: Catalog.cpp:1778
std::vector< int > ChunkKey
Definition: types.h:36
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:266
const DBMetadata currentDB_
Definition: Catalog.h:721
#define CHUNK_KEY_DB_IDX
Definition: types.h:38
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:722
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
TableDescriptor * getMutableMetadataForTableUnlocked(int table_id) const
Definition: Catalog.cpp:1896
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:39
int getDatabaseId() const
Definition: Catalog.h:326
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
Definition: DataMgr.cpp:649
void deleteChunksWithPrefix(const ChunkKey &keyPrefix)
Definition: DataMgr.cpp:522
void closeFileMgr(const int32_t db_id, const int32_t tb_id)
#define CHECK(condition)
Definition: Logger.h:291
static void clearExternalCaches(bool for_update, const TableDescriptor *td, const int current_db_id)
Definition: Execute.h:438
void refreshDictionaryCachesForTableUnlocked(const TableDescriptor &td)
Definition: Catalog.cpp:4209

+ Here is the call graph for this function:

bool Catalog_Namespace::Catalog::isInfoSchemaDb ( ) const

Definition at line 6389 of file Catalog.cpp.

References shared::kInfoSchemaDbName, and setup::name.

6389  {
6390  return name() == shared::kInfoSchemaDbName;
6391 }
const std::string kInfoSchemaDbName
std::string name() const
Definition: Catalog.h:348
std::string Catalog_Namespace::Catalog::name ( ) const
inline

Definition at line 348 of file Catalog.h.

References Catalog_Namespace::DBMetadata::dbName, and getCurrentDB().

Referenced by anonymous_namespace{Calcite.cpp}::check_db_access(), AlterTableAlterColumnCommandRecoveryMgr::getRecoveryFiles(), recordOwnershipOfObjectsInObjectPermissions(), AlterTableAlterColumnCommandRecoveryMgr::resolveIncompleteAlterColumnCommands(), boost::serialization::save_construct_data(), and lockmgr::TableLockMgrImpl< TableDataLockMgr >::validateAndGetExistingTableId().

348 { return getCurrentDB().dbName; }
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:265

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::vector< DBObject > Catalog_Namespace::Catalog::parseDashboardObjects ( const std::string &  view_meta,
const int &  user_id 
)
protected

Definition at line 1694 of file Catalog.cpp.

References DBObjectKey::dbId, logger::INFO, LOG, DBObjectKey::objectId, parse_underlying_dashboard_objects(), DBObjectKey::permissionType, AccessPrivileges::SELECT_FROM_TABLE, AccessPrivileges::SELECT_FROM_VIEW, TableDBObjectType, run_benchmark_import::type, and ViewDBObjectType.

1695  {
1696  std::vector<DBObject> objects;
1697  DBObjectKey key;
1698  key.dbId = currentDB_.dbId;
1699  auto _key_place = [&key](auto type, auto id) {
1700  key.permissionType = type;
1701  key.objectId = id;
1702  return key;
1703  };
1704  for (auto object_name : parse_underlying_dashboard_objects(view_meta)) {
1705  auto td = getMetadataForTable(object_name, false);
1706  if (!td) {
1707  // Parsed object source is not present in current database
1708  // LOG the info and ignore
1709  LOG(INFO) << "Ignoring dashboard source Table/View: " << object_name
1710  << " no longer exists in current DB.";
1711  continue;
1712  }
1713  // Dashboard source can be Table or View
1714  const auto object_type = td->isView ? ViewDBObjectType : TableDBObjectType;
1715  const auto priv = td->isView ? AccessPrivileges::SELECT_FROM_VIEW
1717  objects.emplace_back(_key_place(object_type, td->tableId), priv, user_id);
1718  objects.back().setObjectType(td->isView ? ViewDBObjectType : TableDBObjectType);
1719  objects.back().setName(td->tableName);
1720  }
1721  return objects;
1722 }
#define LOG(tag)
Definition: Logger.h:285
const DBMetadata currentDB_
Definition: Catalog.h:721
int32_t objectId
Definition: DBObject.h:55
static const AccessPrivileges SELECT_FROM_TABLE
Definition: DBObject.h:160
std::vector< std::string > parse_underlying_dashboard_objects(const std::string &meta)
static const AccessPrivileges SELECT_FROM_VIEW
Definition: DBObject.h:180
int32_t dbId
Definition: DBObject.h:54
int32_t permissionType
Definition: DBObject.h:53
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::quoteIfRequired ( const std::string &  column_name) const
private

Definition at line 5867 of file Catalog.cpp.

References Catalog_Namespace::contains_spaces(), Catalog_Namespace::contains_sql_reserved_chars(), get_quoted_string(), and Catalog_Namespace::is_reserved_sql_keyword().

5867  {
5868  if (is_reserved_sql_keyword(column_name) || contains_spaces(column_name) ||
5869  contains_sql_reserved_chars(column_name)) {
5870  return get_quoted_string(column_name, '"', '"');
5871  } else {
5872  return column_name;
5873  }
5874 }
bool contains_spaces(std::string_view str)
returns true if the string contains one or more spaces
Definition: Catalog.cpp:5548
bool contains_sql_reserved_chars(std::string_view str, std::string_view chars="`~!@#$%^&*()-=+[{]}\\|;:'\",<.>/?")
returns true if the string contains one or more OmniSci SQL reserved characters
Definition: Catalog.cpp:5555
std::string get_quoted_string(const std::string &filename, char quote, char escape)
Quote a string while escaping any existing quotes in the string.
bool is_reserved_sql_keyword(std::string_view str)
returns true if the string equals an OmniSci SQL reserved keyword
Definition: Catalog.cpp:5562

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::reassignOwners ( const std::set< std::string > &  old_owners,
const std::string &  new_owner 
)

Reassigns database object ownership from a set of users (old owners) to another user (new owner).

Parameters
old_owners- users whose database object ownership will be reassigned to a new user
new_owner- user who will own reassigned database objects

Definition at line 6161 of file Catalog.cpp.

References Catalog_Namespace::anonymous_namespace{Catalog.cpp}::add_db_object(), AccessPrivileges::ALL_DASHBOARD, AccessPrivileges::ALL_SERVER, AccessPrivileges::ALL_TABLE, AccessPrivileges::ALL_VIEW, CHECK, CHECK_EQ, CHECK_NE, shared::contains(), DashboardDBObjectType, g_enable_fsi, ServerDBObjectType, TableDBObjectType, to_string(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::validate_and_get_user_id(), and ViewDBObjectType.

6162  {
6163  CHECK(!old_owners.empty());
6164  int32_t new_owner_id = validate_and_get_user_id(new_owner);
6165  std::map<int32_t, std::string> old_owners_user_name_by_id;
6166  std::set<int32_t> old_owner_ids;
6167  for (const auto& old_owner : old_owners) {
6168  auto old_owner_id = validate_and_get_user_id(old_owner);
6169  if (old_owner_id != new_owner_id) {
6170  old_owner_ids.emplace(old_owner_id);
6171  old_owners_user_name_by_id[old_owner_id] = old_owner;
6172  }
6173  }
6174 
6175  // An empty set after the above loop implies reassignment to the same user (i.e. all
6176  // users in the old_owners set is the same as new_owner). Do nothing in this case.
6177  if (old_owner_ids.empty()) {
6178  return;
6179  }
6180 
6181  std::map<int32_t, std::vector<DBObject>> old_owner_db_objects;
6182  {
6183  cat_write_lock write_lock(this);
6184  cat_sqlite_lock sqlite_lock(getObjForLock());
6185  sqliteConnector_.query("BEGIN TRANSACTION");
6186  try {
6187  for (const auto old_user_id : old_owner_ids) {
6189  "UPDATE mapd_tables SET userid = ? WHERE userid = ?",
6190  std::vector<std::string>{std::to_string(new_owner_id),
6191  std::to_string(old_user_id)});
6192 
6194  "UPDATE mapd_dashboards SET userid = ? WHERE userid = ?",
6195  std::vector<std::string>{std::to_string(new_owner_id),
6196  std::to_string(old_user_id)});
6197 
6198  if (g_enable_fsi) {
6200  "UPDATE omnisci_foreign_servers SET owner_user_id = ? "
6201  "WHERE owner_user_id = ?",
6202  std::vector<std::string>{std::to_string(new_owner_id),
6203  std::to_string(old_user_id)});
6204  }
6205  }
6206 
6207  for (const auto& [table_name, td] : tableDescriptorMap_) {
6208  if (shared::contains(old_owner_ids, td->userId)) {
6209  if (td->isView) {
6210  add_db_object(td->tableName,
6212  td->userId,
6214  old_owner_db_objects);
6215  } else {
6216  add_db_object(td->tableName,
6218  td->userId,
6220  old_owner_db_objects);
6221  }
6222  td->userId = new_owner_id;
6223  }
6224  }
6225 
6226  DashboardDescriptorMap new_owner_dashboard_map;
6227  for (auto it = dashboardDescriptorMap_.begin();
6228  it != dashboardDescriptorMap_.end();) {
6229  if (auto dashboard = it->second;
6230  shared::contains(old_owner_ids, dashboard->userId)) {
6231  DBObject db_object{dashboard->dashboardId, DBObjectType::DashboardDBObjectType};
6232  db_object.setPrivileges(AccessPrivileges::ALL_DASHBOARD);
6233  old_owner_db_objects[dashboard->userId].emplace_back(db_object);
6234 
6235  // Dashboards in the dashboardDescriptorMap_ use keys with the format
6236  // "{user id}:{dashboard name}". Ensure that map entries are replaced
6237  // with the new owner's user id.
6238  std::string old_key{std::to_string(dashboard->userId) + ":" +
6239  dashboard->dashboardName};
6240  CHECK_EQ(it->first, old_key);
6241  std::string new_key{std::to_string(new_owner_id) + ":" +
6242  dashboard->dashboardName};
6243  CHECK(dashboardDescriptorMap_.find(new_key) == dashboardDescriptorMap_.end());
6244  new_owner_dashboard_map[new_key] = dashboard;
6245  dashboard->userId = new_owner_id;
6246  dashboard->user = new_owner;
6247  it = dashboardDescriptorMap_.erase(it);
6248  } else {
6249  it++;
6250  }
6251  }
6252  dashboardDescriptorMap_.merge(new_owner_dashboard_map);
6253 
6254  if (g_enable_fsi) {
6255  for (const auto& [server_name, server] : foreignServerMap_) {
6256  if (shared::contains(old_owner_ids, server->user_id)) {
6257  add_db_object(server->name,
6259  server->user_id,
6261  old_owner_db_objects);
6262  server->user_id = new_owner_id;
6263  }
6264  }
6265  }
6266 
6267  // Ensure new owner is set in the DB objects.
6268  for (auto& [old_owner_id, db_objects] : old_owner_db_objects) {
6269  for (auto& db_object : db_objects) {
6270  db_object.loadKey(*this);
6271  CHECK_EQ(db_object.getOwner(), new_owner_id);
6272  const auto& object_key = db_object.getObjectKey();
6273  CHECK_EQ(object_key.dbId, getDatabaseId());
6274  CHECK_NE(object_key.objectId, -1);
6275  }
6276  }
6277  } catch (std::exception& e) {
6278  sqliteConnector_.query("ROLLBACK TRANSACTION");
6280  old_owners_user_name_by_id, old_owner_db_objects, new_owner_id);
6281  throw;
6282  }
6283  sqliteConnector_.query("END TRANSACTION");
6284  }
6285 
6286  try {
6288  old_owner_db_objects, new_owner_id, *this);
6289  } catch (std::exception& e) {
6290  restoreOldOwners(old_owners_user_name_by_id, old_owner_db_objects, new_owner_id);
6291  throw;
6292  }
6293 }
bool contains(const T &container, const U &element)
Definition: misc.h:195
#define CHECK_EQ(x, y)
Definition: Logger.h:301
void add_db_object(const std::string &object_name, DBObjectType object_type, int32_t user_id, const AccessPrivileges &privileges, std::map< int32_t, std::vector< DBObject >> &db_objects)
Definition: Catalog.cpp:6150
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
void reassignObjectOwners(const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects, int32_t new_owner_id, const Catalog_Namespace::Catalog &catalog)
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:712
std::string to_string(char const *&&v)
static const AccessPrivileges ALL_VIEW
Definition: DBObject.h:177
void restoreOldOwnersInMemory(const std::map< int32_t, std::string > &old_owners_user_name_by_id, const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects, int32_t new_owner_id)
Definition: Catalog.cpp:6344
static SysCatalog & instance()
Definition: SysCatalog.h:343
#define CHECK_NE(x, y)
Definition: Logger.h:302
void restoreOldOwners(const std::map< int32_t, std::string > &old_owners_user_name_by_id, const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects, int32_t new_owner_id)
Definition: Catalog.cpp:6295
int getDatabaseId() const
Definition: Catalog.h:326
static const AccessPrivileges ALL_SERVER
Definition: DBObject.h:187
ForeignServerMap foreignServerMap_
Definition: Catalog.h:715
std::map< std::string, std::shared_ptr< DashboardDescriptor >> DashboardDescriptorMap
Definition: Types.h:44
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
static const AccessPrivileges ALL_DASHBOARD
Definition: DBObject.h:169
static const AccessPrivileges ALL_TABLE
Definition: DBObject.h:157
#define CHECK(condition)
Definition: Logger.h:291
int32_t validate_and_get_user_id(const std::string &user_name)
Definition: Catalog.cpp:6119
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
bool g_enable_fsi
Definition: Catalog.cpp:96
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:707

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::recordOwnershipOfObjectsInObjectPermissions ( )
protected

Definition at line 797 of file Catalog.cpp.

References AccessPrivileges::ALL_DASHBOARD, AccessPrivileges::ALL_DATABASE, AccessPrivileges::ALL_SERVER, AccessPrivileges::ALL_TABLE, AccessPrivileges::ALL_VIEW, CHECK, currentDB_, DashboardDBObjectType, DatabaseDBObjectType, DBObjectKey::dbId, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, Catalog_Namespace::DBMetadata::dbOwner, logger::ERROR, SqliteConnector::getData(), SqliteConnector::getNumRows(), getObjForLock(), Catalog_Namespace::SysCatalog::instance(), LOG, name(), DBObjectKey::objectId, DBObjectKey::permissionType, Catalog_Namespace::SysCatalog::populateRoleDbObjects(), SqliteConnector::query(), SqliteConnector::query_with_text_params(), ServerDBObjectType, DBObject::setObjectKey(), DBObject::setOwner(), DBObject::setPrivileges(), sqliteConnector_, TableDBObjectType, to_string(), run_benchmark_import::type, and ViewDBObjectType.

Referenced by CheckAndExecuteMigrations().

797  {
798  cat_sqlite_lock sqlite_lock(getObjForLock());
799  sqliteConnector_.query("BEGIN TRANSACTION");
800  std::vector<DBObject> objects;
801  try {
803  "SELECT name FROM sqlite_master WHERE type='table' AND "
804  "name='mapd_record_ownership_marker'");
805  // check if mapd catalog - marker exists
806  if (sqliteConnector_.getNumRows() != 0 && currentDB_.dbId == 1) {
807  // already done
808  sqliteConnector_.query("END TRANSACTION");
809  return;
810  }
811  // check if different catalog - marker exists
812  else if (sqliteConnector_.getNumRows() != 0 && currentDB_.dbId != 1) {
813  sqliteConnector_.query("SELECT dummy FROM mapd_record_ownership_marker");
814  // Check if migration is being performed on existing non mapd catalogs
815  // Older non mapd dbs will have table but no record in them
816  if (sqliteConnector_.getNumRows() != 0) {
817  // already done
818  sqliteConnector_.query("END TRANSACTION");
819  return;
820  }
821  }
822  // marker not exists - create one
823  else {
824  sqliteConnector_.query("CREATE TABLE mapd_record_ownership_marker (dummy integer)");
825  }
826 
827  DBMetadata db;
828  CHECK(SysCatalog::instance().getMetadataForDB(currentDB_.dbName, db));
829  // place dbId as a refernce for migration being performed
831  "INSERT INTO mapd_record_ownership_marker (dummy) VALUES (?1)",
832  std::vector<std::string>{std::to_string(db.dbOwner)});
833 
834  static const std::map<const DBObjectType, const AccessPrivileges>
835  object_level_all_privs_lookup{
841 
842  // grant owner all permissions on DB
843  DBObjectKey key;
844  key.dbId = currentDB_.dbId;
845  auto _key_place = [&key](auto type) {
846  key.permissionType = type;
847  return key;
848  };
849  for (auto& it : object_level_all_privs_lookup) {
850  objects.emplace_back(_key_place(it.first), it.second, db.dbOwner);
851  objects.back().setName(currentDB_.dbName);
852  }
853 
854  {
855  // other users tables and views
856  string tableQuery(
857  "SELECT tableid, name, userid, isview FROM mapd_tables WHERE userid > 0");
858  sqliteConnector_.query(tableQuery);
859  size_t numRows = sqliteConnector_.getNumRows();
860  for (size_t r = 0; r < numRows; ++r) {
861  int32_t tableid = sqliteConnector_.getData<int>(r, 0);
862  std::string tableName = sqliteConnector_.getData<string>(r, 1);
863  int32_t ownerid = sqliteConnector_.getData<int>(r, 2);
864  bool isview = sqliteConnector_.getData<bool>(r, 3);
865 
868  DBObjectKey key;
869  key.dbId = currentDB_.dbId;
870  key.objectId = tableid;
871  key.permissionType = type;
872 
873  DBObject obj(tableName, type);
874  obj.setObjectKey(key);
875  obj.setOwner(ownerid);
876  obj.setPrivileges(isview ? AccessPrivileges::ALL_VIEW
878 
879  objects.push_back(obj);
880  }
881  }
882 
883  {
884  // other users dashboards
885  string tableQuery("SELECT id, name, userid FROM mapd_dashboards WHERE userid > 0");
886  sqliteConnector_.query(tableQuery);
887  size_t numRows = sqliteConnector_.getNumRows();
888  for (size_t r = 0; r < numRows; ++r) {
889  int32_t dashId = sqliteConnector_.getData<int>(r, 0);
890  std::string dashName = sqliteConnector_.getData<string>(r, 1);
891  int32_t ownerid = sqliteConnector_.getData<int>(r, 2);
892 
894  DBObjectKey key;
895  key.dbId = currentDB_.dbId;
896  key.objectId = dashId;
897  key.permissionType = type;
898 
899  DBObject obj(dashName, type);
900  obj.setObjectKey(key);
901  obj.setOwner(ownerid);
902  obj.setPrivileges(AccessPrivileges::ALL_DASHBOARD);
903 
904  objects.push_back(obj);
905  }
906  }
907  } catch (const std::exception& e) {
908  sqliteConnector_.query("ROLLBACK TRANSACTION");
909  throw;
910  }
911  sqliteConnector_.query("END TRANSACTION");
912 
913  // now apply the objects to the syscat to track the permisisons
914  // moved outside transaction to avoid lock in sqlite
915  try {
917  } catch (const std::exception& e) {
918  LOG(ERROR) << " Issue during migration of DB " << name() << " issue was " << e.what();
919  throw std::runtime_error(" Issue during migration of DB " + name() + " issue was " +
920  e.what());
921  // will need to remove the mapd_record_ownership_marker table and retry
922  }
923 }
T getData(const int row, const int col)
static const AccessPrivileges ALL_DATABASE
Definition: DBObject.h:151
virtual void query_with_text_params(std::string const &query_only)
DBObjectType
Definition: DBObject.h:40
#define LOG(tag)
Definition: Logger.h:285
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
virtual void query(const std::string &queryString)
int32_t objectId
Definition: DBObject.h:55
std::string to_string(char const *&&v)
static const AccessPrivileges ALL_VIEW
Definition: DBObject.h:177
std::string name() const
Definition: Catalog.h:348
static SysCatalog & instance()
Definition: SysCatalog.h:343
static const AccessPrivileges ALL_SERVER
Definition: DBObject.h:187
int32_t dbId
Definition: DBObject.h:54
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
static const AccessPrivileges ALL_DASHBOARD
Definition: DBObject.h:169
static const AccessPrivileges ALL_TABLE
Definition: DBObject.h:157
#define CHECK(condition)
Definition: Logger.h:291
int32_t permissionType
Definition: DBObject.h:53
void populateRoleDbObjects(const std::vector< DBObject > &objects)
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool Catalog_Namespace::Catalog::recreateSystemTableIfUpdated ( foreign_storage::ForeignTable foreign_table,
const std::list< ColumnDescriptor > &  columns 
)
private

Definition at line 6898 of file Catalog.cpp.

References CHECK, foreign_storage::ForeignTable::foreign_server, logger::INFO, LOG, foreign_storage::ForeignServer::name, foreign_storage::OptionsContainer::options, and TableDescriptor::tableName.

6899  {
6900  auto stored_td = getMetadataForTable(foreign_table.tableName, false);
6901  bool should_recreate{false};
6902  if (stored_td) {
6903  auto stored_foreign_table =
6904  dynamic_cast<const foreign_storage::ForeignTable*>(stored_td);
6905  CHECK(stored_foreign_table);
6906  if (stored_foreign_table->foreign_server->name !=
6907  foreign_table.foreign_server->name ||
6908  stored_foreign_table->options != foreign_table.options) {
6909  should_recreate = true;
6910  } else {
6911  auto stored_columns =
6912  getAllColumnMetadataForTable(stored_td->tableId, false, false, false);
6913  if (stored_columns.size() != columns.size()) {
6914  should_recreate = true;
6915  } else {
6916  auto it_1 = stored_columns.begin();
6917  auto it_2 = columns.begin();
6918  for (; it_1 != stored_columns.end() && it_2 != columns.end(); it_1++, it_2++) {
6919  // Need a custom comparison here since we don't care if the dictionary comp
6920  // param has changed (comp can change because the column was assigned a
6921  // dictionary, whereas before it was just a compression number).
6922  if ((*it_1)->columnName != it_2->columnName ||
6923  (*it_1)->columnType.get_type() != it_2->columnType.get_type() ||
6924  (*it_1)->columnType.get_subtype() != it_2->columnType.get_subtype() ||
6925  (*it_1)->columnType.get_dimension() != it_2->columnType.get_dimension() ||
6926  (*it_1)->columnType.get_scale() != it_2->columnType.get_scale() ||
6927  (*it_1)->columnType.get_notnull() != it_2->columnType.get_notnull() ||
6928  (*it_1)->columnType.get_compression() !=
6929  it_2->columnType.get_compression() ||
6930  (*it_1)->columnType.get_size() != it_2->columnType.get_size()) {
6931  should_recreate = true;
6932  break;
6933  }
6934  }
6935  }
6936  }
6937  } else {
6938  should_recreate = true;
6939  }
6940  if (should_recreate) {
6941  if (stored_td) {
6942  LOG(INFO) << "Dropping existing \"" << foreign_table.tableName
6943  << "\" system table.";
6944  deleteTableCatalogMetadata(stored_td, {stored_td});
6945  }
6946  LOG(INFO) << "Creating a new \"" << foreign_table.tableName << "\" system table.";
6947  createTable(foreign_table, columns, {}, true);
6948  }
6949  return should_recreate;
6950 }
std::string tableName
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:2935
#define LOG(tag)
Definition: Logger.h:285
void deleteTableCatalogMetadata(const TableDescriptor *logical_table, const std::vector< const TableDescriptor * > &physical_tables)
Definition: Catalog.cpp:4310
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:2172
const ForeignServer * foreign_server
Definition: ForeignTable.h:57
#define CHECK(condition)
Definition: Logger.h:291
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
void Catalog_Namespace::Catalog::refreshDictionaryCachesForTableUnlocked ( const TableDescriptor td)
private

Definition at line 4209 of file Catalog.cpp.

References ColumnDescriptor::columnType, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), kENCODING_DICT, TableDescriptor::nColumns, and TableDescriptor::tableId.

4209  {
4210  for (auto col_id = 0; col_id < td.nColumns; ++col_id) {
4211  if (auto it = columnDescriptorMapById_.find({td.tableId, col_id});
4212  it != columnDescriptorMapById_.end()) {
4213  auto cd = it->second;
4214  auto dict_id = cd->columnType.get_comp_param();
4215  if (cd->columnType.get_compression() == kENCODING_DICT && dict_id) {
4216  DictRef dict_ref(currentDB_.dbId, dict_id);
4217  if (auto dict_it = dictDescriptorMapByRef_.find(dict_ref);
4218  dict_it != dictDescriptorMapByRef_.end()) {
4219  // getMetadataForDict() will only reload if the stringDict is null.
4220  dict_it->second->stringDict = nullptr;
4221  }
4222  getMetadataForDict(dict_id, true);
4223  }
4224  }
4225  }
4226 }
const DBMetadata currentDB_
Definition: Catalog.h:721
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:711
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:710
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1904

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::reloadCatalogMetadata ( const std::map< int32_t, std::string > &  user_name_by_user_id)
private

Definition at line 1190 of file Catalog.cpp.

References Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_user_id_to_user_name_map().

1191  {
1192  cat_write_lock write_lock(this);
1193  cat_sqlite_lock sqlite_lock(getObjForLock());
1195 }
void reloadCatalogMetadataUnlocked(const std::map< int32_t, std::string > &user_name_by_user_id)
Definition: Catalog.cpp:1198
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
std::map< int32_t, std::string > get_user_id_to_user_name_map()
Definition: Catalog.cpp:1042

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::reloadCatalogMetadataUnlocked ( const std::map< int32_t, std::string > &  user_name_by_user_id)
private

Definition at line 1198 of file Catalog.cpp.

References CHECK_EQ, CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, g_enable_fsi, lockmgr::TableSchemaLockMgr::instance(), and shared::kCatalogDirectoryName.

1199  {
1201 
1203  currentDB_.dbName);
1204 
1205  // Notice when tables or columns have been created, dropped, or changed by other nodes.
1206  // Needed so that users will see reasonably-correct lists of what objects exist.
1207 
1208  // Load the list of table ID's that exist on disk storage.
1209  std::set<int> cluster_table_ids;
1210  std::string tableQuery("SELECT tableid from mapd_tables");
1211  sqliteConnector_.query(tableQuery);
1212  auto numRows = sqliteConnector_.getNumRows();
1213  for (size_t r = 0; r < numRows; ++r) {
1214  const auto table_id = sqliteConnector_.getData<int>(r, 0);
1215  cluster_table_ids.insert(table_id);
1216  }
1217 
1218  // Ignore any table ID's locked by other threads on this node.
1219  // Those other threads are already handling any necessary reloading for those tables.
1220  std::set<int> ignored_table_ids;
1221  for (ChunkKey const& k : lockmgr::TableSchemaLockMgr::instance().getLockedTables()) {
1222  CHECK_EQ(k.size(), 2U);
1223  if (k[CHUNK_KEY_DB_IDX] != currentDB_.dbId) {
1224  continue;
1225  }
1226  ignored_table_ids.insert(k[CHUNK_KEY_TABLE_IDX]);
1227  }
1228 
1229  // For this node's Catalog cache:
1230  // Newly created table schemas created by other nodes need to be loaded.
1231  // Unlocked table schemas might have been renamed by other nodes; just reload them all.
1232  // Deleted table schemas still in this node's cache need to be flushed.
1233  std::set<int> reload_table_ids;
1234  for (auto const& cluster_table_id : cluster_table_ids) {
1235  if (ignored_table_ids.find(cluster_table_id) == ignored_table_ids.end()) {
1236  reload_table_ids.insert(cluster_table_id);
1237  }
1238  }
1239  for (auto const& [cached_table_id, td] : tableDescriptorMapById_) {
1240  if (cluster_table_ids.find(cached_table_id) == cluster_table_ids.end()) {
1241  reload_table_ids.insert(cached_table_id);
1242  }
1243  }
1244 
1245  // Reload tables.
1246  for (auto const& reload_table_id : reload_table_ids) {
1247  reloadTableMetadataUnlocked(reload_table_id);
1248  }
1249 
1251 
1252  dashboardDescriptorMap_.clear();
1253  linkDescriptorMap_.clear();
1254  linkDescriptorMapById_.clear();
1255  foreignServerMap_.clear();
1256  foreignServerMapById_.clear();
1257  custom_expr_map_by_id_.clear();
1258 
1259  if (g_enable_fsi) {
1261  }
1262 
1264  buildDashboardsMapUnlocked(user_name_by_user_id);
1267 
1268  // Notify Calcite about the reloaded database.
1269  if (calciteMgr_) {
1270  calciteMgr_->updateMetadata(currentDB_.dbName, {});
1271  }
1272 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
std::vector< int > ChunkKey
Definition: types.h:36
static TableSchemaLockMgr & instance()
Definition: LockMgr.h:40
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
#define CHUNK_KEY_DB_IDX
Definition: types.h:38
virtual void query(const std::string &queryString)
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:712
void reset(const std::string &path)
void buildDashboardsMapUnlocked(const std::map< int32_t, std::string > &user_name_by_user_id)
Definition: Catalog.cpp:1417
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:39
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
void buildCustomExpressionsMapUnlocked()
Definition: Catalog.cpp:1514
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:716
ForeignServerMap foreignServerMap_
Definition: Catalog.h:715
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:725
const std::string kCatalogDirectoryName
void reloadTableMetadataUnlocked(int table_id)
Definition: Catalog.cpp:1100
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:717
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:714
bool g_enable_fsi
Definition: Catalog.cpp:96
virtual size_t getNumRows() const
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:713

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::reloadDictionariesFromDiskUnlocked ( )
private

Definition at line 5170 of file Catalog.cpp.

References g_base_path, shared::kDataDirectoryName, and to_string().

Referenced by reloadTableMetadataUnlocked().

5170  {
5171  std::string dictQuery(
5172  "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
5173  sqliteConnector_.query(dictQuery);
5174  auto numRows = sqliteConnector_.getNumRows();
5175  for (size_t r = 0; r < numRows; ++r) {
5176  auto dictId = sqliteConnector_.getData<int>(r, 0);
5177  auto dictName = sqliteConnector_.getData<string>(r, 1);
5178  auto dictNBits = sqliteConnector_.getData<int>(r, 2);
5179  auto is_shared = sqliteConnector_.getData<bool>(r, 3);
5180  auto refcount = sqliteConnector_.getData<int>(r, 4);
5181  auto fname = g_base_path + "/" + shared::kDataDirectoryName + "/DB_" +
5182  std::to_string(currentDB_.dbId) + "_DICT_" + std::to_string(dictId);
5183  DictRef dict_ref(currentDB_.dbId, dictId);
5184  DictDescriptor dd(dict_ref, dictName, dictNBits, is_shared, refcount, fname, false);
5185  if (auto it = dictDescriptorMapByRef_.find(dict_ref);
5186  it == dictDescriptorMapByRef_.end()) {
5187  dictDescriptorMapByRef_[dict_ref] = std::make_unique<DictDescriptor>(dd);
5188  } else {
5189  *it->second = dd;
5190  }
5191  }
5192 }
const std::string kDataDirectoryName
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:711
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
std::string g_base_path
Definition: SysCatalog.cpp:62
Descriptor for a dictionary for a string columne.
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::reloadForeignTableUnlocked ( foreign_storage::ForeignTable foreign_table)
private

Definition at line 5142 of file Catalog.cpp.

References CHECK, CHECK_EQ, CHECK_NE, shared::contains(), foreign_storage::ForeignServer::data_wrapper_type, foreign_storage::ForeignTable::foreign_server, g_enable_fsi, foreign_storage::DataWrapperType::IN_MEMORY_DATA_WRAPPERS, TableDescriptor::is_in_memory_system_table, TableDescriptor::is_system_table, foreign_storage::ForeignTable::last_refresh_time, foreign_storage::ForeignTable::next_refresh_time, foreign_storage::OptionsContainer::populateOptionsMap(), TableDescriptor::tableId, TableDescriptor::tableName, and to_string().

5142  {
5144  CHECK_NE(foreign_table.tableId, 0)
5145  << "reloadForeignTable expects a table with valid id";
5147  "SELECT server_id, options, last_refresh_time, next_refresh_time from "
5148  "omnisci_foreign_tables WHERE table_id == " +
5149  std::to_string(foreign_table.tableId));
5150  auto num_rows = sqliteConnector_.getNumRows();
5151  CHECK_EQ(num_rows, 1U) << "Expected single entry in omnisci_foreign_tables for table'"
5152  << foreign_table.tableName << "', instead got " << num_rows;
5153  const auto server_id = sqliteConnector_.getData<int32_t>(0, 0);
5154  const auto& options = sqliteConnector_.getData<std::string>(0, 1);
5155  const auto last_refresh_time = sqliteConnector_.getData<int64_t>(0, 2);
5156  const auto next_refresh_time = sqliteConnector_.getData<int64_t>(0, 3);
5157 
5158  foreign_table.foreign_server = foreignServerMapById_[server_id].get();
5159  CHECK(foreign_table.foreign_server);
5160  foreign_table.populateOptionsMap(options);
5161  foreign_table.last_refresh_time = last_refresh_time;
5162  foreign_table.next_refresh_time = next_refresh_time;
5163  if (foreign_table.is_system_table) {
5164  foreign_table.is_in_memory_system_table =
5166  foreign_table.foreign_server->data_wrapper_type);
5167  }
5168 }
bool contains(const T &container, const U &element)
Definition: misc.h:195
#define CHECK_EQ(x, y)
Definition: Logger.h:301
T getData(const int row, const int col)
std::string tableName
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
static constexpr std::array< char const *, 5 > IN_MEMORY_DATA_WRAPPERS
bool is_in_memory_system_table
#define CHECK_NE(x, y)
Definition: Logger.h:302
void populateOptionsMap(OptionsMap &&options_map, bool clear=false)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:716
const ForeignServer * foreign_server
Definition: ForeignTable.h:57
#define CHECK(condition)
Definition: Logger.h:291
bool g_enable_fsi
Definition: Catalog.cpp:96
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::reloadTableMetadata ( int  table_id)
private

Definition at line 1093 of file Catalog.cpp.

References getObjForLock(), and reloadTableMetadataUnlocked().

1093  {
1094  cat_write_lock write_lock(this);
1095  cat_sqlite_lock sqlite_lock(getObjForLock());
1096  reloadTableMetadataUnlocked(table_id);
1097 }
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
void reloadTableMetadataUnlocked(int table_id)
Definition: Catalog.cpp:1100
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::reloadTableMetadataUnlocked ( int  table_id)
private

Definition at line 1100 of file Catalog.cpp.

References CHECK_EQ, columnDescriptorMapById_, currentDB_, dataMgr_, Catalog_Namespace::DBMetadata::dbId, deletedColumnPerTable_, TableDescriptor::hasDeletedCol, TableDescriptor::nColumns, reloadDictionariesFromDiskUnlocked(), removeFromColumnMap(), tableDescriptorMap_, tableDescriptorMapById_, TableDescriptor::tableName, and to_upper().

Referenced by reloadTableMetadata().

1100  {
1101  // Reload dictionaries first.
1102  // TODO(sy): Does dictionary reloading really belong here?
1103  // We don't have dictionary locks in the system but maybe we need them.
1105 
1106  // Delete this table's metadata from the in-memory cache before reloading.
1107  TableDescriptor* original_td = nullptr;
1108  std::list<ColumnDescriptor*> original_cds;
1109  if (auto it1 = tableDescriptorMapById_.find(table_id);
1110  it1 != tableDescriptorMapById_.end()) {
1111  original_td = it1->second;
1112  if (dynamic_cast<foreign_storage::ForeignTable*>(original_td)) {
1113  // If have a foreign table then we need to destroy the local data wrapper or it will
1114  // contain (potentially invalid) cached data. This needs to be done before we
1115  // remove the table descriptors.
1116  dataMgr_->removeTableRelatedDS(currentDB_.dbId, table_id);
1117  } else {
1118  dataMgr_->removeMutableTableDiskCacheData(currentDB_.dbId, table_id);
1119  }
1120  tableDescriptorMapById_.erase(it1);
1121  if (auto it2 = tableDescriptorMap_.find(to_upper(original_td->tableName));
1122  it2 != tableDescriptorMap_.end()) {
1123  CHECK_EQ(original_td, it2->second);
1124  tableDescriptorMap_.erase(it2);
1125  }
1126  if (original_td->hasDeletedCol) {
1127  const auto ret = deletedColumnPerTable_.erase(original_td);
1128  CHECK_EQ(ret, size_t(1));
1129  }
1130  for (int column_id = 0; column_id < original_td->nColumns; ++column_id) {
1131  if (auto it3 = columnDescriptorMapById_.find({table_id, column_id});
1132  it3 != columnDescriptorMapById_.end()) {
1133  ColumnDescriptor* original_cd = it3->second;
1134  original_cds.push_back(original_cd);
1135  removeFromColumnMap(original_cd);
1136  }
1137  }
1138  }
1139 
1140  TableDescriptor* td;
1141  try {
1142  td = createTableFromDiskUnlocked(table_id);
1143  } catch (const NoTableFoundException& e) {
1144  // No entry found for table on disk. Another node may have deleted it.
1145  return;
1146  }
1147 
1148  if (auto tableDescIt = tableDescriptorMapById_.find(table_id);
1149  tableDescIt != tableDescriptorMapById_.end()) {
1150  tableDescIt->second->fragmenter = nullptr;
1151  delete tableDescIt->second;
1152  }
1153 
1154  // Reload the column descriptors.
1155  auto cds = sqliteGetColumnsForTableUnlocked(table_id);
1156 
1157  // Store the descriptors into the cache.
1158  if (original_td) {
1159  td->mutex_ = original_td->mutex_; // TODO(sy): Unnecessary?
1160  delete original_td;
1161  original_td = nullptr;
1162  }
1163  for (ColumnDescriptor* original_cd : original_cds) {
1164  delete original_cd;
1165  }
1166  original_cds.clear();
1168  tableDescriptorMapById_[td->tableId] = td;
1169  int32_t skip_physical_cols = 0;
1170  for (ColumnDescriptor* cd : cds) {
1171  addToColumnMap(cd);
1172 
1173  if (skip_physical_cols <= 0) {
1174  skip_physical_cols = cd->columnType.get_physical_cols();
1175  }
1176 
1177  if (cd->isDeletedCol) {
1178  td->hasDeletedCol = true;
1179  setDeletedColumnUnlocked(td, cd);
1180  } else if (cd->columnType.is_geometry() || skip_physical_cols-- <= 0) {
1181  td->columnIdBySpi_.push_back(cd->columnId);
1182  }
1183  }
1184 
1185  // Notify Calcite about the reloaded table.
1186  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
1187 }
void removeFromColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6992
#define CHECK_EQ(x, y)
Definition: Logger.h:301
std::string tableName
const DBMetadata currentDB_
Definition: Catalog.h:721
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:722
TableDescriptor * createTableFromDiskUnlocked(int32_t table_id)
Definition: Catalog.cpp:5243
std::vector< int > columnIdBySpi_
DeletedColumnPerTableMap deletedColumnPerTable_
Definition: Catalog.h:774
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:710
std::shared_ptr< std::mutex > mutex_
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3945
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
specifies the content in-memory of a row in the column metadata table
std::string to_upper(const std::string &str)
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:725
std::list< ColumnDescriptor * > sqliteGetColumnsForTableUnlocked(int32_t table_id)
Definition: Catalog.cpp:5194
void reloadDictionariesFromDiskUnlocked()
Definition: Catalog.cpp:5170
void addToColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6984
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:707

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::removeChunks ( const int  table_id) const
private

Definition at line 4272 of file Catalog.cpp.

References Data_Namespace::CPU_LEVEL, and Data_Namespace::GPU_LEVEL.

4272  {
4273  removeFragmenterForTable(table_id);
4274 
4275  // remove the chunks from in memory structures
4276  ChunkKey chunkKey = {currentDB_.dbId, table_id};
4277 
4278  dataMgr_->deleteChunksWithPrefix(chunkKey, MemoryLevel::CPU_LEVEL);
4279  dataMgr_->deleteChunksWithPrefix(chunkKey, MemoryLevel::GPU_LEVEL);
4280 }
std::vector< int > ChunkKey
Definition: types.h:36
const DBMetadata currentDB_
Definition: Catalog.h:721
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:722
void removeFragmenterForTable(const int table_id) const
Definition: Catalog.cpp:4260
void Catalog_Namespace::Catalog::removeColumnDescriptor ( const ColumnDescriptor cd)
private

Definition at line 2686 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnId, gpu_enabled::sort(), and ColumnDescriptor::tableId.

2686  {
2687  if (!cd) {
2688  return;
2689  }
2690 
2691  auto tabDescIt = tableDescriptorMapById_.find(cd->tableId);
2692  CHECK(tableDescriptorMapById_.end() != tabDescIt);
2693  auto td = tabDescIt->second;
2694  auto& cd_by_spi = td->columnIdBySpi_;
2695  cd_by_spi.erase(std::remove(cd_by_spi.begin(), cd_by_spi.end(), cd->columnId),
2696  cd_by_spi.end());
2697  delete cd;
2698  std::sort(cd_by_spi.begin(), cd_by_spi.end());
2699 }
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::removeDiskCachedDataForMutableTable ( int32_t  table_id) const
private
void Catalog_Namespace::Catalog::removeFragmenterForTable ( const int  table_id) const

Definition at line 4260 of file Catalog.cpp.

References CHECK.

Referenced by anonymous_namespace{DBHandler.cpp}::check_and_reset_in_memory_system_table(), foreign_storage::refresh_foreign_table_unlocked(), and TableOptimizer::vacuumDeletedRows().

4260  {
4261  cat_write_lock write_lock(this);
4262  auto td = getMetadataForTable(table_id, false);
4263  if (td->fragmenter != nullptr) {
4264  auto tableDescIt = tableDescriptorMapById_.find(table_id);
4265  CHECK(tableDescIt != tableDescriptorMapById_.end());
4266  tableDescIt->second->fragmenter = nullptr;
4267  CHECK(td->fragmenter == nullptr);
4268  }
4269 }
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::removeFromColumnMap ( ColumnDescriptor cd)
private

Definition at line 6992 of file Catalog.cpp.

References ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::is_dict_encoded_type(), ColumnDescriptor::tableId, and to_upper().

Referenced by reloadTableMetadataUnlocked().

6992  {
6993  if (cd->columnType.is_dict_encoded_type()) {
6994  dict_columns_by_table_id_[cd->tableId].erase(cd);
6995  }
6998 }
TableDictColumnsMap dict_columns_by_table_id_
Definition: Catalog.h:718
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:709
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:710
bool is_dict_encoded_type() const
Definition: sqltypes.h:653
std::string to_upper(const std::string &str)
SQLTypeInfo columnType
std::string columnName
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::removeTableFromMap ( const std::string &  tableName,
const int  tableId,
const bool  is_on_error = false 
)
protected

Definition at line 1601 of file Catalog.cpp.

References CHECK, CHECK_EQ, CHECK_GE, ColumnDescriptor::columnName, ColumnDescriptor::columnType, Data_Namespace::CPU_LEVEL, TableDescriptor::fragmenter, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), TableDescriptor::hasDeletedCol, INJECT_TIMER, kENCODING_DICT, TableDescriptor::persistenceLevel, File_Namespace::renameForDelete(), and to_upper().

1603  {
1604  cat_write_lock write_lock(this);
1605  TableDescriptorMapById::iterator tableDescIt = tableDescriptorMapById_.find(tableId);
1606  if (tableDescIt == tableDescriptorMapById_.end()) {
1607  throw TableNotFoundException(tableName, currentDB_.dbName, " Cannot remove.");
1608  }
1609 
1610  TableDescriptor* td = tableDescIt->second;
1611 
1612  if (td->hasDeletedCol) {
1613  const auto ret = deletedColumnPerTable_.erase(td);
1614  CHECK_EQ(ret, size_t(1));
1615  }
1616 
1617  tableDescriptorMapById_.erase(tableDescIt);
1618  tableDescriptorMap_.erase(to_upper(tableName));
1619  td->fragmenter = nullptr;
1620  dict_columns_by_table_id_.erase(tableId);
1621 
1623  delete td;
1624 
1625  std::unique_ptr<StringDictionaryClient> client;
1626  if (SysCatalog::instance().isAggregator()) {
1627  CHECK(!string_dict_hosts_.empty());
1628  DictRef dict_ref(currentDB_.dbId, -1);
1629  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1630  }
1631 
1632  // delete all column descriptors for the table
1633  // no more link columnIds to sequential indexes!
1634  for (auto cit = columnDescriptorMapById_.begin();
1635  cit != columnDescriptorMapById_.end();) {
1636  if (tableId != std::get<0>(cit->first)) {
1637  ++cit;
1638  } else {
1639  int i = std::get<1>(cit++->first);
1640  ColumnIdKey cidKey(tableId, i);
1641  ColumnDescriptorMapById::iterator colDescIt = columnDescriptorMapById_.find(cidKey);
1642  ColumnDescriptor* cd = colDescIt->second;
1643  columnDescriptorMapById_.erase(colDescIt);
1644  ColumnKey cnameKey(tableId, to_upper(cd->columnName));
1645  columnDescriptorMap_.erase(cnameKey);
1646  const int dictId = cd->columnType.get_comp_param();
1647  // Dummy dictionaries created for a shard of a logical table have the id set to
1648  // zero.
1649  if (cd->columnType.get_compression() == kENCODING_DICT && dictId) {
1650  INJECT_TIMER(removingDicts);
1651  DictRef dict_ref(currentDB_.dbId, dictId);
1652  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
1653  // If we're removing this table due to an error, it is possible that the string
1654  // dictionary reference was never populated. Don't crash, just continue cleaning
1655  // up the TableDescriptor and ColumnDescriptors
1656  if (!is_on_error) {
1657  CHECK(dictIt != dictDescriptorMapByRef_.end());
1658  } else {
1659  if (dictIt == dictDescriptorMapByRef_.end()) {
1660  continue;
1661  }
1662  }
1663  const auto& dd = dictIt->second;
1664  CHECK_GE(dd->refcount, 1);
1665  --dd->refcount;
1666  if (!dd->refcount) {
1667  dd->stringDict.reset();
1668  if (!isTemp) {
1669  File_Namespace::renameForDelete(dd->dictFolderPath);
1670  }
1671  if (client) {
1672  client->drop(dict_ref);
1673  }
1674  dictDescriptorMapByRef_.erase(dictIt);
1675  }
1676  }
1677 
1678  delete cd;
1679  }
1680  }
1681 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
TableDictColumnsMap dict_columns_by_table_id_
Definition: Catalog.h:718
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:709
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:724
const DBMetadata currentDB_
Definition: Catalog.h:721
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:711
#define CHECK_GE(x, y)
Definition: Logger.h:306
DeletedColumnPerTableMap deletedColumnPerTable_
Definition: Catalog.h:774
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:710
static SysCatalog & instance()
Definition: SysCatalog.h:343
#define INJECT_TIMER(DESC)
Definition: measure.h:96
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
specifies the content in-memory of a row in the column metadata table
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
std::string to_upper(const std::string &str)
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:399
Data_Namespace::MemoryLevel persistenceLevel
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:402
#define CHECK(condition)
Definition: Logger.h:291
SQLTypeInfo columnType
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:210
std::string columnName
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:707
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::renameColumn ( const TableDescriptor td,
const ColumnDescriptor cd,
const std::string &  newColumnName 
)

Definition at line 4621 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::get_physical_cols(), TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and to_upper().

Referenced by AlterTableAlterColumnCommandRecoveryMgr::rollback().

4623  {
4624  cat_write_lock write_lock(this);
4625  cat_sqlite_lock sqlite_lock(getObjForLock());
4626  sqliteConnector_.query("BEGIN TRANSACTION");
4627  try {
4628  for (int i = 0; i <= cd->columnType.get_physical_cols(); ++i) {
4629  auto cdx = getMetadataForColumn(td->tableId, cd->columnId + i);
4630  CHECK(cdx);
4631  std::string new_column_name = cdx->columnName;
4632  new_column_name.replace(0, cd->columnName.size(), newColumnName);
4634  "UPDATE mapd_columns SET name = ? WHERE tableid = ? AND columnid = ?",
4635  std::vector<std::string>{new_column_name,
4636  std::to_string(td->tableId),
4637  std::to_string(cdx->columnId)});
4638  }
4639  } catch (std::exception& e) {
4640  sqliteConnector_.query("ROLLBACK TRANSACTION");
4641  throw;
4642  }
4643  sqliteConnector_.query("END TRANSACTION");
4644  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
4645  for (int i = 0; i <= cd->columnType.get_physical_cols(); ++i) {
4646  auto cdx = getMetadataForColumn(td->tableId, cd->columnId + i);
4647  CHECK(cdx);
4648  ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.find(
4649  std::make_tuple(td->tableId, to_upper(cdx->columnName)));
4650  CHECK(columnDescIt != columnDescriptorMap_.end());
4651  ColumnDescriptor* changeCd = columnDescIt->second;
4652  changeCd->columnName.replace(0, cd->columnName.size(), newColumnName);
4653  columnDescriptorMap_.erase(columnDescIt); // erase entry under old name
4654  columnDescriptorMap_[std::make_tuple(td->tableId, to_upper(changeCd->columnName))] =
4655  changeCd;
4656  }
4657  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
4658 }
std::string tableName
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:709
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
int get_physical_cols() const
Definition: sqltypes.h:430
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
specifies the content in-memory of a row in the column metadata table
std::string to_upper(const std::string &str)
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:725
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
std::string columnName

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::renameForeignServer ( const std::string &  server_name,
const std::string &  name 
)

Rename a Foreign Server.

Parameters
server_name- Name of the foreign server whose name will be changed
name- New name of server

Definition at line 3438 of file Catalog.cpp.

References CHECK, and setup::name.

3439  {
3440  cat_write_lock write_lock(this);
3441  auto foreign_server_it = foreignServerMap_.find(server_name);
3442  CHECK(foreign_server_it != foreignServerMap_.end());
3443  setForeignServerProperty(server_name, "name", name);
3444  auto foreign_server_shared = foreign_server_it->second;
3445  foreign_server_shared->name = name;
3446  foreignServerMap_[name] = foreign_server_shared;
3447  foreignServerMap_.erase(foreign_server_it);
3448 }
std::string name() const
Definition: Catalog.h:348
ForeignServerMap foreignServerMap_
Definition: Catalog.h:715
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
void setForeignServerProperty(const std::string &server_name, const std::string &property, const std::string &value)
Definition: Catalog.cpp:5310
void Catalog_Namespace::Catalog::renameLegacyDataWrappers ( )
protected

Definition at line 705 of file Catalog.cpp.

References SqliteConnector::getNumRows(), getObjForLock(), logger::INFO, LOG, MAPD_VERSION, SqliteConnector::query(), SqliteConnector::query_with_text_params(), sqliteConnector_, and to_string().

Referenced by CheckAndExecuteMigrations().

705  {
706  // TODO: Move common migration logic to a shared function.
707  cat_sqlite_lock sqlite_lock(getObjForLock());
708  sqliteConnector_.query("BEGIN TRANSACTION");
709  try {
711  "select name from sqlite_master WHERE type='table' AND "
712  "name='mapd_version_history'");
713  static const std::string migration_name{"rename_legacy_data_wrappers"};
714  if (sqliteConnector_.getNumRows() == 0) {
716  "CREATE TABLE mapd_version_history(version integer, migration_history text "
717  "unique)");
718  } else {
720  "select * from mapd_version_history where migration_history = "
721  "'" +
722  migration_name + "'");
723  if (sqliteConnector_.getNumRows() != 0) {
724  // Migration already done.
725  sqliteConnector_.query("END TRANSACTION");
726  return;
727  }
728  }
729  LOG(INFO) << "Executing " << migration_name << " migration.";
730 
731  // Update legacy data wrapper names
733  // clang-format off
734  std::map<std::string, std::string> old_to_new_wrapper_names{
735  {"OMNISCI_CSV", DataWrapperType::CSV},
736  {"OMNISCI_PARQUET", DataWrapperType::PARQUET},
737  {"OMNISCI_REGEX_PARSER", DataWrapperType::REGEX_PARSER},
738  {"OMNISCI_INTERNAL_CATALOG", DataWrapperType::INTERNAL_CATALOG},
739  {"INTERNAL_OMNISCI_MEMORY_STATS", DataWrapperType::INTERNAL_MEMORY_STATS},
740  {"INTERNAL_OMNISCI_STORAGE_STATS", DataWrapperType::INTERNAL_STORAGE_STATS}
741  };
742  // clang-format on
743 
744  for (const auto& [old_wrapper_name, new_wrapper_name] : old_to_new_wrapper_names) {
746  "UPDATE omnisci_foreign_servers SET data_wrapper_type = ? WHERE "
747  "data_wrapper_type = ?",
748  std::vector<std::string>{new_wrapper_name, old_wrapper_name});
749  }
750 
751  // Record migration.
753  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
754  std::vector<std::string>{std::to_string(MAPD_VERSION), migration_name});
755  LOG(INFO) << migration_name << " migration completed.";
756  } catch (std::exception& e) {
757  sqliteConnector_.query("ROLLBACK TRANSACTION");
758  throw;
759  }
760  sqliteConnector_.query("END TRANSACTION");
761 }
virtual void query_with_text_params(std::string const &query_only)
#define LOG(tag)
Definition: Logger.h:285
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
Encapsulates an enumeration of foreign data wrapper type strings.
static const int32_t MAPD_VERSION
Definition: release.h:32
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::renamePhysicalTable ( const TableDescriptor td,
const std::string &  newTableName 
)
protected

Definition at line 4378 of file Catalog.cpp.

References CHECK, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and to_upper().

4378  {
4379  cat_write_lock write_lock(this);
4380  cat_sqlite_lock sqlite_lock(getObjForLock());
4381 
4382  sqliteConnector_.query("BEGIN TRANSACTION");
4383  try {
4385  "UPDATE mapd_tables SET name = ? WHERE tableid = ?",
4386  std::vector<std::string>{newTableName, std::to_string(td->tableId)});
4387  } catch (std::exception& e) {
4388  sqliteConnector_.query("ROLLBACK TRANSACTION");
4389  throw;
4390  }
4391  sqliteConnector_.query("END TRANSACTION");
4392  TableDescriptorMap::iterator tableDescIt =
4394  CHECK(tableDescIt != tableDescriptorMap_.end());
4395  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
4396  // Get table descriptor to change it
4397  TableDescriptor* changeTd = tableDescIt->second;
4398  changeTd->tableName = newTableName;
4399  tableDescriptorMap_.erase(tableDescIt); // erase entry under old name
4400  tableDescriptorMap_[to_upper(newTableName)] = changeTd;
4401  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
4402 }
std::string tableName
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
std::string to_upper(const std::string &str)
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:725
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:707

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::renamePhysicalTables ( std::vector< std::pair< std::string, std::string >> &  names,
std::vector< int > &  tableIds 
)
protected

Definition at line 4444 of file Catalog.cpp.

References CHECK, TableDescriptor::tableName, to_string(), and to_upper().

4446  {
4447  cat_write_lock write_lock(this);
4448  cat_sqlite_lock sqlite_lock(getObjForLock());
4449 
4450  // execute the SQL query
4451  for (size_t i = 0; i < names.size(); i++) {
4452  int tableId = tableIds[i];
4453  const std::string& newTableName = names[i].second;
4455  "UPDATE mapd_tables SET name = ? WHERE tableid = ?",
4456  std::vector<std::string>{newTableName, std::to_string(tableId)});
4457  }
4458 
4459  // reset the table descriptors, give Calcite a kick
4460  for (size_t i = 0; i < names.size(); i++) {
4461  const auto& [curTableName, newTableName] = names[i];
4462 
4463  TableDescriptorMap::iterator tableDescIt =
4464  tableDescriptorMap_.find(to_upper(curTableName));
4465  CHECK(tableDescIt != tableDescriptorMap_.end());
4466  calciteMgr_->updateMetadata(currentDB_.dbName, curTableName);
4467 
4468  // Get table descriptor to change it
4469  TableDescriptor* changeTd = tableDescIt->second;
4470  changeTd->tableName = newTableName;
4471  tableDescriptorMap_.erase(tableDescIt); // erase entry under old name
4472  tableDescriptorMap_[to_upper(newTableName)] = changeTd;
4473  calciteMgr_->updateMetadata(currentDB_.dbName, curTableName);
4474  }
4475 }
std::string tableName
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
std::string to_string(char const *&&v)
std::string to_upper(const std::string &str)
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:725
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:707

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::renameTable ( const TableDescriptor td,
const std::string &  newTableName 
)

Definition at line 4404 of file Catalog.cpp.

References CHECK, DBObjectKey::dbId, DBObjectKey::objectId, DBObjectKey::permissionType, Grantee::renameDbObject(), TableDBObjectType, and TableDescriptor::tableId.

4404  {
4405  {
4406  cat_write_lock write_lock(this);
4407  cat_sqlite_lock sqlite_lock(getObjForLock());
4408  // rename all corresponding physical tables if this is a logical table
4409  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
4410  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
4411  const auto physicalTables = physicalTableIt->second;
4412  CHECK(!physicalTables.empty());
4413  for (size_t i = 0; i < physicalTables.size(); i++) {
4414  int32_t physical_tb_id = physicalTables[i];
4415  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
4416  CHECK(phys_td);
4417  std::string newPhysTableName =
4418  generatePhysicalTableName(newTableName, static_cast<int32_t>(i + 1));
4419  renamePhysicalTable(phys_td, newPhysTableName);
4420  }
4421  }
4422  renamePhysicalTable(td, newTableName);
4423  }
4424  {
4425  DBObject object(newTableName, TableDBObjectType);
4426  // update table name in direct and effective priv map
4427  DBObjectKey key;
4428  key.dbId = currentDB_.dbId;
4429  key.objectId = td->tableId;
4430  key.permissionType = static_cast<int>(DBObjectType::TableDBObjectType);
4431  object.setObjectKey(key);
4432  auto objdescs = SysCatalog::instance().getMetadataForObject(
4433  currentDB_.dbId, static_cast<int>(DBObjectType::TableDBObjectType), td->tableId);
4434  for (auto obj : objdescs) {
4435  Grantee* grnt = SysCatalog::instance().getGrantee(obj->roleName);
4436  if (grnt) {
4437  grnt->renameDbObject(object);
4438  }
4439  }
4441  }
4442 }
const DBMetadata currentDB_
Definition: Catalog.h:721
Grantee * getGrantee(const std::string &name) const
int32_t objectId
Definition: DBObject.h:55
static SysCatalog & instance()
Definition: SysCatalog.h:343
void renameObjectsInDescriptorMap(DBObject &object, const Catalog_Namespace::Catalog &cat)
int32_t dbId
Definition: DBObject.h:54
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
std::string generatePhysicalTableName(const std::string &logicalTableName, const size_t shardNumber)
Definition: Catalog.cpp:5085
int32_t permissionType
Definition: DBObject.h:53
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
void renamePhysicalTable(const TableDescriptor *td, const std::string &newTableName)
Definition: Catalog.cpp:4378
std::vector< ObjectRoleDescriptor * > getMetadataForObject(int32_t dbId, int32_t dbType, int32_t objectId) const
virtual void renameDbObject(const DBObject &object)
Definition: Grantee.cpp:121
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:727

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::renameTableDirectories ( const std::string &  temp_data_dir,
const std::vector< std::string > &  target_paths,
const std::string &  name_prefix 
) const
private
void Catalog_Namespace::Catalog::renameTables ( const std::vector< std::pair< std::string, std::string >> &  names)

Definition at line 4503 of file Catalog.cpp.

References CHECK, CHECK_EQ, DBObjectKey::dbId, DBObjectKey::objectId, DBObjectKey::permissionType, Grantee::renameDbObject(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::replace_cached_table_name(), TableDBObjectType, TableDescriptor::tableId, and TableDescriptor::tableName.

4504  {
4505  // tableId of all tables being renamed
4506  // ... in matching order to 'names'
4507  std::vector<int> tableIds;
4508 
4509  // (sorted & unique) list of tables ids for locking
4510  // (with names index of src in case of error)
4511  // <tableId, strIndex>
4512  // std::map is by definition/implementation sorted
4513  // std::map current usage below tests to avoid over-write
4514  std::map<int, size_t> uniqueOrderedTableIds;
4515 
4516  // mapping of modified tables names -> tableId
4517  std::map<std::string, int> cachedTableMap;
4518 
4519  // -------- Setup --------
4520 
4521  // gather tableIds pre-execute; build maps
4522  for (size_t i = 0; i < names.size(); i++) {
4523  const auto& [curTableName, newTableName] = names[i];
4524 
4525  // make sure the table being renamed exists,
4526  // or will exist when executed in 'name' order
4527  auto td = getCachedTableDescriptor(cachedTableMap, curTableName);
4528  CHECK(td);
4529 
4530  tableIds.push_back(td->tableId);
4531  if (uniqueOrderedTableIds.find(td->tableId) == uniqueOrderedTableIds.end()) {
4532  // don't overwrite as it should map to the first names index 'i'
4533  uniqueOrderedTableIds[td->tableId] = i;
4534  }
4535  replace_cached_table_name(cachedTableMap, curTableName, newTableName, td->tableId);
4536  }
4537 
4538  CHECK_EQ(tableIds.size(), names.size());
4539 
4540  // The outer Stmt created a write lock before calling the catalog rename table
4541  // -> TODO: might want to sort out which really should set the lock :
4542  // the comment in the outer scope indicates it should be in here
4543  // but it's not clear if the access done there *requires* it out there
4544  //
4545  // Lock tables pre-execute (may/will be in different order than rename occurs)
4546  // const auto execute_write_lock = heavyai::unique_lock<heavyai::shared_mutex>(
4547  // *legacylockmgr::LockMgr<heavyai::shared_mutex, bool>::getMutex(
4548  // legacylockmgr::ExecutorOuterLock, true));
4549 
4550  // acquire the locks for all tables being renamed
4552  for (auto& idPair : uniqueOrderedTableIds) {
4553  const std::string& tableName = names[idPair.second].first;
4554  tableLocks.emplace_back(
4557  *this, tableName, false)));
4558  }
4559 
4560  // -------- Rename --------
4561 
4562  {
4563  cat_write_lock write_lock(this);
4564  // collect all (tables + physical tables) into a single list
4565  std::vector<std::pair<std::string, std::string>> allNames;
4566  std::vector<int> allTableIds;
4567 
4568  for (size_t i = 0; i < names.size(); i++) {
4569  int tableId = tableIds[i];
4570  const auto& [curTableName, newTableName] = names[i];
4571 
4572  // rename all corresponding physical tables if this is a logical table
4573  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(tableId);
4574  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
4575  const auto physicalTables = physicalTableIt->second;
4576  CHECK(!physicalTables.empty());
4577  for (size_t k = 0; k < physicalTables.size(); k++) {
4578  int32_t physical_tb_id = physicalTables[k];
4579  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
4580  CHECK(phys_td);
4581  std::string newPhysTableName = generatePhysicalTableName(newTableName, (k + 1));
4582  allNames.emplace_back(phys_td->tableName, newPhysTableName);
4583  allTableIds.push_back(phys_td->tableId);
4584  }
4585  }
4586  allNames.emplace_back(curTableName, newTableName);
4587  allTableIds.push_back(tableId);
4588  }
4589 
4590  // rename all tables in one transaction
4591  execInTransaction(&Catalog::renamePhysicalTables, allNames, allTableIds);
4592  }
4593 
4594  // now update the SysCatalog
4595  for (size_t i = 0; i < names.size(); i++) {
4596  int tableId = tableIds[i];
4597  const std::string& newTableName = names[i].second;
4598  {
4599  // update table name in direct and effective priv map
4600  DBObjectKey key;
4601  key.dbId = currentDB_.dbId;
4602  key.objectId = tableId;
4603  key.permissionType = static_cast<int>(DBObjectType::TableDBObjectType);
4604 
4605  DBObject object(newTableName, TableDBObjectType);
4606  object.setObjectKey(key);
4607 
4608  auto objdescs = SysCatalog::instance().getMetadataForObject(
4609  currentDB_.dbId, static_cast<int>(DBObjectType::TableDBObjectType), tableId);
4610  for (auto obj : objdescs) {
4611  Grantee* grnt = SysCatalog::instance().getGrantee(obj->roleName);
4612  if (grnt) {
4613  grnt->renameDbObject(object);
4614  }
4615  }
4617  }
4618  }
4619 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
std::vector< std::unique_ptr< lockmgr::AbstractLockContainer< const TableDescriptor * >>> LockedTableDescriptors
Definition: LockMgr.h:272
std::string tableName
const DBMetadata currentDB_
Definition: Catalog.h:721
const TableDescriptor * getCachedTableDescriptor(const std::map< std::string, int > &cached_table_map, const std::string &cur_table_name)
Definition: Catalog.cpp:4480
Grantee * getGrantee(const std::string &name) const
int32_t objectId
Definition: DBObject.h:55
static SysCatalog & instance()
Definition: SysCatalog.h:343
void renameObjectsInDescriptorMap(DBObject &object, const Catalog_Namespace::Catalog &cat)
void execInTransaction(F &&f, Args &&...args)
Definition: Catalog.cpp:7002
void replace_cached_table_name(std::map< std::string, int > &cachedTableMap, const std::string &curTableName, const std::string &newTableName, int tableId)
Definition: Catalog.cpp:4491
int32_t dbId
Definition: DBObject.h:54
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
void renamePhysicalTables(std::vector< std::pair< std::string, std::string >> &names, std::vector< int > &tableIds)
Definition: Catalog.cpp:4444
#define CHECK(condition)
Definition: Logger.h:291
std::string generatePhysicalTableName(const std::string &logicalTableName, const size_t shardNumber)
Definition: Catalog.cpp:5085
int32_t permissionType
Definition: DBObject.h:53
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
std::vector< ObjectRoleDescriptor * > getMetadataForObject(int32_t dbId, int32_t dbType, int32_t objectId) const
virtual void renameDbObject(const DBObject &object)
Definition: Grantee.cpp:121
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:727

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::replaceDashboard ( DashboardDescriptor vd)

Definition at line 4724 of file Catalog.cpp.

References CHECK, DashboardDescriptor::dashboardId, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, DashboardDescriptor::dashboardSystemRoleName, logger::ERROR, generate_dashboard_system_rolename(), DashboardDescriptor::imageHash, LOG, to_string(), Catalog_Namespace::sqlite_lock< T >::unlock(), Catalog_Namespace::write_lock< T >::unlock(), DashboardDescriptor::updateTime, and DashboardDescriptor::userId.

4724  {
4725  cat_write_lock write_lock(this);
4726  cat_sqlite_lock sqlite_lock(getObjForLock());
4727 
4729  sqliteConnector_.query("BEGIN TRANSACTION");
4730  try {
4732  "SELECT id FROM mapd_dashboards WHERE id = ?",
4733  std::vector<std::string>{std::to_string(vd.dashboardId)});
4734  if (sqliteConnector_.getNumRows() > 0) {
4736  "UPDATE mapd_dashboards SET name = ?, state = ?, image_hash = ?, metadata = "
4737  "?, userid = ?, update_time = datetime('now') where id = ? ",
4738  std::vector<std::string>{vd.dashboardName,
4739  vd.dashboardState,
4740  vd.imageHash,
4741  vd.dashboardMetadata,
4742  std::to_string(vd.userId),
4744  } else {
4745  LOG(ERROR) << "Error replacing dashboard id " << vd.dashboardId
4746  << " does not exist in db";
4747  throw runtime_error("Error replacing dashboard id " +
4748  std::to_string(vd.dashboardId) + " does not exist in db");
4749  }
4750  } catch (std::exception& e) {
4751  sqliteConnector_.query("ROLLBACK TRANSACTION");
4752  throw;
4753  }
4754  sqliteConnector_.query("END TRANSACTION");
4755 
4756  bool found{false};
4757  for (auto descp : dashboardDescriptorMap_) {
4758  auto dash = descp.second.get();
4759  if (dash->dashboardId == vd.dashboardId) {
4760  found = true;
4761  auto viewDescIt = dashboardDescriptorMap_.find(std::to_string(dash->userId) + ":" +
4762  dash->dashboardName);
4763  if (viewDescIt ==
4764  dashboardDescriptorMap_.end()) { // check to make sure view exists
4765  LOG(ERROR) << "No metadata for dashboard for user " << dash->userId
4766  << " dashboard " << dash->dashboardName << " does not exist in map";
4767  throw runtime_error("No metadata for dashboard for user " +
4768  std::to_string(dash->userId) + " dashboard " +
4769  dash->dashboardName + " does not exist in map");
4770  }
4771  dashboardDescriptorMap_.erase(viewDescIt);
4772  break;
4773  }
4774  }
4775  if (!found) {
4776  LOG(ERROR) << "Error replacing dashboard id " << vd.dashboardId
4777  << " does not exist in map";
4778  throw runtime_error("Error replacing dashboard id " + std::to_string(vd.dashboardId) +
4779  " does not exist in map");
4780  }
4781 
4782  // now reload the object
4784  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM "
4785  "mapd_dashboards "
4786  "WHERE id = ?",
4787  std::vector<std::string>{std::to_string(vd.dashboardId)});
4788  vd.updateTime = sqliteConnector_.getData<string>(0, 1);
4792  sqlite_lock.unlock();
4793  write_lock.unlock();
4794  if (!isInfoSchemaDb()) {
4795  // NOTE(wamsi): Transactionally unsafe
4798  }
4799 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
#define LOG(tag)
Definition: Logger.h:285
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
virtual void query(const std::string &queryString)
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:712
std::string to_string(char const *&&v)
std::string dashboardSystemRoleName
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1724
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1688
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
bool isInfoSchemaDb() const
Definition: Catalog.cpp:6389
#define CHECK(condition)
Definition: Logger.h:291
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
auto getSqlitePtr() const
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::resetTableEpochFloor ( const int  logicalTableId) const

Definition at line 5040 of file Catalog.cpp.

Referenced by AlterTableAlterColumnCommandRecoveryMgr::cleanup().

5040  {
5041  cat_read_lock read_lock(this);
5042  const auto td = getMetadataForTable(logicalTableId, false);
5043  const auto shards = getPhysicalTablesDescriptors(td, false);
5044  for (const auto shard : shards) {
5045  getDataMgr().resetTableEpochFloor(getCurrentDB().dbId, shard->tableId);
5046  }
5047 }
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:266
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:121
void resetTableEpochFloor(const int32_t db_id, const int32_t tb_id)
Definition: DataMgr.cpp:642
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:265
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4869
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::restoreOldOwners ( const std::map< int32_t, std::string > &  old_owners_user_name_by_id,
const std::map< int32_t, std::vector< DBObject >> &  old_owner_db_objects,
int32_t  new_owner_id 
)
private

Definition at line 6295 of file Catalog.cpp.

References CHECK, CHECK_GT, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::convert_object_owners_map_to_string(), DashboardDBObjectType, logger::FATAL, g_enable_fsi, LOG, ServerDBObjectType, TableDBObjectType, to_string(), UNREACHABLE, and ViewDBObjectType.

6298  {
6299  cat_write_lock write_lock(this);
6300  cat_sqlite_lock sqlite_lock(getObjForLock());
6301  sqliteConnector_.query("BEGIN TRANSACTION");
6302  try {
6303  for (const auto& [old_owner_id, db_objects] : old_owner_db_objects) {
6304  for (const auto& db_object : db_objects) {
6305  auto object_id = db_object.getObjectKey().objectId;
6306  CHECK_GT(object_id, 0);
6307  std::vector<std::string> query_params{std::to_string(old_owner_id),
6308  std::to_string(new_owner_id),
6309  std::to_string(object_id)};
6310  auto object_type = db_object.getType();
6311  if (object_type == DBObjectType::TableDBObjectType ||
6312  object_type == DBObjectType::ViewDBObjectType) {
6314  "UPDATE mapd_tables SET userid = ? WHERE userid = ? AND tableid = ?",
6315  query_params);
6316  } else if (object_type == DBObjectType::DashboardDBObjectType) {
6318  "UPDATE mapd_dashboards SET userid = ? WHERE userid = ? AND id = ?",
6319  query_params);
6320  } else if (object_type == DBObjectType::ServerDBObjectType) {
6323  "UPDATE omnisci_foreign_servers SET owner_user_id = ? "
6324  "WHERE owner_user_id = ? AND id = ?",
6325  query_params);
6326  } else {
6327  UNREACHABLE() << "Unexpected DB object type: " << static_cast<int>(object_type);
6328  }
6329  }
6330  }
6332  old_owners_user_name_by_id, old_owner_db_objects, new_owner_id);
6333  } catch (std::exception& e) {
6334  sqliteConnector_.query("ROLLBACK TRANSACTION");
6335  LOG(FATAL)
6336  << "Unable to restore database objects ownership after an error occurred. "
6337  "Database object ownership information may be in an inconsistent state. " +
6339  getDatabaseId(), new_owner_id, old_owner_db_objects);
6340  }
6341  sqliteConnector_.query("END TRANSACTION");
6342 }
virtual void query_with_text_params(std::string const &query_only)
#define LOG(tag)
Definition: Logger.h:285
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
#define UNREACHABLE()
Definition: Logger.h:338
virtual void query(const std::string &queryString)
#define CHECK_GT(x, y)
Definition: Logger.h:305
std::string to_string(char const *&&v)
void restoreOldOwnersInMemory(const std::map< int32_t, std::string > &old_owners_user_name_by_id, const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects, int32_t new_owner_id)
Definition: Catalog.cpp:6344
int getDatabaseId() const
Definition: Catalog.h:326
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
std::string convert_object_owners_map_to_string(int32_t db_id, int32_t new_owner_id, const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects)
Definition: Catalog.cpp:6127
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
bool g_enable_fsi
Definition: Catalog.cpp:96

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::restoreOldOwnersInMemory ( const std::map< int32_t, std::string > &  old_owners_user_name_by_id,
const std::map< int32_t, std::vector< DBObject >> &  old_owner_db_objects,
int32_t  new_owner_id 
)
private

Definition at line 6344 of file Catalog.cpp.

References CHECK, DashboardDBObjectType, ServerDBObjectType, TableDBObjectType, to_string(), UNREACHABLE, and ViewDBObjectType.

6347  {
6348  for (const auto& [old_owner_id, db_objects] : old_owner_db_objects) {
6349  for (const auto& db_object : db_objects) {
6350  auto object_id = db_object.getObjectKey().objectId;
6351  auto object_type = db_object.getType();
6352  if (object_type == DBObjectType::TableDBObjectType ||
6353  object_type == DBObjectType::ViewDBObjectType) {
6354  auto it = tableDescriptorMapById_.find(object_id);
6355  CHECK(it != tableDescriptorMapById_.end());
6356  CHECK(it->second);
6357  it->second->userId = old_owner_id;
6358  } else if (object_type == DBObjectType::DashboardDBObjectType) {
6359  auto it = dashboardDescriptorMap_.find(std::to_string(new_owner_id) + ":" +
6360  db_object.getName());
6361  CHECK(it != dashboardDescriptorMap_.end());
6362  CHECK(it->second);
6363  it->second->userId = old_owner_id;
6364  auto user_name_it = old_owners_user_name_by_id.find(old_owner_id);
6365  CHECK(user_name_it != old_owners_user_name_by_id.end());
6366  it->second->user = user_name_it->second;
6367  dashboardDescriptorMap_[std::to_string(old_owner_id) + ":" +
6368  db_object.getName()] = it->second;
6369  dashboardDescriptorMap_.erase(it);
6370  } else if (object_type == DBObjectType::ServerDBObjectType) {
6371  auto it = foreignServerMapById_.find(object_id);
6372  CHECK(it != foreignServerMapById_.end());
6373  CHECK(it->second);
6374  it->second->user_id = old_owner_id;
6375  } else {
6376  UNREACHABLE() << "Unexpected DB object type: " << static_cast<int>(object_type);
6377  }
6378  }
6379  }
6380 }
#define UNREACHABLE()
Definition: Logger.h:338
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:712
std::string to_string(char const *&&v)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:716
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::rollLegacy ( const bool  forward)

Definition at line 2718 of file Catalog.cpp.

References CHECK.

2718  {
2719  cat_write_lock write_lock(this);
2720  std::set<const TableDescriptor*> tds;
2721 
2722  for (const auto& cdr : columnDescriptorsForRoll) {
2723  auto ocd = cdr.first;
2724  auto ncd = cdr.second;
2725  CHECK(ocd || ncd);
2726  auto tabDescIt = tableDescriptorMapById_.find((ncd ? ncd : ocd)->tableId);
2727  CHECK(tableDescriptorMapById_.end() != tabDescIt);
2728  auto td = tabDescIt->second;
2729  auto& vc = td->columnIdBySpi_;
2730  if (forward) {
2731  if (ocd) {
2732  if (nullptr == ncd ||
2733  ncd->columnType.get_comp_param() != ocd->columnType.get_comp_param()) {
2735  }
2736 
2737  vc.erase(std::remove(vc.begin(), vc.end(), ocd->columnId), vc.end());
2738 
2739  delete ocd;
2740  }
2741  if (ncd) {
2742  // append columnId if its new and not phy geo
2743  if (vc.end() == std::find(vc.begin(), vc.end(), ncd->columnId)) {
2744  if (!ncd->isGeoPhyCol) {
2745  vc.push_back(ncd->columnId);
2746  }
2747  }
2748  }
2749  tds.insert(td);
2750  } else {
2751  if (ocd) {
2752  addToColumnMap(ocd);
2753  }
2754  // roll back the dict of new column
2755  if (ncd) {
2756  removeFromColumnMap(ncd);
2757  if (nullptr == ocd ||
2758  ocd->columnType.get_comp_param() != ncd->columnType.get_comp_param()) {
2760  }
2761  delete ncd;
2762  }
2763  }
2764  }
2765  columnDescriptorsForRoll.clear();
2766 
2767  if (forward) {
2768  for (const auto td : tds) {
2769  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
2770  }
2771  }
2772 }
void removeFromColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6992
const DBMetadata currentDB_
Definition: Catalog.h:721
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:725
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
void delDictionaryNontransactional(const ColumnDescriptor &cd)
Definition: Catalog.cpp:2280
#define CHECK(condition)
Definition: Logger.h:291
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:739
void addToColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:6984
void Catalog_Namespace::Catalog::serializeTableJsonUnlocked ( const TableDescriptor td,
const std::list< ColumnDescriptor > &  cds 
) const
protected

Definition at line 3180 of file Catalog.cpp.

References CHECK, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), TableDescriptor::tableId, TableDescriptor::tableName, and VLOG.

3181  {
3182  // relies on the catalog write lock
3183  using namespace rapidjson;
3184 
3185  VLOG(1) << "Serializing temporary table " << td->tableName << " to JSON for Calcite.";
3186 
3187  const auto db_name = currentDB_.dbName;
3188  const auto file_path = table_json_filepath(basePath_, db_name);
3189 
3190  Document d;
3191  if (boost::filesystem::exists(file_path)) {
3192  // look for an existing file for this database
3193  std::ifstream reader(file_path.string());
3194  CHECK(reader.is_open());
3195  IStreamWrapper json_read_wrapper(reader);
3196  d.ParseStream(json_read_wrapper);
3197  } else {
3198  d.SetObject();
3199  }
3200  CHECK(d.IsObject());
3201  CHECK(!d.HasMember(StringRef(td->tableName.c_str())));
3202 
3203  Value table(kObjectType);
3204  table.AddMember(
3205  "name", Value().SetString(StringRef(td->tableName.c_str())), d.GetAllocator());
3206  table.AddMember("id", Value().SetInt(td->tableId), d.GetAllocator());
3207  table.AddMember("columns", Value(kArrayType), d.GetAllocator());
3208 
3209  for (const auto& cd : cds) {
3210  Value column(kObjectType);
3211  column.AddMember(
3212  "name", Value().SetString(StringRef(cd.columnName)), d.GetAllocator());
3213  column.AddMember("coltype",
3214  Value().SetInt(static_cast<int>(cd.columnType.get_type())),
3215  d.GetAllocator());
3216  column.AddMember("colsubtype",
3217  Value().SetInt(static_cast<int>(cd.columnType.get_subtype())),
3218  d.GetAllocator());
3219  column.AddMember("compression",
3220  Value().SetInt(static_cast<int>(cd.columnType.get_compression())),
3221  d.GetAllocator());
3222  column.AddMember("comp_param",
3223  Value().SetInt(static_cast<int>(cd.columnType.get_comp_param())),
3224  d.GetAllocator());
3225  column.AddMember("size",
3226  Value().SetInt(static_cast<int>(cd.columnType.get_size())),
3227  d.GetAllocator());
3228  column.AddMember(
3229  "coldim", Value().SetInt(cd.columnType.get_dimension()), d.GetAllocator());
3230  column.AddMember(
3231  "colscale", Value().SetInt(cd.columnType.get_scale()), d.GetAllocator());
3232  column.AddMember(
3233  "is_notnull", Value().SetBool(cd.columnType.get_notnull()), d.GetAllocator());
3234  column.AddMember("is_systemcol", Value().SetBool(cd.isSystemCol), d.GetAllocator());
3235  column.AddMember("is_virtualcol", Value().SetBool(cd.isVirtualCol), d.GetAllocator());
3236  column.AddMember("is_deletedcol", Value().SetBool(cd.isDeletedCol), d.GetAllocator());
3237  table["columns"].PushBack(column, d.GetAllocator());
3238  }
3239  d.AddMember(StringRef(td->tableName.c_str()), table, d.GetAllocator());
3240 
3241  // Overwrite the existing file
3242  std::ofstream writer(file_path.string(), std::ios::trunc | std::ios::out);
3243  CHECK(writer.is_open());
3244  OStreamWrapper json_wrapper(writer);
3245 
3246  Writer<OStreamWrapper> json_writer(json_wrapper);
3247  d.Accept(json_writer);
3248  writer.close();
3249 }
std::string tableName
const DBMetadata currentDB_
Definition: Catalog.h:721
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:162
#define CHECK(condition)
Definition: Logger.h:291
#define VLOG(n)
Definition: Logger.h:388

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::setCalciteMgr ( const std::shared_ptr< Calcite > &  new_calcite_mgr)
inline

Definition at line 268 of file Catalog.h.

References calciteMgr_.

268  {
269  // Used for testing.
270  calciteMgr_ = new_calcite_mgr;
271  }
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:725
void Catalog_Namespace::Catalog::setColumnDictionary ( ColumnDescriptor cd,
std::list< DictDescriptor > &  dds,
const TableDescriptor td,
bool  is_logical_table,
bool  use_temp_dictionary = false 
)
protected

Definition at line 4056 of file Catalog.cpp.

References ColumnDescriptor::columnName, ColumnDescriptor::columnType, g_base_path, SQLTypeInfo::get_comp_param(), SQLTypeInfo::is_array(), shared::kDataDirectoryName, SQLTypeInfo::set_comp_param(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::set_dict_key(), SQLTypeInfo::set_size(), TableDescriptor::tableName, and to_string().

4060  {
4061  cat_write_lock write_lock(this);
4062 
4063  std::string dictName{"Initial_key"};
4064  int dictId{0};
4065  std::string folderPath;
4066  if (is_logical_table) {
4067  cat_sqlite_lock sqlite_lock(getObjForLock());
4068 
4070  "INSERT INTO mapd_dictionaries (name, nbits, is_shared, refcount) VALUES (?, ?, "
4071  "?, 1)",
4072  std::vector<std::string>{
4073  dictName, std::to_string(cd.columnType.get_comp_param()), "0"});
4075  "SELECT dictid FROM mapd_dictionaries WHERE name = ?", dictName);
4076  dictId = sqliteConnector_.getData<int>(0, 0);
4077  dictName = td.tableName + "_" + cd.columnName + "_dict" + std::to_string(dictId);
4079  "UPDATE mapd_dictionaries SET name = ? WHERE name = 'Initial_key'", dictName);
4080  folderPath = g_base_path + "/" + shared::kDataDirectoryName + "/DB_" +
4081  std::to_string(currentDB_.dbId) + "_DICT_" + std::to_string(dictId);
4082  }
4084  dictId,
4085  dictName,
4087  false,
4088  1,
4089  folderPath,
4090  use_temp_dictionary);
4091  dds.push_back(dd);
4092  if (!cd.columnType.is_array()) {
4094  }
4095  cd.columnType.set_comp_param(dictId);
4096  set_dict_key(cd);
4097 }
void set_size(int s)
Definition: sqltypes.h:476
const std::string kDataDirectoryName
T getData(const int row, const int col)
std::string tableName
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
std::string to_string(char const *&&v)
std::string g_base_path
Definition: SysCatalog.cpp:62
void set_comp_param(int p)
Definition: sqltypes.h:480
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:402
Descriptor for a dictionary for a string columne.
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
std::string columnName
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
bool is_array() const
Definition: sqltypes.h:583

+ Here is the call graph for this function:

bool Catalog_Namespace::Catalog::setColumnSharedDictionary ( ColumnDescriptor cd,
std::list< ColumnDescriptor > &  cdd,
std::list< DictDescriptor > &  dds,
const TableDescriptor  td,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs 
)
protected

Definition at line 3987 of file Catalog.cpp.

References CHECK, CHECK_GE, ColumnDescriptor::columnName, ColumnDescriptor::columnType, compress_reference_path(), SQLTypeInfo::get_comp_param(), table_is_temporary(), TableDescriptor::tableName, and to_string().

3992  {
3993  cat_write_lock write_lock(this);
3994  cat_sqlite_lock sqlite_lock(getObjForLock());
3995 
3996  if (shared_dict_defs.empty()) {
3997  return false;
3998  }
3999  for (const auto& shared_dict_def : shared_dict_defs) {
4000  // check if the current column is a referencing column
4001  const auto& column = shared_dict_def.get_column();
4002  if (cd.columnName == column) {
4003  if (!shared_dict_def.get_foreign_table().compare(td.tableName)) {
4004  // Dictionaries are being shared in table to be created
4005  const auto& ref_column = shared_dict_def.get_foreign_column();
4006  auto colIt =
4007  std::find_if(cdd.begin(), cdd.end(), [ref_column](const ColumnDescriptor it) {
4008  return !ref_column.compare(it.columnName);
4009  });
4010  CHECK(colIt != cdd.end());
4011  cd.columnType = colIt->columnType;
4012 
4013  const int dict_id = colIt->columnType.get_comp_param();
4014  CHECK_GE(dict_id, 1);
4015  auto dictIt = std::find_if(
4016  dds.begin(), dds.end(), [this, dict_id](const DictDescriptor it) {
4017  return it.dictRef.dbId == this->currentDB_.dbId &&
4018  it.dictRef.dictId == dict_id;
4019  });
4020  if (dictIt != dds.end()) {
4021  // There exists dictionary definition of a dictionary column
4022  CHECK_GE(dictIt->refcount, 1);
4023  ++dictIt->refcount;
4024  if (!table_is_temporary(&td)) {
4025  // Persist reference count
4027  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
4028  {std::to_string(dict_id)});
4029  }
4030  } else {
4031  // The dictionary is referencing a column which is referencing a column in
4032  // diffrent table
4033  auto root_dict_def = compress_reference_path(shared_dict_def, shared_dict_defs);
4034  addReferenceToForeignDict(cd, root_dict_def, !table_is_temporary(&td));
4035  }
4036  } else {
4037  const auto& foreign_table_name = shared_dict_def.get_foreign_table();
4038  const auto foreign_td = getMetadataForTable(foreign_table_name, false);
4039  if (table_is_temporary(foreign_td)) {
4040  if (!table_is_temporary(&td)) {
4041  throw std::runtime_error(
4042  "Only temporary tables can share dictionaries with other temporary "
4043  "tables.");
4044  }
4045  addReferenceToForeignDict(cd, shared_dict_def, false);
4046  } else {
4047  addReferenceToForeignDict(cd, shared_dict_def, !table_is_temporary(&td));
4048  }
4049  }
4050  return true;
4051  }
4052  }
4053  return false;
4054 }
const Parser::SharedDictionaryDef compress_reference_path(Parser::SharedDictionaryDef cur_node, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
std::string tableName
void addReferenceToForeignDict(ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def, const bool persist_reference)
Definition: Catalog.cpp:3965
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
#define CHECK_GE(x, y)
Definition: Logger.h:306
std::string to_string(char const *&&v)
specifies the content in-memory of a row in the column metadata table
bool table_is_temporary(const TableDescriptor *const td)
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:402
#define CHECK(condition)
Definition: Logger.h:291
Descriptor for a dictionary for a string columne.
SQLTypeInfo columnType
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
std::string columnName

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::setDeletedColumn ( const TableDescriptor td,
const ColumnDescriptor cd 
)

Definition at line 3940 of file Catalog.cpp.

3940  {
3941  cat_write_lock write_lock(this);
3942  setDeletedColumnUnlocked(td, cd);
3943 }
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3945
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
void Catalog_Namespace::Catalog::setDeletedColumnUnlocked ( const TableDescriptor td,
const ColumnDescriptor cd 
)
private

Definition at line 3945 of file Catalog.cpp.

References CHECK.

3946  {
3947  const auto it_ok = deletedColumnPerTable_.emplace(td, cd);
3948  CHECK(it_ok.second);
3949 }
DeletedColumnPerTableMap deletedColumnPerTable_
Definition: Catalog.h:774
#define CHECK(condition)
Definition: Logger.h:291
void Catalog_Namespace::Catalog::setForeignServerDataWrapper ( const std::string &  server_name,
const std::string &  data_wrapper 
)

Set the data wrapper of a Foreign Server.

Parameters
server_name- Name of the foreign server whose data wrapper will be set
data_wrapper- Data wrapper to use

Definition at line 3397 of file Catalog.cpp.

References CHECK, foreign_storage::ForeignServer::data_wrapper_type, to_upper(), and foreign_storage::ForeignServer::validate().

3398  {
3399  cat_write_lock write_lock(this);
3400  auto data_wrapper_type = to_upper(data_wrapper);
3401  // update in-memory server
3402  foreign_storage::ForeignServer* foreign_server =
3403  foreignServerMap_.find(server_name)->second.get();
3404  CHECK(foreign_server);
3405  std::string saved_data_wrapper_type = foreign_server->data_wrapper_type;
3406  foreign_server->data_wrapper_type = data_wrapper_type;
3407  try {
3408  foreign_server->validate();
3409  } catch (const std::exception& e) {
3410  // validation did not succeed:
3411  // revert to saved data_wrapper_type & throw exception
3412  foreign_server->data_wrapper_type = saved_data_wrapper_type;
3413  throw;
3414  }
3415  setForeignServerProperty(server_name, "data_wrapper_type", data_wrapper_type);
3416 }
ForeignServerMap foreignServerMap_
Definition: Catalog.h:715
std::string to_upper(const std::string &str)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
void setForeignServerProperty(const std::string &server_name, const std::string &property, const std::string &value)
Definition: Catalog.cpp:5310

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::setForeignServerOptions ( const std::string &  server_name,
const std::string &  options 
)

Set the options of a Foreign Server.

Parameters
server_name- Name of the foreign server whose options will be set
options- Options to set

Definition at line 3418 of file Catalog.cpp.

References CHECK, foreign_storage::OptionsContainer::options, foreign_storage::OptionsContainer::populateOptionsMap(), and foreign_storage::ForeignServer::validate().

3419  {
3420  cat_write_lock write_lock(this);
3421  // update in-memory server
3422  foreign_storage::ForeignServer* foreign_server =
3423  foreignServerMap_.find(server_name)->second.get();
3424  CHECK(foreign_server);
3425  auto saved_options = foreign_server->options;
3426  foreign_server->populateOptionsMap(options, true);
3427  try {
3428  foreign_server->validate();
3429  } catch (const std::exception& e) {
3430  // validation did not succeed:
3431  // revert to saved options & throw exception
3432  foreign_server->options = saved_options;
3433  throw;
3434  }
3435  setForeignServerProperty(server_name, "options", options);
3436 }
void populateOptionsMap(OptionsMap &&options_map, bool clear=false)
ForeignServerMap foreignServerMap_
Definition: Catalog.h:715
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
void setForeignServerProperty(const std::string &server_name, const std::string &property, const std::string &value)
Definition: Catalog.cpp:5310

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::setForeignServerProperty ( const std::string &  server_name,
const std::string &  property,
const std::string &  value 
)
private

Definition at line 5310 of file Catalog.cpp.

References CHECK_EQ, and to_string().

5312  {
5313  cat_sqlite_lock sqlite_lock(getObjForLock());
5315  "SELECT id from omnisci_foreign_servers where name = ?",
5316  std::vector<std::string>{server_name});
5317  auto num_rows = sqliteConnector_.getNumRows();
5318  if (num_rows > 0) {
5319  CHECK_EQ(size_t(1), num_rows);
5320  auto server_id = sqliteConnector_.getData<int32_t>(0, 0);
5322  "UPDATE omnisci_foreign_servers SET " + property + " = ? WHERE id = ?",
5323  std::vector<std::string>{value, std::to_string(server_id)});
5324  } else {
5325  throw std::runtime_error{"Can not change property \"" + property +
5326  "\" for foreign server." + " Foreign server \"" +
5327  server_name + "\" is not found."};
5328  }
5329 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
std::string to_string(char const *&&v)
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::setForeignTableOptions ( const std::string &  table_name,
foreign_storage::OptionsMap options_map,
bool  clear_existing_options = true 
)

Set the options of a Foreign Table.

Parameters
table_name- Name of the foreign table whose options will be set
options- Options to set

Definition at line 5827 of file Catalog.cpp.

5829  {
5830  cat_write_lock write_lock(this);
5831  // update in-memory table
5832  auto foreign_table = getForeignTableUnlocked(table_name);
5833  auto saved_options = foreign_table->options;
5834  foreign_table->populateOptionsMap(std::move(options_map), clear_existing_options);
5835  try {
5836  foreign_table->validateOptionValues();
5837  } catch (const std::exception& e) {
5838  // validation did not succeed:
5839  // revert to saved options & throw exception
5840  foreign_table->options = saved_options;
5841  throw;
5842  }
5844  foreign_table, "options", foreign_table->getOptionsAsJsonString());
5845 }
void setForeignTableProperty(const foreign_storage::ForeignTable *table, const std::string &property, const std::string &value)
Definition: Catalog.cpp:5847
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
foreign_storage::ForeignTable * getForeignTableUnlocked(const std::string &tableName) const
Definition: Catalog.cpp:1818
void Catalog_Namespace::Catalog::setForeignTableProperty ( const foreign_storage::ForeignTable table,
const std::string &  property,
const std::string &  value 
)
private

Definition at line 5847 of file Catalog.cpp.

References CHECK_EQ, TableDescriptor::tableId, TableDescriptor::tableName, and to_string().

5849  {
5850  cat_sqlite_lock sqlite_lock(getObjForLock());
5852  "SELECT table_id from omnisci_foreign_tables where table_id = ?",
5853  std::vector<std::string>{std::to_string(table->tableId)});
5854  auto num_rows = sqliteConnector_.getNumRows();
5855  if (num_rows > 0) {
5856  CHECK_EQ(size_t(1), num_rows);
5858  "UPDATE omnisci_foreign_tables SET " + property + " = ? WHERE table_id = ?",
5859  std::vector<std::string>{value, std::to_string(table->tableId)});
5860  } else {
5861  throw std::runtime_error{"Can not change property \"" + property +
5862  "\" for foreign table." + " Foreign table \"" +
5863  table->tableName + "\" is not found."};
5864  }
5865 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
std::string tableName
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
std::string to_string(char const *&&v)
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::setForReload ( const int32_t  tableId)

Definition at line 5365 of file Catalog.cpp.

5365  {
5366  const auto td = getMetadataForTable(tableId);
5367  for (const auto shard : getPhysicalTablesDescriptors(td)) {
5368  const auto tableEpoch = getTableEpoch(currentDB_.dbId, shard->tableId);
5369  setTableEpoch(currentDB_.dbId, shard->tableId, tableEpoch);
5370  }
5371 }
const DBMetadata currentDB_
Definition: Catalog.h:721
int32_t getTableEpoch(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:3579
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4869
void setTableEpoch(const int db_id, const int table_id, const int new_epoch)
Definition: Catalog.cpp:3647
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
void Catalog_Namespace::Catalog::setMaxRollbackEpochs ( const int32_t  table_id,
const int32_t  max_rollback_epochs 
)

Definition at line 3732 of file Catalog.cpp.

References CHECK, File_Namespace::FileMgrParams::epoch, logger::INFO, LOG, File_Namespace::FileMgrParams::max_rollback_epochs, and TableDescriptorUpdateParams::max_rollback_epochs.

3733  {
3734  // Must be called from AlterTableParamStmt or other method that takes executor and
3735  // TableSchema locks
3736  if (max_rollback_epochs <= -1) {
3737  throw std::runtime_error("Cannot set max_rollback_epochs < 0.");
3738  }
3739  const auto td = getMetadataForTable(
3740  table_id, false); // Deep copy as there will be gap between read and write locks
3741  CHECK(td); // Existence should have already been checked in
3742  // ParserNode::AlterTableParmStmt
3743  TableDescriptorUpdateParams table_update_params(td);
3744  table_update_params.max_rollback_epochs = max_rollback_epochs;
3745  if (table_update_params == td) { // Operator is overloaded to test for equality
3746  LOG(INFO) << "Setting max_rollback_epochs for table " << table_id
3747  << " to existing value, skipping operation";
3748  return;
3749  }
3750  File_Namespace::FileMgrParams file_mgr_params;
3751  file_mgr_params.epoch = -1; // Use existing epoch
3752  file_mgr_params.max_rollback_epochs = max_rollback_epochs;
3753  setTableFileMgrParams(table_id, file_mgr_params);
3754  alterTableMetadata(td, table_update_params);
3755 }
void setTableFileMgrParams(const int table_id, const File_Namespace::FileMgrParams &file_mgr_params)
Definition: Catalog.cpp:3794
#define LOG(tag)
Definition: Logger.h:285
void alterTableMetadata(const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
Definition: Catalog.cpp:3707
#define CHECK(condition)
Definition: Logger.h:291
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
void Catalog_Namespace::Catalog::setMaxRows ( const int32_t  table_id,
const int64_t  max_rows 
)

Definition at line 3757 of file Catalog.cpp.

References CHECK, logger::INFO, LOG, and TableDescriptorUpdateParams::max_rows.

3757  {
3758  if (max_rows < 0) {
3759  throw std::runtime_error("Max rows cannot be a negative number.");
3760  }
3761  const auto td = getMetadataForTable(table_id);
3762  CHECK(td);
3763  TableDescriptorUpdateParams table_update_params(td);
3764  table_update_params.max_rows = max_rows;
3765  if (table_update_params == td) {
3766  LOG(INFO) << "Max rows value of " << max_rows
3767  << " is the same as the existing value. Skipping update.";
3768  return;
3769  }
3770  alterTableMetadata(td, table_update_params);
3771  CHECK(td->fragmenter);
3772  td->fragmenter->dropFragmentsToSize(max_rows);
3773 }
#define LOG(tag)
Definition: Logger.h:285
void alterTableMetadata(const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
Definition: Catalog.cpp:3707
#define CHECK(condition)
Definition: Logger.h:291
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
void Catalog_Namespace::Catalog::setTableEpoch ( const int  db_id,
const int  table_id,
const int  new_epoch 
)

Definition at line 3647 of file Catalog.cpp.

References CHECK, Data_Namespace::DISK_LEVEL, File_Namespace::FileMgrParams::epoch, logger::INFO, LOG, and File_Namespace::FileMgrParams::max_rollback_epochs.

Referenced by AlterTableAlterColumnCommandRecoveryMgr::recoverAlterTableAlterColumnFromFile(), and TableArchiver::restoreTable().

3647  {
3648  LOG(INFO) << "Set table epoch db:" << db_id << " Table ID " << table_id
3649  << " back to new epoch " << new_epoch;
3650  const auto td = getMetadataForTable(table_id, false);
3651  if (!td) {
3652  std::stringstream table_not_found_error_message;
3653  table_not_found_error_message << "Table (" << db_id << "," << table_id
3654  << ") not found";
3655  throw std::runtime_error(table_not_found_error_message.str());
3656  }
3657  if (td->persistenceLevel != Data_Namespace::MemoryLevel::DISK_LEVEL) {
3658  std::stringstream is_temp_table_error_message;
3659  is_temp_table_error_message << "Cannot set epoch on temporary table";
3660  throw std::runtime_error(is_temp_table_error_message.str());
3661  }
3662 
3663  File_Namespace::FileMgrParams file_mgr_params;
3664  file_mgr_params.epoch = new_epoch;
3665  file_mgr_params.max_rollback_epochs = td->maxRollbackEpochs;
3666 
3667  const auto physical_tables = getPhysicalTablesDescriptors(td, false);
3668  CHECK(!physical_tables.empty());
3669  for (const auto table : physical_tables) {
3670  auto table_id = table->tableId;
3671  LOG(INFO) << "Set sharded table epoch db:" << db_id << " Table ID " << table_id
3672  << " back to new epoch " << new_epoch;
3673  // Should have table lock from caller so safe to do this after, avoids
3674  // having to repopulate data on error
3675  removeChunks(table_id);
3676  dataMgr_->getGlobalFileMgr()->setFileMgrParams(db_id, table_id, file_mgr_params);
3677  }
3678 }
#define LOG(tag)
Definition: Logger.h:285
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:722
void removeChunks(const int table_id) const
Definition: Catalog.cpp:4272
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4869
#define CHECK(condition)
Definition: Logger.h:291
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::setTableEpochs ( const int32_t  db_id,
const std::vector< TableEpochInfo > &  table_epochs 
) const

Definition at line 3849 of file Catalog.cpp.

References CHECK, File_Namespace::FileMgrParams::epoch, logger::INFO, LOG, and File_Namespace::FileMgrParams::max_rollback_epochs.

Referenced by UpdelRoll::cancelUpdate(), and import_export::Loader::setTableEpochs().

3850  {
3851  const auto td = getMetadataForTable(table_epochs[0].table_id, false);
3852  CHECK(td);
3853  File_Namespace::FileMgrParams file_mgr_params;
3854  file_mgr_params.max_rollback_epochs = td->maxRollbackEpochs;
3855 
3856  for (const auto& table_epoch_info : table_epochs) {
3857  removeChunks(table_epoch_info.table_id);
3858  file_mgr_params.epoch = table_epoch_info.table_epoch;
3859  dataMgr_->getGlobalFileMgr()->setFileMgrParams(
3860  db_id, table_epoch_info.table_id, file_mgr_params);
3861  LOG(INFO) << "Set table epoch for db id: " << db_id
3862  << ", table id: " << table_epoch_info.table_id
3863  << ", back to epoch: " << table_epoch_info.table_epoch;
3864  }
3865 }
#define LOG(tag)
Definition: Logger.h:285
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:722
void removeChunks(const int table_id) const
Definition: Catalog.cpp:4272
#define CHECK(condition)
Definition: Logger.h:291
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::setTableEpochsLogExceptions ( const int32_t  db_id,
const std::vector< TableEpochInfo > &  table_epochs 
) const

Definition at line 3885 of file Catalog.cpp.

References logger::ERROR, LOG, and Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_epochs_to_string().

Referenced by UpdelRoll::commitUpdate(), TableOptimizer::vacuumDeletedRows(), and TableOptimizer::vacuumFragmentsAboveMinSelectivity().

3887  {
3888  try {
3889  setTableEpochs(db_id, table_epochs);
3890  } catch (std::exception& e) {
3891  LOG(ERROR) << "An error occurred when attempting to set table epochs. DB id: "
3892  << db_id << ", Table epochs: " << table_epochs_to_string(table_epochs)
3893  << ", Error: " << e.what();
3894  }
3895 }
#define LOG(tag)
Definition: Logger.h:285
std::string table_epochs_to_string(const std::vector< TableEpochInfo > &table_epochs)
Definition: Catalog.cpp:3868
void setTableEpochs(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3849

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::setTableFileMgrParams ( const int  table_id,
const File_Namespace::FileMgrParams file_mgr_params 
)
private

Definition at line 3794 of file Catalog.cpp.

References CHECK, and Data_Namespace::DISK_LEVEL.

3796  {
3797  // Expects parent to have write lock
3798  const auto td = getMetadataForTable(table_id, false);
3799  const auto db_id = this->getDatabaseId();
3800  if (!td) {
3801  std::stringstream table_not_found_error_message;
3802  table_not_found_error_message << "Table (" << db_id << "," << table_id
3803  << ") not found";
3804  throw std::runtime_error(table_not_found_error_message.str());
3805  }
3806  if (td->persistenceLevel != Data_Namespace::MemoryLevel::DISK_LEVEL) {
3807  std::stringstream is_temp_table_error_message;
3808  is_temp_table_error_message << "Cannot set storage params on temporary table";
3809  throw std::runtime_error(is_temp_table_error_message.str());
3810  }
3811 
3812  const auto physical_tables = getPhysicalTablesDescriptors(td, false);
3813  CHECK(!physical_tables.empty());
3814  for (const auto table : physical_tables) {
3815  auto table_id = table->tableId;
3816  removeChunks(table_id);
3817  dataMgr_->getGlobalFileMgr()->setFileMgrParams(db_id, table_id, file_mgr_params);
3818  }
3819 }
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:722
void removeChunks(const int table_id) const
Definition: Catalog.cpp:4272
int getDatabaseId() const
Definition: Catalog.h:326
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4869
#define CHECK(condition)
Definition: Logger.h:291
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
void Catalog_Namespace::Catalog::setUncappedTableEpoch ( const std::string &  table_name)

Definition at line 3776 of file Catalog.cpp.

References CHECK, Executor::clearExternalCaches(), File_Namespace::FileMgrParams::max_rollback_epochs, TableDescriptorUpdateParams::max_rollback_epochs, to_upper(), and Catalog_Namespace::write_lock< T >::unlock().

3776  {
3777  cat_write_lock write_lock(this);
3778  auto td_entry = tableDescriptorMap_.find(to_upper(table_name));
3779  CHECK(td_entry != tableDescriptorMap_.end());
3780  auto td = td_entry->second;
3781 
3783 
3784  TableDescriptorUpdateParams table_update_params(td);
3785  table_update_params.max_rollback_epochs = -1;
3786  write_lock.unlock();
3787 
3788  alterTableMetadata(td, table_update_params);
3789  File_Namespace::FileMgrParams file_mgr_params;
3790  file_mgr_params.max_rollback_epochs = -1;
3791  setTableFileMgrParams(td->tableId, file_mgr_params);
3792 }
void setTableFileMgrParams(const int table_id, const File_Namespace::FileMgrParams &file_mgr_params)
Definition: Catalog.cpp:3794
int getDatabaseId() const
Definition: Catalog.h:326
std::string to_upper(const std::string &str)
void alterTableMetadata(const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
Definition: Catalog.cpp:3707
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
static void clearExternalCaches(bool for_update, const TableDescriptor *td, const int current_db_id)
Definition: Execute.h:438
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:707

+ Here is the call graph for this function:

std::list< ColumnDescriptor * > Catalog_Namespace::Catalog::sqliteGetColumnsForTableUnlocked ( int32_t  table_id)
private

Definition at line 5194 of file Catalog.cpp.

References Catalog_Namespace::anonymous_namespace{Catalog.cpp}::set_dict_key(), and to_string().

5194  {
5195  std::list<ColumnDescriptor*> cds;
5196  // TODO(Misiu): Change ColumnDescriptorMap_ to use smartpointers. Right now we use
5197  // temporary smartpointers that release their memory once initialized.
5198  std::list<std::unique_ptr<ColumnDescriptor>> smart_cds;
5199  std::string columnQuery(
5200  "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
5201  "is_notnull, compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
5202  "virtual_expr, is_deletedcol, default_value from mapd_columns WHERE tableid = " +
5203  std::to_string(table_id) + " ORDER BY tableid, columnid");
5204  sqliteConnector_.query(columnQuery);
5205  auto numRows = sqliteConnector_.getNumRows();
5206  int32_t skip_physical_cols = 0;
5207  for (size_t r = 0; r < numRows; ++r) {
5208  std::unique_ptr<ColumnDescriptor> cd = std::make_unique<ColumnDescriptor>();
5209  cd->tableId = sqliteConnector_.getData<int>(r, 0);
5210  cd->columnId = sqliteConnector_.getData<int>(r, 1);
5211  cd->columnName = sqliteConnector_.getData<string>(r, 2);
5212  cd->columnType.set_type((SQLTypes)sqliteConnector_.getData<int>(r, 3));
5213  cd->columnType.set_subtype((SQLTypes)sqliteConnector_.getData<int>(r, 4));
5214  cd->columnType.set_dimension(sqliteConnector_.getData<int>(r, 5));
5215  cd->columnType.set_scale(sqliteConnector_.getData<int>(r, 6));
5216  cd->columnType.set_notnull(sqliteConnector_.getData<bool>(r, 7));
5217  cd->columnType.set_compression((EncodingType)sqliteConnector_.getData<int>(r, 8));
5218  cd->columnType.set_comp_param(sqliteConnector_.getData<int>(r, 9));
5219  cd->columnType.set_size(sqliteConnector_.getData<int>(r, 10));
5220  cd->chunks = sqliteConnector_.getData<string>(r, 11);
5221  cd->isSystemCol = sqliteConnector_.getData<bool>(r, 12);
5222  cd->isVirtualCol = sqliteConnector_.getData<bool>(r, 13);
5223  cd->virtualExpr = sqliteConnector_.getData<string>(r, 14);
5224  cd->isDeletedCol = sqliteConnector_.getData<bool>(r, 15);
5225  if (sqliteConnector_.isNull(r, 16)) {
5226  cd->default_value = std::nullopt;
5227  } else {
5228  cd->default_value = std::make_optional(sqliteConnector_.getData<string>(r, 16));
5229  }
5230  cd->isGeoPhyCol = skip_physical_cols-- > 0;
5231  cd->db_id = getDatabaseId();
5232  set_dict_key(*cd);
5233  smart_cds.emplace_back(std::move(cd));
5234  }
5235  // Once we have correctly initialized all columns, release their ownership as we
5236  // currently handle them as free pointers.
5237  for (auto& cd : smart_cds) {
5238  cds.emplace_back(cd.release());
5239  }
5240  return cds;
5241 }
T getData(const int row, const int col)
SQLTypes
Definition: sqltypes.h:65
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
EncodingType
Definition: sqltypes.h:240
int getDatabaseId() const
Definition: Catalog.h:326
bool isNull(const int row, const int col) const
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::truncateTable ( const TableDescriptor td)

Definition at line 4135 of file Catalog.cpp.

4135  {
4136  // truncate all corresponding physical tables
4137  const auto physical_tables = getPhysicalTablesDescriptors(td);
4138  for (const auto table : physical_tables) {
4139  doTruncateTable(table);
4140  }
4141 }
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4869
void doTruncateTable(const TableDescriptor *td)
Definition: Catalog.cpp:4143
void Catalog_Namespace::Catalog::updateCustomExpression ( int32_t  custom_expression_id,
const std::string &  expression_json 
)

Updates the custom expression for the given id with the given expression json string.

Parameters
custom_expression_id- id of custom expression to update
expression_json- expression json string to be set

Definition at line 6041 of file Catalog.cpp.

References CHECK_EQ, and to_string().

6042  {
6043  cat_write_lock write_lock(this);
6044  cat_sqlite_lock sqlite_lock(getObjForLock());
6045  auto it = custom_expr_map_by_id_.find(custom_expression_id);
6046  if (it == custom_expr_map_by_id_.end() || it->second->is_deleted) {
6047  throw std::runtime_error{"Custom expression with id \"" +
6048  std::to_string(custom_expression_id) + "\" does not exist."};
6049  }
6050  auto old_expression_json = it->second->expression_json;
6051  sqliteConnector_.query("BEGIN TRANSACTION");
6052  try {
6054  "SELECT id FROM omnisci_custom_expressions WHERE id = ?",
6055  std::vector<std::string>{std::to_string(custom_expression_id)});
6056  CHECK_EQ(sqliteConnector_.getNumRows(), static_cast<size_t>(1));
6058  "UPDATE omnisci_custom_expressions SET expression_json = ? WHERE id = ?",
6059  std::vector<std::string>{expression_json, std::to_string(custom_expression_id)});
6060  it->second->expression_json = expression_json;
6061  } catch (std::exception& e) {
6062  sqliteConnector_.query("ROLLBACK TRANSACTION");
6063  it->second->expression_json = old_expression_json;
6064  throw;
6065  }
6066  sqliteConnector_.query("END TRANSACTION");
6067 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:717
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::updateCustomExpressionsSchema ( )
protected

Definition at line 763 of file Catalog.cpp.

References getCustomExpressionsSchema(), getObjForLock(), SqliteConnector::query(), and sqliteConnector_.

Referenced by CheckAndExecuteMigrations().

763  {
764  cat_sqlite_lock sqlite_lock(getObjForLock());
765  sqliteConnector_.query("BEGIN TRANSACTION");
766  try {
768  } catch (const std::exception& e) {
769  sqliteConnector_.query("ROLLBACK TRANSACTION");
770  throw;
771  }
772  sqliteConnector_.query("END TRANSACTION");
773 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
static const std::string getCustomExpressionsSchema(bool if_not_exists=false)
Definition: Catalog.cpp:790
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::updateDefaultColumnValues ( )
protected

Definition at line 555 of file Catalog.cpp.

References logger::ERROR, SqliteConnector::getData(), SqliteConnector::getNumRows(), getObjForLock(), logger::INFO, LOG, SqliteConnector::query(), and sqliteConnector_.

Referenced by CheckAndExecuteMigrations().

555  {
556  cat_sqlite_lock sqlite_lock(getObjForLock());
557  sqliteConnector_.query("BEGIN TRANSACTION");
558  try {
559  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_columns)");
560  std::vector<std::string> cols;
561  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
562  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
563  }
564  if (std::find(cols.begin(), cols.end(), std::string("default_value")) == cols.end()) {
565  LOG(INFO) << "Adding support for default values to mapd_columns";
566  sqliteConnector_.query("ALTER TABLE mapd_columns ADD default_value TEXT");
567  }
568  } catch (std::exception& e) {
569  sqliteConnector_.query("ROLLBACK TRANSACTION");
570  LOG(ERROR) << "Failed to make metadata update for default values` support";
571  throw;
572  }
573  sqliteConnector_.query("END TRANSACTION");
574 }
T getData(const int row, const int col)
#define LOG(tag)
Definition: Logger.h:285
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::updateDeletedColumnIndicator ( )
protected

Definition at line 528 of file Catalog.cpp.

References Catalog_Namespace::DEFAULT_INITIAL_VERSION, SqliteConnector::getData(), SqliteConnector::getNumRows(), getObjForLock(), logger::INFO, LOG, SqliteConnector::query(), sqliteConnector_, and to_string().

Referenced by CheckAndExecuteMigrations().

528  {
529  cat_sqlite_lock sqlite_lock(getObjForLock());
530  sqliteConnector_.query("BEGIN TRANSACTION");
531  try {
532  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_columns)");
533  std::vector<std::string> cols;
534  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
535  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
536  }
537  if (std::find(cols.begin(), cols.end(), std::string("version_num")) == cols.end()) {
538  LOG(INFO) << "Updating mapd_columns updateDeletedColumnIndicator";
539  // need to add new version info
540  string queryString("ALTER TABLE mapd_columns ADD version_num BIGINT DEFAULT " +
542  sqliteConnector_.query(queryString);
543  // need to add new column to table defintion to indicate deleted column, column used
544  // as bitmap for deleted rows.
546  "ALTER TABLE mapd_columns ADD is_deletedcol boolean default 0 ");
547  }
548  } catch (std::exception& e) {
549  sqliteConnector_.query("ROLLBACK TRANSACTION");
550  throw;
551  }
552  sqliteConnector_.query("END TRANSACTION");
553 }
T getData(const int row, const int col)
#define LOG(tag)
Definition: Logger.h:285
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
const int DEFAULT_INITIAL_VERSION
Definition: Catalog.cpp:110
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::updateDictionaryNames ( )
protected

Definition at line 579 of file Catalog.cpp.

References currentDB_, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, Catalog_Namespace::DEFAULT_INITIAL_VERSION, logger::ERROR, g_base_path, SqliteConnector::getData(), SqliteConnector::getNumRows(), getObjForLock(), logger::INFO, shared::kDataDirectoryName, LOG, SqliteConnector::query(), run_benchmark_import::result, sqliteConnector_, and to_string().

Referenced by CheckAndExecuteMigrations().

579  {
580  cat_sqlite_lock sqlite_lock(getObjForLock());
581  if (currentDB_.dbName.length() == 0) {
582  // updateDictionaryNames dbName length is zero nothing to do here
583  return;
584  }
585  sqliteConnector_.query("BEGIN TRANSACTION");
586  try {
587  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_dictionaries)");
588  std::vector<std::string> cols;
589  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
590  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
591  }
592  if (std::find(cols.begin(), cols.end(), std::string("version_num")) == cols.end()) {
593  // No version number
594  // need to rename dictionaries
595  string dictQuery("SELECT dictid, name from mapd_dictionaries");
596  sqliteConnector_.query(dictQuery);
597  size_t numRows = sqliteConnector_.getNumRows();
598  for (size_t r = 0; r < numRows; ++r) {
599  int dictId = sqliteConnector_.getData<int>(r, 0);
600  std::string dictName = sqliteConnector_.getData<string>(r, 1);
601 
602  std::string oldName = g_base_path + "/" + shared::kDataDirectoryName + "/" +
603  currentDB_.dbName + "_" + dictName;
604  std::string newName = g_base_path + "/" + shared::kDataDirectoryName + "/DB_" +
605  std::to_string(currentDB_.dbId) + "_DICT_" +
606  std::to_string(dictId);
607 
608  int result = rename(oldName.c_str(), newName.c_str());
609 
610  if (result == 0) {
611  LOG(INFO) << "Dictionary upgrade: successfully renamed " << oldName << " to "
612  << newName;
613  } else {
614  LOG(ERROR) << "Failed to rename old dictionary directory " << oldName << " to "
615  << newName + " dbname '" << currentDB_.dbName << "' error code "
616  << std::to_string(result);
617  }
618  }
619  // need to add new version info
620  string queryString("ALTER TABLE mapd_dictionaries ADD version_num BIGINT DEFAULT " +
622  sqliteConnector_.query(queryString);
623  }
624  } catch (std::exception& e) {
625  sqliteConnector_.query("ROLLBACK TRANSACTION");
626  throw;
627  }
628  sqliteConnector_.query("END TRANSACTION");
629 }
const std::string kDataDirectoryName
T getData(const int row, const int col)
#define LOG(tag)
Definition: Logger.h:285
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
std::string g_base_path
Definition: SysCatalog.cpp:62
const int DEFAULT_INITIAL_VERSION
Definition: Catalog.cpp:110
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::updateDictionarySchema ( )
protected

Definition at line 673 of file Catalog.cpp.

References SqliteConnector::getData(), SqliteConnector::getNumRows(), getObjForLock(), SqliteConnector::query(), and sqliteConnector_.

Referenced by CheckAndExecuteMigrations().

673  {
674  cat_sqlite_lock sqlite_lock(getObjForLock());
675  sqliteConnector_.query("BEGIN TRANSACTION");
676  try {
677  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_dictionaries)");
678  std::vector<std::string> cols;
679  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
680  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
681  }
682  if (std::find(cols.begin(), cols.end(), std::string("refcount")) == cols.end()) {
683  sqliteConnector_.query("ALTER TABLE mapd_dictionaries ADD refcount DEFAULT 1");
684  }
685  } catch (std::exception& e) {
686  sqliteConnector_.query("ROLLBACK TRANSACTION");
687  throw;
688  }
689  sqliteConnector_.query("END TRANSACTION");
690 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::updateFixlenArrayColumns ( )
protected

Definition at line 332 of file Catalog.cpp.

References SqliteConnector::getNumRows(), getObjForLock(), logger::INFO, kARRAY, LOG, MAPD_VERSION, SqliteConnector::query(), SqliteConnector::query_with_text_params(), sqliteConnector_, and to_string().

Referenced by CheckAndExecuteMigrations().

332  {
333  cat_sqlite_lock sqlite_lock(getObjForLock());
334  sqliteConnector_.query("BEGIN TRANSACTION");
335  try {
337  "select name from sqlite_master WHERE type='table' AND "
338  "name='mapd_version_history'");
339  if (sqliteConnector_.getNumRows() == 0) {
341  "CREATE TABLE mapd_version_history(version integer, migration_history text "
342  "unique)");
343  } else {
345  "select * from mapd_version_history where migration_history = "
346  "'notnull_fixlen_arrays'");
347  if (sqliteConnector_.getNumRows() != 0) {
348  // legacy fixlen arrays had migrated
349  // no need for further execution
350  sqliteConnector_.query("END TRANSACTION");
351  return;
352  }
353  }
354  // Insert check for migration
356  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
357  std::vector<std::string>{std::to_string(MAPD_VERSION), "notnull_fixlen_arrays"});
358  LOG(INFO) << "Updating mapd_columns, legacy fixlen arrays";
359  // Upating all fixlen array columns
360  string queryString("UPDATE mapd_columns SET is_notnull=1 WHERE coltype=" +
361  std::to_string(kARRAY) + " AND size>0;");
362  sqliteConnector_.query(queryString);
363  } catch (std::exception& e) {
364  sqliteConnector_.query("ROLLBACK TRANSACTION");
365  throw;
366  }
367  sqliteConnector_.query("END TRANSACTION");
368 }
virtual void query_with_text_params(std::string const &query_only)
#define LOG(tag)
Definition: Logger.h:285
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
static const int32_t MAPD_VERSION
Definition: release.h:32
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::updateForeignTableRefreshTimes ( const int32_t  table_id)

Updates the last and next (if applicable) refresh times of the foreign table with the given table id.

Parameters
table_id- id of table to apply updates to

Definition at line 5805 of file Catalog.cpp.

References CHECK, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_next_refresh_time(), foreign_storage::RefreshTimeCalculator::getCurrentTime(), and to_string().

Referenced by foreign_storage::refresh_foreign_table_unlocked().

5805  {
5806  cat_write_lock write_lock(this);
5807  cat_sqlite_lock sqlite_lock(getObjForLock());
5808  CHECK(tableDescriptorMapById_.find(table_id) != tableDescriptorMapById_.end());
5809  auto table_descriptor = tableDescriptorMapById_.find(table_id)->second;
5810  CHECK(table_descriptor);
5811  auto foreign_table = dynamic_cast<foreign_storage::ForeignTable*>(table_descriptor);
5812  CHECK(foreign_table);
5813  auto last_refresh_time = foreign_storage::RefreshTimeCalculator::getCurrentTime();
5814  auto next_refresh_time = get_next_refresh_time(*foreign_table);
5816  "UPDATE omnisci_foreign_tables SET last_refresh_time = ?, next_refresh_time = ? "
5817  "WHERE table_id = ?",
5818  std::vector<std::string>{std::to_string(last_refresh_time),
5819  std::to_string(next_refresh_time),
5820  std::to_string(foreign_table->tableId)});
5821  foreign_table->last_refresh_time = last_refresh_time;
5822  foreign_table->next_refresh_time = next_refresh_time;
5823 }
int64_t get_next_refresh_time(const foreign_storage::ForeignTable &foreign_table)
Definition: Catalog.cpp:2922
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
std::string to_string(char const *&&v)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:122
#define CHECK(condition)
Definition: Logger.h:291
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::updateForeignTablesInMapUnlocked ( )
private

Definition at line 5112 of file Catalog.cpp.

References CHECK, shared::contains(), g_enable_fsi, and foreign_storage::DataWrapperType::IN_MEMORY_DATA_WRAPPERS.

5112  {
5115  "SELECT table_id, server_id, options, last_refresh_time, next_refresh_time from "
5116  "omnisci_foreign_tables");
5117  auto num_rows = sqliteConnector_.getNumRows();
5118  for (size_t r = 0; r < num_rows; r++) {
5119  const auto table_id = sqliteConnector_.getData<int32_t>(r, 0);
5120  const auto server_id = sqliteConnector_.getData<int32_t>(r, 1);
5121  const auto& options = sqliteConnector_.getData<std::string>(r, 2);
5122  const auto last_refresh_time = sqliteConnector_.getData<int64_t>(r, 3);
5123  const auto next_refresh_time = sqliteConnector_.getData<int64_t>(r, 4);
5124 
5125  CHECK(tableDescriptorMapById_.find(table_id) != tableDescriptorMapById_.end());
5126  auto foreign_table =
5127  dynamic_cast<foreign_storage::ForeignTable*>(tableDescriptorMapById_[table_id]);
5128  CHECK(foreign_table);
5129  foreign_table->foreign_server = foreignServerMapById_[server_id].get();
5130  CHECK(foreign_table->foreign_server);
5131  foreign_table->populateOptionsMap(options);
5132  foreign_table->last_refresh_time = last_refresh_time;
5133  foreign_table->next_refresh_time = next_refresh_time;
5134  if (foreign_table->is_system_table) {
5135  foreign_table->is_in_memory_system_table =
5137  foreign_table->foreign_server->data_wrapper_type);
5138  }
5139  }
5140 }
bool contains(const T &container, const U &element)
Definition: misc.h:195
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
static constexpr std::array< char const *, 5 > IN_MEMORY_DATA_WRAPPERS
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:716
#define CHECK(condition)
Definition: Logger.h:291
bool g_enable_fsi
Definition: Catalog.cpp:96
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::updateFrontendViewAndLinkUsers ( )
protected

Definition at line 470 of file Catalog.cpp.

References SqliteConnector::getNumRows(), getObjForLock(), SqliteConnector::query(), and sqliteConnector_.

Referenced by CheckAndExecuteMigrations().

470  {
471  cat_sqlite_lock sqlite_lock(getObjForLock());
472  sqliteConnector_.query("BEGIN TRANSACTION");
473  try {
474  sqliteConnector_.query("UPDATE mapd_links SET userid = 0 WHERE userid IS NULL");
475  // check table still exists
477  "SELECT name FROM sqlite_master WHERE type='table' AND "
478  "name='mapd_frontend_views'");
479  if (sqliteConnector_.getNumRows() == 0) {
480  // table does not exists
481  // no need to migrate
482  sqliteConnector_.query("END TRANSACTION");
483  return;
484  }
486  "UPDATE mapd_frontend_views SET userid = 0 WHERE userid IS NULL");
487  } catch (const std::exception& e) {
488  sqliteConnector_.query("ROLLBACK TRANSACTION");
489  throw;
490  }
491  sqliteConnector_.query("END TRANSACTION");
492 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::updateFrontendViewSchema ( )
protected

Definition at line 412 of file Catalog.cpp.

References SqliteConnector::getData(), SqliteConnector::getNumRows(), getObjForLock(), SqliteConnector::query(), and sqliteConnector_.

Referenced by CheckAndExecuteMigrations().

412  {
413  cat_sqlite_lock sqlite_lock(getObjForLock());
414  sqliteConnector_.query("BEGIN TRANSACTION");
415  try {
416  // check table still exists
418  "SELECT name FROM sqlite_master WHERE type='table' AND "
419  "name='mapd_frontend_views'");
420  if (sqliteConnector_.getNumRows() == 0) {
421  // table does not exists
422  // no need to migrate
423  sqliteConnector_.query("END TRANSACTION");
424  return;
425  }
426  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_frontend_views)");
427  std::vector<std::string> cols;
428  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
429  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
430  }
431  if (std::find(cols.begin(), cols.end(), std::string("image_hash")) == cols.end()) {
432  sqliteConnector_.query("ALTER TABLE mapd_frontend_views ADD image_hash text");
433  }
434  if (std::find(cols.begin(), cols.end(), std::string("update_time")) == cols.end()) {
435  sqliteConnector_.query("ALTER TABLE mapd_frontend_views ADD update_time timestamp");
436  }
437  if (std::find(cols.begin(), cols.end(), std::string("view_metadata")) == cols.end()) {
438  sqliteConnector_.query("ALTER TABLE mapd_frontend_views ADD view_metadata text");
439  }
440  } catch (std::exception& e) {
441  sqliteConnector_.query("ROLLBACK TRANSACTION");
442  throw;
443  }
444  sqliteConnector_.query("END TRANSACTION");
445 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::updateFrontendViewsToDashboards ( )
protected

Definition at line 129 of file Catalog.cpp.

References SqliteConnector::getNumRows(), getObjForLock(), SqliteConnector::query(), and sqliteConnector_.

Referenced by CheckAndExecuteMigrations().

129  {
130  cat_sqlite_lock sqlite_lock(getObjForLock());
131  sqliteConnector_.query("BEGIN TRANSACTION");
132  try {
134  "SELECT name FROM sqlite_master WHERE type='table' AND name='mapd_dashboards'");
135  if (sqliteConnector_.getNumRows() != 0) {
136  // already done
137  sqliteConnector_.query("END TRANSACTION");
138  return;
139  }
141  "CREATE TABLE mapd_dashboards (id integer primary key autoincrement, name text , "
142  "userid integer references mapd_users, state text, image_hash text, update_time "
143  "timestamp, "
144  "metadata text, UNIQUE(userid, name) )");
145  // now copy content from old table to new table
147  "insert into mapd_dashboards (id, name , "
148  "userid, state, image_hash, update_time , "
149  "metadata) "
150  "SELECT viewid , name , userid, view_state, image_hash, update_time, "
151  "view_metadata "
152  "from mapd_frontend_views");
153  } catch (const std::exception& e) {
154  sqliteConnector_.query("ROLLBACK TRANSACTION");
155  throw;
156  }
157  sqliteConnector_.query("END TRANSACTION");
158 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::updateFsiSchemas ( )
protected

Definition at line 692 of file Catalog.cpp.

References getForeignServerSchema(), getForeignTableSchema(), getObjForLock(), SqliteConnector::query(), and sqliteConnector_.

Referenced by CheckAndExecuteMigrations().

692  {
693  cat_sqlite_lock sqlite_lock(getObjForLock());
694  sqliteConnector_.query("BEGIN TRANSACTION");
695  try {
698  } catch (std::exception& e) {
699  sqliteConnector_.query("ROLLBACK TRANSACTION");
700  throw;
701  }
702  sqliteConnector_.query("END TRANSACTION");
703 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
static const std::string getForeignTableSchema(bool if_not_exists=false)
Definition: Catalog.cpp:782
static const std::string getForeignServerSchema(bool if_not_exists=false)
Definition: Catalog.cpp:775
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::updateGeoColumns ( )
protected

Definition at line 370 of file Catalog.cpp.

References SqliteConnector::getNumRows(), getObjForLock(), logger::INFO, kLINESTRING, kMULTILINESTRING, kMULTIPOINT, kMULTIPOLYGON, kPOINT, kPOLYGON, LOG, MAPD_VERSION, SqliteConnector::query(), SqliteConnector::query_with_text_params(), sqliteConnector_, and to_string().

Referenced by CheckAndExecuteMigrations().

370  {
371  cat_sqlite_lock sqlite_lock(getObjForLock());
372  sqliteConnector_.query("BEGIN TRANSACTION");
373  try {
375  "select name from sqlite_master WHERE type='table' AND "
376  "name='mapd_version_history'");
377  if (sqliteConnector_.getNumRows() == 0) {
379  "CREATE TABLE mapd_version_history(version integer, migration_history text "
380  "unique)");
381  } else {
383  "select * from mapd_version_history where migration_history = "
384  "'notnull_geo_columns'");
385  if (sqliteConnector_.getNumRows() != 0) {
386  // legacy geo columns had migrated
387  // no need for further execution
388  sqliteConnector_.query("END TRANSACTION");
389  return;
390  }
391  }
392  // Insert check for migration
394  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
395  std::vector<std::string>{std::to_string(MAPD_VERSION), "notnull_geo_columns"});
396  LOG(INFO) << "Updating mapd_columns, legacy geo columns";
397  // Upating all geo columns
398  string queryString(
399  "UPDATE mapd_columns SET is_notnull=1 WHERE coltype=" + std::to_string(kPOINT) +
400  " OR coltype=" + std::to_string(kMULTIPOINT) + " OR coltype=" +
402  " OR coltype=" + std::to_string(kPOLYGON) +
403  " OR coltype=" + std::to_string(kMULTIPOLYGON) + ";");
404  sqliteConnector_.query(queryString);
405  } catch (std::exception& e) {
406  sqliteConnector_.query("ROLLBACK TRANSACTION");
407  throw;
408  }
409  sqliteConnector_.query("END TRANSACTION");
410 }
virtual void query_with_text_params(std::string const &query_only)
#define LOG(tag)
Definition: Logger.h:285
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
static const int32_t MAPD_VERSION
Definition: release.h:32
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::updateInColumnMap ( ColumnDescriptor cd,
ColumnDescriptor old_cd 
)
private

Definition at line 6960 of file Catalog.cpp.

References ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_checked_table_name(), shared::get_from_map(), SQLTypeInfo::is_dict_encoded_type(), ColumnDescriptor::tableId, and to_upper().

6960  {
6962  ColumnKey{cd->tableId, to_upper(cd->columnName)}) = cd;
6964  cd;
6965  auto dict_it = dict_columns_by_table_id_.find(cd->tableId);
6966  if (dict_it != dict_columns_by_table_id_.end()) {
6967  auto& set = dict_it->second;
6968  for (auto it = set.begin(); it != set.end(); ++it) {
6969  if ((*it)->columnId == cd->columnId) {
6970  set.erase(it);
6971  break;
6972  }
6973  }
6974  }
6975  if (cd->columnType.is_dict_encoded_type()) {
6976  dict_columns_by_table_id_[cd->tableId].emplace(cd);
6977  }
6978 
6979  removeColumnDescriptor(old_cd);
6980  addColumnDescriptor(cd);
6981  calciteMgr_->updateMetadata(currentDB_.dbName, get_checked_table_name(this, cd));
6982 }
TableDictColumnsMap dict_columns_by_table_id_
Definition: Catalog.h:718
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:709
const DBMetadata currentDB_
Definition: Catalog.h:721
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:710
void removeColumnDescriptor(const ColumnDescriptor *cd)
Definition: Catalog.cpp:2686
bool is_dict_encoded_type() const
Definition: sqltypes.h:653
std::string get_checked_table_name(const Catalog *catalog, const ColumnDescriptor *cd)
Definition: Catalog.cpp:6953
std::string to_upper(const std::string &str)
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:725
V & get_from_map(std::map< K, V, comp > &map, const K &key)
Definition: misc.h:61
void addColumnDescriptor(const ColumnDescriptor *cd)
Definition: Catalog.cpp:2701
SQLTypeInfo columnType
std::string columnName
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::updateLeaf ( const LeafHostInfo string_dict_host)
void Catalog_Namespace::Catalog::updateLinkSchema ( )
protected

Definition at line 447 of file Catalog.cpp.

References SqliteConnector::getData(), SqliteConnector::getNumRows(), getObjForLock(), SqliteConnector::query(), and sqliteConnector_.

Referenced by CheckAndExecuteMigrations().

447  {
448  cat_sqlite_lock sqlite_lock(getObjForLock());
449  sqliteConnector_.query("BEGIN TRANSACTION");
450  try {
452  "CREATE TABLE IF NOT EXISTS mapd_links (linkid integer primary key, userid "
453  "integer references mapd_users, "
454  "link text unique, view_state text, update_time timestamp, view_metadata text)");
455  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_links)");
456  std::vector<std::string> cols;
457  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
458  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
459  }
460  if (std::find(cols.begin(), cols.end(), std::string("view_metadata")) == cols.end()) {
461  sqliteConnector_.query("ALTER TABLE mapd_links ADD view_metadata text");
462  }
463  } catch (const std::exception& e) {
464  sqliteConnector_.query("ROLLBACK TRANSACTION");
465  throw;
466  }
467  sqliteConnector_.query("END TRANSACTION");
468 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::updateLogicalToPhysicalTableLinkSchema ( )
protected

Definition at line 631 of file Catalog.cpp.

References getObjForLock(), SqliteConnector::query(), and sqliteConnector_.

Referenced by CheckAndExecuteMigrations().

631  {
632  cat_sqlite_lock sqlite_lock(getObjForLock());
633  sqliteConnector_.query("BEGIN TRANSACTION");
634  try {
636  "CREATE TABLE IF NOT EXISTS mapd_logical_to_physical("
637  "logical_table_id integer, physical_table_id integer)");
638  } catch (const std::exception& e) {
639  sqliteConnector_.query("ROLLBACK TRANSACTION");
640  throw;
641  }
642  sqliteConnector_.query("END TRANSACTION");
643 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::updateLogicalToPhysicalTableMap ( const int32_t  logical_tb_id)
protected

Definition at line 645 of file Catalog.cpp.

References CHECK, getObjForLock(), logicalToPhysicalTableMapById_, SqliteConnector::query(), SqliteConnector::query_with_text_params(), sqliteConnector_, and to_string().

645  {
646  /* this proc inserts/updates all pairs of (logical_tb_id, physical_tb_id) in
647  * sqlite mapd_logical_to_physical table for given logical_tb_id as needed
648  */
649 
650  cat_sqlite_lock sqlite_lock(getObjForLock());
651  sqliteConnector_.query("BEGIN TRANSACTION");
652  try {
653  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
654  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
655  const auto physicalTables = physicalTableIt->second;
656  CHECK(!physicalTables.empty());
657  for (size_t i = 0; i < physicalTables.size(); i++) {
658  int32_t physical_tb_id = physicalTables[i];
660  "INSERT OR REPLACE INTO mapd_logical_to_physical (logical_table_id, "
661  "physical_table_id) VALUES (?1, ?2)",
662  std::vector<std::string>{std::to_string(logical_tb_id),
663  std::to_string(physical_tb_id)});
664  }
665  }
666  } catch (std::exception& e) {
667  sqliteConnector_.query("ROLLBACK TRANSACTION");
668  throw;
669  }
670  sqliteConnector_.query("END TRANSACTION");
671 }
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
#define CHECK(condition)
Definition: Logger.h:291
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:727

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::updatePageSize ( )
protected

Definition at line 498 of file Catalog.cpp.

References currentDB_, Catalog_Namespace::DBMetadata::dbName, Catalog_Namespace::DEFAULT_INITIAL_VERSION, SqliteConnector::getData(), SqliteConnector::getNumRows(), getObjForLock(), logger::INFO, LOG, SqliteConnector::query(), sqliteConnector_, and to_string().

Referenced by CheckAndExecuteMigrations().

498  {
499  cat_sqlite_lock sqlite_lock(getObjForLock());
500  if (currentDB_.dbName.length() == 0) {
501  // updateDictionaryNames dbName length is zero nothing to do here
502  return;
503  }
504  sqliteConnector_.query("BEGIN TRANSACTION");
505  try {
506  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_tables)");
507  std::vector<std::string> cols;
508  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
509  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
510  }
511  if (std::find(cols.begin(), cols.end(), std::string("version_num")) == cols.end()) {
512  LOG(INFO) << "Updating mapd_tables updatePageSize";
513  // No version number
514  // need to update the defaul tpagesize to old correct value
515  sqliteConnector_.query("UPDATE mapd_tables SET frag_page_size = 2097152 ");
516  // need to add new version info
517  string queryString("ALTER TABLE mapd_tables ADD version_num BIGINT DEFAULT " +
519  sqliteConnector_.query(queryString);
520  }
521  } catch (std::exception& e) {
522  sqliteConnector_.query("ROLLBACK TRANSACTION");
523  throw;
524  }
525  sqliteConnector_.query("END TRANSACTION");
526 }
T getData(const int row, const int col)
#define LOG(tag)
Definition: Logger.h:285
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
const int DEFAULT_INITIAL_VERSION
Definition: Catalog.cpp:110
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::updateTableDescriptorSchema ( )
protected

Definition at line 265 of file Catalog.cpp.

References DEFAULT_MAX_CHUNK_SIZE, SqliteConnector::getData(), SqliteConnector::getNumRows(), getObjForLock(), shared::kRootUserId, SqliteConnector::query(), sqliteConnector_, and to_string().

Referenced by CheckAndExecuteMigrations().

265  {
266  cat_sqlite_lock sqlite_lock(getObjForLock());
267  sqliteConnector_.query("BEGIN TRANSACTION");
268  try {
269  sqliteConnector_.query("PRAGMA TABLE_INFO(mapd_tables)");
270  std::vector<std::string> cols;
271  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
272  cols.push_back(sqliteConnector_.getData<std::string>(i, 1));
273  }
274  if (std::find(cols.begin(), cols.end(), std::string("max_chunk_size")) ==
275  cols.end()) {
276  string queryString("ALTER TABLE mapd_tables ADD max_chunk_size BIGINT DEFAULT " +
278  sqliteConnector_.query(queryString);
279  }
280  if (std::find(cols.begin(), cols.end(), std::string("shard_column_id")) ==
281  cols.end()) {
282  string queryString("ALTER TABLE mapd_tables ADD shard_column_id BIGINT DEFAULT " +
283  std::to_string(0));
284  sqliteConnector_.query(queryString);
285  }
286  if (std::find(cols.begin(), cols.end(), std::string("shard")) == cols.end()) {
287  string queryString("ALTER TABLE mapd_tables ADD shard BIGINT DEFAULT " +
288  std::to_string(-1));
289  sqliteConnector_.query(queryString);
290  }
291  if (std::find(cols.begin(), cols.end(), std::string("num_shards")) == cols.end()) {
292  string queryString("ALTER TABLE mapd_tables ADD num_shards BIGINT DEFAULT " +
293  std::to_string(0));
294  sqliteConnector_.query(queryString);
295  }
296  if (std::find(cols.begin(), cols.end(), std::string("key_metainfo")) == cols.end()) {
297  string queryString("ALTER TABLE mapd_tables ADD key_metainfo TEXT DEFAULT '[]'");
298  sqliteConnector_.query(queryString);
299  }
300  if (std::find(cols.begin(), cols.end(), std::string("userid")) == cols.end()) {
301  string queryString("ALTER TABLE mapd_tables ADD userid integer DEFAULT " +
303  sqliteConnector_.query(queryString);
304  }
305  if (std::find(cols.begin(), cols.end(), std::string("sort_column_id")) ==
306  cols.end()) {
308  "ALTER TABLE mapd_tables ADD sort_column_id INTEGER DEFAULT 0");
309  }
310  if (std::find(cols.begin(), cols.end(), std::string("storage_type")) == cols.end()) {
311  string queryString("ALTER TABLE mapd_tables ADD storage_type TEXT DEFAULT ''");
312  sqliteConnector_.query(queryString);
313  }
314  if (std::find(cols.begin(), cols.end(), std::string("max_rollback_epochs")) ==
315  cols.end()) {
316  string queryString("ALTER TABLE mapd_tables ADD max_rollback_epochs INT DEFAULT " +
317  std::to_string(-1));
318  sqliteConnector_.query(queryString);
319  }
320  if (std::find(cols.begin(), cols.end(), std::string("is_system_table")) ==
321  cols.end()) {
322  string queryString("ALTER TABLE mapd_tables ADD is_system_table BOOLEAN DEFAULT 0");
323  sqliteConnector_.query(queryString);
324  }
325  } catch (std::exception& e) {
326  sqliteConnector_.query("ROLLBACK TRANSACTION");
327  throw;
328  }
329  sqliteConnector_.query("END TRANSACTION");
330 }
const int kRootUserId
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
#define DEFAULT_MAX_CHUNK_SIZE
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
const Catalog * getObjForLock()
Definition: Catalog.cpp:257
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:123
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::updateViewsInMapUnlocked ( )
private

Definition at line 1405 of file Catalog.cpp.

1405  {
1406  std::string viewQuery("SELECT tableid, sql FROM mapd_views");
1407  sqliteConnector_.query(viewQuery);
1408  auto numRows = sqliteConnector_.getNumRows();
1409  for (size_t r = 0; r < numRows; ++r) {
1410  auto tableId = sqliteConnector_.getData<int>(r, 0);
1411  auto td = tableDescriptorMapById_[tableId];
1412  td->viewSQL = sqliteConnector_.getData<string>(r, 1);
1413  td->fragmenter = nullptr;
1414  }
1415 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
virtual size_t getNumRows() const
void Catalog_Namespace::Catalog::updateViewUnlocked ( TableDescriptor td)
private

Definition at line 1395 of file Catalog.cpp.

References CHECK_EQ, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and TableDescriptor::viewSQL.

1395  {
1396  std::string viewQuery("SELECT sql FROM mapd_views where tableid = " +
1397  std::to_string(td.tableId));
1398  sqliteConnector_.query(viewQuery);
1399  auto num_rows = sqliteConnector_.getNumRows();
1400  CHECK_EQ(num_rows, 1U) << "Expected single entry in mapd_views for view '"
1401  << td.tableName << "', instead got " << num_rows;
1402  td.viewSQL = sqliteConnector_.getData<string>(0, 0);
1403 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
T getData(const int row, const int col)
std::string tableName
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
std::string viewSQL
virtual size_t getNumRows() const

+ Here is the call graph for this function:

bool Catalog_Namespace::Catalog::validateNonExistentTableOrView ( const std::string &  name,
const bool  if_not_exists 
)

Validates that a table or view with given name does not already exist. An exception is thrown if a table or view with given name already exists and "if_not_exists" is false.

Parameters
name- Name of table or view whose existence is checked
if_not_exists- flag indicating whether or not existence of a table or view with given name is an exception
Returns
true if table or view with name does not exist. Otherwise, return false

Definition at line 5772 of file Catalog.cpp.

5773  {
5774  if (getMetadataForTable(name, false)) {
5775  if (if_not_exists) {
5776  return false;
5777  }
5778  throw std::runtime_error("Table or View with name \"" + name + "\" already exists.");
5779  }
5780  return true;
5781 }
std::string name() const
Definition: Catalog.h:348
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

Friends And Related Function Documentation

template<class T >
friend class lockmgr::TableLockMgrImpl
friend

Definition at line 750 of file Catalog.h.

Member Data Documentation

std::string Catalog_Namespace::Catalog::basePath_
protected

Definition at line 706 of file Catalog.h.

Referenced by getCatalogBasePath(), and ~Catalog().

std::shared_ptr<Calcite> Catalog_Namespace::Catalog::calciteMgr_
protected

Definition at line 725 of file Catalog.h.

Referenced by getCalciteMgr(), and setCalciteMgr().

constexpr const char* Catalog_Namespace::Catalog::CATALOG_SERVER_NAME {"system_catalog_server"}
staticprivate

Definition at line 892 of file Catalog.h.

ColumnDescriptorMap Catalog_Namespace::Catalog::columnDescriptorMap_
protected

Definition at line 709 of file Catalog.h.

Referenced by ~Catalog().

ColumnDescriptorMapById Catalog_Namespace::Catalog::columnDescriptorMapById_
protected

Definition at line 710 of file Catalog.h.

Referenced by reloadTableMetadataUnlocked().

ColumnDescriptorsForRoll Catalog_Namespace::Catalog::columnDescriptorsForRoll
protected

Definition at line 739 of file Catalog.h.

CustomExpressionMapById Catalog_Namespace::Catalog::custom_expr_map_by_id_
protected

Definition at line 717 of file Catalog.h.

DashboardDescriptorMap Catalog_Namespace::Catalog::dashboardDescriptorMap_
protected

Definition at line 712 of file Catalog.h.

std::shared_ptr<Data_Namespace::DataMgr> Catalog_Namespace::Catalog::dataMgr_
protected

Definition at line 722 of file Catalog.h.

Referenced by getDataMgr(), and reloadTableMetadataUnlocked().

std::unique_ptr<heavyai::DistributedSharedMutex> Catalog_Namespace::Catalog::dcatalogMutex_
mutable

Definition at line 906 of file Catalog.h.

DeletedColumnPerTableMap Catalog_Namespace::Catalog::deletedColumnPerTable_
private

Definition at line 774 of file Catalog.h.

Referenced by reloadTableMetadataUnlocked().

TableDictColumnsMap Catalog_Namespace::Catalog::dict_columns_by_table_id_
protected

Definition at line 718 of file Catalog.h.

DictDescriptorMapById Catalog_Namespace::Catalog::dictDescriptorMapByRef_
protected

Definition at line 711 of file Catalog.h.

Referenced by buildDictionaryMapUnlocked().

std::unique_ptr<heavyai::DistributedSharedMutex> Catalog_Namespace::Catalog::dsqliteMutex_
mutable

Definition at line 907 of file Catalog.h.

constexpr const char* Catalog_Namespace::Catalog::EXECUTOR_STATS_SERVER_NAME {"system_executor_stats_server"}
staticprivate

Definition at line 895 of file Catalog.h.

ForeignServerMap Catalog_Namespace::Catalog::foreignServerMap_
protected

Definition at line 715 of file Catalog.h.

ForeignServerMapById Catalog_Namespace::Catalog::foreignServerMapById_
protected

Definition at line 716 of file Catalog.h.

bool Catalog_Namespace::Catalog::initialized_ = false

Definition at line 914 of file Catalog.h.

Referenced by getObjForLock().

constexpr std::array<const char*, 6> Catalog_Namespace::Catalog::INTERNAL_SERVERS
staticprivate
LinkDescriptorMap Catalog_Namespace::Catalog::linkDescriptorMap_
protected

Definition at line 713 of file Catalog.h.

LinkDescriptorMapById Catalog_Namespace::Catalog::linkDescriptorMapById_
protected

Definition at line 714 of file Catalog.h.

LogicalToPhysicalTableMapById Catalog_Namespace::Catalog::logicalToPhysicalTableMapById_
protected

Definition at line 727 of file Catalog.h.

Referenced by updateLogicalToPhysicalTableMap().

constexpr const char* Catalog_Namespace::Catalog::LOGS_SERVER_NAME {"system_logs_server"}
staticprivate

Definition at line 897 of file Catalog.h.

constexpr const char* Catalog_Namespace::Catalog::MEMORY_STATS_SERVER_NAME {"system_memory_stats_server"}
staticprivate

Definition at line 893 of file Catalog.h.

constexpr const char* Catalog_Namespace::Catalog::ML_METADATA_SERVER_NAME {"ml_metadata_server"}
staticprivate

Definition at line 896 of file Catalog.h.

int Catalog_Namespace::Catalog::nextTempDictId_
protected

Definition at line 731 of file Catalog.h.

int Catalog_Namespace::Catalog::nextTempTableId_
protected

Definition at line 730 of file Catalog.h.

const std::string Catalog_Namespace::Catalog::physicalTableNameTag_
staticprotected

Definition at line 729 of file Catalog.h.

heavyai::shared_mutex Catalog_Namespace::Catalog::sharedMutex_
mutable

Definition at line 909 of file Catalog.h.

std::mutex Catalog_Namespace::Catalog::sqliteMutex_
mutable

Definition at line 908 of file Catalog.h.

constexpr const char* Catalog_Namespace::Catalog::STORAGE_STATS_SERVER_NAME {"system_storage_stats_server"}
staticprivate

Definition at line 894 of file Catalog.h.

const std::vector<LeafHostInfo> Catalog_Namespace::Catalog::string_dict_hosts_
protected

Definition at line 724 of file Catalog.h.

TableDescriptorMap Catalog_Namespace::Catalog::tableDescriptorMap_
protected

Definition at line 707 of file Catalog.h.

Referenced by reloadTableMetadataUnlocked(), and ~Catalog().

TableDescriptorMapById Catalog_Namespace::Catalog::tableDescriptorMapById_
protected
std::atomic<std::thread::id> Catalog_Namespace::Catalog::thread_holding_sqlite_lock
mutable

Definition at line 910 of file Catalog.h.

std::atomic<std::thread::id> Catalog_Namespace::Catalog::thread_holding_write_lock
mutable

Definition at line 911 of file Catalog.h.

thread_local bool Catalog_Namespace::Catalog::thread_holds_read_lock = false
static

Definition at line 913 of file Catalog.h.


The documentation for this class was generated from the following files: