OmniSciDB  a5dc49c757
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Catalog_Namespace::Catalog Class Referencefinal

class for a per-database catalog. also includes metadata for the current database and the current user. More...

#include <Catalog.h>

+ Collaboration diagram for Catalog_Namespace::Catalog:

Classes

struct  NoTableFoundException
 

Public Member Functions

 Catalog (const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
 Constructor - takes basePath to already extant data directory for writing. More...
 
 Catalog ()
 Constructor builds a hollow catalog used during constructor of other catalogs. More...
 
 ~Catalog ()
 Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite. More...
 
void createTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
 
void createShardedTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
int32_t createDashboard (DashboardDescriptor &vd)
 
void replaceDashboard (DashboardDescriptor &vd)
 
std::string createLink (LinkDescriptor &ld, size_t min_length)
 
void dropTable (const TableDescriptor *td)
 
void truncateTable (const TableDescriptor *td)
 
void renameTable (const TableDescriptor *td, const std::string &newTableName)
 
void renameTables (const std::vector< std::pair< std::string, std::string >> &names)
 
void renameColumn (const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
 
int getNextAddedColumnId (const TableDescriptor &td)
 
void addColumn (const TableDescriptor &td, ColumnDescriptor &cd)
 
void addColumnTransactional (const TableDescriptor &td, ColumnDescriptor &cd)
 
void alterColumnTypeTransactional (const ColumnDescriptor &cd)
 
void dropColumnTransactional (const TableDescriptor &td, const ColumnDescriptor &cd)
 
void dropColumnNontransactional (const TableDescriptor &td, const ColumnDescriptor &cd)
 
void dropColumn (const TableDescriptor &td, const ColumnDescriptor &cd)
 
void dropColumnPolicies (const TableDescriptor &td, const ColumnDescriptor &cd)
 
void invalidateCachesForTable (const int table_id)
 
void removeFragmenterForTable (const int table_id) const
 
const std::map< int, const
ColumnDescriptor * > 
getDictionaryToColumnMapping ()
 
const TableDescriptorgetMetadataForTable (const std::string &tableName, const bool populateFragmenter=true) const
 Returns a pointer to a const TableDescriptor struct matching the provided tableName. More...
 
const TableDescriptorgetMetadataForTable (int tableId, bool populateFragmenter=true) const
 
std::optional< std::string > getTableName (int32_t table_id) const
 
std::optional< int32_t > getTableId (const std::string &table_name) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, const std::string &colName) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, int columnId) const
 
const std::optional< std::string > getColumnName (int table_id, int column_id) const
 
const int getColumnIdBySpi (const int tableId, const size_t spi) const
 
const ColumnDescriptorgetMetadataForColumnBySpi (const int tableId, const size_t spi) const
 
const DashboardDescriptorgetMetadataForDashboard (const std::string &userId, const std::string &dashName) const
 
const DashboardDescriptorgetMetadataForDashboard (const int32_t dashboard_id) const
 
void deleteMetadataForDashboards (const std::vector< int32_t > ids, const UserMetadata &user)
 
const LinkDescriptorgetMetadataForLink (const std::string &link) const
 
const LinkDescriptorgetMetadataForLink (int linkId) const
 
const
foreign_storage::ForeignTable
getForeignTable (const std::string &tableName) const
 
const
foreign_storage::ForeignTable
getForeignTable (int table_id) const
 
std::list< const
ColumnDescriptor * > 
getAllColumnMetadataForTable (const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular table specified by table id. More...
 
std::list< const
TableDescriptor * > 
getAllTableMetadata () const
 
std::vector< TableDescriptorgetAllTableMetadataCopy () const
 
std::list< const
DashboardDescriptor * > 
getAllDashboardsMetadata () const
 
std::vector< DashboardDescriptorgetAllDashboardsMetadataForSysTable () const
 
const DBMetadatagetCurrentDB () const
 
Data_Namespace::DataMgrgetDataMgr () const
 
std::shared_ptr< CalcitegetCalciteMgr () const
 
void setCalciteMgr (const std::shared_ptr< Calcite > &new_calcite_mgr)
 
const std::string & getCatalogBasePath () const
 
const DictDescriptorgetMetadataForDict (int dict_ref, bool loadDict=true) const
 
const std::vector< LeafHostInfo > & getStringDictionaryHosts () const
 
const ColumnDescriptorgetShardColumnMetadataForTable (const TableDescriptor *td) const
 
std::vector< const
TableDescriptor * > 
getPhysicalTablesDescriptors (const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
 
std::vector< std::pair
< int32_t, int32_t > > 
getAllPersistedTableAndShardIds () const
 
std::vector< std::string > getTableNamesForUser (const UserMetadata &user, const GetTablesType get_tables_type) const
 
std::vector< TableMetadatagetTablesMetadataForUser (const UserMetadata &user_metadata, const GetTablesType get_tables_type, const std::string &filter_table_name) const
 
int32_t getTableEpoch (const int32_t db_id, const int32_t table_id) const
 
void setTableEpoch (const int db_id, const int table_id, const int new_epoch)
 
void setMaxRollbackEpochs (const int32_t table_id, const int32_t max_rollback_epochs)
 
void setMaxRows (const int32_t table_id, const int64_t max_rows)
 
std::vector< TableEpochInfogetTableEpochs (const int32_t db_id, const int32_t table_id) const
 
void setTableEpochs (const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
 
void setTableEpochsLogExceptions (const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
 
int getDatabaseId () const
 
SqliteConnectorgetSqliteConnector ()
 
void rollLegacy (const bool forward)
 
void getDictionary (const ColumnDescriptor &cd, std::map< int, StringDictionary * > &stringDicts)
 
size_t getTotalMemorySizeForDictionariesForDatabase () const
 
DictRef addDictionaryTransactional (ColumnDescriptor &cd)
 
void delDictionaryTransactional (const ColumnDescriptor &cd)
 
std::list< const DictDescriptor * > getAllDictionariesWithColumnInName (const ColumnDescriptor *cd)
 
const bool checkMetadataForDeletedRecs (const TableDescriptor *td, int column_id) const
 
const ColumnDescriptorgetDeletedColumn (const TableDescriptor *td) const
 
const ColumnDescriptorgetDeletedColumnIfRowsDeleted (const TableDescriptor *td) const
 
void setDeletedColumn (const TableDescriptor *td, const ColumnDescriptor *cd)
 
int getLogicalTableId (const int physicalTableId) const
 
void checkpoint (const int logicalTableId) const
 
void checkpointWithAutoRollback (const int logical_table_id) const
 
void resetTableEpochFloor (const int logicalTableId) const
 
std::string name () const
 
void eraseDbMetadata ()
 
void eraseDbPhysicalData ()
 
void eraseTablePhysicalData (const TableDescriptor *td)
 
void setForReload (const int32_t tableId)
 
std::vector< std::string > getTableDataDirectories (const TableDescriptor *td) const
 
std::vector< std::string > getTableDictDirectories (const TableDescriptor *td) const
 
std::set< std::string > getTableDictDirectoryPaths (int32_t table_id) const
 
std::string getColumnDictDirectory (const ColumnDescriptor *cd, bool file_name_only=true) const
 
std::string dumpSchema (const TableDescriptor *td) const
 
std::string dumpCreateTable (const TableDescriptor *td, bool multiline_formatting=true, bool dump_defaults=false) const
 
std::optional< std::string > dumpCreateTable (int32_t table_id, bool multiline_formatting=true, bool dump_defaults=false) const
 
std::string dumpCreateServer (const std::string &name, bool multiline_formatting=true) const
 
void createForeignServer (std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
 
const
foreign_storage::ForeignServer
getForeignServer (const std::string &server_name) const
 
const std::unique_ptr< const
foreign_storage::ForeignServer
getForeignServerFromStorage (const std::string &server_name)
 
const std::unique_ptr< const
foreign_storage::ForeignTable
getForeignTableFromStorage (int table_id)
 
void changeForeignServerOwner (const std::string &server_name, const int new_owner_id)
 
void setForeignServerDataWrapper (const std::string &server_name, const std::string &data_wrapper)
 
void setForeignServerOptions (const std::string &server_name, const std::string &options)
 
void renameForeignServer (const std::string &server_name, const std::string &name)
 
void dropForeignServer (const std::string &server_name)
 
std::vector< const
foreign_storage::ForeignTable * > 
getAllForeignTablesForForeignServer (const int32_t foreign_server_id)
 Get all of the foreign tables for associated with a foreign server id. More...
 
void getForeignServersForUser (const rapidjson::Value *filters, const UserMetadata &user, std::vector< const foreign_storage::ForeignServer * > &results)
 
void createDefaultServersIfNotExists ()
 
bool validateNonExistentTableOrView (const std::string &name, const bool if_not_exists)
 
std::vector< const
TableDescriptor * > 
getAllForeignTablesForRefresh () const
 
void updateForeignTableRefreshTimes (const int32_t table_id)
 
void setForeignTableOptions (const std::string &table_name, foreign_storage::OptionsMap &options_map, bool clear_existing_options=true)
 
void updateLeaf (const LeafHostInfo &string_dict_host)
 
void setUncappedTableEpoch (const std::string &table_name)
 
int32_t createCustomExpression (std::unique_ptr< CustomExpression > custom_expression)
 
const CustomExpressiongetCustomExpression (int32_t custom_expression_id) const
 
const std::unique_ptr< const
CustomExpression
getCustomExpressionFromStorage (int32_t custom_expression_id)
 
std::vector< const
CustomExpression * > 
getCustomExpressionsForUser (const UserMetadata &user) const
 
void updateCustomExpression (int32_t custom_expression_id, const std::string &expression_json)
 
void deleteCustomExpressions (const std::vector< int32_t > &custom_expression_ids, bool do_soft_delete)
 
void reassignOwners (const std::set< std::string > &old_owners, const std::string &new_owner)
 
bool isInfoSchemaDb () const
 
bool checkDropRenderGroupColumnsMigration ()
 

Static Public Member Functions

static void expandGeoColumn (const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
 
static const std::string getForeignTableSchema (bool if_not_exists=false)
 
static const std::string getForeignServerSchema (bool if_not_exists=false)
 
static const std::string getCustomExpressionsSchema (bool if_not_exists=false)
 

Public Attributes

std::unique_ptr
< heavyai::DistributedSharedMutex
dcatalogMutex_
 
std::unique_ptr
< heavyai::DistributedSharedMutex
dsqliteMutex_
 
std::mutex sqliteMutex_
 
heavyai::shared_mutex sharedMutex_
 
std::atomic< std::thread::id > thread_holding_sqlite_lock
 
std::atomic< std::thread::id > thread_holding_write_lock
 
bool initialized_ = false
 

Static Public Attributes

static thread_local bool thread_holds_read_lock = false
 

Protected Types

using ColumnDescriptorsForRoll = std::vector< std::pair< ColumnDescriptor *, ColumnDescriptor * >>
 

Protected Member Functions

void CheckAndExecuteMigrations ()
 
void CheckAndExecuteMigrationsPostBuildMaps ()
 
void updateDictionaryNames ()
 
void updateTableDescriptorSchema ()
 
void updateFixlenArrayColumns ()
 
void updateGeoColumns ()
 
void updateFrontendViewSchema ()
 
void updateLinkSchema ()
 
void updateFrontendViewAndLinkUsers ()
 
void updateLogicalToPhysicalTableLinkSchema ()
 
void updateLogicalToPhysicalTableMap (const int32_t logical_tb_id)
 
void updateDictionarySchema ()
 
void updatePageSize ()
 
void updateDeletedColumnIndicator ()
 
void updateDefaultColumnValues ()
 
void updateFrontendViewsToDashboards ()
 
void updateCustomExpressionsSchema ()
 
void updateFsiSchemas ()
 
void renameLegacyDataWrappers ()
 
void recordOwnershipOfObjectsInObjectPermissions ()
 
void checkDateInDaysColumnMigration ()
 
void createDashboardSystemRoles ()
 
void buildMaps ()
 
void addTableToMap (const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
 
void addReferenceToForeignDict (ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def, const bool persist_reference)
 
bool setColumnSharedDictionary (ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
void setColumnDictionary (ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, bool is_logical_table, bool use_temp_dictionary=false)
 
void addFrontendViewToMap (DashboardDescriptor &vd)
 
void addFrontendViewToMapNoLock (DashboardDescriptor &vd)
 
void addLinkToMap (LinkDescriptor &ld)
 
void removeTableFromMap (const std::string &tableName, const int tableId, const bool is_on_error=false)
 
void eraseTableMetadata (const TableDescriptor *td)
 
void executeDropTableSqliteQueries (const TableDescriptor *td)
 
void doTruncateTable (const TableDescriptor *td)
 
void renamePhysicalTable (const TableDescriptor *td, const std::string &newTableName)
 
void renamePhysicalTables (std::vector< std::pair< std::string, std::string >> &names, std::vector< int > &tableIds)
 
void instantiateFragmenter (TableDescriptor *td) const
 
void getAllColumnMetadataForTableImpl (const TableDescriptor *td, std::list< const ColumnDescriptor * > &colDescs, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 
std::string calculateSHA1 (const std::string &data)
 
std::string generatePhysicalTableName (const std::string &logicalTableName, const size_t shardNumber)
 
std::vector< DBObjectparseDashboardObjects (const std::string &view_meta, const int &user_id)
 
void createOrUpdateDashboardSystemRole (const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
 
const int getColumnIdBySpiUnlocked (const int table_id, const size_t spi) const
 
void serializeTableJsonUnlocked (const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
 
void dropTableFromJsonUnlocked (const std::string &table_name) const
 

Protected Attributes

std::string basePath_
 
TableDescriptorMap tableDescriptorMap_
 
TableDescriptorMapById tableDescriptorMapById_
 
ColumnDescriptorMap columnDescriptorMap_
 
ColumnDescriptorMapById columnDescriptorMapById_
 
DictDescriptorMapById dictDescriptorMapByRef_
 
DashboardDescriptorMap dashboardDescriptorMap_
 
LinkDescriptorMap linkDescriptorMap_
 
LinkDescriptorMapById linkDescriptorMapById_
 
ForeignServerMap foreignServerMap_
 
ForeignServerMapById foreignServerMapById_
 
CustomExpressionMapById custom_expr_map_by_id_
 
TableDictColumnsMap dict_columns_by_table_id_
 
SqliteConnector sqliteConnector_
 
const DBMetadata currentDB_
 
std::shared_ptr
< Data_Namespace::DataMgr
dataMgr_
 
const std::vector< LeafHostInfostring_dict_hosts_
 
std::shared_ptr< CalcitecalciteMgr_
 
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
 
int nextTempTableId_
 
int nextTempDictId_
 
ColumnDescriptorsForRoll columnDescriptorsForRoll
 

Static Protected Attributes

static const std::string physicalTableNameTag_
 

Private Member Functions

void addColumnDescriptor (const ColumnDescriptor *cd)
 
void removeColumnDescriptor (const ColumnDescriptor *cd)
 
void delDictionaryNontransactional (const ColumnDescriptor &cd)
 
void addColumnNontransactional (const TableDescriptor &td, ColumnDescriptor &cd)
 
DictRef addDictionaryNontransactional (ColumnDescriptor &cd)
 
void buildDictionaryMapUnlocked ()
 
void reloadTableMetadata (int table_id)
 
void reloadTableMetadataUnlocked (int table_id)
 
TableDescriptorcreateTableFromDiskUnlocked (int32_t table_id)
 
void reloadDictionariesFromDiskUnlocked ()
 
std::list< ColumnDescriptor * > sqliteGetColumnsForTableUnlocked (int32_t table_id)
 
void reloadForeignTableUnlocked (foreign_storage::ForeignTable &foreign_table)
 
void reloadCatalogMetadata (const std::map< int32_t, std::string > &user_name_by_user_id)
 
void reloadCatalogMetadataUnlocked (const std::map< int32_t, std::string > &user_name_by_user_id)
 
void refreshDictionaryCachesForTableUnlocked (const TableDescriptor &td)
 
void buildTablesMapUnlocked ()
 
void buildColumnsMapUnlocked ()
 
void updateViewsInMapUnlocked ()
 
void updateViewUnlocked (TableDescriptor &)
 
void buildDashboardsMapUnlocked (const std::map< int32_t, std::string > &user_name_by_user_id)
 
void buildLinksMapUnlocked ()
 
void buildLogicalToPhysicalMapUnlocked ()
 
void updateForeignTablesInMapUnlocked ()
 
void gatherAdditionalInfo (std::vector< std::string > &additional_info, std::set< std::string > &shared_dict_column_names, const TableDescriptor *td) const
 
std::string quoteIfRequired (const std::string &column_name) const
 
void adjustAlteredTableFiles (const std::string &temp_data_dir, const std::unordered_map< int, int > &all_column_ids_map) const
 
void renameTableDirectories (const std::string &temp_data_dir, const std::vector< std::string > &target_paths, const std::string &name_prefix) const
 
void buildForeignServerMapUnlocked ()
 
void setForeignServerProperty (const std::string &server_name, const std::string &property, const std::string &value)
 
void setForeignTableProperty (const foreign_storage::ForeignTable *table, const std::string &property, const std::string &value)
 
void alterPhysicalTableMetadata (const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
 
void alterTableMetadata (const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
 
void setTableFileMgrParams (const int table_id, const File_Namespace::FileMgrParams &file_mgr_params)
 
bool filterTableByTypeAndUser (const TableDescriptor *td, const UserMetadata &user_metadata, const GetTablesType get_tables_type) const
 
TableDescriptorgetMutableMetadataForTableUnlocked (int table_id) const
 
TableDescriptorgetMutableMetadataForTableUnlocked (const std::string &table_name) const
 
void createForeignServerNoLocks (std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
 
foreign_storage::ForeignTablegetForeignTableUnlocked (const std::string &tableName) const
 
const CataloggetObjForLock ()
 
void removeChunks (const int table_id) const
 
void buildCustomExpressionsMapUnlocked ()
 
std::unique_ptr< CustomExpressiongetCustomExpressionFromConnector (size_t row)
 
void restoreOldOwners (const std::map< int32_t, std::string > &old_owners_user_name_by_id, const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects, int32_t new_owner_id)
 
void restoreOldOwnersInMemory (const std::map< int32_t, std::string > &old_owners_user_name_by_id, const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects, int32_t new_owner_id)
 
void conditionallyInitializeSystemObjects ()
 
void initializeSystemServers ()
 
void initializeSystemTables ()
 
void initializeUsersSystemTable ()
 
void initializeDatabasesSystemTable ()
 
void initializePermissionsSystemTable ()
 
void initializeRolesSystemTable ()
 
void initializeTablesSystemTable ()
 
void initializeDashboardsSystemTable ()
 
void initializeRoleAssignmentsSystemTable ()
 
void initializeMemorySummarySystemTable ()
 
void initializeMemoryDetailsSystemTable ()
 
void initializeStorageDetailsSystemTable ()
 
void initializeExecutorResourcePoolSummarySystemTable ()
 
void initializeMLModelMetadataSystemTable ()
 
void initializeServerLogsSystemTables ()
 
void initializeRequestLogsSystemTables ()
 
void initializeWebServerLogsSystemTables ()
 
void initializeWebServerAccessLogsSystemTables ()
 
void createSystemTableServer (const std::string &server_name, const std::string &data_wrapper_type, const foreign_storage::OptionsMap &options={})
 
std::pair
< foreign_storage::ForeignTable,
std::list< ColumnDescriptor > > 
getSystemTableSchema (const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name, bool is_in_memory_system_table)
 
bool recreateSystemTableIfUpdated (foreign_storage::ForeignTable &foreign_table, const std::list< ColumnDescriptor > &columns)
 
void setDeletedColumnUnlocked (const TableDescriptor *td, const ColumnDescriptor *cd)
 
void updateInColumnMap (ColumnDescriptor *cd, ColumnDescriptor *old_cd)
 
void addToColumnMap (ColumnDescriptor *cd)
 
void removeFromColumnMap (ColumnDescriptor *cd)
 
void deleteTableCatalogMetadata (const TableDescriptor *logical_table, const std::vector< const TableDescriptor * > &physical_tables)
 
std::string dumpCreateTableUnlocked (const TableDescriptor *td, bool multiline_formatting, bool dump_defaults) const
 
void removeDiskCachedDataForMutableTable (int32_t table_id) const
 
template<typename F , typename... Args>
void execInTransaction (F &&f, Args &&...args)
 
const TableDescriptorgetCachedTableDescriptor (const std::map< std::string, int > &cached_table_map, const std::string &cur_table_name)
 

Private Attributes

DeletedColumnPerTableMap deletedColumnPerTable_
 

Static Private Attributes

static constexpr const char * CATALOG_SERVER_NAME {"system_catalog_server"}
 
static constexpr const char * MEMORY_STATS_SERVER_NAME {"system_memory_stats_server"}
 
static constexpr const char * STORAGE_STATS_SERVER_NAME {"system_storage_stats_server"}
 
static constexpr const char * EXECUTOR_STATS_SERVER_NAME {"system_executor_stats_server"}
 
static constexpr const char * ML_METADATA_SERVER_NAME {"ml_metadata_server"}
 
static constexpr const char * LOGS_SERVER_NAME {"system_logs_server"}
 
static constexpr std::array
< const char *, 6 > 
INTERNAL_SERVERS
 

Friends

template<class T >
class lockmgr::TableLockMgrImpl
 

Detailed Description

class for a per-database catalog. also includes metadata for the current database and the current user.

Catalog

Definition at line 143 of file Catalog.h.

Member Typedef Documentation

Definition at line 738 of file Catalog.h.

Constructor & Destructor Documentation

Catalog_Namespace::Catalog::Catalog ( const std::string &  basePath,
const DBMetadata curDB,
std::shared_ptr< Data_Namespace::DataMgr dataMgr,
const std::vector< LeafHostInfo > &  string_dict_hosts,
std::shared_ptr< Calcite calcite,
bool  is_new_db 
)

Constructor - takes basePath to already extant data directory for writing.

Parameters
basePathdirectory path for writing catalog
dbNamename of the database
fragmenterFragmenter object metadata - expects for this directory to already exist
Catalog_Namespace::Catalog::Catalog ( )

Constructor builds a hollow catalog used during constructor of other catalogs.

Definition at line 174 of file Catalog.cpp.

174 {}
Catalog_Namespace::Catalog::~Catalog ( )

Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite.

Definition at line 234 of file Catalog.cpp.

References basePath_, columnDescriptorMap_, currentDB_, Catalog_Namespace::DBMetadata::dbName, g_serialize_temp_tables, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and tableDescriptorMap_.

234  {
235  // cat_write_lock write_lock(this);
236 
237  // must clean up heap-allocated TableDescriptor and ColumnDescriptor structs
238  for (TableDescriptorMap::iterator tableDescIt = tableDescriptorMap_.begin();
239  tableDescIt != tableDescriptorMap_.end();
240  ++tableDescIt) {
241  tableDescIt->second->fragmenter = nullptr;
242  delete tableDescIt->second;
243  }
244 
245  // TableDescriptorMapById points to the same descriptors. No need to delete
246 
247  for (ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.begin();
248  columnDescIt != columnDescriptorMap_.end();
249  ++columnDescIt) {
250  delete columnDescIt->second;
251  }
252 
253  // ColumnDescriptorMapById points to the same descriptors. No need to delete
254 
256  boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
257  }
258 }
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:709
const DBMetadata currentDB_
Definition: Catalog.h:721
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:165
bool g_serialize_temp_tables
Definition: Catalog.cpp:109
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:707

+ Here is the call graph for this function:

Member Function Documentation

void Catalog_Namespace::Catalog::addColumn ( const TableDescriptor td,
ColumnDescriptor cd 
)

Definition at line 2538 of file Catalog.cpp.

References ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::db_id, ColumnDescriptor::default_value, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, kENCODING_DICT, TableDescriptor::nShards, TableDescriptor::shard, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), and ColumnDescriptor::virtualExpr.

2538  {
2539  // caller must handle sqlite/chunk transaction TOGETHER
2540  cd.tableId = td.tableId;
2541  cd.db_id = getDatabaseId();
2542  if (td.nShards > 0 && td.shard < 0) {
2543  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
2544  auto shard_cd = cd;
2545  addColumn(*shard, shard_cd);
2546  }
2547  }
2550  }
2551 
2552  using BindType = SqliteConnector::BindType;
2553  std::vector<BindType> types(17, BindType::TEXT);
2554  if (!cd.default_value.has_value()) {
2555  types[16] = BindType::NULL_TYPE;
2556  }
2558  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
2559  "colscale, is_notnull, "
2560  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
2561  "is_deletedcol, default_value) "
2562  "VALUES (?, "
2563  "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
2564  "?, ?, ?, "
2565  "?, "
2566  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2567  std::vector<std::string>{std::to_string(td.tableId),
2568  std::to_string(td.tableId),
2569  cd.columnName,
2578  "",
2581  cd.virtualExpr,
2583  cd.default_value.value_or("NULL")},
2584  types);
2585 
2587  "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
2588  std::vector<std::string>{std::to_string(td.tableId)});
2589 
2591  "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
2592  std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
2593  cd.columnId = sqliteConnector_.getData<int>(0, 0);
2594 
2595  ++tableDescriptorMapById_[td.tableId]->nColumns;
2596  auto ncd = new ColumnDescriptor(cd);
2597  addToColumnMap(ncd);
2598  columnDescriptorsForRoll.emplace_back(nullptr, ncd);
2599 }
std::string virtualExpr
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:392
DictRef addDictionaryNontransactional(ColumnDescriptor &cd)
Definition: Catalog.cpp:2252
HOST DEVICE int get_size() const
Definition: sqltypes.h:403
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
void addColumn(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:2538
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
HOST DEVICE int get_scale() const
Definition: sqltypes.h:396
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:391
std::string to_string(char const *&&v)
int getDatabaseId() const
Definition: Catalog.h:326
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
specifies the content in-memory of a row in the column metadata table
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4879
std::optional< std::string > default_value
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:399
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:393
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:402
SQLTypeInfo columnType
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:398
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:739
std::string columnName
void addToColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:7007

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addColumnDescriptor ( const ColumnDescriptor cd)
private

Definition at line 2711 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnId, ColumnDescriptor::isGeoPhyCol, gpu_enabled::sort(), and ColumnDescriptor::tableId.

2711  {
2712  if (!cd || cd->isGeoPhyCol) {
2713  return;
2714  }
2715 
2716  auto tabDescIt = tableDescriptorMapById_.find(cd->tableId);
2717  CHECK(tableDescriptorMapById_.end() != tabDescIt);
2718  auto td = tabDescIt->second;
2719  auto& cd_by_spi = td->columnIdBySpi_;
2720 
2721  if (cd_by_spi.end() == std::find(cd_by_spi.begin(), cd_by_spi.end(), cd->columnId)) {
2722  cd_by_spi.push_back(cd->columnId);
2723  }
2724  std::sort(cd_by_spi.begin(), cd_by_spi.end());
2725 }
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addColumnNontransactional ( const TableDescriptor td,
ColumnDescriptor cd 
)
private

Definition at line 2472 of file Catalog.cpp.

References ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::db_id, ColumnDescriptor::default_value, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, kENCODING_DICT, TableDescriptor::nShards, TableDescriptor::shard, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and ColumnDescriptor::virtualExpr.

2472  {
2473  cat_write_lock write_lock(this);
2474  cat_sqlite_lock sqlite_lock(this);
2475  cd.tableId = td.tableId;
2476  cd.db_id = getDatabaseId();
2477  if (td.nShards > 0 && td.shard < 0) {
2478  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
2479  auto shard_cd = cd;
2480  addColumnNontransactional(*shard, shard_cd);
2481  }
2482  }
2485  }
2486 
2487  using BindType = SqliteConnector::BindType;
2488  std::vector<BindType> types(17, BindType::TEXT);
2489  if (!cd.default_value.has_value()) {
2490  types[16] = BindType::NULL_TYPE;
2491  }
2493  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
2494  "colscale, is_notnull, "
2495  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
2496  "is_deletedcol, default_value) "
2497  "VALUES (?, "
2498  "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
2499  "?, ?, ?, "
2500  "?, "
2501  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2502  std::vector<std::string>{std::to_string(td.tableId),
2503  std::to_string(td.tableId),
2504  cd.columnName,
2513  "",
2516  cd.virtualExpr,
2518  cd.default_value.value_or("NULL")},
2519  types);
2520 
2522  "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
2523  std::vector<std::string>{std::to_string(td.tableId)});
2524 
2526  "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
2527  std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
2528  cd.columnId = sqliteConnector_.getData<int>(0, 0);
2529 
2530  ++tableDescriptorMapById_[td.tableId]->nColumns;
2531  auto ncd = new ColumnDescriptor(cd);
2532  addToColumnMap(ncd);
2533  addColumnDescriptor(ncd);
2534  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
2535 }
std::string virtualExpr
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:392
DictRef addDictionaryNontransactional(ColumnDescriptor &cd)
Definition: Catalog.cpp:2252
HOST DEVICE int get_size() const
Definition: sqltypes.h:403
T getData(const int row, const int col)
std::string tableName
virtual void query_with_text_params(std::string const &query_only)
void addColumnNontransactional(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:2472
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
HOST DEVICE int get_scale() const
Definition: sqltypes.h:396
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:391
std::string to_string(char const *&&v)
int getDatabaseId() const
Definition: Catalog.h:326
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
specifies the content in-memory of a row in the column metadata table
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4879
std::optional< std::string > default_value
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:725
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:399
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:393
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:402
void addColumnDescriptor(const ColumnDescriptor *cd)
Definition: Catalog.cpp:2711
SQLTypeInfo columnType
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:398
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:126
std::string columnName
void addToColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:7007

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addColumnTransactional ( const TableDescriptor td,
ColumnDescriptor cd 
)

Definition at line 2459 of file Catalog.cpp.

2459  {
2460  cat_write_lock write_lock(this);
2461  cat_sqlite_lock sqlite_lock(this);
2462  sqliteConnector_.query("BEGIN TRANSACTION");
2463  try {
2464  addColumnNontransactional(td, cd);
2465  } catch (std::exception& e) {
2466  sqliteConnector_.query("ROLLBACK TRANSACTION");
2467  throw;
2468  }
2469  sqliteConnector_.query("END TRANSACTION");
2470 }
void addColumnNontransactional(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:2472
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:126
DictRef Catalog_Namespace::Catalog::addDictionaryNontransactional ( ColumnDescriptor cd)
private

Definition at line 2252 of file Catalog.cpp.

References CHECK, DictDescriptor::dictFolderPath, and ColumnDescriptor::tableId.

2252  {
2253  cat_write_lock write_lock(this);
2254  const auto& td = *tableDescriptorMapById_[cd.tableId];
2255  list<DictDescriptor> dds;
2256  setColumnDictionary(cd, dds, td, true);
2257  auto& dd = dds.back();
2258  CHECK(dd.dictRef.dictId);
2259 
2260  std::unique_ptr<StringDictionaryClient> client;
2261  if (!string_dict_hosts_.empty()) {
2262  client.reset(new StringDictionaryClient(
2263  string_dict_hosts_.front(), DictRef(currentDB_.dbId, -1), true));
2264  }
2265  if (client) {
2266  client->create(dd.dictRef, dd.dictIsTemp);
2267  }
2268 
2269  DictDescriptor* new_dd = new DictDescriptor(dd);
2270  dictDescriptorMapByRef_[dd.dictRef].reset(new_dd);
2271  if (!dd.dictIsTemp) {
2272  boost::filesystem::create_directory(new_dd->dictFolderPath);
2273  }
2274  return dd.dictRef;
2275 }
std::string dictFolderPath
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:724
const DBMetadata currentDB_
Definition: Catalog.h:721
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:711
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, bool is_logical_table, bool use_temp_dictionary=false)
Definition: Catalog.cpp:4066
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
struct dict_ref_t DictRef
Definition: DictRef.h:54
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
#define CHECK(condition)
Definition: Logger.h:291
Descriptor for a dictionary for a string columne.
DictRef Catalog_Namespace::Catalog::addDictionaryTransactional ( ColumnDescriptor cd)

Definition at line 2237 of file Catalog.cpp.

2237  {
2238  cat_write_lock write_lock(this);
2239  cat_sqlite_lock sqlite_lock(this);
2240  DictRef ref{};
2241  sqliteConnector_.query("BEGIN TRANSACTION");
2242  try {
2244  } catch (std::exception& e) {
2245  sqliteConnector_.query("ROLLBACK TRANSACTION");
2246  throw;
2247  }
2248  sqliteConnector_.query("END TRANSACTION");
2249  return ref;
2250 }
DictRef addDictionaryNontransactional(ColumnDescriptor &cd)
Definition: Catalog.cpp:2252
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:126
void Catalog_Namespace::Catalog::addFrontendViewToMap ( DashboardDescriptor vd)
protected

Definition at line 1686 of file Catalog.cpp.

1686  {
1687  cat_write_lock write_lock(this);
1689 }
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1691
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
void Catalog_Namespace::Catalog::addFrontendViewToMapNoLock ( DashboardDescriptor vd)
protected

Definition at line 1691 of file Catalog.cpp.

References DashboardDescriptor::dashboardName, to_string(), and DashboardDescriptor::userId.

1691  {
1692  cat_write_lock write_lock(this);
1694  std::make_shared<DashboardDescriptor>(vd);
1695 }
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:712
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addLinkToMap ( LinkDescriptor ld)
protected

Definition at line 1773 of file Catalog.cpp.

References LinkDescriptor::link, LinkDescriptor::linkId, and to_string().

1773  {
1774  cat_write_lock write_lock(this);
1775  LinkDescriptor* new_ld = new LinkDescriptor();
1776  *new_ld = ld;
1778  linkDescriptorMapById_[ld.linkId] = new_ld;
1779 }
const DBMetadata currentDB_
Definition: Catalog.h:721
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:714
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:713

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addReferenceToForeignDict ( ColumnDescriptor referencing_column,
Parser::SharedDictionaryDef  shared_dict_def,
const bool  persist_reference 
)
protected

Definition at line 3975 of file Catalog.cpp.

References CHECK, CHECK_GE, ColumnDescriptor::columnType, SQLTypeInfo::get_comp_param(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_foreign_col(), and to_string().

3977  {
3978  cat_write_lock write_lock(this);
3979  const auto foreign_ref_col = get_foreign_col(*this, shared_dict_def);
3980  CHECK(foreign_ref_col);
3981  referencing_column.columnType = foreign_ref_col->columnType;
3982  const int dict_id = referencing_column.columnType.get_comp_param();
3983  const DictRef dict_ref(currentDB_.dbId, dict_id);
3984  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
3985  CHECK(dictIt != dictDescriptorMapByRef_.end());
3986  const auto& dd = dictIt->second;
3987  CHECK_GE(dd->refcount, 1);
3988  ++dd->refcount;
3989  if (persist_reference) {
3990  cat_sqlite_lock sqlite_lock(getObjForLock());
3992  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
3993  {std::to_string(dict_id)});
3994  }
3995 }
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:711
#define CHECK_GE(x, y)
Definition: Logger.h:306
const ColumnDescriptor * get_foreign_col(const Catalog &cat, const Parser::SharedDictionaryDef &shared_dict_def)
Definition: Catalog.cpp:3963
std::string to_string(char const *&&v)
const Catalog * getObjForLock()
Definition: Catalog.cpp:260
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:402
#define CHECK(condition)
Definition: Logger.h:291
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:126

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addTableToMap ( const TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::list< DictDescriptor > &  dicts 
)
protected

Definition at line 1545 of file Catalog.cpp.

References anonymous_namespace{Utm.h}::a, CHECK, TableDescriptor::columnIdBySpi_, DictDescriptor::dictFolderPath, dict_ref_t::dictId, TableDescriptor::hasDeletedCol, TableDescriptor::mutex_, gpu_enabled::sort(), TableDescriptor::tableId, TableDescriptor::tableName, and to_upper().

1547  {
1548  cat_write_lock write_lock(this);
1549  TableDescriptor* new_td;
1550 
1551  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
1552  if (foreign_table) {
1553  auto new_foreign_table = new foreign_storage::ForeignTable();
1554  *new_foreign_table = *foreign_table;
1555  new_td = new_foreign_table;
1556  } else {
1557  new_td = new TableDescriptor();
1558  *new_td = *td;
1559  }
1560 
1561  new_td->mutex_ = std::make_shared<std::mutex>();
1562  tableDescriptorMap_[to_upper(td->tableName)] = new_td;
1563  tableDescriptorMapById_[td->tableId] = new_td;
1564  for (auto cd : columns) {
1565  ColumnDescriptor* new_cd = new ColumnDescriptor();
1566  *new_cd = cd;
1567  addToColumnMap(new_cd);
1568 
1569  // Add deleted column to the map
1570  if (cd.isDeletedCol) {
1571  CHECK(new_td->hasDeletedCol);
1572  setDeletedColumnUnlocked(new_td, new_cd);
1573  }
1574  }
1575 
1576  std::sort(new_td->columnIdBySpi_.begin(),
1577  new_td->columnIdBySpi_.end(),
1578  [](const size_t a, const size_t b) -> bool { return a < b; });
1579  // TODO(sy): Why does addTableToMap() sort columnIdBySpi_ but not insert into it while
1580  // buildColumnsMapUnlocked() does both?
1581 
1582  std::unique_ptr<StringDictionaryClient> client;
1583  DictRef dict_ref(currentDB_.dbId, -1);
1584  if (!string_dict_hosts_.empty()) {
1585  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1586  }
1587  for (auto dd : dicts) {
1588  if (!dd.dictRef.dictId) {
1589  // Dummy entry created for a shard of a logical table, nothing to do.
1590  continue;
1591  }
1592  dict_ref.dictId = dd.dictRef.dictId;
1593  if (client) {
1594  client->create(dict_ref, dd.dictIsTemp);
1595  }
1596  DictDescriptor* new_dd = new DictDescriptor(dd);
1597  dictDescriptorMapByRef_[dict_ref].reset(new_dd);
1598  if (!dd.dictIsTemp) {
1599  boost::filesystem::create_directory(new_dd->dictFolderPath);
1600  }
1601  }
1602 }
std::string dictFolderPath
std::string tableName
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:724
const DBMetadata currentDB_
Definition: Catalog.h:721
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:711
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
std::vector< int > columnIdBySpi_
std::shared_ptr< std::mutex > mutex_
constexpr double a
Definition: Utm.h:32
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3955
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
specifies the content in-memory of a row in the column metadata table
std::string to_upper(const std::string &str)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
#define CHECK(condition)
Definition: Logger.h:291
Descriptor for a dictionary for a string columne.
void addToColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:7007
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:707

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addToColumnMap ( ColumnDescriptor cd)
private

Definition at line 7007 of file Catalog.cpp.

References ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::is_dict_encoded_type(), ColumnDescriptor::tableId, and to_upper().

7007  {
7010  if (cd->columnType.is_dict_encoded_type()) {
7011  dict_columns_by_table_id_[cd->tableId].emplace(cd);
7012  }
7013 }
TableDictColumnsMap dict_columns_by_table_id_
Definition: Catalog.h:718
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:709
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:710
bool is_dict_encoded_type() const
Definition: sqltypes.h:655
std::string to_upper(const std::string &str)
SQLTypeInfo columnType
std::string columnName
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::adjustAlteredTableFiles ( const std::string &  temp_data_dir,
const std::unordered_map< int, int > &  all_column_ids_map 
) const
private
void Catalog_Namespace::Catalog::alterColumnTypeTransactional ( const ColumnDescriptor cd)

Definition at line 2393 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::default_value, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), ColumnDescriptor::tableId, to_string(), and to_upper().

Referenced by AlterTableAlterColumnCommandRecoveryMgr::rollback().

2393  {
2394  cat_write_lock write_lock(this);
2395  cat_sqlite_lock sqlite_lock(this);
2396 
2397  sqliteConnector_.query("BEGIN TRANSACTION");
2398  try {
2399  const auto table_id = cd.tableId;
2400 
2401  auto catalog_cd = getMetadataForColumn(table_id, cd.columnId);
2402 
2403  CHECK(catalog_cd) << " can not alter non existing column";
2404 
2405  using BindType = SqliteConnector::BindType;
2406  std::vector<BindType> types(11, BindType::TEXT);
2407  if (!cd.default_value.has_value()) {
2408  types[8] = BindType::NULL_TYPE;
2409  }
2411  "UPDATE mapd_columns SET "
2412  "coltype = ?,"
2413  "colsubtype = ?,"
2414  "coldim = ?,"
2415  "colscale = ?,"
2416  "is_notnull = ?,"
2417  "compression = ?,"
2418  "comp_param = ?,"
2419  "size = ?,"
2420  "default_value = ? "
2421  "WHERE tableid = ? and columnid = ?",
2422  std::vector<std::string>{std::to_string(cd.columnType.get_type()),
2430  cd.default_value.value_or("NULL"),
2431  std::to_string(table_id),
2432  std::to_string(cd.columnId)},
2433 
2434  types);
2435 
2436  auto ncd = new ColumnDescriptor(cd);
2437 
2438  ColumnDescriptorMap::iterator columnDescIt =
2440  CHECK(columnDescIt != columnDescriptorMap_.end());
2441  auto ocd = columnDescIt->second;
2442 
2443  updateInColumnMap(ncd, ocd);
2444  } catch (std::exception& e) {
2445  sqliteConnector_.query("ROLLBACK TRANSACTION");
2446  throw;
2447  }
2448  sqliteConnector_.query("END TRANSACTION");
2449 }
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:392
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
HOST DEVICE int get_size() const
Definition: sqltypes.h:403
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:709
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
HOST DEVICE int get_scale() const
Definition: sqltypes.h:396
virtual void query(const std::string &queryString)
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:391
std::string to_string(char const *&&v)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
specifies the content in-memory of a row in the column metadata table
void updateInColumnMap(ColumnDescriptor *cd, ColumnDescriptor *old_cd)
Definition: Catalog.cpp:6983
std::string to_upper(const std::string &str)
std::optional< std::string > default_value
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:399
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:393
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:402
#define CHECK(condition)
Definition: Logger.h:291
SQLTypeInfo columnType
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:398
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:126
std::string columnName

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::alterPhysicalTableMetadata ( const TableDescriptor td,
const TableDescriptorUpdateParams table_update_params 
)
private

Definition at line 3690 of file Catalog.cpp.

References CHECK, TableDescriptorUpdateParams::max_rollback_epochs, TableDescriptorUpdateParams::max_rows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::tableId, and to_string().

3692  {
3693  // Only called from parent alterTableParamMetadata, expect already to have catalog and
3694  // sqlite write locks
3695 
3696  // Sqlite transaction should have already been begun in parent alterTableCatalogMetadata
3697 
3699  CHECK(mutable_td);
3700  if (td->maxRollbackEpochs != table_update_params.max_rollback_epochs) {
3702  "UPDATE mapd_tables SET max_rollback_epochs = ? WHERE tableid = ?",
3703  std::vector<std::string>{std::to_string(table_update_params.max_rollback_epochs),
3704  std::to_string(td->tableId)});
3705  mutable_td->maxRollbackEpochs = table_update_params.max_rollback_epochs;
3706  }
3707 
3708  if (td->maxRows != table_update_params.max_rows) {
3710  "UPDATE mapd_tables SET max_rows = ? WHERE tableid = ?",
3711  std::vector<std::string>{std::to_string(table_update_params.max_rows),
3712  std::to_string(td->tableId)});
3713  mutable_td->maxRows = table_update_params.max_rows;
3714  }
3715 }
int32_t maxRollbackEpochs
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
TableDescriptor * getMutableMetadataForTableUnlocked(int table_id) const
Definition: Catalog.cpp:1899
std::string to_string(char const *&&v)
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::alterTableMetadata ( const TableDescriptor td,
const TableDescriptorUpdateParams table_update_params 
)
private

Definition at line 3717 of file Catalog.cpp.

References CHECK, logger::FATAL, LOG, TableDescriptor::tableId, and TableDescriptor::tableName.

3718  {
3719  cat_write_lock write_lock(this);
3720  cat_sqlite_lock sqlite_lock(getObjForLock());
3721  sqliteConnector_.query("BEGIN TRANSACTION");
3722  try {
3723  const auto physical_table_it = logicalToPhysicalTableMapById_.find(td->tableId);
3724  if (physical_table_it != logicalToPhysicalTableMapById_.end()) {
3725  const auto physical_tables = physical_table_it->second;
3726  CHECK(!physical_tables.empty());
3727  for (size_t i = 0; i < physical_tables.size(); i++) {
3728  int32_t physical_tb_id = physical_tables[i];
3729  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id, false);
3730  CHECK(phys_td);
3731  alterPhysicalTableMetadata(phys_td, table_update_params);
3732  }
3733  }
3734  alterPhysicalTableMetadata(td, table_update_params);
3735  } catch (std::exception& e) {
3736  sqliteConnector_.query("ROLLBACK TRANSACTION");
3737  LOG(FATAL) << "Table '" << td->tableName << "' catalog update failed";
3738  }
3739  sqliteConnector_.query("END TRANSACTION");
3740 }
std::string tableName
#define LOG(tag)
Definition: Logger.h:285
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
void alterPhysicalTableMetadata(const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
Definition: Catalog.cpp:3690
const Catalog * getObjForLock()
Definition: Catalog.cpp:260
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
#define CHECK(condition)
Definition: Logger.h:291
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:126
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:727
void Catalog_Namespace::Catalog::buildColumnsMapUnlocked ( )
private

Definition at line 1336 of file Catalog.cpp.

References anonymous_namespace{Utm.h}::a, CHECK, ColumnDescriptor::chunks, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::db_id, ColumnDescriptor::default_value, SQLTypeInfo::get_physical_cols(), SQLTypeInfo::is_geometry(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_compression(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::set_dict_key(), SQLTypeInfo::set_dimension(), SQLTypeInfo::set_notnull(), SQLTypeInfo::set_scale(), SQLTypeInfo::set_size(), SQLTypeInfo::set_subtype(), SQLTypeInfo::set_type(), gpu_enabled::sort(), ColumnDescriptor::tableId, and ColumnDescriptor::virtualExpr.

1336  {
1337  std::string columnQuery(
1338  "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
1339  "is_notnull, compression, comp_param, "
1340  "size, chunks, is_systemcol, is_virtualcol, virtual_expr, is_deletedcol, "
1341  "default_value from "
1342  "mapd_columns ORDER BY tableid, "
1343  "columnid");
1344  sqliteConnector_.query(columnQuery);
1345  auto numRows = sqliteConnector_.getNumRows();
1346  int32_t skip_physical_cols = 0;
1347  for (size_t r = 0; r < numRows; ++r) {
1348  ColumnDescriptor* cd = new ColumnDescriptor();
1349  cd->tableId = sqliteConnector_.getData<int>(r, 0);
1350  cd->columnId = sqliteConnector_.getData<int>(r, 1);
1351  cd->columnName = sqliteConnector_.getData<string>(r, 2);
1355  cd->columnType.set_scale(sqliteConnector_.getData<int>(r, 6));
1359  cd->columnType.set_size(sqliteConnector_.getData<int>(r, 10));
1360  cd->chunks = sqliteConnector_.getData<string>(r, 11);
1361  cd->isSystemCol = sqliteConnector_.getData<bool>(r, 12);
1362  cd->isVirtualCol = sqliteConnector_.getData<bool>(r, 13);
1363  cd->virtualExpr = sqliteConnector_.getData<string>(r, 14);
1364  cd->isDeletedCol = sqliteConnector_.getData<bool>(r, 15);
1365  if (sqliteConnector_.isNull(r, 16)) {
1366  cd->default_value = std::nullopt;
1367  } else {
1368  cd->default_value = std::make_optional(sqliteConnector_.getData<string>(r, 16));
1369  }
1370  cd->isGeoPhyCol = skip_physical_cols > 0;
1371  cd->db_id = getDatabaseId();
1372  set_dict_key(*cd);
1373  addToColumnMap(cd);
1374 
1375  if (skip_physical_cols <= 0) {
1376  skip_physical_cols = cd->columnType.get_physical_cols();
1377  }
1378 
1379  auto td_itr = tableDescriptorMapById_.find(cd->tableId);
1380  CHECK(td_itr != tableDescriptorMapById_.end());
1381 
1382  if (cd->isDeletedCol) {
1383  td_itr->second->hasDeletedCol = true;
1384  setDeletedColumnUnlocked(td_itr->second, cd);
1385  } else if (cd->columnType.is_geometry() || skip_physical_cols-- <= 0) {
1386  tableDescriptorMapById_[cd->tableId]->columnIdBySpi_.push_back(cd->columnId);
1387  }
1388  }
1389 
1390  // sort columnIdBySpi_ based on columnId
1391  for (auto& tit : tableDescriptorMapById_) {
1392  std::sort(tit.second->columnIdBySpi_.begin(),
1393  tit.second->columnIdBySpi_.end(),
1394  [](const size_t a, const size_t b) -> bool { return a < b; });
1395  }
1396 }
std::string virtualExpr
void set_compression(EncodingType c)
Definition: sqltypes.h:481
void set_size(int s)
Definition: sqltypes.h:478
T getData(const int row, const int col)
SQLTypes
Definition: sqltypes.h:65
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:471
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
virtual void query(const std::string &queryString)
std::string chunks
constexpr double a
Definition: Utm.h:32
EncodingType
Definition: sqltypes.h:240
int get_physical_cols() const
Definition: sqltypes.h:432
void set_scale(int s)
Definition: sqltypes.h:475
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3955
int getDatabaseId() const
Definition: Catalog.h:326
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
specifies the content in-memory of a row in the column metadata table
void set_comp_param(int p)
Definition: sqltypes.h:482
std::optional< std::string > default_value
bool isNull(const int row, const int col) const
void set_dimension(int d)
Definition: sqltypes.h:472
void set_notnull(bool n)
Definition: sqltypes.h:477
#define CHECK(condition)
Definition: Logger.h:291
bool is_geometry() const
Definition: sqltypes.h:597
SQLTypeInfo columnType
std::string columnName
void addToColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:7007
virtual size_t getNumRows() const
HOST DEVICE void set_type(SQLTypes t)
Definition: sqltypes.h:470

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildCustomExpressionsMapUnlocked ( )
private

Definition at line 1517 of file Catalog.cpp.

1517  {
1519  "SELECT id, name, expression_json, data_source_type, data_source_id, "
1520  "is_deleted "
1521  "FROM omnisci_custom_expressions");
1522  auto num_rows = sqliteConnector_.getNumRows();
1523  for (size_t row = 0; row < num_rows; row++) {
1524  auto custom_expr = getCustomExpressionFromConnector(row);
1525  custom_expr_map_by_id_[custom_expr->id] = std::move(custom_expr);
1526  }
1527 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
std::unique_ptr< CustomExpression > getCustomExpressionFromConnector(size_t row)
Definition: Catalog.cpp:1529
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:717
virtual size_t getNumRows() const
void Catalog_Namespace::Catalog::buildDashboardsMapUnlocked ( const std::map< int32_t, std::string > &  user_name_by_user_id)
private

Definition at line 1420 of file Catalog.cpp.

References generate_dashboard_system_rolename(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_user_name_from_id(), and to_string().

1421  {
1422  std::string frontendViewQuery(
1423  "SELECT id, state, name, image_hash, strftime('%Y-%m-%dT%H:%M:%SZ', update_time), "
1424  "userid, "
1425  "metadata "
1426  "FROM mapd_dashboards");
1427  sqliteConnector_.query(frontendViewQuery);
1428  auto numRows = sqliteConnector_.getNumRows();
1429  for (size_t r = 0; r < numRows; ++r) {
1430  auto vd = std::make_shared<DashboardDescriptor>();
1431  vd->dashboardId = sqliteConnector_.getData<int>(r, 0);
1432  vd->dashboardState = sqliteConnector_.getData<string>(r, 1);
1433  vd->dashboardName = sqliteConnector_.getData<string>(r, 2);
1434  vd->imageHash = sqliteConnector_.getData<string>(r, 3);
1435  vd->updateTime = sqliteConnector_.getData<string>(r, 4);
1436  vd->userId = sqliteConnector_.getData<int>(r, 5);
1437  vd->dashboardMetadata = sqliteConnector_.getData<string>(r, 6);
1438  vd->user = get_user_name_from_id(vd->userId, user_name_by_user_id);
1439  vd->dashboardSystemRoleName = generate_dashboard_system_rolename(
1441  dashboardDescriptorMap_[std::to_string(vd->userId) + ":" + vd->dashboardName] = vd;
1442  }
1443 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
virtual void query(const std::string &queryString)
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:712
std::string to_string(char const *&&v)
std::string get_user_name_from_id(int32_t id, const std::map< int32_t, std::string > &user_name_by_user_id)
Definition: Catalog.cpp:1054
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildDictionaryMapUnlocked ( )
private

Definition at line 1075 of file Catalog.cpp.

References currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, g_base_path, SqliteConnector::getData(), SqliteConnector::getNumRows(), shared::kDataDirectoryName, SqliteConnector::query(), sqliteConnector_, and to_string().

1075  {
1076  std::string dictQuery(
1077  "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
1078  sqliteConnector_.query(dictQuery);
1079  auto numRows = sqliteConnector_.getNumRows();
1080  for (size_t r = 0; r < numRows; ++r) {
1081  auto dictId = sqliteConnector_.getData<int>(r, 0);
1082  auto dictName = sqliteConnector_.getData<string>(r, 1);
1083  auto dictNBits = sqliteConnector_.getData<int>(r, 2);
1084  auto is_shared = sqliteConnector_.getData<bool>(r, 3);
1085  auto refcount = sqliteConnector_.getData<int>(r, 4);
1086  auto fname = g_base_path + "/" + shared::kDataDirectoryName + "/DB_" +
1087  std::to_string(currentDB_.dbId) + "_DICT_" + std::to_string(dictId);
1088  DictRef dict_ref(currentDB_.dbId, dictId);
1089  auto dd = new DictDescriptor(
1090  dict_ref, dictName, dictNBits, is_shared, refcount, fname, false);
1091  dictDescriptorMapByRef_[dict_ref].reset(dd);
1092  }
1093 }
const std::string kDataDirectoryName
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:711
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
std::string g_base_path
Definition: SysCatalog.cpp:62
Descriptor for a dictionary for a string columne.
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildForeignServerMapUnlocked ( )
private

Definition at line 5102 of file Catalog.cpp.

References CHECK, and g_enable_fsi.

5102  {
5105  "SELECT id, name, data_wrapper_type, options, owner_user_id, creation_time FROM "
5106  "omnisci_foreign_servers");
5107  auto num_rows = sqliteConnector_.getNumRows();
5108 
5109  for (size_t row = 0; row < num_rows; row++) {
5110  auto foreign_server = std::make_shared<foreign_storage::ForeignServer>(
5111  sqliteConnector_.getData<int>(row, 0),
5112  sqliteConnector_.getData<std::string>(row, 1),
5113  sqliteConnector_.getData<std::string>(row, 2),
5114  sqliteConnector_.getData<std::string>(row, 3),
5115  sqliteConnector_.getData<std::int32_t>(row, 4),
5116  sqliteConnector_.getData<std::int32_t>(row, 5));
5117  foreignServerMap_[foreign_server->name] = foreign_server;
5118  foreignServerMapById_[foreign_server->id] = foreign_server;
5119  }
5120 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:716
ForeignServerMap foreignServerMap_
Definition: Catalog.h:715
#define CHECK(condition)
Definition: Logger.h:291
bool g_enable_fsi
Definition: Catalog.cpp:96
virtual size_t getNumRows() const
void Catalog_Namespace::Catalog::buildLinksMapUnlocked ( )
private

Definition at line 1445 of file Catalog.cpp.

References to_string().

1445  {
1446  std::string linkQuery(
1447  "SELECT linkid, userid, link, view_state, strftime('%Y-%m-%dT%H:%M:%SZ', "
1448  "update_time), view_metadata "
1449  "FROM mapd_links");
1450  sqliteConnector_.query(linkQuery);
1451  auto numRows = sqliteConnector_.getNumRows();
1452  for (size_t r = 0; r < numRows; ++r) {
1453  auto ld = new LinkDescriptor();
1454  ld->linkId = sqliteConnector_.getData<int>(r, 0);
1455  ld->userId = sqliteConnector_.getData<int>(r, 1);
1456  ld->link = sqliteConnector_.getData<string>(r, 2);
1457  ld->viewState = sqliteConnector_.getData<string>(r, 3);
1458  ld->updateTime = sqliteConnector_.getData<string>(r, 4);
1459  ld->viewMetadata = sqliteConnector_.getData<string>(r, 5);
1461  linkDescriptorMapById_[ld->linkId] = ld;
1462  }
1463 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:714
virtual size_t getNumRows() const
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:713

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildLogicalToPhysicalMapUnlocked ( )
private

Definition at line 1465 of file Catalog.cpp.

References CHECK.

1465  {
1466  /* rebuild map linking logical tables to corresponding physical ones */
1467  std::string logicalToPhysicalTableMapQuery(
1468  "SELECT logical_table_id, physical_table_id "
1469  "FROM mapd_logical_to_physical");
1470  sqliteConnector_.query(logicalToPhysicalTableMapQuery);
1471  auto numRows = sqliteConnector_.getNumRows();
1472  for (size_t r = 0; r < numRows; ++r) {
1473  auto logical_tb_id = sqliteConnector_.getData<int>(r, 0);
1474  auto physical_tb_id = sqliteConnector_.getData<int>(r, 1);
1475  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
1476  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
1477  /* add new entity to the map logicalToPhysicalTableMapById_ */
1478  std::vector<int32_t> physicalTables{physical_tb_id};
1479  const auto it_ok =
1480  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
1481  CHECK(it_ok.second);
1482  } else {
1483  /* update map logicalToPhysicalTableMapById_ */
1484  physicalTableIt->second.push_back(physical_tb_id);
1485  }
1486  }
1487 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
#define CHECK(condition)
Definition: Logger.h:291
virtual size_t getNumRows() const
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:727
void Catalog_Namespace::Catalog::buildMaps ( )
protected

Definition at line 1492 of file Catalog.cpp.

References g_enable_fsi, and Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_user_id_to_user_name_map().

1492  {
1493  // Get all user id to username mapping here in order to avoid making a call to
1494  // SysCatalog (and attempting to acquire SysCatalog locks) while holding locks for this
1495  // catalog.
1496  const auto user_name_by_user_id = get_user_id_to_user_name_map();
1497 
1498  cat_write_lock write_lock(this);
1499  cat_sqlite_lock sqlite_lock(getObjForLock());
1500 
1503 
1504  if (g_enable_fsi) {
1507  }
1508 
1511  buildDashboardsMapUnlocked(user_name_by_user_id);
1515 }
void buildLogicalToPhysicalMapUnlocked()
Definition: Catalog.cpp:1465
void buildDashboardsMapUnlocked(const std::map< int32_t, std::string > &user_name_by_user_id)
Definition: Catalog.cpp:1420
void buildCustomExpressionsMapUnlocked()
Definition: Catalog.cpp:1517
const Catalog * getObjForLock()
Definition: Catalog.cpp:260
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
void updateForeignTablesInMapUnlocked()
Definition: Catalog.cpp:5122
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:126
bool g_enable_fsi
Definition: Catalog.cpp:96
std::map< int32_t, std::string > get_user_id_to_user_name_map()
Definition: Catalog.cpp:1045

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildTablesMapUnlocked ( )
private

Definition at line 1277 of file Catalog.cpp.

References logger::FATAL, StorageType::FOREIGN_TABLE, TableDescriptor::fragmenter, TableDescriptor::fragments, TableDescriptor::fragPageSize, TableDescriptor::fragType, TableDescriptor::hasDeletedCol, TableDescriptor::is_system_table, TableDescriptor::isView, TableDescriptor::keyMetainfo, LOG, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nColumns, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::storageType, TableDescriptor::tableId, TableDescriptor::tableName, to_upper(), and TableDescriptor::userId.

1277  {
1278  std::string tableQuery(
1279  "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
1280  "max_chunk_size, frag_page_size, "
1281  "max_rows, partitions, shard_column_id, shard, num_shards, key_metainfo, userid, "
1282  "sort_column_id, storage_type, max_rollback_epochs, is_system_table "
1283  "from mapd_tables");
1284  sqliteConnector_.query(tableQuery);
1285  auto numRows = sqliteConnector_.getNumRows();
1286  for (size_t r = 0; r < numRows; ++r) {
1287  TableDescriptor* td;
1288  const auto& storage_type = sqliteConnector_.getData<string>(r, 17);
1289  if (!storage_type.empty() && storage_type != StorageType::FOREIGN_TABLE) {
1290  const auto table_id = sqliteConnector_.getData<int>(r, 0);
1291  const auto& table_name = sqliteConnector_.getData<string>(r, 1);
1292  LOG(FATAL) << "Unable to read Catalog metadata: storage type is currently not a "
1293  "supported table option (table "
1294  << table_name << " [" << table_id << "] in database "
1295  << currentDB_.dbName << ").";
1296  }
1297 
1298  if (storage_type == StorageType::FOREIGN_TABLE) {
1299  td = new foreign_storage::ForeignTable();
1300  } else {
1301  td = new TableDescriptor();
1302  }
1303 
1304  td->storageType = storage_type;
1305  td->tableId = sqliteConnector_.getData<int>(r, 0);
1306  td->tableName = sqliteConnector_.getData<string>(r, 1);
1307  td->nColumns = sqliteConnector_.getData<int>(r, 2);
1308  td->isView = sqliteConnector_.getData<bool>(r, 3);
1309  td->fragments = sqliteConnector_.getData<string>(r, 4);
1310  td->fragType =
1312  td->maxFragRows = sqliteConnector_.getData<int>(r, 6);
1313  td->maxChunkSize = sqliteConnector_.getData<int64_t>(r, 7);
1314  td->fragPageSize = sqliteConnector_.getData<int>(r, 8);
1315  td->maxRows = sqliteConnector_.getData<int64_t>(r, 9);
1316  td->partitions = sqliteConnector_.getData<string>(r, 10);
1317  td->shardedColumnId = sqliteConnector_.getData<int>(r, 11);
1318  td->shard = sqliteConnector_.getData<int>(r, 12);
1319  td->nShards = sqliteConnector_.getData<int>(r, 13);
1320  td->keyMetainfo = sqliteConnector_.getData<string>(r, 14);
1321  td->userId = sqliteConnector_.getData<int>(r, 15);
1322  td->sortedColumnId =
1323  sqliteConnector_.isNull(r, 16) ? 0 : sqliteConnector_.getData<int>(r, 16);
1324  if (!td->isView) {
1325  td->fragmenter = nullptr;
1326  }
1327  td->maxRollbackEpochs = sqliteConnector_.getData<int>(r, 18);
1328  td->is_system_table = sqliteConnector_.getData<bool>(r, 19);
1329  td->hasDeletedCol = false;
1330 
1332  tableDescriptorMapById_[td->tableId] = td;
1333  }
1334 }
int32_t maxRollbackEpochs
std::string partitions
T getData(const int row, const int col)
std::string tableName
#define LOG(tag)
Definition: Logger.h:285
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
std::string storageType
virtual void query(const std::string &queryString)
std::string fragments
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
std::string keyMetainfo
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
std::string to_upper(const std::string &str)
bool isNull(const int row, const int col) const
Fragmenter_Namespace::FragmenterType fragType
static constexpr char const * FOREIGN_TABLE
virtual size_t getNumRows() const
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:707

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::calculateSHA1 ( const std::string &  data)
protected

Definition at line 4811 of file Catalog.cpp.

4811  {
4812  boost::uuids::detail::sha1 sha1;
4813  unsigned int digest[5];
4814  sha1.process_bytes(data.c_str(), data.length());
4815  sha1.get_digest(digest);
4816  std::stringstream ss;
4817  for (size_t i = 0; i < 5; i++) {
4818  ss << std::hex << digest[i];
4819  }
4820  return ss.str();
4821 }
void Catalog_Namespace::Catalog::changeForeignServerOwner ( const std::string &  server_name,
const int  new_owner_id 
)

Change the owner of a Foreign Server to a new owner.

Parameters
server_name- Name of the foreign server whose owner to change
new_owner_id- New owner's user id

Definition at line 3396 of file Catalog.cpp.

References CHECK, to_string(), and foreign_storage::ForeignServer::user_id.

3397  {
3398  cat_write_lock write_lock(this);
3399  foreign_storage::ForeignServer* foreign_server =
3400  foreignServerMap_.find(server_name)->second.get();
3401  CHECK(foreign_server);
3402  setForeignServerProperty(server_name, "owner_user_id", std::to_string(new_owner_id));
3403  // update in-memory server
3404  foreign_server->user_id = new_owner_id;
3405 }
std::string to_string(char const *&&v)
ForeignServerMap foreignServerMap_
Definition: Catalog.h:715
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
#define CHECK(condition)
Definition: Logger.h:291
void setForeignServerProperty(const std::string &server_name, const std::string &property, const std::string &value)
Definition: Catalog.cpp:5320

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::CheckAndExecuteMigrations ( )
protected

Definition at line 1015 of file Catalog.cpp.

References g_enable_fsi, getObjForLock(), recordOwnershipOfObjectsInObjectPermissions(), renameLegacyDataWrappers(), updateCustomExpressionsSchema(), updateDefaultColumnValues(), updateDeletedColumnIndicator(), updateDictionaryNames(), updateDictionarySchema(), updateFixlenArrayColumns(), updateFrontendViewAndLinkUsers(), updateFrontendViewSchema(), updateFrontendViewsToDashboards(), updateFsiSchemas(), updateGeoColumns(), updateLinkSchema(), updateLogicalToPhysicalTableLinkSchema(), updatePageSize(), and updateTableDescriptorSchema().

1015  {
1016  cat_write_lock write_lock(this);
1017  cat_sqlite_lock sqlite_lock(getObjForLock());
1020  updateGeoColumns();
1023  updateLinkSchema();
1027  updatePageSize();
1031  if (g_enable_fsi) {
1032  updateFsiSchemas();
1034  }
1037 }
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:473
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:800
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:132
const Catalog * getObjForLock()
Definition: Catalog.cpp:260
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
void updateCustomExpressionsSchema()
Definition: Catalog.cpp:766
void updateLogicalToPhysicalTableLinkSchema()
Definition: Catalog.cpp:634
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:126
bool g_enable_fsi
Definition: Catalog.cpp:96

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::CheckAndExecuteMigrationsPostBuildMaps ( )
protected

Definition at line 1039 of file Catalog.cpp.

References checkDateInDaysColumnMigration(), and createDashboardSystemRoles().

1039  {
1042 }
void checkDateInDaysColumnMigration()
Definition: Catalog.cpp:928

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::checkDateInDaysColumnMigration ( )
protected

Definition at line 928 of file Catalog.cpp.

References getCurrentDB(), getObjForLock(), migrations::MigrationMgr::migrateDateInDaysMetadata(), sqliteConnector_, and tableDescriptorMapById_.

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

928  {
929  cat_sqlite_lock sqlite_lock(getObjForLock());
932 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:265
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
const Catalog * getObjForLock()
Definition: Catalog.cpp:260
static void migrateDateInDaysMetadata(const Catalog_Namespace::TableDescriptorMapById &table_descriptors_by_id, const int database_id, Catalog_Namespace::Catalog *cat, SqliteConnector &sqlite)
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:126

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool Catalog_Namespace::Catalog::checkDropRenderGroupColumnsMigration ( )

Definition at line 934 of file Catalog.cpp.

References migrations::MigrationMgr::dropRenderGroupColumns(), and tableDescriptorMapById_.

Referenced by Catalog_Namespace::SysCatalog::checkDropRenderGroupColumnsMigration().

934  {
935  // do not take cat_sqlite_lock here as Catalog functions do that themselves
937 }
static bool dropRenderGroupColumns(const Catalog_Namespace::TableDescriptorMapById &table_descriptors_by_id, Catalog_Namespace::Catalog *cat)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const bool Catalog_Namespace::Catalog::checkMetadataForDeletedRecs ( const TableDescriptor td,
int  column_id 
) const

Definition at line 3913 of file Catalog.cpp.

References CHECK, and TableDescriptor::fragmenter.

3914  {
3915  // check if there are rows deleted by examining the deletedColumn metadata
3916  CHECK(td);
3917  auto fragmenter = td->fragmenter;
3918  if (fragmenter) {
3919  return fragmenter->hasDeletedRows(delete_column_id);
3920  } else {
3921  return false;
3922  }
3923 }
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
#define CHECK(condition)
Definition: Logger.h:291
void Catalog_Namespace::Catalog::checkpoint ( const int  logicalTableId) const

Definition at line 5032 of file Catalog.cpp.

Referenced by UpdelRoll::commitUpdate(), TableOptimizer::vacuumDeletedRows(), and TableOptimizer::vacuumFragmentsAboveMinSelectivity().

5032  {
5033  const auto td = getMetadataForTable(logicalTableId);
5034  const auto shards = getPhysicalTablesDescriptors(td);
5035  for (const auto shard : shards) {
5036  getDataMgr().checkpoint(getCurrentDB().dbId, shard->tableId);
5037  }
5038 }
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:266
void checkpoint(const int db_id, const int tb_id)
Definition: DataMgr.cpp:634
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:265
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4879
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::checkpointWithAutoRollback ( const int  logical_table_id) const

Definition at line 5040 of file Catalog.cpp.

Referenced by AlterTableAlterColumnCommandRecoveryMgr::checkpoint(), Fragmenter_Namespace::LocalInsertConnector::checkpoint(), import_export::Loader::checkpoint(), StorageIOFacility::TransactionParameters::finalizeTransaction(), and TableOptimizer::vacuumFragmentsAboveMinSelectivity().

5040  {
5041  auto table_epochs = getTableEpochs(getDatabaseId(), logical_table_id);
5042  try {
5043  checkpoint(logical_table_id);
5044  } catch (...) {
5045  setTableEpochsLogExceptions(getDatabaseId(), table_epochs);
5046  throw;
5047  }
5048 }
int getDatabaseId() const
Definition: Catalog.h:326
void checkpoint(const int logicalTableId) const
Definition: Catalog.cpp:5032
void setTableEpochsLogExceptions(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3895
std::vector< TableEpochInfo > getTableEpochs(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:3831

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::conditionallyInitializeSystemObjects ( )
private

Definition at line 6392 of file Catalog.cpp.

References g_enable_system_tables.

6392  {
6396  }
6397 }
bool g_enable_system_tables
Definition: SysCatalog.cpp:64
bool isInfoSchemaDb() const
Definition: Catalog.cpp:6399
int32_t Catalog_Namespace::Catalog::createCustomExpression ( std::unique_ptr< CustomExpression custom_expression)

Creates a new custom expression.

Parameters
custom_expression- unique pointer to struct containing custom expression details.
Returns
id of created custom expression

Definition at line 5954 of file Catalog.cpp.

References CHECK, CHECK_EQ, CHECK_GT, and to_string().

5955  {
5956  cat_write_lock write_lock(this);
5957  cat_sqlite_lock sqlite_lock(getObjForLock());
5958  sqliteConnector_.query("BEGIN TRANSACTION");
5959  int32_t custom_expression_id{-1};
5960  try {
5961  auto data_source_type_str =
5962  CustomExpression::dataSourceTypeToString(custom_expression->data_source_type);
5963  auto data_source_id_str = std::to_string(custom_expression->data_source_id);
5964  std::string custom_expr_select_query{
5965  "SELECT id FROM omnisci_custom_expressions WHERE name = ? and data_source_type = "
5966  "? and data_source_id = ? and is_deleted = ?"};
5967  std::vector<std::string> custom_expr_select_params{custom_expression->name,
5968  data_source_type_str,
5969  data_source_id_str,
5970  std::to_string(false)};
5971  sqliteConnector_.query_with_text_params(custom_expr_select_query,
5972  custom_expr_select_params);
5973  if (sqliteConnector_.getNumRows() > 0) {
5974  throw std::runtime_error{
5975  "A custom expression with the given "
5976  "name and data source already exists."};
5977  }
5979  "INSERT INTO omnisci_custom_expressions(name, expression_json, "
5980  "data_source_type, data_source_id, is_deleted) VALUES (?,?,?,?,?)",
5981  std::vector<std::string>{custom_expression->name,
5982  custom_expression->expression_json,
5983  data_source_type_str,
5984  data_source_id_str,
5985  std::to_string(false)});
5986  sqliteConnector_.query_with_text_params(custom_expr_select_query,
5987  custom_expr_select_params);
5988  CHECK_EQ(sqliteConnector_.getNumRows(), static_cast<size_t>(1));
5989  custom_expression->id = sqliteConnector_.getData<int32_t>(0, 0);
5990  custom_expression_id = custom_expression->id;
5991  CHECK(custom_expr_map_by_id_.find(custom_expression->id) ==
5992  custom_expr_map_by_id_.end());
5993  custom_expr_map_by_id_[custom_expression->id] = std::move(custom_expression);
5994  } catch (std::exception& e) {
5995  sqliteConnector_.query("ROLLBACK TRANSACTION");
5996  throw;
5997  }
5998  sqliteConnector_.query("END TRANSACTION");
5999  CHECK_GT(custom_expression_id, 0);
6000  return custom_expression_id;
6001 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
#define CHECK_GT(x, y)
Definition: Logger.h:305
std::string to_string(char const *&&v)
static std::string dataSourceTypeToString(DataSourceType type_enum)
const Catalog * getObjForLock()
Definition: Catalog.cpp:260
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
#define CHECK(condition)
Definition: Logger.h:291
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:717
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:126
virtual size_t getNumRows() const

+ Here is the call graph for this function:

int32_t Catalog_Namespace::Catalog::createDashboard ( DashboardDescriptor vd)

Definition at line 4670 of file Catalog.cpp.

References DashboardDescriptor::dashboardId, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, DashboardDescriptor::dashboardSystemRoleName, generate_dashboard_system_rolename(), DashboardDescriptor::imageHash, to_string(), Catalog_Namespace::sqlite_lock< T >::unlock(), Catalog_Namespace::write_lock< T >::unlock(), DashboardDescriptor::updateTime, and DashboardDescriptor::userId.

4670  {
4671  cat_write_lock write_lock(this);
4672  cat_sqlite_lock sqlite_lock(getObjForLock());
4673  sqliteConnector_.query("BEGIN TRANSACTION");
4674  try {
4675  // TODO(andrew): this should be an upsert
4677  "SELECT id FROM mapd_dashboards WHERE name = ? and userid = ?",
4678  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
4679  if (sqliteConnector_.getNumRows() > 0) {
4681  "UPDATE mapd_dashboards SET state = ?, image_hash = ?, metadata = ?, "
4682  "update_time = "
4683  "datetime('now') where name = ? "
4684  "and userid = ?",
4685  std::vector<std::string>{vd.dashboardState,
4686  vd.imageHash,
4687  vd.dashboardMetadata,
4688  vd.dashboardName,
4689  std::to_string(vd.userId)});
4690  } else {
4692  "INSERT INTO mapd_dashboards (name, state, image_hash, metadata, "
4693  "update_time, "
4694  "userid) "
4695  "VALUES "
4696  "(?,?,?,?, "
4697  "datetime('now'), ?)",
4698  std::vector<std::string>{vd.dashboardName,
4699  vd.dashboardState,
4700  vd.imageHash,
4701  vd.dashboardMetadata,
4702  std::to_string(vd.userId)});
4703  }
4704  } catch (std::exception& e) {
4705  sqliteConnector_.query("ROLLBACK TRANSACTION");
4706  throw;
4707  }
4708  sqliteConnector_.query("END TRANSACTION");
4709 
4710  // now get the auto generated dashboardId
4711  try {
4713  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_dashboards "
4714  "WHERE name = ? and userid = ?",
4715  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
4716  vd.dashboardId = sqliteConnector_.getData<int>(0, 0);
4717  vd.updateTime = sqliteConnector_.getData<std::string>(0, 1);
4718  } catch (std::exception& e) {
4719  throw;
4720  }
4724  sqlite_lock.unlock();
4725  write_lock.unlock();
4726  if (!isInfoSchemaDb()) {
4727  // NOTE(wamsi): Transactionally unsafe
4730  }
4731  return vd.dashboardId;
4732 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
virtual void query(const std::string &queryString)
void addFrontendViewToMap(DashboardDescriptor &vd)
Definition: Catalog.cpp:1686
std::string to_string(char const *&&v)
std::string dashboardSystemRoleName
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1727
const Catalog * getObjForLock()
Definition: Catalog.cpp:260
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
bool isInfoSchemaDb() const
Definition: Catalog.cpp:6399
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:126
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createDashboardSystemRoles ( )
protected

Definition at line 939 of file Catalog.cpp.

References createOrUpdateDashboardSystemRole(), currentDB_, Catalog_Namespace::DBMetadata::dbId, logger::ERROR, generate_dashboard_system_rolename(), SqliteConnector::getData(), Catalog_Namespace::SysCatalog::getGranteesOfSharedDashboards(), SqliteConnector::getNumRows(), getObjForLock(), Catalog_Namespace::SysCatalog::getRoleGrantee(), Catalog_Namespace::SysCatalog::grantRoleBatch(), logger::INFO, Catalog_Namespace::SysCatalog::instance(), LOG, MAPD_VERSION, SqliteConnector::query(), SqliteConnector::query_with_text_params(), run_benchmark_import::result, sqliteConnector_, and to_string().

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

939  {
940  std::unordered_map<std::string, std::pair<int, std::string>> dashboards;
941  std::vector<std::string> dashboard_ids;
942  static const std::string migration_name{"dashboard_roles_migration"};
943  {
944  cat_sqlite_lock sqlite_lock(getObjForLock());
945  sqliteConnector_.query("BEGIN TRANSACTION");
946  try {
947  // migration_history should be present in all catalogs by now
948  // if not then would be created before this migration
950  "select * from mapd_version_history where migration_history = '" +
951  migration_name + "'");
952  if (sqliteConnector_.getNumRows() != 0) {
953  // no need for further execution
954  sqliteConnector_.query("END TRANSACTION");
955  return;
956  }
957  LOG(INFO) << "Performing dashboard internal roles Migration.";
958  sqliteConnector_.query("select id, userid, metadata from mapd_dashboards");
959  for (size_t i = 0; i < sqliteConnector_.getNumRows(); ++i) {
962  sqliteConnector_.getData<string>(i, 0)))) {
963  // Successfully created roles during previous migration/crash
964  // No need to include them
965  continue;
966  }
967  dashboards[sqliteConnector_.getData<string>(i, 0)] = std::make_pair(
968  sqliteConnector_.getData<int>(i, 1), sqliteConnector_.getData<string>(i, 2));
969  dashboard_ids.push_back(sqliteConnector_.getData<string>(i, 0));
970  }
971  } catch (const std::exception& e) {
972  sqliteConnector_.query("ROLLBACK TRANSACTION");
973  throw;
974  }
975  sqliteConnector_.query("END TRANSACTION");
976  }
977  // All current grantees with shared dashboards.
978  const auto active_grantees =
980 
981  try {
982  // NOTE(wamsi): Transactionally unsafe
983  for (auto dash : dashboards) {
984  createOrUpdateDashboardSystemRole(dash.second.second,
985  dash.second.first,
987  std::to_string(currentDB_.dbId), dash.first));
988  auto result = active_grantees.find(dash.first);
989  if (result != active_grantees.end()) {
992  dash.first)},
993  result->second);
994  }
995  }
996  cat_sqlite_lock sqlite_lock(getObjForLock());
997  // check if this has already been completed
999  "select * from mapd_version_history where migration_history = '" +
1000  migration_name + "'");
1001  if (sqliteConnector_.getNumRows() != 0) {
1002  return;
1003  }
1005  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
1006  std::vector<std::string>{std::to_string(MAPD_VERSION), migration_name});
1007  } catch (const std::exception& e) {
1008  LOG(ERROR) << "Failed to create dashboard system roles during migration: "
1009  << e.what();
1010  throw;
1011  }
1012  LOG(INFO) << "Successfully created dashboard system roles during migration.";
1013 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
#define LOG(tag)
Definition: Logger.h:285
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
void grantRoleBatch(const std::vector< std::string > &roles, const std::vector< std::string > &grantees)
static SysCatalog & instance()
Definition: SysCatalog.h:343
static const int32_t MAPD_VERSION
Definition: release.h:32
Role * getRoleGrantee(const std::string &name) const
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1727
const Catalog * getObjForLock()
Definition: Catalog.cpp:260
std::unordered_map< std::string, std::vector< std::string > > getGranteesOfSharedDashboards(const std::vector< std::string > &dashboard_ids)
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:126
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createDefaultServersIfNotExists ( )

Creates default local file servers (if they don't already exist).

Definition at line 5341 of file Catalog.cpp.

References CHECK, foreign_storage::DataWrapperType::CSV, g_enable_fsi, shared::kRootUserId, foreign_storage::AbstractFileStorageDataWrapper::LOCAL_FILE_STORAGE_TYPE, foreign_storage::DataWrapperType::PARQUET, foreign_storage::DataWrapperType::REGEX_PARSER, and foreign_storage::AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY.

5341  {
5346 
5347  auto local_csv_server = std::make_unique<foreign_storage::ForeignServer>(
5348  "default_local_delimited",
5350  options,
5352  local_csv_server->validate();
5353  createForeignServerNoLocks(std::move(local_csv_server), true);
5354 
5355 #ifdef ENABLE_IMPORT_PARQUET
5356  auto local_parquet_server = std::make_unique<foreign_storage::ForeignServer>(
5357  "default_local_parquet",
5359  options,
5361  local_parquet_server->validate();
5362  createForeignServerNoLocks(std::move(local_parquet_server), true);
5363 #endif
5364 
5365  auto local_regex_parser_server = std::make_unique<foreign_storage::ForeignServer>(
5366  "default_local_regex_parsed",
5368  options,
5370  local_regex_parser_server->validate();
5371  createForeignServerNoLocks(std::move(local_regex_parser_server), true);
5372 }
const int kRootUserId
static constexpr char const * REGEX_PARSER
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:3301
#define CHECK(condition)
Definition: Logger.h:291
static constexpr char const * CSV
std::map< std::string, std::string, std::less<>> OptionsMap
bool g_enable_fsi
Definition: Catalog.cpp:96
static constexpr char const * PARQUET
void Catalog_Namespace::Catalog::createForeignServer ( std::unique_ptr< foreign_storage::ForeignServer foreign_server,
bool  if_not_exists 
)

Creates a new foreign server DB object.

Parameters
foreign_server- unique pointer to struct containing foreign server details
if_not_exists- flag indicating whether or not an attempt to create a new foreign server should occur if a server with the same name already exists. An exception is thrown if this flag is set to "false" and an attempt is made to create a pre-existing foreign server

Definition at line 3293 of file Catalog.cpp.

3295  {
3296  cat_write_lock write_lock(this);
3297  cat_sqlite_lock sqlite_lock(getObjForLock());
3298  createForeignServerNoLocks(std::move(foreign_server), if_not_exists);
3299 }
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:3301
const Catalog * getObjForLock()
Definition: Catalog.cpp:260
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:126
void Catalog_Namespace::Catalog::createForeignServerNoLocks ( std::unique_ptr< foreign_storage::ForeignServer foreign_server,
bool  if_not_exists 
)
private

Same as createForeignServer() but without acquiring locks. This should only be called from within a function/code block that already acquires appropriate locks.

Definition at line 3301 of file Catalog.cpp.

References CHECK, CHECK_EQ, setup::name, and to_string().

3303  {
3304  const auto& name = foreign_server->name;
3305 
3307  "SELECT name from omnisci_foreign_servers where name = ?",
3308  std::vector<std::string>{name});
3309 
3310  if (sqliteConnector_.getNumRows() == 0) {
3311  foreign_server->creation_time = std::time(nullptr);
3313  "INSERT INTO omnisci_foreign_servers (name, data_wrapper_type, owner_user_id, "
3314  "creation_time, "
3315  "options) "
3316  "VALUES (?, ?, ?, ?, ?)",
3317  std::vector<std::string>{name,
3318  foreign_server->data_wrapper_type,
3319  std::to_string(foreign_server->user_id),
3320  std::to_string(foreign_server->creation_time),
3321  foreign_server->getOptionsAsJsonString()});
3323  "SELECT id from omnisci_foreign_servers where name = ?",
3324  std::vector<std::string>{name});
3325  CHECK_EQ(sqliteConnector_.getNumRows(), size_t(1));
3326  foreign_server->id = sqliteConnector_.getData<int32_t>(0, 0);
3327  std::shared_ptr<foreign_storage::ForeignServer> foreign_server_shared =
3328  std::move(foreign_server);
3330  << "Attempting to insert a foreign server into foreign server map that already "
3331  "exists.";
3332  foreignServerMap_[name] = foreign_server_shared;
3333  foreignServerMapById_[foreign_server_shared->id] = foreign_server_shared;
3334  } else if (!if_not_exists) {
3335  throw std::runtime_error{"A foreign server with name \"" + foreign_server->name +
3336  "\" already exists."};
3337  }
3338 
3339  const auto& server_it = foreignServerMap_.find(name);
3340  CHECK(server_it != foreignServerMap_.end());
3341  CHECK(foreignServerMapById_.find(server_it->second->id) != foreignServerMapById_.end());
3342 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
std::string to_string(char const *&&v)
std::string name() const
Definition: Catalog.h:348
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:716
ForeignServerMap foreignServerMap_
Definition: Catalog.h:715
#define CHECK(condition)
Definition: Logger.h:291
virtual size_t getNumRows() const

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::createLink ( LinkDescriptor ld,
size_t  min_length 
)

Definition at line 4823 of file Catalog.cpp.

References LinkDescriptor::link, LinkDescriptor::linkId, to_string(), LinkDescriptor::updateTime, LinkDescriptor::userId, LinkDescriptor::viewMetadata, and LinkDescriptor::viewState.

4823  {
4824  cat_write_lock write_lock(this);
4825  cat_sqlite_lock sqlite_lock(getObjForLock());
4826  sqliteConnector_.query("BEGIN TRANSACTION");
4827  try {
4829  .substr(0, 8);
4831  "SELECT linkid FROM mapd_links WHERE link = ? and userid = ?",
4832  std::vector<std::string>{ld.link, std::to_string(ld.userId)});
4833  if (sqliteConnector_.getNumRows() > 0) {
4835  "UPDATE mapd_links SET update_time = datetime('now') WHERE userid = ? AND "
4836  "link = ?",
4837  std::vector<std::string>{std::to_string(ld.userId), ld.link});
4838  } else {
4840  "INSERT INTO mapd_links (userid, link, view_state, view_metadata, "
4841  "update_time) VALUES (?,?,?,?, datetime('now'))",
4842  std::vector<std::string>{
4843  std::to_string(ld.userId), ld.link, ld.viewState, ld.viewMetadata});
4844  }
4845  // now get the auto generated dashid
4847  "SELECT linkid, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_links "
4848  "WHERE link = ?",
4849  ld.link);
4850  ld.linkId = sqliteConnector_.getData<int>(0, 0);
4851  ld.updateTime = sqliteConnector_.getData<std::string>(0, 1);
4852  } catch (std::exception& e) {
4853  sqliteConnector_.query("ROLLBACK TRANSACTION");
4854  throw;
4855  }
4856  sqliteConnector_.query("END TRANSACTION");
4857  addLinkToMap(ld);
4858  return ld.link;
4859 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
std::string calculateSHA1(const std::string &data)
Definition: Catalog.cpp:4811
void addLinkToMap(LinkDescriptor &ld)
Definition: Catalog.cpp:1773
const Catalog * getObjForLock()
Definition: Catalog.cpp:260
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:126
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createOrUpdateDashboardSystemRole ( const std::string &  view_meta,
const int32_t &  user_id,
const std::string &  dash_role_name 
)
protected

Definition at line 1727 of file Catalog.cpp.

References Grantee::findDbObject(), Grantee::getDbObjects(), lockmgr::instance(), TableDBObjectType, and ViewDBObjectType.

Referenced by createDashboardSystemRoles().

1729  {
1730  auto objects = parseDashboardObjects(view_meta, user_id);
1731  Role* rl = SysCatalog::instance().getRoleGrantee(dash_role_name);
1732  if (!rl) {
1733  // Dashboard role does not exist
1734  // create role and grant privileges
1735  // NOTE(wamsi): Transactionally unsafe
1737  dash_role_name, /*user_private_role=*/false, /*is_temporary=*/false);
1738  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1739  } else {
1740  // Dashboard system role already exists
1741  // Add/remove privileges on objects
1742  std::set<DBObjectKey> revoke_keys;
1743  auto ex_objects = rl->getDbObjects(true);
1744  for (auto key : *ex_objects | boost::adaptors::map_keys) {
1745  if (key.permissionType != TableDBObjectType &&
1746  key.permissionType != ViewDBObjectType) {
1747  continue;
1748  }
1749  bool found = false;
1750  for (auto obj : objects) {
1751  found = key == obj.getObjectKey() ? true : false;
1752  if (found) {
1753  break;
1754  }
1755  }
1756  if (!found) {
1757  revoke_keys.insert(key);
1758  }
1759  }
1760  for (auto& key : revoke_keys) {
1761  // revoke privs on object since the object is no
1762  // longer used by the dashboard as source
1763  // NOTE(wamsi): Transactionally unsafe
1765  dash_role_name, *rl->findDbObject(key, true), *this);
1766  }
1767  // Update privileges on remaining objects
1768  // NOTE(wamsi): Transactionally unsafe
1769  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1770  }
1771 }
void revokeDBObjectPrivileges(const std::string &grantee, const DBObject &object, const Catalog_Namespace::Catalog &catalog)
void createRole(const std::string &roleName, const bool user_private_role, const bool is_temporary=false)
Definition: Grantee.h:81
DBObject * findDbObject(const DBObjectKey &objectKey, bool only_direct) const
Definition: Grantee.cpp:85
static SysCatalog & instance()
Definition: SysCatalog.h:343
Role * getRoleGrantee(const std::string &name) const
void grantDBObjectPrivilegesBatch(const std::vector< std::string > &grantees, const std::vector< DBObject > &objects, const Catalog_Namespace::Catalog &catalog)
const DBObjectMap * getDbObjects(bool only_direct) const
Definition: Grantee.h:56
std::vector< DBObject > parseDashboardObjects(const std::string &view_meta, const int &user_id)
Definition: Catalog.cpp:1697

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createShardedTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs 
)

Definition at line 4109 of file Catalog.cpp.

References CHECK, TableDescriptor::nShards, TableDescriptor::shard, table_is_temporary(), TableDescriptor::tableId, and TableDescriptor::tableName.

4112  {
4113  /* create logical table */
4114  TableDescriptor* tdl = &td;
4115  createTable(*tdl, cols, shared_dict_defs, true); // create logical table
4116  int32_t logical_tb_id = tdl->tableId;
4117  std::string logical_table_name = tdl->tableName;
4118 
4119  /* create physical tables and link them to the logical table */
4120  std::vector<int32_t> physicalTables;
4121  for (int32_t i = 1; i <= td.nShards; i++) {
4122  TableDescriptor* tdp = &td;
4123  tdp->tableName = generatePhysicalTableName(logical_table_name, i);
4124  tdp->shard = i - 1;
4125  createTable(*tdp, cols, shared_dict_defs, false); // create physical table
4126  int32_t physical_tb_id = tdp->tableId;
4127 
4128  /* add physical table to the vector of physical tables */
4129  physicalTables.push_back(physical_tb_id);
4130  }
4131 
4132  if (!physicalTables.empty()) {
4133  cat_write_lock write_lock(this);
4134  /* add logical to physical tables correspondence to the map */
4135  const auto it_ok =
4136  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
4137  CHECK(it_ok.second);
4138  /* update sqlite mapd_logical_to_physical in sqlite database */
4139  if (!table_is_temporary(&td)) {
4140  updateLogicalToPhysicalTableMap(logical_tb_id);
4141  }
4142  }
4143 }
std::string tableName
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:2945
bool table_is_temporary(const TableDescriptor *const td)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
#define CHECK(condition)
Definition: Logger.h:291
std::string generatePhysicalTableName(const std::string &logicalTableName, const size_t shardNumber)
Definition: Catalog.cpp:5095
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:727
void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id)
Definition: Catalog.cpp:648

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createSystemTableServer ( const std::string &  server_name,
const std::string &  data_wrapper_type,
const foreign_storage::OptionsMap options = {} 
)
private

Definition at line 6856 of file Catalog.cpp.

References logger::INFO, shared::kRootUserId, LOG, and run_benchmark_import::tables.

6858  {
6859  auto server = std::make_unique<foreign_storage::ForeignServer>(
6860  server_name, data_wrapper_type, options, shared::kRootUserId);
6861  server->validate();
6862  auto stored_server = getForeignServer(server_name);
6863  if (stored_server && stored_server->options != server->options) {
6864  // Drop all tables for server before dropping server.
6865  auto tables = getAllForeignTablesForForeignServer(stored_server->id);
6866  for (const auto table : tables) {
6867  LOG(INFO) << "Dropping existing \"" << table->tableName << "\" system table for \""
6868  << server_name << "\" foreign server.";
6869  dropTable(table);
6870  }
6871  LOG(INFO) << "Dropping existing \"" << server_name
6872  << "\" system table foreign server.";
6873  dropForeignServer(server_name);
6874  stored_server = nullptr;
6875  }
6876  if (!stored_server) {
6877  LOG(INFO) << "Creating a new \"" << server_name << "\" system table foreign server.";
6878  createForeignServer(std::move(server), true);
6879  }
6880 }
const int kRootUserId
void createForeignServer(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:3293
const foreign_storage::ForeignServer * getForeignServer(const std::string &server_name) const
Definition: Catalog.cpp:3344
#define LOG(tag)
Definition: Logger.h:285
std::vector< const foreign_storage::ForeignTable * > getAllForeignTablesForForeignServer(const int32_t foreign_server_id)
Get all of the foreign tables for associated with a foreign server id.
Definition: Catalog.cpp:3641
void dropForeignServer(const std::string &server_name)
Definition: Catalog.cpp:3460
void dropTable(const TableDescriptor *td)
Definition: Catalog.cpp:4292
void Catalog_Namespace::Catalog::createTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs,
bool  isLogicalTable 
)

Definition at line 2945 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnId, TableDescriptor::columnIdBySpi_, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::default_value, dict_ref_t::dictId, Data_Namespace::DISK_LEVEL, StorageType::FOREIGN_TABLE, TableDescriptor::fragPageSize, TableDescriptor::fragType, g_enable_fsi, g_serialize_temp_tables, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_next_refresh_time(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), TableDescriptor::hasDeletedCol, SQLTypeInfo::is_array(), TableDescriptor::is_system_table, ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, kBIGINT, kBOOLEAN, kENCODING_DICT, TableDescriptor::keyMetainfo, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nColumns, foreign_storage::ForeignTable::next_refresh_time, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, SQLTypeInfo::set_comp_param(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::set_dict_key(), SQLTypeInfo::set_size(), TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::storageType, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), TableDescriptor::userId, TableDescriptor::viewSQL, and ColumnDescriptor::virtualExpr.

2949  {
2950  cat_write_lock write_lock(this);
2951  list<ColumnDescriptor> cds = cols;
2952  list<DictDescriptor> dds;
2953  std::set<std::string> toplevel_column_names;
2954  list<ColumnDescriptor> columns;
2955 
2956  if (!td.storageType.empty() &&
2959  throw std::runtime_error("Only temporary tables can be backed by foreign storage.");
2960  }
2961  dataMgr_->getForeignStorageInterface()->prepareTable(getCurrentDB().dbId, td, cds);
2962  }
2963 
2964  for (auto cd : cds) {
2965  if (cd.columnName == "rowid") {
2966  throw std::runtime_error(
2967  "Cannot create column with name rowid. rowid is a system defined column.");
2968  }
2969  columns.push_back(cd);
2970  toplevel_column_names.insert(cd.columnName);
2971  if (cd.columnType.is_geometry()) {
2972  expandGeoColumn(cd, columns);
2973  }
2974  }
2975  cds.clear();
2976 
2977  ColumnDescriptor cd;
2978  // add row_id column -- Must be last column in the table
2979  cd.columnName = "rowid";
2980  cd.isSystemCol = true;
2981  cd.columnType = SQLTypeInfo(kBIGINT, true);
2982 #ifdef MATERIALIZED_ROWID
2983  cd.isVirtualCol = false;
2984 #else
2985  cd.isVirtualCol = true;
2986  cd.virtualExpr = "MAPD_FRAG_ID * MAPD_ROWS_PER_FRAG + MAPD_FRAG_ROW_ID";
2987 #endif
2988  columns.push_back(cd);
2989  toplevel_column_names.insert(cd.columnName);
2990 
2991  if (td.hasDeletedCol) {
2992  ColumnDescriptor cd_del;
2993  cd_del.columnName = "$deleted$";
2994  cd_del.isSystemCol = true;
2995  cd_del.isVirtualCol = false;
2996  cd_del.columnType = SQLTypeInfo(kBOOLEAN, true);
2997  cd_del.isDeletedCol = true;
2998 
2999  columns.push_back(cd_del);
3000  }
3001 
3002  for (auto& column : columns) {
3003  column.db_id = getDatabaseId();
3004  }
3005 
3006  td.nColumns = columns.size();
3007  // TODO(sy): don't take disk locks or touch sqlite connector for temporary tables
3008  cat_sqlite_lock sqlite_lock(getObjForLock());
3009  sqliteConnector_.query("BEGIN TRANSACTION");
3011  try {
3013  R"(INSERT INTO mapd_tables (name, userid, ncolumns, isview, fragments, frag_type, max_frag_rows, max_chunk_size, frag_page_size, max_rows, partitions, shard_column_id, shard, num_shards, sort_column_id, storage_type, max_rollback_epochs, is_system_table, key_metainfo) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?))",
3014  std::vector<std::string>{td.tableName,
3015  std::to_string(td.userId),
3017  std::to_string(td.isView),
3018  "",
3023  std::to_string(td.maxRows),
3024  td.partitions,
3026  std::to_string(td.shard),
3027  std::to_string(td.nShards),
3029  td.storageType,
3032  td.keyMetainfo});
3033 
3034  // now get the auto generated tableid
3036  "SELECT tableid FROM mapd_tables WHERE name = ?", td.tableName);
3037  td.tableId = sqliteConnector_.getData<int>(0, 0);
3038  int colId = 1;
3039  for (auto cd : columns) {
3041  const bool is_foreign_col =
3042  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
3043  if (!is_foreign_col) {
3044  // Ideally we would like to not persist string dictionaries for system tables,
3045  // since system table content can be highly dynamic and string dictionaries
3046  // are not currently vacuumed. However, in distributed this causes issues
3047  // when the cluster is out of sync (when the agg resets but leaves persist) so
3048  // for the sake of testing we need to leave this as normal dictionaries until
3049  // we solve the distributed issue.
3050  auto use_temp_dictionary = false; // td.is_system_table;
3051  setColumnDictionary(cd, dds, td, isLogicalTable, use_temp_dictionary);
3052  }
3053  }
3054 
3055  if (toplevel_column_names.count(cd.columnName)) {
3056  if (!cd.isGeoPhyCol) {
3057  td.columnIdBySpi_.push_back(colId);
3058  }
3059  }
3060 
3061  using BindType = SqliteConnector::BindType;
3062  std::vector<BindType> types(17, BindType::TEXT);
3063  if (!cd.default_value.has_value()) {
3064  types[16] = BindType::NULL_TYPE;
3065  }
3067  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, "
3068  "coldim, colscale, is_notnull, "
3069  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
3070  "virtual_expr, is_deletedcol, default_value) "
3071  "VALUES (?, ?, ?, ?, ?, "
3072  "?, "
3073  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
3074  std::vector<std::string>{std::to_string(td.tableId),
3075  std::to_string(colId),
3076  cd.columnName,
3085  "",
3088  cd.virtualExpr,
3090  cd.default_value.value_or("NULL")},
3091  types);
3092  cd.tableId = td.tableId;
3093  cd.columnId = colId++;
3094  cds.push_back(cd);
3095  }
3096  if (td.isView) {
3098  "INSERT INTO mapd_views (tableid, sql) VALUES (?,?)",
3099  std::vector<std::string>{std::to_string(td.tableId), td.viewSQL});
3100  }
3102  auto& foreign_table = dynamic_cast<foreign_storage::ForeignTable&>(td);
3103  foreign_table.next_refresh_time = get_next_refresh_time(foreign_table);
3105  "INSERT INTO omnisci_foreign_tables (table_id, server_id, options, "
3106  "last_refresh_time, next_refresh_time) VALUES (?, ?, ?, ?, ?)",
3107  std::vector<std::string>{std::to_string(foreign_table.tableId),
3108  std::to_string(foreign_table.foreign_server->id),
3109  foreign_table.getOptionsAsJsonString(),
3110  std::to_string(foreign_table.last_refresh_time),
3111  std::to_string(foreign_table.next_refresh_time)});
3112  }
3113  } catch (std::exception& e) {
3114  sqliteConnector_.query("ROLLBACK TRANSACTION");
3115  throw;
3116  }
3117  } else { // Temporary table
3118  td.tableId = nextTempTableId_++;
3119  int colId = 1;
3120  for (auto cd : columns) {
3122  const bool is_foreign_col =
3123  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
3124 
3125  if (!is_foreign_col) {
3126  // Create a new temporary dictionary
3127  std::string fileName("");
3128  std::string folderPath("");
3130  nextTempDictId_++;
3131  DictDescriptor dd(dict_ref,
3132  fileName,
3134  false,
3135  1,
3136  folderPath,
3137  true); // Is dictName (2nd argument) used?
3138  dds.push_back(dd);
3139  if (!cd.columnType.is_array()) {
3141  }
3142  cd.columnType.set_comp_param(dict_ref.dictId);
3143  set_dict_key(cd);
3144  }
3145  }
3146  if (toplevel_column_names.count(cd.columnName)) {
3147  if (!cd.isGeoPhyCol) {
3148  td.columnIdBySpi_.push_back(colId);
3149  }
3150  }
3151  cd.tableId = td.tableId;
3152  cd.columnId = colId++;
3153  cds.push_back(cd);
3154  }
3155 
3157  serializeTableJsonUnlocked(&td, cds);
3158  }
3159  }
3160 
3161  try {
3162  auto cache = dataMgr_->getPersistentStorageMgr()->getDiskCache();
3163  if (cache) {
3164  CHECK(!cache->hasCachedMetadataForKeyPrefix({getCurrentDB().dbId, td.tableId}))
3165  << "Disk cache at " + cache->getCacheDirectory()
3166  << " contains preexisting data for new table. Please "
3167  "delete or clear cache before continuing";
3168  }
3169 
3170  addTableToMap(&td, cds, dds);
3171  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
3172  if (!td.storageType.empty() && td.storageType != StorageType::FOREIGN_TABLE) {
3173  dataMgr_->getForeignStorageInterface()->registerTable(this, td, cds);
3174  }
3175  } catch (std::exception& e) {
3176  sqliteConnector_.query("ROLLBACK TRANSACTION");
3177  removeTableFromMap(td.tableName, td.tableId, true);
3178  throw;
3179  }
3180  sqliteConnector_.query("END TRANSACTION");
3181 
3183  write_lock.unlock();
3184  sqlite_lock.unlock();
3186  true); // cause instantiateFragmenter() to be called
3187  }
3188 }
int32_t maxRollbackEpochs
void serializeTableJsonUnlocked(const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
Definition: Catalog.cpp:3190
int64_t get_next_refresh_time(const foreign_storage::ForeignTable &foreign_table)
Definition: Catalog.cpp:2932
std::string virtualExpr
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:392
void set_size(int s)
Definition: sqltypes.h:478
std::string partitions
HOST DEVICE int get_size() const
Definition: sqltypes.h:403
T getData(const int row, const int col)
std::string tableName
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
HOST DEVICE int get_scale() const
Definition: sqltypes.h:396
std::string storageType
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, bool is_logical_table, bool use_temp_dictionary=false)
Definition: Catalog.cpp:4066
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:722
virtual void query(const std::string &queryString)
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:391
std::vector< int > columnIdBySpi_
std::string to_string(char const *&&v)
void addTableToMap(const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
Definition: Catalog.cpp:1545
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:265
int getDatabaseId() const
Definition: Catalog.h:326
specifies the content in-memory of a row in the column metadata table
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:2784
std::string keyMetainfo
bool g_serialize_temp_tables
Definition: Catalog.cpp:109
void set_comp_param(int p)
Definition: sqltypes.h:482
std::optional< std::string > default_value
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:725
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:399
Fragmenter_Namespace::FragmenterType fragType
Data_Namespace::MemoryLevel persistenceLevel
const Catalog * getObjForLock()
Definition: Catalog.cpp:260
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:393
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1604
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:402
bool setColumnSharedDictionary(ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:3997
#define CHECK(condition)
Definition: Logger.h:291
Descriptor for a dictionary for a string columne.
std::string viewSQL
SQLTypeInfo columnType
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:398
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:126
static constexpr char const * FOREIGN_TABLE
bool g_enable_fsi
Definition: Catalog.cpp:96
std::string columnName
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
bool is_array() const
Definition: sqltypes.h:585

+ Here is the call graph for this function:

TableDescriptor * Catalog_Namespace::Catalog::createTableFromDiskUnlocked ( int32_t  table_id)
private

Definition at line 5253 of file Catalog.cpp.

References logger::FATAL, StorageType::FOREIGN_TABLE, LOG, and to_string().

5253  {
5254  std::string query(
5255  "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
5256  "max_chunk_size, frag_page_size, max_rows, partitions, shard_column_id, shard, "
5257  "num_shards, key_metainfo, userid, sort_column_id, storage_type, "
5258  "max_rollback_epochs, is_system_table from mapd_tables WHERE tableid = " +
5259  std::to_string(table_id));
5260  sqliteConnector_.query(query);
5261  auto numRows = sqliteConnector_.getNumRows();
5262  if (!numRows) {
5263  throw NoTableFoundException(table_id);
5264  }
5265 
5266  const auto& storage_type = sqliteConnector_.getData<string>(0, 17);
5267  if (!storage_type.empty() && storage_type != StorageType::FOREIGN_TABLE) {
5268  const auto& table_name = sqliteConnector_.getData<string>(0, 1);
5269  LOG(FATAL) << "Unable to read Catalog metadata: storage type is currently not a "
5270  "supported table option (table "
5271  << table_name << " [" << table_id << "] in database " << currentDB_.dbName
5272  << ").";
5273  }
5274 
5275  // TODO(Misiu): Get rid of manual memory allocation and use smart pointers for
5276  // TableDecriptorMap_. Currently we use a smartpointer to cleanup if we catch
5277  // exceptions during initialization and then release ownership into the existing system.
5278  std::unique_ptr<TableDescriptor> td;
5279  td = (storage_type == StorageType::FOREIGN_TABLE)
5280  ? std::make_unique<foreign_storage::ForeignTable>()
5281  : std::make_unique<TableDescriptor>();
5282 
5283  td->tableId = sqliteConnector_.getData<int>(0, 0);
5284  td->tableName = sqliteConnector_.getData<string>(0, 1);
5285  td->nColumns = sqliteConnector_.getData<int>(0, 2);
5286  td->isView = sqliteConnector_.getData<bool>(0, 3);
5287  td->fragments = sqliteConnector_.getData<string>(0, 4);
5288  td->fragType = static_cast<Fragmenter_Namespace::FragmenterType>(
5289  sqliteConnector_.getData<int>(0, 5));
5290  td->maxFragRows = sqliteConnector_.getData<int>(0, 6);
5291  td->maxChunkSize = sqliteConnector_.getData<int64_t>(0, 7);
5292  td->fragPageSize = sqliteConnector_.getData<int>(0, 8);
5293  td->maxRows = sqliteConnector_.getData<int64_t>(0, 9);
5294  td->partitions = sqliteConnector_.getData<string>(0, 10);
5295  td->shardedColumnId = sqliteConnector_.getData<int>(0, 11);
5296  td->shard = sqliteConnector_.getData<int>(0, 12);
5297  td->nShards = sqliteConnector_.getData<int>(0, 13);
5298  td->keyMetainfo = sqliteConnector_.getData<string>(0, 14);
5299  td->userId = sqliteConnector_.getData<int>(0, 15);
5300  td->sortedColumnId =
5301  sqliteConnector_.isNull(0, 16) ? 0 : sqliteConnector_.getData<int>(0, 16);
5302  td->storageType = storage_type;
5303  td->maxRollbackEpochs = sqliteConnector_.getData<int>(0, 18);
5304  td->is_system_table = sqliteConnector_.getData<bool>(0, 19);
5305  td->hasDeletedCol = false;
5306 
5307  if (td->isView) {
5308  updateViewUnlocked(*td);
5309  } else {
5310  td->fragmenter = nullptr;
5311  }
5312 
5313  if (auto ftd = dynamic_cast<foreign_storage::ForeignTable*>(td.get())) {
5315  }
5316 
5317  return td.release();
5318 }
void updateViewUnlocked(TableDescriptor &)
Definition: Catalog.cpp:1398
T getData(const int row, const int col)
#define LOG(tag)
Definition: Logger.h:285
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
void reloadForeignTableUnlocked(foreign_storage::ForeignTable &foreign_table)
Definition: Catalog.cpp:5152
bool isNull(const int row, const int col) const
static constexpr char const * FOREIGN_TABLE
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::delDictionaryNontransactional ( const ColumnDescriptor cd)
private

Definition at line 2290 of file Catalog.cpp.

References CHECK, CHECK_GT, ColumnDescriptor::columnType, g_base_path, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::is_string(), SQLTypeInfo::is_string_array(), shared::kDataDirectoryName, kENCODING_DICT, File_Namespace::renameForDelete(), ColumnDescriptor::tableId, to_string(), and VLOG.

2290  {
2291  cat_write_lock write_lock(this);
2292  cat_sqlite_lock sqlite_lock(getObjForLock());
2293  if (!(cd.columnType.is_string() || cd.columnType.is_string_array())) {
2294  return;
2295  }
2296  if (!(cd.columnType.get_compression() == kENCODING_DICT)) {
2297  return;
2298  }
2299  const auto dictId = cd.columnType.get_comp_param();
2300  CHECK_GT(dictId, 0);
2301  // decrement and zero check dict ref count
2302  const auto td = getMetadataForTable(cd.tableId, false);
2303  CHECK(td);
2305  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
2306  std::to_string(dictId));
2308  "SELECT refcount FROM mapd_dictionaries WHERE dictid = ?", std::to_string(dictId));
2309  const auto refcount = sqliteConnector_.getData<int>(0, 0);
2310  VLOG(3) << "Dictionary " << dictId << "from dropped table has reference count "
2311  << refcount;
2312  if (refcount > 0) {
2313  return;
2314  }
2315  const DictRef dictRef(currentDB_.dbId, dictId);
2316  sqliteConnector_.query_with_text_param("DELETE FROM mapd_dictionaries WHERE dictid = ?",
2317  std::to_string(dictId));
2319  "/DB_" + std::to_string(currentDB_.dbId) + "_DICT_" +
2320  std::to_string(dictId));
2321 
2322  std::unique_ptr<StringDictionaryClient> client;
2323  if (!string_dict_hosts_.empty()) {
2324  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dictRef, true));
2325  }
2326  if (client) {
2327  client->drop(dictRef);
2328  }
2329 
2330  dictDescriptorMapByRef_.erase(dictRef);
2331 }
const std::string kDataDirectoryName
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:724
const DBMetadata currentDB_
Definition: Catalog.h:721
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:711
#define CHECK_GT(x, y)
Definition: Logger.h:305
std::string to_string(char const *&&v)
std::string g_base_path
Definition: SysCatalog.cpp:62
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:399
const Catalog * getObjForLock()
Definition: Catalog.cpp:260
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:402
#define CHECK(condition)
Definition: Logger.h:291
SQLTypeInfo columnType
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
bool is_string() const
Definition: sqltypes.h:561
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:210
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:126
bool is_string_array() const
Definition: sqltypes.h:564
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
#define VLOG(n)
Definition: Logger.h:388

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::delDictionaryTransactional ( const ColumnDescriptor cd)

Definition at line 2277 of file Catalog.cpp.

Referenced by AlterTableAlterColumnCommandRecoveryMgr::cleanupDeleteDictionaries(), and AlterTableAlterColumnCommandRecoveryMgr::rollback().

2277  {
2278  cat_write_lock write_lock(this);
2279  cat_sqlite_lock sqlite_lock(this);
2280  sqliteConnector_.query("BEGIN TRANSACTION");
2281  try {
2283  } catch (std::exception& e) {
2284  sqliteConnector_.query("ROLLBACK TRANSACTION");
2285  throw;
2286  }
2287  sqliteConnector_.query("END TRANSACTION");
2288 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
void delDictionaryNontransactional(const ColumnDescriptor &cd)
Definition: Catalog.cpp:2290
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:126

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::deleteCustomExpressions ( const std::vector< int32_t > &  custom_expression_ids,
bool  do_soft_delete 
)

Deletes custom expressions with the given ids.

Parameters
custom_expression_ids- ids of custom expressions to delete
do_soft_delete- flag indicating whether or not to do a soft delete

Definition at line 6079 of file Catalog.cpp.

References CHECK, CHECK_EQ, join(), and to_string().

6080  {
6081  cat_write_lock write_lock(this);
6082  cat_sqlite_lock sqlite_lock(getObjForLock());
6083 
6084  std::vector<int32_t> invalid_ids;
6085  for (const auto id : custom_expression_ids) {
6086  if (custom_expr_map_by_id_.find(id) == custom_expr_map_by_id_.end()) {
6087  invalid_ids.emplace_back(id);
6088  }
6089  }
6090  if (!invalid_ids.empty()) {
6091  throw std::runtime_error{"Custom expressions with ids: " + join(invalid_ids, ",") +
6092  " do not exist."};
6093  }
6094  sqliteConnector_.query("BEGIN TRANSACTION");
6095  try {
6096  for (const auto id : custom_expression_ids) {
6098  "SELECT id FROM omnisci_custom_expressions WHERE id = ?",
6099  std::vector<std::string>{std::to_string(id)});
6100  CHECK_EQ(sqliteConnector_.getNumRows(), static_cast<size_t>(1));
6101  if (do_soft_delete) {
6103  "UPDATE omnisci_custom_expressions SET is_deleted = ? WHERE id = ?",
6104  std::vector<std::string>{std::to_string(true), std::to_string(id)});
6105  } else {
6107  "DELETE FROM omnisci_custom_expressions WHERE id = ?",
6108  std::vector<std::string>{std::to_string(id)});
6109  }
6110  }
6111 
6112  for (const auto id : custom_expression_ids) {
6113  if (do_soft_delete) {
6114  auto it = custom_expr_map_by_id_.find(id);
6115  CHECK(it != custom_expr_map_by_id_.end());
6116  it->second->is_deleted = true;
6117  } else {
6118  custom_expr_map_by_id_.erase(id);
6119  }
6120  }
6121  } catch (std::exception& e) {
6122  sqliteConnector_.query("ROLLBACK TRANSACTION");
6123  throw;
6124  }
6125  sqliteConnector_.query("END TRANSACTION");
6126 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
std::string join(T const &container, std::string const &delim)
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
const Catalog * getObjForLock()
Definition: Catalog.cpp:260
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
#define CHECK(condition)
Definition: Logger.h:291
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:717
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:126
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::deleteMetadataForDashboards ( const std::vector< int32_t >  ids,
const UserMetadata user 
)

Definition at line 2014 of file Catalog.cpp.

References DashboardDBObjectType, AccessPrivileges::DELETE_DASHBOARD, lockmgr::instance(), and to_string().

2015  {
2016  std::stringstream invalid_ids, restricted_ids;
2017 
2018  for (int32_t dashboard_id : dashboard_ids) {
2019  if (!getMetadataForDashboard(dashboard_id)) {
2020  invalid_ids << (!invalid_ids.str().empty() ? ", " : "") << dashboard_id;
2021  continue;
2022  }
2023  DBObject object(dashboard_id, DashboardDBObjectType);
2024  object.loadKey(*this);
2025  object.setPrivileges(AccessPrivileges::DELETE_DASHBOARD);
2026  std::vector<DBObject> privs = {object};
2027  if (!SysCatalog::instance().checkPrivileges(user, privs)) {
2028  restricted_ids << (!restricted_ids.str().empty() ? ", " : "") << dashboard_id;
2029  }
2030  }
2031 
2032  if (invalid_ids.str().size() > 0 || restricted_ids.str().size() > 0) {
2033  std::stringstream error_message;
2034  error_message << "Delete dashboard(s) failed with error(s):";
2035  if (invalid_ids.str().size() > 0) {
2036  error_message << "\nDashboard id: " << invalid_ids.str()
2037  << " - Dashboard id does not exist";
2038  }
2039  if (restricted_ids.str().size() > 0) {
2040  error_message
2041  << "\nDashboard id: " << restricted_ids.str()
2042  << " - User should be either owner of dashboard or super user to delete it";
2043  }
2044  throw std::runtime_error(error_message.str());
2045  }
2046  std::vector<DBObject> dash_objs;
2047 
2048  for (int32_t dashboard_id : dashboard_ids) {
2049  dash_objs.emplace_back(dashboard_id, DashboardDBObjectType);
2050  }
2051  // BE-5245: Transactionally unsafe (like other combined Catalog/Syscatalog operations)
2053  {
2054  cat_write_lock write_lock(this);
2055  cat_sqlite_lock sqlite_lock(getObjForLock());
2056 
2057  sqliteConnector_.query("BEGIN TRANSACTION");
2058  try {
2059  for (int32_t dashboard_id : dashboard_ids) {
2060  auto dash = getMetadataForDashboard(dashboard_id);
2061  // Dash should still exist if revokeDBObjectPrivileges passed but throw and
2062  // rollback if already deleted
2063  if (!dash) {
2064  throw std::runtime_error(
2065  std::string("Delete dashboard(s) failed with error(s):\nDashboard id: ") +
2066  std::to_string(dashboard_id) + " - Dashboard id does not exist ");
2067  }
2068  std::string user_id = std::to_string(dash->userId);
2069  std::string dash_name = dash->dashboardName;
2070  auto viewDescIt = dashboardDescriptorMap_.find(user_id + ":" + dash_name);
2071  dashboardDescriptorMap_.erase(viewDescIt);
2073  "DELETE FROM mapd_dashboards WHERE name = ? and userid = ?",
2074  std::vector<std::string>{dash_name, user_id});
2075  }
2076  } catch (std::exception& e) {
2077  sqliteConnector_.query("ROLLBACK TRANSACTION");
2078  throw;
2079  }
2080  sqliteConnector_.query("END TRANSACTION");
2081  }
2082 }
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:712
std::string to_string(char const *&&v)
static SysCatalog & instance()
Definition: SysCatalog.h:343
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const
const Catalog * getObjForLock()
Definition: Catalog.cpp:260
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
static const AccessPrivileges DELETE_DASHBOARD
Definition: DBObject.h:173
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:126
void revokeDBObjectPrivilegesFromAllBatch(std::vector< DBObject > &objects, Catalog *catalog)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::deleteTableCatalogMetadata ( const TableDescriptor logical_table,
const std::vector< const TableDescriptor * > &  physical_tables 
)
private

Definition at line 4320 of file Catalog.cpp.

References TableDescriptor::tableId, and to_string().

4322  {
4323  cat_write_lock write_lock(this);
4324  cat_sqlite_lock sqlite_lock(getObjForLock());
4325  sqliteConnector_.query("BEGIN TRANSACTION");
4326  try {
4327  // remove corresponding record from the logicalToPhysicalTableMap in sqlite database
4329  "DELETE FROM mapd_logical_to_physical WHERE logical_table_id = ?",
4330  std::to_string(logical_table->tableId));
4331  logicalToPhysicalTableMapById_.erase(logical_table->tableId);
4332  for (auto table : physical_tables) {
4333  eraseTableMetadata(table);
4334  }
4335  } catch (std::exception& e) {
4336  sqliteConnector_.query("ROLLBACK TRANSACTION");
4337  throw;
4338  }
4339  sqliteConnector_.query("END TRANSACTION");
4340 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
const Catalog * getObjForLock()
Definition: Catalog.cpp:260
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
void eraseTableMetadata(const TableDescriptor *td)
Definition: Catalog.cpp:4342
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:126
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:727

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::doTruncateTable ( const TableDescriptor td)
protected

Definition at line 4153 of file Catalog.cpp.

References CHECK, CHECK_GE, ColumnDescriptor::columnType, Data_Namespace::CPU_LEVEL, dict_ref_t::dictId, DictDescriptor::dictIsTemp, DictDescriptor::dictRef, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), Data_Namespace::GPU_LEVEL, lockmgr::instance(), kENCODING_DICT, File_Namespace::renameForDelete(), ColumnDescriptor::tableId, and TableDescriptor::tableId.

4153  {
4154  // must destroy fragmenter before deleteChunks is called.
4156 
4157  const int tableId = td->tableId;
4158  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
4159  // assuming deleteChunksWithPrefix is atomic
4160  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
4161  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
4162 
4163  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
4164 
4165  cat_write_lock write_lock(this);
4166  std::unique_ptr<StringDictionaryClient> client;
4167  if (SysCatalog::instance().isAggregator()) {
4168  CHECK(!string_dict_hosts_.empty());
4169  DictRef dict_ref(currentDB_.dbId, -1);
4170  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
4171  }
4172  // clean up any dictionaries
4173  // delete all column descriptors for the table
4174  for (const auto& columnDescriptor : columnDescriptorMapById_) {
4175  auto cd = columnDescriptor.second;
4176  if (cd->tableId != td->tableId) {
4177  continue;
4178  }
4179  const int dict_id = cd->columnType.get_comp_param();
4180  // Dummy dictionaries created for a shard of a logical table have the id set to zero.
4181  if (cd->columnType.get_compression() == kENCODING_DICT && dict_id) {
4182  const DictRef dict_ref(currentDB_.dbId, dict_id);
4183  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
4184  CHECK(dictIt != dictDescriptorMapByRef_.end());
4185  const auto& dd = dictIt->second;
4186  CHECK_GE(dd->refcount, 1);
4187  // if this is the only table using this dict reset the dict
4188  if (dd->refcount == 1) {
4189  // close the dictionary
4190  dd->stringDict.reset();
4191  File_Namespace::renameForDelete(dd->dictFolderPath);
4192  if (client) {
4193  client->drop(dd->dictRef);
4194  }
4195  if (!dd->dictIsTemp) {
4196  boost::filesystem::create_directory(dd->dictFolderPath);
4197  }
4198  }
4199 
4200  DictDescriptor* new_dd = new DictDescriptor(dd->dictRef,
4201  dd->dictName,
4202  dd->dictNBits,
4203  dd->dictIsShared,
4204  dd->refcount,
4205  dd->dictFolderPath,
4206  dd->dictIsTemp);
4207  dictDescriptorMapByRef_.erase(dictIt);
4208  // now create new Dict -- need to figure out what to do here for temp tables
4209  if (client) {
4210  client->create(new_dd->dictRef, new_dd->dictIsTemp);
4211  }
4212  dictDescriptorMapByRef_[new_dd->dictRef].reset(new_dd);
4214  }
4215  }
4216 }
std::vector< int > ChunkKey
Definition: types.h:36
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:724
const DBMetadata currentDB_
Definition: Catalog.h:721
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:711
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:722
#define CHECK_GE(x, y)
Definition: Logger.h:306
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:710
static SysCatalog & instance()
Definition: SysCatalog.h:343
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1907
int32_t dictId
Definition: DictRef.h:14
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
void removeFragmenterForTable(const int table_id) const
Definition: Catalog.cpp:4270
#define CHECK(condition)
Definition: Logger.h:291
Descriptor for a dictionary for a string columne.
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:210

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropColumn ( const TableDescriptor td,
const ColumnDescriptor cd 
)

Definition at line 2663 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnId, ColumnDescriptor::columnName, TableDescriptor::nShards, TableDescriptor::shard, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), and to_upper().

2663  {
2664  {
2665  cat_write_lock write_lock(this);
2666  cat_sqlite_lock sqlite_lock(getObjForLock());
2667  // caller must handle sqlite/chunk transaction TOGETHER
2669  "DELETE FROM mapd_columns where tableid = ? and columnid = ?",
2670  std::vector<std::string>{std::to_string(td.tableId),
2671  std::to_string(cd.columnId)});
2672 
2674  "UPDATE mapd_tables SET ncolumns = ncolumns - 1 WHERE tableid = ?",
2675  std::vector<std::string>{std::to_string(td.tableId)});
2676 
2677  ColumnDescriptorMap::iterator columnDescIt =
2679  CHECK(columnDescIt != columnDescriptorMap_.end());
2680 
2681  columnDescriptorsForRoll.emplace_back(columnDescIt->second, nullptr);
2682  removeFromColumnMap(columnDescIt->second);
2683  --tableDescriptorMapById_[td.tableId]->nColumns;
2684  }
2685 
2686  // for each shard
2687  if (td.nShards > 0 && td.shard < 0) {
2688  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
2689  const auto shard_cd = getMetadataForColumn(shard->tableId, cd.columnId);
2690  CHECK(shard_cd);
2691  dropColumn(*shard, *shard_cd);
2692  }
2693  }
2694 }
void removeFromColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:7015
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:709
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
void dropColumn(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:2663
std::string to_string(char const *&&v)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4879
std::string to_upper(const std::string &str)
const Catalog * getObjForLock()
Definition: Catalog.cpp:260
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
#define CHECK(condition)
Definition: Logger.h:291
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:126
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:739
std::string columnName

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropColumnNontransactional ( const TableDescriptor td,
const ColumnDescriptor cd 
)

Definition at line 2617 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnId, ColumnDescriptor::columnName, TableDescriptor::nShards, TableDescriptor::shard, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and to_upper().

2618  {
2619  cat_write_lock write_lock(this);
2620  cat_sqlite_lock sqlite_lock(this);
2621 
2623  "DELETE FROM mapd_columns where tableid = ? and columnid = ?",
2624  std::vector<std::string>{std::to_string(td.tableId), std::to_string(cd.columnId)});
2625 
2627  "UPDATE mapd_tables SET ncolumns = ncolumns - 1 WHERE tableid = ?",
2628  std::vector<std::string>{std::to_string(td.tableId)});
2629 
2630  ColumnDescriptorMap::iterator columnDescIt =
2632  CHECK(columnDescIt != columnDescriptorMap_.end());
2633 
2634  auto ocd = columnDescIt->second;
2635  removeFromColumnMap(ocd);
2636  --tableDescriptorMapById_[td.tableId]->nColumns;
2638  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
2639 
2640  // for each shard
2641  if (td.nShards > 0 && td.shard < 0) {
2642  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
2643  const auto shard_cd = getMetadataForColumn(shard->tableId, cd.columnId);
2644  CHECK(shard_cd);
2645  dropColumnNontransactional(*shard, *shard_cd);
2646  }
2647  }
2648 }
void removeFromColumnMap(ColumnDescriptor *cd)
Definition: Catalog.cpp:7015
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
std::string tableName
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:709
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
const DBMetadata currentDB_
Definition: Catalog.h:721
std::string to_string(char const *&&v)
void removeColumnDescriptor(const ColumnDescriptor *cd)
Definition: Catalog.cpp:2696
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:708
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4879
std::string to_upper(const std::string &str)
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:725
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
#define CHECK(condition)
Definition: Logger.h:291
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:126
std::string columnName
void dropColumnNontransactional(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:2617

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropColumnPolicies ( const TableDescriptor td,
const ColumnDescriptor cd 
)

Definition at line 2650 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnId, TableDescriptor::nShards, and TableDescriptor::shard.

2650  {
2651 
2652  // for each shard
2653  if (td.nShards > 0 && td.shard < 0) {
2654  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
2655  const auto shard_cd = getMetadataForColumn(shard->tableId, cd.columnId);
2656  CHECK(shard_cd);
2657  dropColumnPolicies(*shard, *shard_cd);
2658  }
2659  }
2660 }
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4879
void dropColumnPolicies(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:2650
#define CHECK(condition)
Definition: Logger.h:291
void Catalog_Namespace::Catalog::dropColumnTransactional ( const TableDescriptor td,
const ColumnDescriptor cd 
)

Definition at line 2601 of file Catalog.cpp.

Referenced by AlterTableAlterColumnCommandRecoveryMgr::cleanupDropSourceGeoColumns(), and AlterTableAlterColumnCommandRecoveryMgr::rollback().

2602  {
2603  dropColumnPolicies(td, cd);
2604 
2605  cat_write_lock write_lock(this);
2606  cat_sqlite_lock sqlite_lock(this);
2607  sqliteConnector_.query("BEGIN TRANSACTION");
2608  try {
2610  } catch (std::exception& e) {
2611  sqliteConnector_.query("ROLLBACK TRANSACTION");
2612  throw;
2613  }
2614  sqliteConnector_.query("END TRANSACTION");
2615 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
void dropColumnPolicies(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:2650
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:125
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:126
void dropColumnNontransactional(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:2617

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::dropForeignServer ( const std::string &  server_name)

Drops/deletes a foreign server DB object.

Parameters
server_name- Name of foreign server that will be deleted

Definition at line 3460 of file Catalog.cpp.

References CHECK_EQ, and to_string().

3460  {
3461  cat_write_lock write_lock(this);
3462  cat_sqlite_lock sqlite_lock(getObjForLock());
3463 
3465  "SELECT id from omnisci_foreign_servers where name = ?",
3466  std::vector<std::string>{server_name});
3467  auto num_rows = sqliteConnector_.getNumRows();
3468  if (num_rows > 0) {
3469  CHECK_EQ(size_t(1), num_rows);
3470  auto server_id = sqliteConnector_.getData<int32_t>(0, 0);
3472  "SELECT table_id from omnisci_foreign_tables where server_id = ?",
3473  std::to_string(server_id));
3474  if (sqliteConnector_.getNumRows() > 0) {
3475  throw std::runtime_error{"Foreign server \"" + server_name +
3476  "\" is referenced "
3477  "by existing foreign tables and cannot be dropped."};
3478  }
3479  sqliteConnector_.query("BEGIN TRANSACTION");
3480  try {
3482  "DELETE FROM omnisci_foreign_servers WHERE name = ?",
3483  std::vector<std::string>{server_name});
3484  } catch (const std::exception& e) {
3485  sqliteConnector_.query("ROLLBACK TRANSACTION");
3486  throw;
3487  }
3488  sqliteConnector_.query("END TRANSACTION");
3489  foreignServerMap_.erase(server_name);
3490  foreignServerMapById_.erase(server_id);
3491  }
3492 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:720
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)