class for a per-database catalog. also includes metadata for the current database and the current user. More...

#include <Catalog.h>

Collaboration diagram for Catalog_Namespace::Catalog:

Public Member Functions
	Catalog (const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
	Constructor - takes basePath to already extant data directory for writing. More...

	Catalog ()
	Constructor builds a hollow catalog used during constructor of other catalogs. More...

	~Catalog ()
	Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite. More...

void	createTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)

void	createShardedTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)

int32_t	createDashboard (DashboardDescriptor &vd)

void	replaceDashboard (DashboardDescriptor &vd)

std::string	createLink (LinkDescriptor &ld, size_t min_length)

void	dropTable (const TableDescriptor *td)

void	truncateTable (const TableDescriptor *td)

void	renameTable (const TableDescriptor *td, const std::string &newTableName)

void	renameTable (std::vector< std::pair< std::string, std::string >> &names)

void	renameColumn (const TableDescriptor td, const ColumnDescriptor cd, const std::string &newColumnName)

void	addColumn (const TableDescriptor &td, ColumnDescriptor &cd)

void	dropColumn (const TableDescriptor &td, const ColumnDescriptor &cd)

void	invalidateCachesForTable (const int table_id)

void	removeFragmenterForTable (const int table_id) const

const std::map< int, const ColumnDescriptor * >	getDictionaryToColumnMapping ()

const TableDescriptor *	getMetadataForTable (const std::string &tableName, const bool populateFragmenter=true) const
	Returns a pointer to a const TableDescriptor struct matching the provided tableName. More...

const TableDescriptor *	getMetadataForTable (int tableId, bool populateFragmenter=true) const

std::optional< std::string >	getTableName (int32_t table_id) const

std::optional< int32_t >	getTableId (const std::string &table_name) const

const ColumnDescriptor *	getMetadataForColumn (int tableId, const std::string &colName) const

const ColumnDescriptor *	getMetadataForColumn (int tableId, int columnId) const

const std::optional< std::string >	getColumnName (int table_id, int column_id) const

const int	getColumnIdBySpi (const int tableId, const size_t spi) const

const ColumnDescriptor *	getMetadataForColumnBySpi (const int tableId, const size_t spi) const

const DashboardDescriptor *	getMetadataForDashboard (const std::string &userId, const std::string &dashName) const

const DashboardDescriptor *	getMetadataForDashboard (const int32_t dashboard_id) const

void	deleteMetadataForDashboards (const std::vector< int32_t > ids, const UserMetadata &user)

const LinkDescriptor *	getMetadataForLink (const std::string &link) const

const LinkDescriptor *	getMetadataForLink (int linkId) const

const foreign_storage::ForeignTable *	getForeignTable (const std::string &tableName) const

const foreign_storage::ForeignTable *	getForeignTable (int table_id) const

std::list< const ColumnDescriptor * >	getAllColumnMetadataForTable (const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
	Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular table specified by table id. More...

std::list< const TableDescriptor * >	getAllTableMetadata () const

std::vector< TableDescriptor >	getAllTableMetadataCopy () const

std::list< const DashboardDescriptor * >	getAllDashboardsMetadata () const

std::vector< DashboardDescriptor >	getAllDashboardsMetadataCopy () const

const DBMetadata &	getCurrentDB () const

Data_Namespace::DataMgr &	getDataMgr () const

std::shared_ptr< Calcite >	getCalciteMgr () const

const std::string &	getCatalogBasePath () const

const DictDescriptor *	getMetadataForDict (int dict_ref, bool loadDict=true) const

const std::vector< LeafHostInfo > &	getStringDictionaryHosts () const

const ColumnDescriptor *	getShardColumnMetadataForTable (const TableDescriptor *td) const

std::vector< const TableDescriptor * >	getPhysicalTablesDescriptors (const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const

std::vector< std::pair < int32_t, int32_t > >	getAllPersistedTableAndShardIds () const

std::vector< std::string >	getTableNamesForUser (const UserMetadata &user, const GetTablesType get_tables_type) const

std::vector< TableMetadata >	getTablesMetadataForUser (const UserMetadata &user_metadata, const GetTablesType get_tables_type, const std::string &filter_table_name) const

int32_t	getTableEpoch (const int32_t db_id, const int32_t table_id) const

void	setTableEpoch (const int db_id, const int table_id, const int new_epoch)

void	setMaxRollbackEpochs (const int32_t table_id, const int32_t max_rollback_epochs)

void	setMaxRows (const int32_t table_id, const int64_t max_rows)

std::vector< TableEpochInfo >	getTableEpochs (const int32_t db_id, const int32_t table_id) const

void	setTableEpochs (const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const

void	setTableEpochsLogExceptions (const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const

int	getDatabaseId () const

SqliteConnector &	getSqliteConnector ()

void	roll (const bool forward)

DictRef	addDictionary (ColumnDescriptor &cd)

void	delDictionary (const ColumnDescriptor &cd)

void	getDictionary (const ColumnDescriptor &cd, std::map< int, StringDictionary * > &stringDicts)

const bool	checkMetadataForDeletedRecs (const TableDescriptor *td, int column_id) const

const ColumnDescriptor *	getDeletedColumn (const TableDescriptor *td) const

const ColumnDescriptor *	getDeletedColumnIfRowsDeleted (const TableDescriptor *td) const

void	setDeletedColumn (const TableDescriptor td, const ColumnDescriptor cd)

int	getLogicalTableId (const int physicalTableId) const

void	checkpoint (const int logicalTableId) const

void	checkpointWithAutoRollback (const int logical_table_id) const

void	resetTableEpochFloor (const int logicalTableId) const

std::string	name () const

void	eraseDbMetadata ()

void	eraseDbPhysicalData ()

void	eraseTablePhysicalData (const TableDescriptor *td)

void	setForReload (const int32_t tableId)

std::vector< std::string >	getTableDataDirectories (const TableDescriptor *td) const

std::vector< std::string >	getTableDictDirectories (const TableDescriptor *td) const

std::set< std::string >	getTableDictDirectoryPaths (int32_t table_id) const

std::string	getColumnDictDirectory (const ColumnDescriptor *cd, bool file_name_only=true) const

std::string	dumpSchema (const TableDescriptor *td) const

std::string	dumpCreateTable (const TableDescriptor *td, bool multiline_formatting=true, bool dump_defaults=false) const

std::optional< std::string >	dumpCreateTable (int32_t table_id, bool multiline_formatting=true, bool dump_defaults=false) const

std::string	dumpCreateServer (const std::string &name, bool multiline_formatting=true) const

void	createForeignServer (std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)

const foreign_storage::ForeignServer *	getForeignServer (const std::string &server_name) const

const std::unique_ptr< const foreign_storage::ForeignServer >	getForeignServerFromStorage (const std::string &server_name)

const std::unique_ptr< const foreign_storage::ForeignTable >	getForeignTableFromStorage (int table_id)

void	changeForeignServerOwner (const std::string &server_name, const int new_owner_id)

void	setForeignServerDataWrapper (const std::string &server_name, const std::string &data_wrapper)

void	setForeignServerOptions (const std::string &server_name, const std::string &options)

void	renameForeignServer (const std::string &server_name, const std::string &name)

void	dropForeignServer (const std::string &server_name)

std::vector< const foreign_storage::ForeignTable * >	getAllForeignTablesForForeignServer (const int32_t foreign_server_id)
	Get all of the foreign tables for associated with a foreign server id. More...

void	getForeignServersForUser (const rapidjson::Value filters, const UserMetadata &user, std::vector< const foreign_storage::ForeignServer > &results)

void	createDefaultServersIfNotExists ()

bool	validateNonExistentTableOrView (const std::string &name, const bool if_not_exists)

std::vector< const TableDescriptor * >	getAllForeignTablesForRefresh () const

void	updateForeignTableRefreshTimes (const int32_t table_id)

void	setForeignTableOptions (const std::string &table_name, foreign_storage::OptionsMap &options_map, bool clear_existing_options=true)

void	updateLeaf (const LeafHostInfo &string_dict_host)

void	setUncappedTableEpoch (const std::string &table_name)

int32_t	createCustomExpression (std::unique_ptr< CustomExpression > custom_expression)

const CustomExpression *	getCustomExpression (int32_t custom_expression_id) const

const std::unique_ptr< const CustomExpression >	getCustomExpressionFromStorage (int32_t custom_expression_id)

std::vector< const CustomExpression * >	getCustomExpressionsForUser (const UserMetadata &user) const

void	updateCustomExpression (int32_t custom_expression_id, const std::string &expression_json)

void	deleteCustomExpressions (const std::vector< int32_t > &custom_expression_ids, bool do_soft_delete)

void	reassignOwners (const std::set< std::string > &old_owners, const std::string &new_owner)

bool	isInfoSchemaDb () const

Static Public Member Functions
static void	expandGeoColumn (const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)

static const std::string	getForeignTableSchema (bool if_not_exists=false)

static const std::string	getForeignServerSchema (bool if_not_exists=false)

static const std::string	getCustomExpressionsSchema (bool if_not_exists=false)

Public Attributes
std::unique_ptr < heavyai::DistributedSharedMutex >	dcatalogMutex_

std::unique_ptr < heavyai::DistributedSharedMutex >	dsqliteMutex_

std::mutex	sqliteMutex_

heavyai::shared_mutex	sharedMutex_

std::atomic< std::thread::id >	thread_holding_sqlite_lock

std::atomic< std::thread::id >	thread_holding_write_lock

bool	initialized_ = false

Static Public Attributes
static thread_local bool	thread_holds_read_lock = false

Protected Types
using	ColumnDescriptorsForRoll = std::vector< std::pair< ColumnDescriptor , ColumnDescriptor >>

Protected Member Functions
void	CheckAndExecuteMigrations ()

void	CheckAndExecuteMigrationsPostBuildMaps ()

void	updateDictionaryNames ()

void	updateTableDescriptorSchema ()

void	updateFixlenArrayColumns ()

void	updateGeoColumns ()

void	updateFrontendViewSchema ()

void	updateLinkSchema ()

void	updateFrontendViewAndLinkUsers ()

void	updateLogicalToPhysicalTableLinkSchema ()

void	updateLogicalToPhysicalTableMap (const int32_t logical_tb_id)

void	updateDictionarySchema ()

void	updatePageSize ()

void	updateDeletedColumnIndicator ()

void	updateDefaultColumnValues ()

void	updateFrontendViewsToDashboards ()

void	updateCustomExpressionsSchema ()

void	updateFsiSchemas ()

void	renameLegacyDataWrappers ()

void	recordOwnershipOfObjectsInObjectPermissions ()

void	checkDateInDaysColumnMigration ()

void	createDashboardSystemRoles ()

void	buildMaps ()

void	addTableToMap (const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)

void	addReferenceToForeignDict (ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def, const bool persist_reference)

bool	setColumnSharedDictionary (ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)

void	setColumnDictionary (ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, bool is_logical_table, bool use_temp_dictionary=false)

void	addFrontendViewToMap (DashboardDescriptor &vd)

void	addFrontendViewToMapNoLock (DashboardDescriptor &vd)

void	addLinkToMap (LinkDescriptor &ld)

void	removeTableFromMap (const std::string &tableName, const int tableId, const bool is_on_error=false)

void	eraseTableMetadata (const TableDescriptor *td)

void	executeDropTableSqliteQueries (const TableDescriptor *td)

void	doTruncateTable (const TableDescriptor *td)

void	renamePhysicalTable (const TableDescriptor *td, const std::string &newTableName)

void	renamePhysicalTable (std::vector< std::pair< std::string, std::string >> &names, std::vector< int > &tableIds)

void	instantiateFragmenter (TableDescriptor *td) const

void	getAllColumnMetadataForTableImpl (const TableDescriptor td, std::list< const ColumnDescriptor > &colDescs, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const

std::string	calculateSHA1 (const std::string &data)

std::string	generatePhysicalTableName (const std::string &logicalTableName, const int32_t &shardNumber)

std::vector< DBObject >	parseDashboardObjects (const std::string &view_meta, const int &user_id)

void	createOrUpdateDashboardSystemRole (const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)

const int	getColumnIdBySpiUnlocked (const int table_id, const size_t spi) const

void	serializeTableJsonUnlocked (const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const

void	dropTableFromJsonUnlocked (const std::string &table_name) const

Protected Attributes
std::string	basePath_

TableDescriptorMap	tableDescriptorMap_

TableDescriptorMapById	tableDescriptorMapById_

ColumnDescriptorMap	columnDescriptorMap_

ColumnDescriptorMapById	columnDescriptorMapById_

DictDescriptorMapById	dictDescriptorMapByRef_

DashboardDescriptorMap	dashboardDescriptorMap_

LinkDescriptorMap	linkDescriptorMap_

LinkDescriptorMapById	linkDescriptorMapById_

ForeignServerMap	foreignServerMap_

ForeignServerMapById	foreignServerMapById_

CustomExpressionMapById	custom_expr_map_by_id_

TableDictColumnsMap	dict_columns_by_table_id_

SqliteConnector	sqliteConnector_

const DBMetadata	currentDB_

std::shared_ptr < Data_Namespace::DataMgr >	dataMgr_

const std::vector< LeafHostInfo >	string_dict_hosts_

std::shared_ptr< Calcite >	calciteMgr_

LogicalToPhysicalTableMapById	logicalToPhysicalTableMapById_

int	nextTempTableId_

int	nextTempDictId_

ColumnDescriptorsForRoll	columnDescriptorsForRoll

Static Protected Attributes
static const std::string	physicalTableNameTag_

Private Member Functions
void	buildDictionaryMapUnlocked ()

void	reloadTableMetadata (int table_id)

void	reloadTableMetadataUnlocked (int table_id)

void	reloadCatalogMetadata (const std::map< int32_t, std::string > &user_name_by_user_id)

void	reloadCatalogMetadataUnlocked (const std::map< int32_t, std::string > &user_name_by_user_id)

void	refreshDictionaryCachesForTableUnlocked (const TableDescriptor &td)

void	buildTablesMapUnlocked ()

void	buildColumnsMapUnlocked ()

void	updateViewsInMapUnlocked ()

void	updateViewUnlocked (TableDescriptor &)

void	buildDashboardsMapUnlocked (const std::map< int32_t, std::string > &user_name_by_user_id)

void	buildLinksMapUnlocked ()

void	buildLogicalToPhysicalMapUnlocked ()

void	updateForeignTablesInMapUnlocked ()

void	gatherAdditionalInfo (std::vector< std::string > &additional_info, std::set< std::string > &shared_dict_column_names, const TableDescriptor *td) const

std::string	quoteIfRequired (const std::string &column_name) const

void	adjustAlteredTableFiles (const std::string &temp_data_dir, const std::unordered_map< int, int > &all_column_ids_map) const

void	renameTableDirectories (const std::string &temp_data_dir, const std::vector< std::string > &target_paths, const std::string &name_prefix) const

void	buildForeignServerMapUnlocked ()

void	setForeignServerProperty (const std::string &server_name, const std::string &property, const std::string &value)

void	setForeignTableProperty (const foreign_storage::ForeignTable *table, const std::string &property, const std::string &value)

void	alterPhysicalTableMetadata (const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)

void	alterTableMetadata (const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)

void	setTableFileMgrParams (const int table_id, const File_Namespace::FileMgrParams &file_mgr_params)

bool	filterTableByTypeAndUser (const TableDescriptor *td, const UserMetadata &user_metadata, const GetTablesType get_tables_type) const

TableDescriptor *	getMutableMetadataForTableUnlocked (int table_id) const

TableDescriptor *	getMutableMetadataForTableUnlocked (const std::string &table_name) const

void	createForeignServerNoLocks (std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)

foreign_storage::ForeignTable *	getForeignTableUnlocked (const std::string &tableName) const

const Catalog *	getObjForLock ()

void	removeChunks (const int table_id) const

void	buildCustomExpressionsMapUnlocked ()

std::unique_ptr< CustomExpression >	getCustomExpressionFromConnector (size_t row)

void	restoreOldOwners (const std::map< int32_t, std::string > &old_owners_user_name_by_id, const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects, int32_t new_owner_id)

void	restoreOldOwnersInMemory (const std::map< int32_t, std::string > &old_owners_user_name_by_id, const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects, int32_t new_owner_id)

void	conditionallyInitializeSystemObjects ()

void	initializeSystemServers ()

void	initializeSystemTables ()

void	initializeUsersSystemTable ()

void	initializeDatabasesSystemTable ()

void	initializePermissionsSystemTable ()

void	initializeRolesSystemTable ()

void	initializeTablesSystemTable ()

void	initializeDashboardsSystemTable ()

void	initializeRoleAssignmentsSystemTable ()

void	initializeMemorySummarySystemTable ()

void	initializeMemoryDetailsSystemTable ()

void	initializeStorageDetailsSystemTable ()

void	initializeServerLogsSystemTables ()

void	initializeRequestLogsSystemTables ()

void	initializeWebServerLogsSystemTables ()

void	initializeWebServerAccessLogsSystemTables ()

void	createSystemTableServer (const std::string &server_name, const std::string &data_wrapper_type, const foreign_storage::OptionsMap &options={})

std::pair < foreign_storage::ForeignTable, std::list< ColumnDescriptor > >	getSystemTableSchema (const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name, bool is_in_memory_system_table)

bool	recreateSystemTableIfUpdated (foreign_storage::ForeignTable &foreign_table, const std::list< ColumnDescriptor > &columns)

void	setDeletedColumnUnlocked (const TableDescriptor td, const ColumnDescriptor cd)

void	addToColumnMap (ColumnDescriptor *cd)

void	removeFromColumnMap (ColumnDescriptor *cd)

void	deleteTableCatalogMetadata (const TableDescriptor logical_table, const std::vector< const TableDescriptor > &physical_tables)

std::string	dumpCreateTableUnlocked (const TableDescriptor *td, bool multiline_formatting, bool dump_defaults) const

Private Attributes
DeletedColumnPerTableMap	deletedColumnPerTable_

Static Private Attributes
static constexpr const char *	CATALOG_SERVER_NAME {"system_catalog_server"}

static constexpr const char *	MEMORY_STATS_SERVER_NAME {"system_memory_stats_server"}

static constexpr const char *	STORAGE_STATS_SERVER_NAME {"system_storage_stats_server"}

static constexpr const char *	LOGS_SERVER_NAME {"system_logs_server"}

static constexpr std::array < const char *, 4 >	INTERNAL_SERVERS

Friends
template<class T >
class	lockmgr::TableLockMgrImpl

Detailed Description

class for a per-database catalog. also includes metadata for the current database and the current user.

Catalog

Definition at line 132 of file Catalog.h.

Member Typedef Documentation

using Catalog_Namespace::Catalog::ColumnDescriptorsForRoll = std::vector<std::pair<ColumnDescriptor*, ColumnDescriptor*>>

protected

Definition at line 710 of file Catalog.h.

Constructor & Destructor Documentation

Catalog_Namespace::Catalog::Catalog	(	const std::string &	basePath,
		const DBMetadata &	curDB,
		std::shared_ptr< Data_Namespace::DataMgr >	dataMgr,
		const std::vector< LeafHostInfo > &	string_dict_hosts,
		std::shared_ptr< Calcite >	calcite,
		bool	is_new_db
	)

Constructor - takes basePath to already extant data directory for writing.

Parameters

basePath	directory path for writing catalog
dbName	name of the database
fragmenter	Fragmenter object metadata - expects for this directory to already exist

Catalog_Namespace::Catalog::Catalog ( )

Constructor builds a hollow catalog used during constructor of other catalogs.

Definition at line 171 of file Catalog.cpp.

171 {}

Catalog_Namespace::Catalog::~Catalog ( )

Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite.

Definition at line 231 of file Catalog.cpp.

References basePath_, columnDescriptorMap_, currentDB_, Catalog_Namespace::DBMetadata::dbName, g_serialize_temp_tables, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and tableDescriptorMap_.

                   {
   // cat_write_lock write_lock(this);
 
   // must clean up heap-allocated TableDescriptor and ColumnDescriptor structs
   for (TableDescriptorMap::iterator tableDescIt = tableDescriptorMap_.begin();
        tableDescIt != tableDescriptorMap_.end();
        ++tableDescIt) {
     tableDescIt->second->fragmenter = nullptr;
     delete tableDescIt->second;
   }
 
   // TableDescriptorMapById points to the same descriptors.  No need to delete
 
   for (ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.begin();
        columnDescIt != columnDescriptorMap_.end();
        ++columnDescIt) {
     delete columnDescIt->second;
   }
 
   // ColumnDescriptorMapById points to the same descriptors.  No need to delete
 
   if (g_serialize_temp_tables) {
     boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
   }
 }

Here is the call graph for this function:

Member Function Documentation

void Catalog_Namespace::Catalog::addColumn	(	const TableDescriptor &	td,
		ColumnDescriptor &	cd
	)

Definition at line 2418 of file Catalog.cpp.

References ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::db_id, ColumnDescriptor::default_value, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, kENCODING_DICT, TableDescriptor::nShards, TableDescriptor::shard, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), and ColumnDescriptor::virtualExpr.

                                                                        {
   // caller must handle sqlite/chunk transaction TOGETHER
   cd.tableId = td.tableId;
   cd.db_id = getDatabaseId();
   if (td.nShards > 0 && td.shard < 0) {
     for (const auto shard : getPhysicalTablesDescriptors(&td)) {
       auto shard_cd = cd;
       addColumn(*shard, shard_cd);
     }
   }
   if (cd.columnType.get_compression() == kENCODING_DICT) {
     addDictionary(cd);
   }
 
   using BindType = SqliteConnector::BindType;
   std::vector<BindType> types(17, BindType::TEXT);
   if (!cd.default_value.has_value()) {
     types[16] = BindType::NULL_TYPE;
   }
   sqliteConnector_.query_with_text_params(
       "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
       "colscale, is_notnull, "
       "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
       "is_deletedcol, default_value) "
       "VALUES (?, "
       "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
       "?, ?, ?, "
       "?, "
       "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
       std::vector<std::string>{std::to_string(td.tableId),
                                std::to_string(td.tableId),
                                cd.columnName,
                                std::to_string(cd.columnType.get_type()),
                                std::to_string(cd.columnType.get_subtype()),
                                std::to_string(cd.columnType.get_dimension()),
                                std::to_string(cd.columnType.get_scale()),
                                std::to_string(cd.columnType.get_notnull()),
                                std::to_string(cd.columnType.get_compression()),
                                std::to_string(cd.columnType.get_comp_param()),
                                std::to_string(cd.columnType.get_size()),
                                "",
                                std::to_string(cd.isSystemCol),
                                std::to_string(cd.isVirtualCol),
                                cd.virtualExpr,
                                std::to_string(cd.isDeletedCol),
                                cd.default_value.value_or("NULL")},
       types);
 
   sqliteConnector_.query_with_text_params(
       "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
       std::vector<std::string>{std::to_string(td.tableId)});
 
   sqliteConnector_.query_with_text_params(
       "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
       std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
   cd.columnId = sqliteConnector_.getData<int>(0, 0);
 
   ++tableDescriptorMapById_[td.tableId]->nColumns;
   auto ncd = new ColumnDescriptor(cd);
   addToColumnMap(ncd);
   columnDescriptorsForRoll.emplace_back(nullptr, ncd);
 }

Here is the call graph for this function:

DictRef Catalog_Namespace::Catalog::addDictionary ( ColumnDescriptor & cd )

Definition at line 2322 of file Catalog.cpp.

References CHECK, DictDescriptor::dictFolderPath, and ColumnDescriptor::tableId.

                                                    {
   cat_write_lock write_lock(this);
   const auto& td = *tableDescriptorMapById_[cd.tableId];
   list<DictDescriptor> dds;
   setColumnDictionary(cd, dds, td, true);
   auto& dd = dds.back();
   CHECK(dd.dictRef.dictId);
 
   std::unique_ptr<StringDictionaryClient> client;
   if (!string_dict_hosts_.empty()) {
     client.reset(new StringDictionaryClient(
         string_dict_hosts_.front(), DictRef(currentDB_.dbId, -1), true));
   }
   if (client) {
     client->create(dd.dictRef, dd.dictIsTemp);
   }
 
   DictDescriptor* new_dd = new DictDescriptor(dd);
   dictDescriptorMapByRef_[dd.dictRef].reset(new_dd);
   if (!dd.dictIsTemp) {
     boost::filesystem::create_directory(new_dd->dictFolderPath);
   }
   return dd.dictRef;
 }

void Catalog_Namespace::Catalog::addFrontendViewToMap ( DashboardDescriptor & vd )

protected

Definition at line 1778 of file Catalog.cpp.

                                                           {
   cat_write_lock write_lock(this);
   addFrontendViewToMapNoLock(vd);
 }

void Catalog_Namespace::Catalog::addFrontendViewToMapNoLock ( DashboardDescriptor & vd )

protected

Definition at line 1783 of file Catalog.cpp.

References DashboardDescriptor::dashboardName, to_string(), and DashboardDescriptor::userId.

                                                                 {
   cat_write_lock write_lock(this);
   dashboardDescriptorMap_[std::to_string(vd.userId) + ":" + vd.dashboardName] =
       std::make_shared<DashboardDescriptor>(vd);
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::addLinkToMap ( LinkDescriptor & ld )

protected

Definition at line 1865 of file Catalog.cpp.

References LinkDescriptor::link, LinkDescriptor::linkId, and to_string().

                                              {
   cat_write_lock write_lock(this);
   LinkDescriptor* new_ld = new LinkDescriptor();
   *new_ld = ld;
   linkDescriptorMap_[std::to_string(currentDB_.dbId) + ld.link] = new_ld;
   linkDescriptorMapById_[ld.linkId] = new_ld;
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::addReferenceToForeignDict	(	ColumnDescriptor &	referencing_column,
		Parser::SharedDictionaryDef	shared_dict_def,
		const bool	persist_reference
	)

protected

Definition at line 3774 of file Catalog.cpp.

References CHECK, CHECK_GE, ColumnDescriptor::columnType, SQLTypeInfo::get_comp_param(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_foreign_col(), and to_string().

                                                                       {
   cat_write_lock write_lock(this);
   const auto foreign_ref_col = get_foreign_col(*this, shared_dict_def);
   CHECK(foreign_ref_col);
   referencing_column.columnType = foreign_ref_col->columnType;
   const int dict_id = referencing_column.columnType.get_comp_param();
   const DictRef dict_ref(currentDB_.dbId, dict_id);
   const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
   CHECK(dictIt != dictDescriptorMapByRef_.end());
   const auto& dd = dictIt->second;
   CHECK_GE(dd->refcount, 1);
   ++dd->refcount;
   if (persist_reference) {
     cat_sqlite_lock sqlite_lock(getObjForLock());
     sqliteConnector_.query_with_text_params(
         "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
         {std::to_string(dict_id)});
   }
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::addTableToMap	(	const TableDescriptor *	td,
		const std::list< ColumnDescriptor > &	columns,
		const std::list< DictDescriptor > &	dicts
	)

protected

Definition at line 1637 of file Catalog.cpp.

References anonymous_namespace{Utm.h}::a, CHECK, TableDescriptor::columnIdBySpi_, DictDescriptor::dictFolderPath, dict_ref_t::dictId, TableDescriptor::hasDeletedCol, ColumnDescriptor::isDeletedCol, TableDescriptor::mutex_, gpu_enabled::sort(), TableDescriptor::tableId, TableDescriptor::tableName, and to_upper().

                                                                {
   cat_write_lock write_lock(this);
   TableDescriptor* new_td;
 
   auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
   if (foreign_table) {
     auto new_foreign_table = new foreign_storage::ForeignTable();
     *new_foreign_table = *foreign_table;
     new_td = new_foreign_table;
   } else {
     new_td = new TableDescriptor();
     *new_td = *td;
   }
 
   new_td->mutex_ = std::make_shared<std::mutex>();
   tableDescriptorMap_[to_upper(td->tableName)] = new_td;
   tableDescriptorMapById_[td->tableId] = new_td;
   for (auto cd : columns) {
     ColumnDescriptor* new_cd = new ColumnDescriptor();
     *new_cd = cd;
     addToColumnMap(new_cd);
 
     // Add deleted column to the map
     if (cd.isDeletedCol) {
       CHECK(new_td->hasDeletedCol);
       setDeletedColumnUnlocked(new_td, new_cd);
     }
   }
 
   std::sort(new_td->columnIdBySpi_.begin(),
             new_td->columnIdBySpi_.end(),
             [](const size_t a, const size_t b) -> bool { return a < b; });
   // TODO(sy): Why does addTableToMap() sort columnIdBySpi_ but not insert into it while
   // buildColumnsMapUnlocked() does both?
 
   std::unique_ptr<StringDictionaryClient> client;
   DictRef dict_ref(currentDB_.dbId, -1);
   if (!string_dict_hosts_.empty()) {
     client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
   }
   for (auto dd : dicts) {
     if (!dd.dictRef.dictId) {
       // Dummy entry created for a shard of a logical table, nothing to do.
       continue;
     }
     dict_ref.dictId = dd.dictRef.dictId;
     if (client) {
       client->create(dict_ref, dd.dictIsTemp);
     }
     DictDescriptor* new_dd = new DictDescriptor(dd);
     dictDescriptorMapByRef_[dict_ref].reset(new_dd);
     if (!dd.dictIsTemp) {
       boost::filesystem::create_directory(new_dd->dictFolderPath);
     }
   }
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::addToColumnMap ( ColumnDescriptor * cd )

private

Definition at line 6561 of file Catalog.cpp.

References ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::is_dict_encoded_type(), ColumnDescriptor::tableId, and to_upper().

                                                  {
   columnDescriptorMap_[ColumnKey{cd->tableId, to_upper(cd->columnName)}] = cd;
   columnDescriptorMapById_[ColumnIdKey{cd->tableId, cd->columnId}] = cd;
   if (cd->columnType.is_dict_encoded_type()) {
     dict_columns_by_table_id_[cd->tableId].emplace(cd);
   }
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::adjustAlteredTableFiles	(	const std::string &	temp_data_dir,
		const std::unordered_map< int, int > &	all_column_ids_map
	)		const

private

void Catalog_Namespace::Catalog::alterPhysicalTableMetadata	(	const TableDescriptor *	td,
		const TableDescriptorUpdateParams &	table_update_params
	)

private

Definition at line 3488 of file Catalog.cpp.

References CHECK, TableDescriptorUpdateParams::max_rollback_epochs, TableDescriptorUpdateParams::max_rows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::tableId, and to_string().

                                                             {
   // Only called from parent alterTableParamMetadata, expect already to have catalog and
   // sqlite write locks
 
   // Sqlite transaction should have already been begun in parent alterTableCatalogMetadata
 
   TableDescriptor* mutable_td = getMutableMetadataForTableUnlocked(td->tableId);
   CHECK(mutable_td);
   if (td->maxRollbackEpochs != table_update_params.max_rollback_epochs) {
     sqliteConnector_.query_with_text_params(
         "UPDATE mapd_tables SET max_rollback_epochs = ? WHERE tableid = ?",
         std::vector<std::string>{std::to_string(table_update_params.max_rollback_epochs),
                                  std::to_string(td->tableId)});
     mutable_td->maxRollbackEpochs = table_update_params.max_rollback_epochs;
   }
 
   if (td->maxRows != table_update_params.max_rows) {
     sqliteConnector_.query_with_text_params(
         "UPDATE mapd_tables SET max_rows = ? WHERE tableid = ?",
         std::vector<std::string>{std::to_string(table_update_params.max_rows),
                                  std::to_string(td->tableId)});
     mutable_td->maxRows = table_update_params.max_rows;
   }
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::alterTableMetadata	(	const TableDescriptor *	td,
		const TableDescriptorUpdateParams &	table_update_params
	)

private

Definition at line 3515 of file Catalog.cpp.

References CHECK, logger::FATAL, LOG, TableDescriptor::tableId, and TableDescriptor::tableName.

                                                                                          {
   cat_write_lock write_lock(this);
   cat_sqlite_lock sqlite_lock(getObjForLock());
   sqliteConnector_.query("BEGIN TRANSACTION");
   try {
     const auto physical_table_it = logicalToPhysicalTableMapById_.find(td->tableId);
     if (physical_table_it != logicalToPhysicalTableMapById_.end()) {
       const auto physical_tables = physical_table_it->second;
       CHECK(!physical_tables.empty());
       for (size_t i = 0; i < physical_tables.size(); i++) {
         int32_t physical_tb_id = physical_tables[i];
         const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id, false);
         CHECK(phys_td);
         alterPhysicalTableMetadata(phys_td, table_update_params);
       }
     }
     alterPhysicalTableMetadata(td, table_update_params);
   } catch (std::exception& e) {
     sqliteConnector_.query("ROLLBACK TRANSACTION");
     LOG(FATAL) << "Table '" << td->tableName << "' catalog update failed";
   }
   sqliteConnector_.query("END TRANSACTION");
 }

void Catalog_Namespace::Catalog::buildColumnsMapUnlocked ( )

private

Definition at line 1428 of file Catalog.cpp.

References anonymous_namespace{Utm.h}::a, CHECK, ColumnDescriptor::chunks, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::db_id, ColumnDescriptor::default_value, SQLTypeInfo::get_physical_cols(), SQLTypeInfo::is_geometry(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_compression(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::set_dict_key(), SQLTypeInfo::set_dimension(), SQLTypeInfo::set_notnull(), SQLTypeInfo::set_scale(), SQLTypeInfo::set_size(), SQLTypeInfo::set_subtype(), SQLTypeInfo::set_type(), gpu_enabled::sort(), ColumnDescriptor::tableId, and ColumnDescriptor::virtualExpr.

                                       {
   std::string columnQuery(
       "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
       "is_notnull, compression, comp_param, "
       "size, chunks, is_systemcol, is_virtualcol, virtual_expr, is_deletedcol, "
       "default_value from "
       "mapd_columns ORDER BY tableid, "
       "columnid");
   sqliteConnector_.query(columnQuery);
   auto numRows = sqliteConnector_.getNumRows();
   int32_t skip_physical_cols = 0;
   for (size_t r = 0; r < numRows; ++r) {
     ColumnDescriptor* cd = new ColumnDescriptor();
     cd->tableId = sqliteConnector_.getData<int>(r, 0);
     cd->columnId = sqliteConnector_.getData<int>(r, 1);
     cd->columnName = sqliteConnector_.getData<string>(r, 2);
     cd->columnType.set_type((SQLTypes)sqliteConnector_.getData<int>(r, 3));
     cd->columnType.set_subtype((SQLTypes)sqliteConnector_.getData<int>(r, 4));
     cd->columnType.set_dimension(sqliteConnector_.getData<int>(r, 5));
     cd->columnType.set_scale(sqliteConnector_.getData<int>(r, 6));
     cd->columnType.set_notnull(sqliteConnector_.getData<bool>(r, 7));
     cd->columnType.set_compression((EncodingType)sqliteConnector_.getData<int>(r, 8));
     cd->columnType.set_comp_param(sqliteConnector_.getData<int>(r, 9));
     cd->columnType.set_size(sqliteConnector_.getData<int>(r, 10));
     cd->chunks = sqliteConnector_.getData<string>(r, 11);
     cd->isSystemCol = sqliteConnector_.getData<bool>(r, 12);
     cd->isVirtualCol = sqliteConnector_.getData<bool>(r, 13);
     cd->virtualExpr = sqliteConnector_.getData<string>(r, 14);
     cd->isDeletedCol = sqliteConnector_.getData<bool>(r, 15);
     if (sqliteConnector_.isNull(r, 16)) {
       cd->default_value = std::nullopt;
     } else {
       cd->default_value = std::make_optional(sqliteConnector_.getData<string>(r, 16));
     }
     cd->isGeoPhyCol = skip_physical_cols > 0;
     cd->db_id = getDatabaseId();
     set_dict_key(*cd);
     addToColumnMap(cd);
 
     if (skip_physical_cols <= 0) {
       skip_physical_cols = cd->columnType.get_physical_cols();
     }
 
     auto td_itr = tableDescriptorMapById_.find(cd->tableId);
     CHECK(td_itr != tableDescriptorMapById_.end());
 
     if (cd->isDeletedCol) {
       td_itr->second->hasDeletedCol = true;
       setDeletedColumnUnlocked(td_itr->second, cd);
     } else if (cd->columnType.is_geometry() || skip_physical_cols-- <= 0) {
       tableDescriptorMapById_[cd->tableId]->columnIdBySpi_.push_back(cd->columnId);
     }
   }
 
   // sort columnIdBySpi_ based on columnId
   for (auto& tit : tableDescriptorMapById_) {
     std::sort(tit.second->columnIdBySpi_.begin(),
               tit.second->columnIdBySpi_.end(),
               [](const size_t a, const size_t b) -> bool { return a < b; });
   }
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildCustomExpressionsMapUnlocked ( )

private

Definition at line 1609 of file Catalog.cpp.

                                                 {
   sqliteConnector_.query(
       "SELECT id, name, expression_json, data_source_type, data_source_id, "
       "is_deleted "
       "FROM omnisci_custom_expressions");
   auto num_rows = sqliteConnector_.getNumRows();
   for (size_t row = 0; row < num_rows; row++) {
     auto custom_expr = getCustomExpressionFromConnector(row);
     custom_expr_map_by_id_[custom_expr->id] = std::move(custom_expr);
   }
 }

void Catalog_Namespace::Catalog::buildDashboardsMapUnlocked ( const std::map< int32_t, std::string > & user_name_by_user_id )

private

Definition at line 1512 of file Catalog.cpp.

References generate_dashboard_system_rolename(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_user_name_from_id(), and to_string().

                                                             {
   std::string frontendViewQuery(
       "SELECT id, state, name, image_hash, strftime('%Y-%m-%dT%H:%M:%SZ', update_time), "
       "userid, "
       "metadata "
       "FROM mapd_dashboards");
   sqliteConnector_.query(frontendViewQuery);
   auto numRows = sqliteConnector_.getNumRows();
   for (size_t r = 0; r < numRows; ++r) {
     auto vd = std::make_shared<DashboardDescriptor>();
     vd->dashboardId = sqliteConnector_.getData<int>(r, 0);
     vd->dashboardState = sqliteConnector_.getData<string>(r, 1);
     vd->dashboardName = sqliteConnector_.getData<string>(r, 2);
     vd->imageHash = sqliteConnector_.getData<string>(r, 3);
     vd->updateTime = sqliteConnector_.getData<string>(r, 4);
     vd->userId = sqliteConnector_.getData<int>(r, 5);
     vd->dashboardMetadata = sqliteConnector_.getData<string>(r, 6);
     vd->user = get_user_name_from_id(vd->userId, user_name_by_user_id);
     vd->dashboardSystemRoleName = generate_dashboard_system_rolename(
         std::to_string(currentDB_.dbId), sqliteConnector_.getData<string>(r, 0));
     dashboardDescriptorMap_[std::to_string(vd->userId) + ":" + vd->dashboardName] = vd;
   }
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildDictionaryMapUnlocked ( )

private

Definition at line 1067 of file Catalog.cpp.

References currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, g_base_path, SqliteConnector::getData(), SqliteConnector::getNumRows(), shared::kDataDirectoryName, SqliteConnector::query(), sqliteConnector_, and to_string().

                                          {
   std::string dictQuery(
       "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
   sqliteConnector_.query(dictQuery);
   auto numRows = sqliteConnector_.getNumRows();
   for (size_t r = 0; r < numRows; ++r) {
     auto dictId = sqliteConnector_.getData<int>(r, 0);
     auto dictName = sqliteConnector_.getData<string>(r, 1);
     auto dictNBits = sqliteConnector_.getData<int>(r, 2);
     auto is_shared = sqliteConnector_.getData<bool>(r, 3);
     auto refcount = sqliteConnector_.getData<int>(r, 4);
     auto fname = g_base_path + "/" + shared::kDataDirectoryName + "/DB_" +
                  std::to_string(currentDB_.dbId) + "_DICT_" + std::to_string(dictId);
     DictRef dict_ref(currentDB_.dbId, dictId);
     auto dd = new DictDescriptor(
         dict_ref, dictName, dictNBits, is_shared, refcount, fname, false);
     dictDescriptorMapByRef_[dict_ref].reset(dd);
   }
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildForeignServerMapUnlocked ( )

private

Definition at line 4923 of file Catalog.cpp.

References CHECK, and g_enable_fsi.

                                             {
   CHECK(g_enable_fsi);
   sqliteConnector_.query(
       "SELECT id, name, data_wrapper_type, options, owner_user_id, creation_time FROM "
       "omnisci_foreign_servers");
   auto num_rows = sqliteConnector_.getNumRows();
 
   for (size_t row = 0; row < num_rows; row++) {
     auto foreign_server = std::make_shared<foreign_storage::ForeignServer>(
         sqliteConnector_.getData<int>(row, 0),
         sqliteConnector_.getData<std::string>(row, 1),
         sqliteConnector_.getData<std::string>(row, 2),
         sqliteConnector_.getData<std::string>(row, 3),
         sqliteConnector_.getData<std::int32_t>(row, 4),
         sqliteConnector_.getData<std::int32_t>(row, 5));
     foreignServerMap_[foreign_server->name] = foreign_server;
     foreignServerMapById_[foreign_server->id] = foreign_server;
   }
 }

void Catalog_Namespace::Catalog::buildLinksMapUnlocked ( )

private

Definition at line 1537 of file Catalog.cpp.

References to_string().

                                     {
   std::string linkQuery(
       "SELECT linkid, userid, link, view_state, strftime('%Y-%m-%dT%H:%M:%SZ', "
       "update_time), view_metadata "
       "FROM mapd_links");
   sqliteConnector_.query(linkQuery);
   auto numRows = sqliteConnector_.getNumRows();
   for (size_t r = 0; r < numRows; ++r) {
     auto ld = new LinkDescriptor();
     ld->linkId = sqliteConnector_.getData<int>(r, 0);
     ld->userId = sqliteConnector_.getData<int>(r, 1);
     ld->link = sqliteConnector_.getData<string>(r, 2);
     ld->viewState = sqliteConnector_.getData<string>(r, 3);
     ld->updateTime = sqliteConnector_.getData<string>(r, 4);
     ld->viewMetadata = sqliteConnector_.getData<string>(r, 5);
     linkDescriptorMap_[std::to_string(currentDB_.dbId) + ld->link] = ld;
     linkDescriptorMapById_[ld->linkId] = ld;
   }
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildLogicalToPhysicalMapUnlocked ( )

private

Definition at line 1557 of file Catalog.cpp.

References CHECK.

                                                 {
   /* rebuild map linking logical tables to corresponding physical ones */
   std::string logicalToPhysicalTableMapQuery(
       "SELECT logical_table_id, physical_table_id "
       "FROM mapd_logical_to_physical");
   sqliteConnector_.query(logicalToPhysicalTableMapQuery);
   auto numRows = sqliteConnector_.getNumRows();
   for (size_t r = 0; r < numRows; ++r) {
     auto logical_tb_id = sqliteConnector_.getData<int>(r, 0);
     auto physical_tb_id = sqliteConnector_.getData<int>(r, 1);
     const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
     if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
       /* add new entity to the map logicalToPhysicalTableMapById_ */
       std::vector<int32_t> physicalTables{physical_tb_id};
       const auto it_ok =
           logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
       CHECK(it_ok.second);
     } else {
       /* update map logicalToPhysicalTableMapById_ */
       physicalTableIt->second.push_back(physical_tb_id);
     }
   }
 }

void Catalog_Namespace::Catalog::buildMaps ( )

protected

Definition at line 1584 of file Catalog.cpp.

References g_enable_fsi, and Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_user_id_to_user_name_map().

                         {
   // Get all user id to username mapping here in order to avoid making a call to
   // SysCatalog (and attempting to acquire SysCatalog locks) while holding locks for this
   // catalog.
   const auto user_name_by_user_id = get_user_id_to_user_name_map();
 
   cat_write_lock write_lock(this);
   cat_sqlite_lock sqlite_lock(getObjForLock());
 
   buildDictionaryMapUnlocked();
   buildTablesMapUnlocked();
 
   if (g_enable_fsi) {
     buildForeignServerMapUnlocked();
     updateForeignTablesInMapUnlocked();
   }
 
   buildColumnsMapUnlocked();
   updateViewsInMapUnlocked();
   buildDashboardsMapUnlocked(user_name_by_user_id);
   buildLinksMapUnlocked();
   buildLogicalToPhysicalMapUnlocked();
   buildCustomExpressionsMapUnlocked();
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::buildTablesMapUnlocked ( )

private

Definition at line 1369 of file Catalog.cpp.

References logger::FATAL, StorageType::FOREIGN_TABLE, TableDescriptor::fragmenter, TableDescriptor::fragments, TableDescriptor::fragPageSize, TableDescriptor::fragType, TableDescriptor::hasDeletedCol, TableDescriptor::is_system_table, TableDescriptor::isView, TableDescriptor::keyMetainfo, LOG, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nColumns, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::storageType, TableDescriptor::tableId, TableDescriptor::tableName, to_upper(), and TableDescriptor::userId.

                                      {
   std::string tableQuery(
       "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
       "max_chunk_size, frag_page_size, "
       "max_rows, partitions, shard_column_id, shard, num_shards, key_metainfo, userid, "
       "sort_column_id, storage_type, max_rollback_epochs, is_system_table "
       "from mapd_tables");
   sqliteConnector_.query(tableQuery);
   auto numRows = sqliteConnector_.getNumRows();
   for (size_t r = 0; r < numRows; ++r) {
     TableDescriptor* td;
     const auto& storage_type = sqliteConnector_.getData<string>(r, 17);
     if (!storage_type.empty() && storage_type != StorageType::FOREIGN_TABLE) {
       const auto table_id = sqliteConnector_.getData<int>(r, 0);
       const auto& table_name = sqliteConnector_.getData<string>(r, 1);
       LOG(FATAL) << "Unable to read Catalog metadata: storage type is currently not a "
                     "supported table option (table "
                  << table_name << " [" << table_id << "] in database "
                  << currentDB_.dbName << ").";
     }
 
     if (storage_type == StorageType::FOREIGN_TABLE) {
       td = new foreign_storage::ForeignTable();
     } else {
       td = new TableDescriptor();
     }
 
     td->storageType = storage_type;
     td->tableId = sqliteConnector_.getData<int>(r, 0);
     td->tableName = sqliteConnector_.getData<string>(r, 1);
     td->nColumns = sqliteConnector_.getData<int>(r, 2);
     td->isView = sqliteConnector_.getData<bool>(r, 3);
     td->fragments = sqliteConnector_.getData<string>(r, 4);
     td->fragType =
         (Fragmenter_Namespace::FragmenterType)sqliteConnector_.getData<int>(r, 5);
     td->maxFragRows = sqliteConnector_.getData<int>(r, 6);
     td->maxChunkSize = sqliteConnector_.getData<int64_t>(r, 7);
     td->fragPageSize = sqliteConnector_.getData<int>(r, 8);
     td->maxRows = sqliteConnector_.getData<int64_t>(r, 9);
     td->partitions = sqliteConnector_.getData<string>(r, 10);
     td->shardedColumnId = sqliteConnector_.getData<int>(r, 11);
     td->shard = sqliteConnector_.getData<int>(r, 12);
     td->nShards = sqliteConnector_.getData<int>(r, 13);
     td->keyMetainfo = sqliteConnector_.getData<string>(r, 14);
     td->userId = sqliteConnector_.getData<int>(r, 15);
     td->sortedColumnId =
         sqliteConnector_.isNull(r, 16) ? 0 : sqliteConnector_.getData<int>(r, 16);
     if (!td->isView) {
       td->fragmenter = nullptr;
     }
     td->maxRollbackEpochs = sqliteConnector_.getData<int>(r, 18);
     td->is_system_table = sqliteConnector_.getData<bool>(r, 19);
     td->hasDeletedCol = false;
 
     tableDescriptorMap_[to_upper(td->tableName)] = td;
     tableDescriptorMapById_[td->tableId] = td;
   }
 }

Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::calculateSHA1 ( const std::string & data )

protected

Definition at line 4632 of file Catalog.cpp.

                                                       {
   boost::uuids::detail::sha1 sha1;
   unsigned int digest[5];
   sha1.process_bytes(data.c_str(), data.length());
   sha1.get_digest(digest);
   std::stringstream ss;
   for (size_t i = 0; i < 5; i++) {
     ss << std::hex << digest[i];
   }
   return ss.str();
 }

void Catalog_Namespace::Catalog::changeForeignServerOwner	(	const std::string &	server_name,
		const int	new_owner_id
	)

Change the owner of a Foreign Server to a new owner.

Parameters

server_name	- Name of the foreign server whose owner to change
new_owner_id	- New owner's user id

Definition at line 3194 of file Catalog.cpp.

References CHECK, to_string(), and foreign_storage::ForeignServer::user_id.

                                                                {
   cat_write_lock write_lock(this);
   foreign_storage::ForeignServer* foreign_server =
       foreignServerMap_.find(server_name)->second.get();
   CHECK(foreign_server);
   setForeignServerProperty(server_name, "owner_user_id", std::to_string(new_owner_id));
   // update in-memory server
   foreign_server->user_id = new_owner_id;
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::CheckAndExecuteMigrations ( )

protected

Definition at line 1007 of file Catalog.cpp.

References g_enable_fsi, getObjForLock(), recordOwnershipOfObjectsInObjectPermissions(), renameLegacyDataWrappers(), updateCustomExpressionsSchema(), updateDefaultColumnValues(), updateDeletedColumnIndicator(), updateDictionaryNames(), updateDictionarySchema(), updateFixlenArrayColumns(), updateFrontendViewAndLinkUsers(), updateFrontendViewSchema(), updateFrontendViewsToDashboards(), updateFsiSchemas(), updateGeoColumns(), updateLinkSchema(), updateLogicalToPhysicalTableLinkSchema(), updatePageSize(), and updateTableDescriptorSchema().

                                         {
   cat_write_lock write_lock(this);
   cat_sqlite_lock sqlite_lock(getObjForLock());
   updateTableDescriptorSchema();
   updateFixlenArrayColumns();
   updateGeoColumns();
   updateFrontendViewAndLinkUsers();
   updateFrontendViewSchema();
   updateLinkSchema();
   updateDictionaryNames();
   updateLogicalToPhysicalTableLinkSchema();
   updateDictionarySchema();
   updatePageSize();
   updateDeletedColumnIndicator();
   updateFrontendViewsToDashboards();
   recordOwnershipOfObjectsInObjectPermissions();
   if (g_enable_fsi) {
     updateFsiSchemas();
     renameLegacyDataWrappers();
   }
   updateCustomExpressionsSchema();
   updateDefaultColumnValues();
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::CheckAndExecuteMigrationsPostBuildMaps ( )

protected

Definition at line 1031 of file Catalog.cpp.

References checkDateInDaysColumnMigration(), and createDashboardSystemRoles().

                                                      {
   checkDateInDaysColumnMigration();
   createDashboardSystemRoles();
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::checkDateInDaysColumnMigration ( )

protected

Definition at line 925 of file Catalog.cpp.

References getCurrentDB(), getObjForLock(), migrations::MigrationMgr::migrateDateInDaysMetadata(), sqliteConnector_, and tableDescriptorMapById_.

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

                                              {
   cat_sqlite_lock sqlite_lock(getObjForLock());
   migrations::MigrationMgr::migrateDateInDaysMetadata(
       tableDescriptorMapById_, getCurrentDB().dbId, this, sqliteConnector_);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

const bool Catalog_Namespace::Catalog::checkMetadataForDeletedRecs	(	const TableDescriptor *	td,
		int	column_id
	)		const

Definition at line 3712 of file Catalog.cpp.

References CHECK, and TableDescriptor::fragmenter.

                                                                             {
   // check if there are rows deleted by examining the deletedColumn metadata
   CHECK(td);
   auto fragmenter = td->fragmenter;
   if (fragmenter) {
     return fragmenter->hasDeletedRows(delete_column_id);
   } else {
     return false;
   }
 }

void Catalog_Namespace::Catalog::checkpoint ( const int logicalTableId ) const

Definition at line 4853 of file Catalog.cpp.

Referenced by UpdelRoll::commitUpdate(), TableOptimizer::vacuumDeletedRows(), and TableOptimizer::vacuumFragmentsAboveMinSelectivity().

                                                        {
   const auto td = getMetadataForTable(logicalTableId);
   const auto shards = getPhysicalTablesDescriptors(td);
   for (const auto shard : shards) {
     getDataMgr().checkpoint(getCurrentDB().dbId, shard->tableId);
   }
 }

Here is the caller graph for this function:

void Catalog_Namespace::Catalog::checkpointWithAutoRollback ( const int logical_table_id ) const

Definition at line 4861 of file Catalog.cpp.

Referenced by Fragmenter_Namespace::LocalInsertConnector::checkpoint(), import_export::Loader::checkpoint(), StorageIOFacility::TransactionParameters::finalizeTransaction(), and TableOptimizer::vacuumFragmentsAboveMinSelectivity().

                                                                          {
   auto table_epochs = getTableEpochs(getDatabaseId(), logical_table_id);
   try {
     checkpoint(logical_table_id);
   } catch (...) {
     setTableEpochsLogExceptions(getDatabaseId(), table_epochs);
     throw;
   }
 }

Here is the caller graph for this function:

void Catalog_Namespace::Catalog::conditionallyInitializeSystemObjects ( )

private

Definition at line 6045 of file Catalog.cpp.

References g_enable_system_tables.

                                                    {
   if (g_enable_system_tables && isInfoSchemaDb()) {
     initializeSystemServers();
     initializeSystemTables();
   }
 }

int32_t Catalog_Namespace::Catalog::createCustomExpression ( std::unique_ptr< CustomExpression > custom_expression )

Creates a new custom expression.

Parameters

custom_expression - unique pointer to struct containing custom expression details.

Returns: id of created custom expression

Definition at line 5607 of file Catalog.cpp.

References CHECK, CHECK_EQ, CHECK_GT, and to_string().

                                                        {
   cat_write_lock write_lock(this);
   cat_sqlite_lock sqlite_lock(getObjForLock());
   sqliteConnector_.query("BEGIN TRANSACTION");
   int32_t custom_expression_id{-1};
   try {
     auto data_source_type_str =
         CustomExpression::dataSourceTypeToString(custom_expression->data_source_type);
     auto data_source_id_str = std::to_string(custom_expression->data_source_id);
     std::string custom_expr_select_query{
         "SELECT id FROM omnisci_custom_expressions WHERE name = ? and data_source_type = "
         "? and data_source_id = ? and is_deleted = ?"};
     std::vector<std::string> custom_expr_select_params{custom_expression->name,
                                                        data_source_type_str,
                                                        data_source_id_str,
                                                        std::to_string(false)};
     sqliteConnector_.query_with_text_params(custom_expr_select_query,
                                             custom_expr_select_params);
     if (sqliteConnector_.getNumRows() > 0) {
       throw std::runtime_error{
           "A custom expression with the given "
           "name and data source already exists."};
     }
     sqliteConnector_.query_with_text_params(
         "INSERT INTO omnisci_custom_expressions(name, expression_json, "
         "data_source_type, data_source_id, is_deleted) VALUES (?,?,?,?,?)",
         std::vector<std::string>{custom_expression->name,
                                  custom_expression->expression_json,
                                  data_source_type_str,
                                  data_source_id_str,
                                  std::to_string(false)});
     sqliteConnector_.query_with_text_params(custom_expr_select_query,
                                             custom_expr_select_params);
     CHECK_EQ(sqliteConnector_.getNumRows(), static_cast<size_t>(1));
     custom_expression->id = sqliteConnector_.getData<int32_t>(0, 0);
     custom_expression_id = custom_expression->id;
     CHECK(custom_expr_map_by_id_.find(custom_expression->id) ==
           custom_expr_map_by_id_.end());
     custom_expr_map_by_id_[custom_expression->id] = std::move(custom_expression);
   } catch (std::exception& e) {
     sqliteConnector_.query("ROLLBACK TRANSACTION");
     throw;
   }
   sqliteConnector_.query("END TRANSACTION");
   CHECK_GT(custom_expression_id, 0);
   return custom_expression_id;
 }

Here is the call graph for this function:

int32_t Catalog_Namespace::Catalog::createDashboard ( DashboardDescriptor & vd )

Definition at line 4491 of file Catalog.cpp.

References DashboardDescriptor::dashboardId, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, DashboardDescriptor::dashboardSystemRoleName, generate_dashboard_system_rolename(), DashboardDescriptor::imageHash, to_string(), Catalog_Namespace::sqlite_lock< T >::unlock(), Catalog_Namespace::write_lock< T >::unlock(), DashboardDescriptor::updateTime, and DashboardDescriptor::userId.

                                                         {
   cat_write_lock write_lock(this);
   cat_sqlite_lock sqlite_lock(getObjForLock());
   sqliteConnector_.query("BEGIN TRANSACTION");
   try {
     // TODO(andrew): this should be an upsert
     sqliteConnector_.query_with_text_params(
         "SELECT id FROM mapd_dashboards WHERE name = ? and userid = ?",
         std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
     if (sqliteConnector_.getNumRows() > 0) {
       sqliteConnector_.query_with_text_params(
           "UPDATE mapd_dashboards SET state = ?, image_hash = ?, metadata = ?, "
           "update_time = "
           "datetime('now') where name = ? "
           "and userid = ?",
           std::vector<std::string>{vd.dashboardState,
                                    vd.imageHash,
                                    vd.dashboardMetadata,
                                    vd.dashboardName,
                                    std::to_string(vd.userId)});
     } else {
       sqliteConnector_.query_with_text_params(
           "INSERT INTO mapd_dashboards (name, state, image_hash, metadata, "
           "update_time, "
           "userid) "
           "VALUES "
           "(?,?,?,?, "
           "datetime('now'), ?)",
           std::vector<std::string>{vd.dashboardName,
                                    vd.dashboardState,
                                    vd.imageHash,
                                    vd.dashboardMetadata,
                                    std::to_string(vd.userId)});
     }
   } catch (std::exception& e) {
     sqliteConnector_.query("ROLLBACK TRANSACTION");
     throw;
   }
   sqliteConnector_.query("END TRANSACTION");
 
   // now get the auto generated dashboardId
   try {
     sqliteConnector_.query_with_text_params(
         "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_dashboards "
         "WHERE name = ? and userid = ?",
         std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
     vd.dashboardId = sqliteConnector_.getData<int>(0, 0);
     vd.updateTime = sqliteConnector_.getData<std::string>(0, 1);
   } catch (std::exception& e) {
     throw;
   }
   vd.dashboardSystemRoleName = generate_dashboard_system_rolename(
       std::to_string(currentDB_.dbId), std::to_string(vd.dashboardId));
   addFrontendViewToMap(vd);
   sqlite_lock.unlock();
   write_lock.unlock();
   if (!isInfoSchemaDb()) {
     // NOTE(wamsi): Transactionally unsafe
     createOrUpdateDashboardSystemRole(
         vd.dashboardMetadata, vd.userId, vd.dashboardSystemRoleName);
   }
   return vd.dashboardId;
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::createDashboardSystemRoles ( )

protected

Definition at line 931 of file Catalog.cpp.

References createOrUpdateDashboardSystemRole(), currentDB_, Catalog_Namespace::DBMetadata::dbId, logger::ERROR, generate_dashboard_system_rolename(), SqliteConnector::getData(), Catalog_Namespace::SysCatalog::getGranteesOfSharedDashboards(), SqliteConnector::getNumRows(), getObjForLock(), Catalog_Namespace::SysCatalog::getRoleGrantee(), Catalog_Namespace::SysCatalog::grantRoleBatch(), logger::INFO, Catalog_Namespace::SysCatalog::instance(), LOG, MAPD_VERSION, SqliteConnector::query(), SqliteConnector::query_with_text_params(), run_benchmark_import::result, sqliteConnector_, and to_string().

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

                                          {
   std::unordered_map<std::string, std::pair<int, std::string>> dashboards;
   std::vector<std::string> dashboard_ids;
   static const std::string migration_name{"dashboard_roles_migration"};
   {
     cat_sqlite_lock sqlite_lock(getObjForLock());
     sqliteConnector_.query("BEGIN TRANSACTION");
     try {
       // migration_history should be present in all catalogs by now
       // if not then would be created before this migration
       sqliteConnector_.query(
           "select * from mapd_version_history where migration_history = '" +
           migration_name + "'");
       if (sqliteConnector_.getNumRows() != 0) {
         // no need for further execution
         sqliteConnector_.query("END TRANSACTION");
         return;
       }
       LOG(INFO) << "Performing dashboard internal roles Migration.";
       sqliteConnector_.query("select id, userid, metadata from mapd_dashboards");
       for (size_t i = 0; i < sqliteConnector_.getNumRows(); ++i) {
         if (SysCatalog::instance().getRoleGrantee(generate_dashboard_system_rolename(
                 std::to_string(currentDB_.dbId),
                 sqliteConnector_.getData<string>(i, 0)))) {
           // Successfully created roles during previous migration/crash
           // No need to include them
           continue;
         }
         dashboards[sqliteConnector_.getData<string>(i, 0)] = std::make_pair(
             sqliteConnector_.getData<int>(i, 1), sqliteConnector_.getData<string>(i, 2));
         dashboard_ids.push_back(sqliteConnector_.getData<string>(i, 0));
       }
     } catch (const std::exception& e) {
       sqliteConnector_.query("ROLLBACK TRANSACTION");
       throw;
     }
     sqliteConnector_.query("END TRANSACTION");
   }
   // All current grantees with shared dashboards.
   const auto active_grantees =
       SysCatalog::instance().getGranteesOfSharedDashboards(dashboard_ids);
 
   try {
     // NOTE(wamsi): Transactionally unsafe
     for (auto dash : dashboards) {
       createOrUpdateDashboardSystemRole(dash.second.second,
                                         dash.second.first,
                                         generate_dashboard_system_rolename(
                                             std::to_string(currentDB_.dbId), dash.first));
       auto result = active_grantees.find(dash.first);
       if (result != active_grantees.end()) {
         SysCatalog::instance().grantRoleBatch(
             {generate_dashboard_system_rolename(std::to_string(currentDB_.dbId),
                                                 dash.first)},
             result->second);
       }
     }
     cat_sqlite_lock sqlite_lock(getObjForLock());
     // check if this has already been completed
     sqliteConnector_.query(
         "select * from mapd_version_history where migration_history = '" +
         migration_name + "'");
     if (sqliteConnector_.getNumRows() != 0) {
       return;
     }
     sqliteConnector_.query_with_text_params(
         "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
         std::vector<std::string>{std::to_string(MAPD_VERSION), migration_name});
   } catch (const std::exception& e) {
     LOG(ERROR) << "Failed to create dashboard system roles during migration: "
                << e.what();
     throw;
   }
   LOG(INFO) << "Successfully created dashboard system roles during migration.";
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createDefaultServersIfNotExists ( )

Creates default local file servers (if they don't already exist).

Definition at line 4994 of file Catalog.cpp.

References CHECK, foreign_storage::DataWrapperType::CSV, g_enable_fsi, shared::kRootUserId, foreign_storage::AbstractFileStorageDataWrapper::LOCAL_FILE_STORAGE_TYPE, foreign_storage::DataWrapperType::PARQUET, foreign_storage::DataWrapperType::REGEX_PARSER, and foreign_storage::AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY.

                                               {
   CHECK(g_enable_fsi);
   foreign_storage::OptionsMap options;
   options[foreign_storage::AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY] =
       foreign_storage::AbstractFileStorageDataWrapper::LOCAL_FILE_STORAGE_TYPE;
 
   auto local_csv_server = std::make_unique<foreign_storage::ForeignServer>(
       "default_local_delimited",
       foreign_storage::DataWrapperType::CSV,
       options,
       shared::kRootUserId);
   local_csv_server->validate();
   createForeignServerNoLocks(std::move(local_csv_server), true);
 
 #ifdef ENABLE_IMPORT_PARQUET
   auto local_parquet_server = std::make_unique<foreign_storage::ForeignServer>(
       "default_local_parquet",
       foreign_storage::DataWrapperType::PARQUET,
       options,
       shared::kRootUserId);
   local_parquet_server->validate();
   createForeignServerNoLocks(std::move(local_parquet_server), true);
 #endif
 
   auto local_regex_parser_server = std::make_unique<foreign_storage::ForeignServer>(
       "default_local_regex_parsed",
       foreign_storage::DataWrapperType::REGEX_PARSER,
       options,
       shared::kRootUserId);
   local_regex_parser_server->validate();
   createForeignServerNoLocks(std::move(local_regex_parser_server), true);
 }

void Catalog_Namespace::Catalog::createForeignServer	(	std::unique_ptr< foreign_storage::ForeignServer >	foreign_server,
		bool	if_not_exists
	)

Creates a new foreign server DB object.

Parameters

foreign_server	- unique pointer to struct containing foreign server details
if_not_exists	- flag indicating whether or not an attempt to create a new foreign server should occur if a server with the same name already exists. An exception is thrown if this flag is set to "false" and an attempt is made to create a pre-existing foreign server

Definition at line 3091 of file Catalog.cpp.

                         {
   cat_write_lock write_lock(this);
   cat_sqlite_lock sqlite_lock(getObjForLock());
   createForeignServerNoLocks(std::move(foreign_server), if_not_exists);
 }

void Catalog_Namespace::Catalog::createForeignServerNoLocks	(	std::unique_ptr< foreign_storage::ForeignServer >	foreign_server,
		bool	if_not_exists
	)

private

Same as createForeignServer() but without acquiring locks. This should only be called from within a function/code block that already acquires appropriate locks.

Definition at line 3099 of file Catalog.cpp.

References CHECK, CHECK_EQ, setup::name, and to_string().

                         {
   const auto& name = foreign_server->name;
 
   sqliteConnector_.query_with_text_params(
       "SELECT name from omnisci_foreign_servers where name = ?",
       std::vector<std::string>{name});
 
   if (sqliteConnector_.getNumRows() == 0) {
     foreign_server->creation_time = std::time(nullptr);
     sqliteConnector_.query_with_text_params(
         "INSERT INTO omnisci_foreign_servers (name, data_wrapper_type, owner_user_id, "
         "creation_time,  "
         "options) "
         "VALUES (?, ?, ?, ?, ?)",
         std::vector<std::string>{name,
                                  foreign_server->data_wrapper_type,
                                  std::to_string(foreign_server->user_id),
                                  std::to_string(foreign_server->creation_time),
                                  foreign_server->getOptionsAsJsonString()});
     sqliteConnector_.query_with_text_params(
         "SELECT id from omnisci_foreign_servers where name = ?",
         std::vector<std::string>{name});
     CHECK_EQ(sqliteConnector_.getNumRows(), size_t(1));
     foreign_server->id = sqliteConnector_.getData<int32_t>(0, 0);
     std::shared_ptr<foreign_storage::ForeignServer> foreign_server_shared =
         std::move(foreign_server);
     CHECK(foreignServerMap_.find(name) == foreignServerMap_.end())
         << "Attempting to insert a foreign server into foreign server map that already "
            "exists.";
     foreignServerMap_[name] = foreign_server_shared;
     foreignServerMapById_[foreign_server_shared->id] = foreign_server_shared;
   } else if (!if_not_exists) {
     throw std::runtime_error{"A foreign server with name \"" + foreign_server->name +
                              "\" already exists."};
   }
 
   const auto& server_it = foreignServerMap_.find(name);
   CHECK(server_it != foreignServerMap_.end());
   CHECK(foreignServerMapById_.find(server_it->second->id) != foreignServerMapById_.end());
 }

Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::createLink	(	LinkDescriptor &	ld,
		size_t	min_length
	)

Definition at line 4644 of file Catalog.cpp.

References LinkDescriptor::link, LinkDescriptor::linkId, to_string(), LinkDescriptor::updateTime, LinkDescriptor::userId, LinkDescriptor::viewMetadata, and LinkDescriptor::viewState.

                                                                    {
   cat_write_lock write_lock(this);
   cat_sqlite_lock sqlite_lock(getObjForLock());
   sqliteConnector_.query("BEGIN TRANSACTION");
   try {
     ld.link = calculateSHA1(ld.viewState + ld.viewMetadata + std::to_string(ld.userId))
                   .substr(0, 8);
     sqliteConnector_.query_with_text_params(
         "SELECT linkid FROM mapd_links WHERE link = ? and userid = ?",
         std::vector<std::string>{ld.link, std::to_string(ld.userId)});
     if (sqliteConnector_.getNumRows() > 0) {
       sqliteConnector_.query_with_text_params(
           "UPDATE mapd_links SET update_time = datetime('now') WHERE userid = ? AND "
           "link = ?",
           std::vector<std::string>{std::to_string(ld.userId), ld.link});
     } else {
       sqliteConnector_.query_with_text_params(
           "INSERT INTO mapd_links (userid, link, view_state, view_metadata, "
           "update_time) VALUES (?,?,?,?, datetime('now'))",
           std::vector<std::string>{
               std::to_string(ld.userId), ld.link, ld.viewState, ld.viewMetadata});
     }
     // now get the auto generated dashid
     sqliteConnector_.query_with_text_param(
         "SELECT linkid, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_links "
         "WHERE link = ?",
         ld.link);
     ld.linkId = sqliteConnector_.getData<int>(0, 0);
     ld.updateTime = sqliteConnector_.getData<std::string>(0, 1);
   } catch (std::exception& e) {
     sqliteConnector_.query("ROLLBACK TRANSACTION");
     throw;
   }
   sqliteConnector_.query("END TRANSACTION");
   addLinkToMap(ld);
   return ld.link;
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::createOrUpdateDashboardSystemRole	(	const std::string &	view_meta,
		const int32_t &	user_id,
		const std::string &	dash_role_name
	)

protected

Definition at line 1819 of file Catalog.cpp.

References Grantee::findDbObject(), Grantee::getDbObjects(), TableDBObjectType, and ViewDBObjectType.

Referenced by createDashboardSystemRoles().

                                                                                  {
   auto objects = parseDashboardObjects(view_meta, user_id);
   Role* rl = SysCatalog::instance().getRoleGrantee(dash_role_name);
   if (!rl) {
     // Dashboard role does not exist
     // create role and grant privileges
     // NOTE(wamsi): Transactionally unsafe
     SysCatalog::instance().createRole(
         dash_role_name, /*user_private_role=*/false, /*is_temporary=*/false);
     SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
   } else {
     // Dashboard system role already exists
     // Add/remove privileges on objects
     std::set<DBObjectKey> revoke_keys;
     auto ex_objects = rl->getDbObjects(true);
     for (auto key : *ex_objects | boost::adaptors::map_keys) {
       if (key.permissionType != TableDBObjectType &&
           key.permissionType != ViewDBObjectType) {
         continue;
       }
       bool found = false;
       for (auto obj : objects) {
         found = key == obj.getObjectKey() ? true : false;
         if (found) {
           break;
         }
       }
       if (!found) {
         revoke_keys.insert(key);
       }
     }
     for (auto& key : revoke_keys) {
       // revoke privs on object since the object is no
       // longer used by the dashboard as source
       // NOTE(wamsi): Transactionally unsafe
       SysCatalog::instance().revokeDBObjectPrivileges(
           dash_role_name, *rl->findDbObject(key, true), *this);
     }
     // Update privileges on remaining objects
     // NOTE(wamsi): Transactionally unsafe
     SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createShardedTable	(	TableDescriptor &	td,
		const std::list< ColumnDescriptor > &	columns,
		const std::vector< Parser::SharedDictionaryDef > &	shared_dict_defs
	)

Definition at line 3908 of file Catalog.cpp.

References CHECK, TableDescriptor::nShards, TableDescriptor::shard, table_is_temporary(), TableDescriptor::tableId, and TableDescriptor::tableName.

                                                                   {
   /* create logical table */
   TableDescriptor* tdl = &td;
   createTable(*tdl, cols, shared_dict_defs, true);  // create logical table
   int32_t logical_tb_id = tdl->tableId;
   std::string logical_table_name = tdl->tableName;
 
   /* create physical tables and link them to the logical table */
   std::vector<int32_t> physicalTables;
   for (int32_t i = 1; i <= td.nShards; i++) {
     TableDescriptor* tdp = &td;
     tdp->tableName = generatePhysicalTableName(logical_table_name, i);
     tdp->shard = i - 1;
     createTable(*tdp, cols, shared_dict_defs, false);  // create physical table
     int32_t physical_tb_id = tdp->tableId;
 
     /* add physical table to the vector of physical tables */
     physicalTables.push_back(physical_tb_id);
   }
 
   if (!physicalTables.empty()) {
     cat_write_lock write_lock(this);
     /* add logical to physical tables correspondence to the map */
     const auto it_ok =
         logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
     CHECK(it_ok.second);
     /* update sqlite mapd_logical_to_physical in sqlite database */
     if (!table_is_temporary(&td)) {
       updateLogicalToPhysicalTableMap(logical_tb_id);
     }
   }
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::createSystemTableServer	(	const std::string &	server_name,
		const std::string &	data_wrapper_type,
		const foreign_storage::OptionsMap &	options = `{}`
	)

private

Definition at line 6442 of file Catalog.cpp.

References logger::INFO, shared::kRootUserId, LOG, and run_benchmark_import::tables.

                                                                                 {
   auto server = std::make_unique<foreign_storage::ForeignServer>(
       server_name, data_wrapper_type, options, shared::kRootUserId);
   server->validate();
   auto stored_server = getForeignServer(server_name);
   if (stored_server && stored_server->options != server->options) {
     // Drop all tables for server before dropping server.
     auto tables = getAllForeignTablesForForeignServer(stored_server->id);
     for (const auto table : tables) {
       LOG(INFO) << "Dropping existing \"" << table->tableName << "\" system table for \""
                 << server_name << "\" foreign server.";
       dropTable(table);
     }
     LOG(INFO) << "Dropping existing \"" << server_name
               << "\" system table foreign server.";
     dropForeignServer(server_name);
     stored_server = nullptr;
   }
   if (!stored_server) {
     LOG(INFO) << "Creating a new \"" << server_name << "\" system table foreign server.";
     createForeignServer(std::move(server), true);
   }
 }

void Catalog_Namespace::Catalog::createTable	(	TableDescriptor &	td,
		const std::list< ColumnDescriptor > &	columns,
		const std::vector< Parser::SharedDictionaryDef > &	shared_dict_defs,
		bool	isLogicalTable
	)

Definition at line 2743 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnId, TableDescriptor::columnIdBySpi_, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::default_value, dict_ref_t::dictId, Data_Namespace::DISK_LEVEL, StorageType::FOREIGN_TABLE, TableDescriptor::fragPageSize, TableDescriptor::fragType, g_enable_fsi, g_serialize_temp_tables, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_next_refresh_time(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), TableDescriptor::hasDeletedCol, SQLTypeInfo::is_array(), SQLTypeInfo::is_geometry(), TableDescriptor::is_system_table, ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, kBIGINT, kBOOLEAN, kENCODING_DICT, TableDescriptor::keyMetainfo, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nColumns, foreign_storage::ForeignTable::next_refresh_time, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, SQLTypeInfo::set_comp_param(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::set_dict_key(), SQLTypeInfo::set_size(), TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::storageType, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), TableDescriptor::userId, TableDescriptor::viewSQL, and ColumnDescriptor::virtualExpr.

                          {
   cat_write_lock write_lock(this);
   list<ColumnDescriptor> cds = cols;
   list<DictDescriptor> dds;
   std::set<std::string> toplevel_column_names;
   list<ColumnDescriptor> columns;
 
   if (!td.storageType.empty() &&
       (!g_enable_fsi || td.storageType != StorageType::FOREIGN_TABLE)) {
     if (td.persistenceLevel == Data_Namespace::MemoryLevel::DISK_LEVEL) {
       throw std::runtime_error("Only temporary tables can be backed by foreign storage.");
     }
     dataMgr_->getForeignStorageInterface()->prepareTable(getCurrentDB().dbId, td, cds);
   }
 
   for (auto cd : cds) {
     if (cd.columnName == "rowid") {
       throw std::runtime_error(
           "Cannot create column with name rowid. rowid is a system defined column.");
     }
     columns.push_back(cd);
     toplevel_column_names.insert(cd.columnName);
     if (cd.columnType.is_geometry()) {
       expandGeoColumn(cd, columns);
     }
   }
   cds.clear();
 
   ColumnDescriptor cd;
   // add row_id column -- Must be last column in the table
   cd.columnName = "rowid";
   cd.isSystemCol = true;
   cd.columnType = SQLTypeInfo(kBIGINT, true);
 #ifdef MATERIALIZED_ROWID
   cd.isVirtualCol = false;
 #else
   cd.isVirtualCol = true;
   cd.virtualExpr = "MAPD_FRAG_ID * MAPD_ROWS_PER_FRAG + MAPD_FRAG_ROW_ID";
 #endif
   columns.push_back(cd);
   toplevel_column_names.insert(cd.columnName);
 
   if (td.hasDeletedCol) {
     ColumnDescriptor cd_del;
     cd_del.columnName = "$deleted$";
     cd_del.isSystemCol = true;
     cd_del.isVirtualCol = false;
     cd_del.columnType = SQLTypeInfo(kBOOLEAN, true);
     cd_del.isDeletedCol = true;
 
     columns.push_back(cd_del);
   }
 
   for (auto& column : columns) {
     column.db_id = getDatabaseId();
   }
 
   td.nColumns = columns.size();
   // TODO(sy): don't take disk locks or touch sqlite connector for temporary tables
   cat_sqlite_lock sqlite_lock(getObjForLock());
   sqliteConnector_.query("BEGIN TRANSACTION");
   if (td.persistenceLevel == Data_Namespace::MemoryLevel::DISK_LEVEL) {
     try {
       sqliteConnector_.query_with_text_params(
           R"(INSERT INTO mapd_tables (name, userid, ncolumns, isview, fragments, frag_type, max_frag_rows, max_chunk_size, frag_page_size, max_rows, partitions, shard_column_id, shard, num_shards, sort_column_id, storage_type, max_rollback_epochs, is_system_table, key_metainfo) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?))",
           std::vector<std::string>{td.tableName,
                                    std::to_string(td.userId),
                                    std::to_string(td.nColumns),
                                    std::to_string(td.isView),
                                    "",
                                    std::to_string(td.fragType),
                                    std::to_string(td.maxFragRows),
                                    std::to_string(td.maxChunkSize),
                                    std::to_string(td.fragPageSize),
                                    std::to_string(td.maxRows),
                                    td.partitions,
                                    std::to_string(td.shardedColumnId),
                                    std::to_string(td.shard),
                                    std::to_string(td.nShards),
                                    std::to_string(td.sortedColumnId),
                                    td.storageType,
                                    std::to_string(td.maxRollbackEpochs),
                                    std::to_string(td.is_system_table),
                                    td.keyMetainfo});
 
       // now get the auto generated tableid
       sqliteConnector_.query_with_text_param(
           "SELECT tableid FROM mapd_tables WHERE name = ?", td.tableName);
       td.tableId = sqliteConnector_.getData<int>(0, 0);
       int colId = 1;
       for (auto cd : columns) {
         if (cd.columnType.get_compression() == kENCODING_DICT) {
           const bool is_foreign_col =
               setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
           if (!is_foreign_col) {
             // Ideally we would like to not persist string dictionaries for system tables,
             // since system table content can be highly dynamic and string dictionaries
             // are not currently vacuumed.  However, in distributed this causes issues
             // when the cluster is out of sync (when the agg resets but leaves persist) so
             // for the sake of testing we need to leave this as normal dictionaries until
             // we solve the distributed issue.
             auto use_temp_dictionary = false;  // td.is_system_table;
             setColumnDictionary(cd, dds, td, isLogicalTable, use_temp_dictionary);
           }
         }
 
         if (toplevel_column_names.count(cd.columnName)) {
           if (!cd.isGeoPhyCol) {
             td.columnIdBySpi_.push_back(colId);
           }
         }
 
         using BindType = SqliteConnector::BindType;
         std::vector<BindType> types(17, BindType::TEXT);
         if (!cd.default_value.has_value()) {
           types[16] = BindType::NULL_TYPE;
         }
         sqliteConnector_.query_with_text_params(
             "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, "
             "coldim, colscale, is_notnull, "
             "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
             "virtual_expr, is_deletedcol, default_value) "
             "VALUES (?, ?, ?, ?, ?, "
             "?, "
             "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
             std::vector<std::string>{std::to_string(td.tableId),
                                      std::to_string(colId),
                                      cd.columnName,
                                      std::to_string(cd.columnType.get_type()),
                                      std::to_string(cd.columnType.get_subtype()),
                                      std::to_string(cd.columnType.get_dimension()),
                                      std::to_string(cd.columnType.get_scale()),
                                      std::to_string(cd.columnType.get_notnull()),
                                      std::to_string(cd.columnType.get_compression()),
                                      std::to_string(cd.columnType.get_comp_param()),
                                      std::to_string(cd.columnType.get_size()),
                                      "",
                                      std::to_string(cd.isSystemCol),
                                      std::to_string(cd.isVirtualCol),
                                      cd.virtualExpr,
                                      std::to_string(cd.isDeletedCol),
                                      cd.default_value.value_or("NULL")},
             types);
         cd.tableId = td.tableId;
         cd.columnId = colId++;
         cds.push_back(cd);
       }
       if (td.isView) {
         sqliteConnector_.query_with_text_params(
             "INSERT INTO mapd_views (tableid, sql) VALUES (?,?)",
             std::vector<std::string>{std::to_string(td.tableId), td.viewSQL});
       }
       if (td.storageType == StorageType::FOREIGN_TABLE) {
         auto& foreign_table = dynamic_cast<foreign_storage::ForeignTable&>(td);
         foreign_table.next_refresh_time = get_next_refresh_time(foreign_table);
         sqliteConnector_.query_with_text_params(
             "INSERT INTO omnisci_foreign_tables (table_id, server_id, options, "
             "last_refresh_time, next_refresh_time) VALUES (?, ?, ?, ?, ?)",
             std::vector<std::string>{std::to_string(foreign_table.tableId),
                                      std::to_string(foreign_table.foreign_server->id),
                                      foreign_table.getOptionsAsJsonString(),
                                      std::to_string(foreign_table.last_refresh_time),
                                      std::to_string(foreign_table.next_refresh_time)});
       }
     } catch (std::exception& e) {
       sqliteConnector_.query("ROLLBACK TRANSACTION");
       throw;
     }
   } else {  // Temporary table
     td.tableId = nextTempTableId_++;
     int colId = 1;
     for (auto cd : columns) {
       if (cd.columnType.get_compression() == kENCODING_DICT) {
         const bool is_foreign_col =
             setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
 
         if (!is_foreign_col) {
           // Create a new temporary dictionary
           std::string fileName("");
           std::string folderPath("");
           DictRef dict_ref(currentDB_.dbId, nextTempDictId_);
           nextTempDictId_++;
           DictDescriptor dd(dict_ref,
                             fileName,
                             cd.columnType.get_comp_param(),
                             false,
                             1,
                             folderPath,
                             true);  // Is dictName (2nd argument) used?
           dds.push_back(dd);
           if (!cd.columnType.is_array()) {
             cd.columnType.set_size(cd.columnType.get_comp_param() / 8);
           }
           cd.columnType.set_comp_param(dict_ref.dictId);
           set_dict_key(cd);
         }
       }
       if (toplevel_column_names.count(cd.columnName)) {
         if (!cd.isGeoPhyCol) {
           td.columnIdBySpi_.push_back(colId);
         }
       }
       cd.tableId = td.tableId;
       cd.columnId = colId++;
       cds.push_back(cd);
     }
 
     if (g_serialize_temp_tables) {
       serializeTableJsonUnlocked(&td, cds);
     }
   }
 
   try {
     auto cache = dataMgr_->getPersistentStorageMgr()->getDiskCache();
     if (cache) {
       CHECK(!cache->hasCachedMetadataForKeyPrefix({getCurrentDB().dbId, td.tableId}))
           << "Disk cache at " + cache->getCacheDirectory()
           << " contains preexisting data for new table.  Please "
              "delete or clear cache before continuing";
     }
 
     addTableToMap(&td, cds, dds);
     calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
     if (!td.storageType.empty() && td.storageType != StorageType::FOREIGN_TABLE) {
       dataMgr_->getForeignStorageInterface()->registerTable(this, td, cds);
     }
   } catch (std::exception& e) {
     sqliteConnector_.query("ROLLBACK TRANSACTION");
     removeTableFromMap(td.tableName, td.tableId, true);
     throw;
   }
   sqliteConnector_.query("END TRANSACTION");
 
   if (td.storageType != StorageType::FOREIGN_TABLE) {
     write_lock.unlock();
     sqlite_lock.unlock();
     getMetadataForTable(td.tableName,
                         true);  // cause instantiateFragmenter() to be called
   }
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::delDictionary ( const ColumnDescriptor & cd )

Definition at line 2347 of file Catalog.cpp.

References CHECK, CHECK_GT, ColumnDescriptor::columnType, g_base_path, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::is_string(), SQLTypeInfo::is_string_array(), shared::kDataDirectoryName, kENCODING_DICT, File_Namespace::renameForDelete(), ColumnDescriptor::tableId, to_string(), and VLOG.

                                                       {
   cat_write_lock write_lock(this);
   cat_sqlite_lock sqlite_lock(getObjForLock());
   if (!(cd.columnType.is_string() || cd.columnType.is_string_array())) {
     return;
   }
   if (!(cd.columnType.get_compression() == kENCODING_DICT)) {
     return;
   }
   const auto dictId = cd.columnType.get_comp_param();
   CHECK_GT(dictId, 0);
   // decrement and zero check dict ref count
   const auto td = getMetadataForTable(cd.tableId, false);
   CHECK(td);
   sqliteConnector_.query_with_text_param(
       "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
       std::to_string(dictId));
   sqliteConnector_.query_with_text_param(
       "SELECT refcount FROM mapd_dictionaries WHERE dictid = ?", std::to_string(dictId));
   const auto refcount = sqliteConnector_.getData<int>(0, 0);
   VLOG(3) << "Dictionary " << dictId << "from dropped table has reference count "
           << refcount;
   if (refcount > 0) {
     return;
   }
   const DictRef dictRef(currentDB_.dbId, dictId);
   sqliteConnector_.query_with_text_param("DELETE FROM mapd_dictionaries WHERE dictid = ?",
                                          std::to_string(dictId));
   File_Namespace::renameForDelete(g_base_path + "/" + shared::kDataDirectoryName +
                                   "/DB_" + std::to_string(currentDB_.dbId) + "_DICT_" +
                                   std::to_string(dictId));
 
   std::unique_ptr<StringDictionaryClient> client;
   if (!string_dict_hosts_.empty()) {
     client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dictRef, true));
   }
   if (client) {
     client->drop(dictRef);
   }
 
   dictDescriptorMapByRef_.erase(dictRef);
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::deleteCustomExpressions	(	const std::vector< int32_t > &	custom_expression_ids,
		bool	do_soft_delete
	)

Deletes custom expressions with the given ids.

Parameters

custom_expression_ids	- ids of custom expressions to delete
do_soft_delete	- flag indicating whether or not to do a soft delete

Definition at line 5732 of file Catalog.cpp.

References CHECK, CHECK_EQ, join(), and to_string().

                                                            {
   cat_write_lock write_lock(this);
   cat_sqlite_lock sqlite_lock(getObjForLock());
 
   std::vector<int32_t> invalid_ids;
   for (const auto id : custom_expression_ids) {
     if (custom_expr_map_by_id_.find(id) == custom_expr_map_by_id_.end()) {
       invalid_ids.emplace_back(id);
     }
   }
   if (!invalid_ids.empty()) {
     throw std::runtime_error{"Custom expressions with ids: " + join(invalid_ids, ",") +
                              " do not exist."};
   }
   sqliteConnector_.query("BEGIN TRANSACTION");
   try {
     for (const auto id : custom_expression_ids) {
       sqliteConnector_.query_with_text_params(
           "SELECT id FROM omnisci_custom_expressions WHERE id = ?",
           std::vector<std::string>{std::to_string(id)});
       CHECK_EQ(sqliteConnector_.getNumRows(), static_cast<size_t>(1));
       if (do_soft_delete) {
         sqliteConnector_.query_with_text_params(
             "UPDATE omnisci_custom_expressions SET is_deleted = ? WHERE id = ?",
             std::vector<std::string>{std::to_string(true), std::to_string(id)});
       } else {
         sqliteConnector_.query_with_text_params(
             "DELETE FROM omnisci_custom_expressions WHERE id = ?",
             std::vector<std::string>{std::to_string(id)});
       }
     }
 
     for (const auto id : custom_expression_ids) {
       if (do_soft_delete) {
         auto it = custom_expr_map_by_id_.find(id);
         CHECK(it != custom_expr_map_by_id_.end());
         it->second->is_deleted = true;
       } else {
         custom_expr_map_by_id_.erase(id);
       }
     }
   } catch (std::exception& e) {
     sqliteConnector_.query("ROLLBACK TRANSACTION");
     throw;
   }
   sqliteConnector_.query("END TRANSACTION");
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::deleteMetadataForDashboards	(	const std::vector< int32_t >	ids,
		const UserMetadata &	user
	)

Definition at line 2106 of file Catalog.cpp.

References DashboardDBObjectType, AccessPrivileges::DELETE_DASHBOARD, and to_string().

                                                                     {
   std::stringstream invalid_ids, restricted_ids;
 
   for (int32_t dashboard_id : dashboard_ids) {
     if (!getMetadataForDashboard(dashboard_id)) {
       invalid_ids << (!invalid_ids.str().empty() ? ", " : "") << dashboard_id;
       continue;
     }
     DBObject object(dashboard_id, DashboardDBObjectType);
     object.loadKey(*this);
     object.setPrivileges(AccessPrivileges::DELETE_DASHBOARD);
     std::vector<DBObject> privs = {object};
     if (!SysCatalog::instance().checkPrivileges(user, privs)) {
       restricted_ids << (!restricted_ids.str().empty() ? ", " : "") << dashboard_id;
     }
   }
 
   if (invalid_ids.str().size() > 0 || restricted_ids.str().size() > 0) {
     std::stringstream error_message;
     error_message << "Delete dashboard(s) failed with error(s):";
     if (invalid_ids.str().size() > 0) {
       error_message << "\nDashboard id: " << invalid_ids.str()
                     << " - Dashboard id does not exist";
     }
     if (restricted_ids.str().size() > 0) {
       error_message
           << "\nDashboard id: " << restricted_ids.str()
           << " - User should be either owner of dashboard or super user to delete it";
     }
     throw std::runtime_error(error_message.str());
   }
   std::vector<DBObject> dash_objs;
 
   for (int32_t dashboard_id : dashboard_ids) {
     dash_objs.emplace_back(dashboard_id, DashboardDBObjectType);
   }
   // BE-5245: Transactionally unsafe (like other combined Catalog/Syscatalog operations)
   SysCatalog::instance().revokeDBObjectPrivilegesFromAllBatch(dash_objs, this);
   {
     cat_write_lock write_lock(this);
     cat_sqlite_lock sqlite_lock(getObjForLock());
 
     sqliteConnector_.query("BEGIN TRANSACTION");
     try {
       for (int32_t dashboard_id : dashboard_ids) {
         auto dash = getMetadataForDashboard(dashboard_id);
         // Dash should still exist if revokeDBObjectPrivileges passed but throw and
         // rollback if already deleted
         if (!dash) {
           throw std::runtime_error(
               std::string("Delete dashboard(s) failed with error(s):\nDashboard id: ") +
               std::to_string(dashboard_id) + " - Dashboard id does not exist ");
         }
         std::string user_id = std::to_string(dash->userId);
         std::string dash_name = dash->dashboardName;
         auto viewDescIt = dashboardDescriptorMap_.find(user_id + ":" + dash_name);
         dashboardDescriptorMap_.erase(viewDescIt);
         sqliteConnector_.query_with_text_params(
             "DELETE FROM mapd_dashboards WHERE name = ? and userid = ?",
             std::vector<std::string>{dash_name, user_id});
       }
     } catch (std::exception& e) {
       sqliteConnector_.query("ROLLBACK TRANSACTION");
       throw;
     }
     sqliteConnector_.query("END TRANSACTION");
   }
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::deleteTableCatalogMetadata	(	const TableDescriptor *	logical_table,
		const std::vector< const TableDescriptor * > &	physical_tables
	)

private

Definition at line 4114 of file Catalog.cpp.

References TableDescriptor::tableId, and to_string().

                                                               {
   cat_write_lock write_lock(this);
   cat_sqlite_lock sqlite_lock(getObjForLock());
   sqliteConnector_.query("BEGIN TRANSACTION");
   try {
     // remove corresponding record from the logicalToPhysicalTableMap in sqlite database
     sqliteConnector_.query_with_text_param(
         "DELETE FROM mapd_logical_to_physical WHERE logical_table_id = ?",
         std::to_string(logical_table->tableId));
     logicalToPhysicalTableMapById_.erase(logical_table->tableId);
     for (auto table : physical_tables) {
       eraseTableMetadata(table);
     }
   } catch (std::exception& e) {
     sqliteConnector_.query("ROLLBACK TRANSACTION");
     throw;
   }
   sqliteConnector_.query("END TRANSACTION");
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::doTruncateTable ( const TableDescriptor * td )

protected

Definition at line 3952 of file Catalog.cpp.

References CHECK, CHECK_GE, ColumnDescriptor::columnType, Data_Namespace::CPU_LEVEL, dict_ref_t::dictId, DictDescriptor::dictIsTemp, DictDescriptor::dictRef, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), Data_Namespace::GPU_LEVEL, kENCODING_DICT, File_Namespace::renameForDelete(), ColumnDescriptor::tableId, and TableDescriptor::tableId.

                                                        {
   // must destroy fragmenter before deleteChunks is called.
   removeFragmenterForTable(td->tableId);
 
   const int tableId = td->tableId;
   ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
   // assuming deleteChunksWithPrefix is atomic
   dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
   dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
 
   dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
 
   cat_write_lock write_lock(this);
   std::unique_ptr<StringDictionaryClient> client;
   if (SysCatalog::instance().isAggregator()) {
     CHECK(!string_dict_hosts_.empty());
     DictRef dict_ref(currentDB_.dbId, -1);
     client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
   }
   // clean up any dictionaries
   // delete all column descriptors for the table
   for (const auto& columnDescriptor : columnDescriptorMapById_) {
     auto cd = columnDescriptor.second;
     if (cd->tableId != td->tableId) {
       continue;
     }
     const int dict_id = cd->columnType.get_comp_param();
     // Dummy dictionaries created for a shard of a logical table have the id set to zero.
     if (cd->columnType.get_compression() == kENCODING_DICT && dict_id) {
       const DictRef dict_ref(currentDB_.dbId, dict_id);
       const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
       CHECK(dictIt != dictDescriptorMapByRef_.end());
       const auto& dd = dictIt->second;
       CHECK_GE(dd->refcount, 1);
       // if this is the only table using this dict reset the dict
       if (dd->refcount == 1) {
         // close the dictionary
         dd->stringDict.reset();
         File_Namespace::renameForDelete(dd->dictFolderPath);
         if (client) {
           client->drop(dd->dictRef);
         }
         if (!dd->dictIsTemp) {
           boost::filesystem::create_directory(dd->dictFolderPath);
         }
       }
 
       DictDescriptor* new_dd = new DictDescriptor(dd->dictRef,
                                                   dd->dictName,
                                                   dd->dictNBits,
                                                   dd->dictIsShared,
                                                   dd->refcount,
                                                   dd->dictFolderPath,
                                                   dd->dictIsTemp);
       dictDescriptorMapByRef_.erase(dictIt);
       // now create new Dict -- need to figure out what to do here for temp tables
       if (client) {
         client->create(new_dd->dictRef, new_dd->dictIsTemp);
       }
       dictDescriptorMapByRef_[new_dd->dictRef].reset(new_dd);
       getMetadataForDict(new_dd->dictRef.dictId);
     }
   }
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropColumn	(	const TableDescriptor &	td,
		const ColumnDescriptor &	cd
	)

Definition at line 2481 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnId, ColumnDescriptor::columnName, TableDescriptor::nShards, TableDescriptor::shard, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), and to_upper().

                                                                               {
   {
     cat_write_lock write_lock(this);
     cat_sqlite_lock sqlite_lock(getObjForLock());
     // caller must handle sqlite/chunk transaction TOGETHER
     sqliteConnector_.query_with_text_params(
         "DELETE FROM mapd_columns where tableid = ? and columnid = ?",
         std::vector<std::string>{std::to_string(td.tableId),
                                  std::to_string(cd.columnId)});
 
     sqliteConnector_.query_with_text_params(
         "UPDATE mapd_tables SET ncolumns = ncolumns - 1 WHERE tableid = ?",
         std::vector<std::string>{std::to_string(td.tableId)});
 
     ColumnDescriptorMap::iterator columnDescIt =
         columnDescriptorMap_.find(ColumnKey(cd.tableId, to_upper(cd.columnName)));
     CHECK(columnDescIt != columnDescriptorMap_.end());
 
     columnDescriptorsForRoll.emplace_back(columnDescIt->second, nullptr);
     removeFromColumnMap(columnDescIt->second);
     --tableDescriptorMapById_[td.tableId]->nColumns;
   }
 
   // for each shard
   if (td.nShards > 0 && td.shard < 0) {
     for (const auto shard : getPhysicalTablesDescriptors(&td)) {
       const auto shard_cd = getMetadataForColumn(shard->tableId, cd.columnId);
       CHECK(shard_cd);
       dropColumn(*shard, *shard_cd);
     }
   }
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropForeignServer ( const std::string & server_name )

Drops/deletes a foreign server DB object.

Parameters

server_name - Name of foreign server that will be deleted

Definition at line 3258 of file Catalog.cpp.

References CHECK_EQ, and to_string().

                                                             {
   cat_write_lock write_lock(this);
   cat_sqlite_lock sqlite_lock(getObjForLock());
 
   sqliteConnector_.query_with_text_params(
       "SELECT id from omnisci_foreign_servers where name = ?",
       std::vector<std::string>{server_name});
   auto num_rows = sqliteConnector_.getNumRows();
   if (num_rows > 0) {
     CHECK_EQ(size_t(1), num_rows);
     auto server_id = sqliteConnector_.getData<int32_t>(0, 0);
     sqliteConnector_.query_with_text_param(
         "SELECT table_id from omnisci_foreign_tables where server_id = ?",
         std::to_string(server_id));
     if (sqliteConnector_.getNumRows() > 0) {
       throw std::runtime_error{"Foreign server \"" + server_name +
                                "\" is referenced "
                                "by existing foreign tables and cannot be dropped."};
     }
     sqliteConnector_.query("BEGIN TRANSACTION");
     try {
       sqliteConnector_.query_with_text_params(
           "DELETE FROM omnisci_foreign_servers WHERE name = ?",
           std::vector<std::string>{server_name});
     } catch (const std::exception& e) {
       sqliteConnector_.query("ROLLBACK TRANSACTION");
       throw;
     }
     sqliteConnector_.query("END TRANSACTION");
     foreignServerMap_.erase(server_name);
     foreignServerMapById_.erase(server_id);
   }
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropTable ( const TableDescriptor * td )

Definition at line 4086 of file Catalog.cpp.

References CHECK, TableDescriptor::isView, TableDBObjectType, TableDescriptor::tableId, TableDescriptor::tableName, and ViewDBObjectType.

Referenced by Catalog_Namespace::anonymous_namespace{Catalog.cpp}::drop_tables().

                                                  {
   SysCatalog::instance().revokeDBObjectPrivilegesFromAll(
       DBObject(td->tableName, td->isView ? ViewDBObjectType : TableDBObjectType), this);
   std::vector<const TableDescriptor*> tables_to_drop;
   {
     cat_read_lock read_lock(this);
     const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
     if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
       // remove all corresponding physical tables if this is a logical table
       const auto physicalTables = physicalTableIt->second;
       CHECK(!physicalTables.empty());
       for (size_t i = 0; i < physicalTables.size(); i++) {
         int32_t physical_tb_id = physicalTables[i];
         const TableDescriptor* phys_td =
             getMutableMetadataForTableUnlocked(physical_tb_id);
         CHECK(phys_td);
         tables_to_drop.emplace_back(phys_td);
       }
     }
     tables_to_drop.emplace_back(td);
   }
 
   for (auto table : tables_to_drop) {
     eraseTablePhysicalData(table);
   }
   deleteTableCatalogMetadata(td, tables_to_drop);
 }

Here is the caller graph for this function:

void Catalog_Namespace::Catalog::dropTableFromJsonUnlocked ( const std::string & table_name ) const

protected

Definition at line 3059 of file Catalog.cpp.

References CHECK, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and VLOG.

                                                                          {
   // relies on the catalog write lock
   using namespace rapidjson;
 
   VLOG(1) << "Dropping temporary table " << table_name << " to JSON for Calcite.";
 
   const auto db_name = currentDB_.dbName;
   const auto file_path = table_json_filepath(basePath_, db_name);
 
   CHECK(boost::filesystem::exists(file_path));
   Document d;
 
   std::ifstream reader(file_path.string());
   CHECK(reader.is_open());
   IStreamWrapper json_read_wrapper(reader);
   d.ParseStream(json_read_wrapper);
 
   CHECK(d.IsObject());
   auto table_name_ref = StringRef(table_name.c_str());
   CHECK(d.HasMember(table_name_ref));
   CHECK(d.RemoveMember(table_name_ref));
 
   // Overwrite the existing file
   std::ofstream writer(file_path.string(), std::ios::trunc | std::ios::out);
   CHECK(writer.is_open());
   OStreamWrapper json_wrapper(writer);
 
   Writer<OStreamWrapper> json_writer(json_wrapper);
   d.Accept(json_writer);
   writer.close();
 }

Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::dumpCreateServer	(	const std::string &	name,
		bool	multiline_formatting = `true`
	)		const

Definition at line 5409 of file Catalog.cpp.

References join().

                                                                        {
   cat_read_lock read_lock(this);
   auto server_it = foreignServerMap_.find(name);
   if (server_it == foreignServerMap_.end()) {
     throw std::runtime_error("Foreign server " + name + " does not exist.");
   }
   auto server = server_it->second.get();
   std::ostringstream os;
   os << "CREATE SERVER " << name << " FOREIGN DATA WRAPPER " << server->data_wrapper_type;
   std::vector<std::string> with_options;
   for (const auto& [option, value] : server->options) {
     with_options.emplace_back(option + "='" + value + "'");
   }
   if (!with_options.empty()) {
     if (!multiline_formatting) {
       os << " ";
     } else {
       os << "\n";
     }
     os << "WITH (" + boost::algorithm::join(with_options, ", ") + ")";
   }
   os << ";";
   return os.str();
 }

Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::dumpCreateTable	(	const TableDescriptor *	td,
		bool	multiline_formatting = `true`,
		bool	dump_defaults = `false`
	)		const

Definition at line 5230 of file Catalog.cpp.

                                                                {
   cat_read_lock read_lock(this);
   return dumpCreateTableUnlocked(td, multiline_formatting, dump_defaults);
 }

std::optional< std::string > Catalog_Namespace::Catalog::dumpCreateTable	(	int32_t	table_id,
		bool	multiline_formatting = `true`,
		bool	dump_defaults = `false`
	)		const

Definition at line 5237 of file Catalog.cpp.

                                                                               {
   cat_read_lock read_lock(this);
   const auto td = getMutableMetadataForTableUnlocked(table_id);
   if (!td) {
     return {};
   }
   return dumpCreateTableUnlocked(td, multiline_formatting, dump_defaults);
 }

std::string Catalog_Namespace::Catalog::dumpCreateTableUnlocked	(	const TableDescriptor *	td,
		bool	multiline_formatting,
		bool	dump_defaults
	)		const

private

Definition at line 5248 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, Data_Namespace::CPU_LEVEL, DEFAULT_FRAGMENT_ROWS, DEFAULT_MAX_CHUNK_SIZE, DEFAULT_MAX_ROLLBACK_EPOCHS, DEFAULT_MAX_ROWS, DEFAULT_PAGE_SIZE, ColumnDescriptor::default_value, TableDescriptor::fragPageSize, g_leaf_count, ColumnDescriptor::getDefaultValueLiteral(), TableDescriptor::hasDeletedCol, TableDescriptor::is_system_table, ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, join(), kCHAR, kENCODING_DICT, kENCODING_GEOINT, kTEXT, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and TableDescriptor::viewSQL.

                                                                        {
   auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
   std::ostringstream os;
 
   if (foreign_table && !td->is_system_table) {
     os << "CREATE FOREIGN TABLE " << td->tableName << " (";
   } else if (!td->isView) {
     os << "CREATE ";
     if (td->persistenceLevel == Data_Namespace::MemoryLevel::CPU_LEVEL) {
       os << "TEMPORARY ";
     }
     os << "TABLE " + td->tableName + " (";
   } else {
     os << "CREATE VIEW " + td->tableName + " AS " << td->viewSQL;
     return os.str();
   }
   // scan column defines
   std::vector<std::string> additional_info;
   std::set<std::string> shared_dict_column_names;
 
   gatherAdditionalInfo(additional_info, shared_dict_column_names, td);
 
   // gather column defines
   const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
   std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
   bool first = true;
   for (const auto cd : cds) {
     if (!(cd->isSystemCol || cd->isVirtualCol)) {
       const auto& ti = cd->columnType;
       if (!first) {
         os << ",";
         if (!multiline_formatting) {
           os << " ";
         }
       } else {
         first = false;
       }
       if (multiline_formatting) {
         os << "\n  ";
       }
       // column name
       os << quoteIfRequired(cd->columnName);
       // CHAR is perculiar... better dump it as TEXT(32) like \d does
       if (ti.get_type() == SQLTypes::kCHAR) {
         os << " "
            << "TEXT";
       } else if (ti.get_subtype() == SQLTypes::kCHAR) {
         os << " "
            << "TEXT[]";
       } else {
         os << " " << ti.get_type_name();
       }
       os << (ti.get_notnull() ? " NOT NULL" : "");
       if (cd->default_value.has_value()) {
         os << " DEFAULT " << cd->getDefaultValueLiteral();
       }
       if (shared_dict_column_names.find(cd->columnName) ==
           shared_dict_column_names.end()) {
         // avoids "Column ... shouldn't specify an encoding, it borrows it
         //    from the referenced column"
         if (ti.is_string() || (ti.is_array() && ti.get_subtype() == kTEXT)) {
           auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
           if (ti.get_compression() == kENCODING_DICT) {
             os << " ENCODING " << ti.get_compression_name() << "(" << (size * 8) << ")";
           } else {
             os << " ENCODING NONE";
           }
         } else if (ti.is_date_in_days() ||
                    (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
           const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
           os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
         } else if (ti.is_geometry()) {
           if (ti.get_compression() == kENCODING_GEOINT) {
             os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
                << ")";
           } else {
             os << " ENCODING NONE";
           }
         }
       }
     }
   }
   // gather SHARED DICTIONARYs
   if (additional_info.size()) {
     std::string comma;
     if (!multiline_formatting) {
       comma = ", ";
     } else {
       comma = ",\n  ";
     }
     os << comma;
     os << boost::algorithm::join(additional_info, comma);
   }
   os << ")";
 
   std::vector<std::string> with_options;
   if (foreign_table && !td->is_system_table) {
     if (multiline_formatting) {
       os << "\n";
     } else {
       os << " ";
     }
     os << "SERVER " << foreign_table->foreign_server->name;
 
     // gather WITH options ...
     for (const auto& [option, value] : foreign_table->options) {
       with_options.emplace_back(option + "='" + value + "'");
     }
   }
 
   if (dump_defaults || td->maxFragRows != DEFAULT_FRAGMENT_ROWS) {
     with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
   }
   if (dump_defaults || td->maxChunkSize != DEFAULT_MAX_CHUNK_SIZE) {
     with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
   }
   if (!foreign_table && (dump_defaults || td->fragPageSize != DEFAULT_PAGE_SIZE)) {
     with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
   }
   if (!foreign_table && (dump_defaults || td->maxRows != DEFAULT_MAX_ROWS)) {
     with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
   }
   if ((dump_defaults || td->maxRollbackEpochs != DEFAULT_MAX_ROLLBACK_EPOCHS) &&
       td->maxRollbackEpochs != -1) {
     with_options.push_back("MAX_ROLLBACK_EPOCHS=" +
                            std::to_string(td->maxRollbackEpochs));
   }
   if (!foreign_table && (dump_defaults || !td->hasDeletedCol)) {
     with_options.emplace_back(td->hasDeletedCol ? "VACUUM='DELAYED'"
                                                 : "VACUUM='IMMEDIATE'");
   }
   if (!foreign_table && !td->partitions.empty()) {
     with_options.push_back("PARTITIONS='" + td->partitions + "'");
   }
   if (!foreign_table && td->nShards > 0) {
     const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
     CHECK(shard_cd);
     with_options.push_back(
         "SHARD_COUNT=" +
         std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
   }
   if (!foreign_table && td->sortedColumnId > 0) {
     const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
     CHECK(sort_cd);
     with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
   }
 
   if (!with_options.empty()) {
     if (!multiline_formatting) {
       os << " ";
     } else {
       os << "\n";
     }
     os << "WITH (" + boost::algorithm::join(with_options, ", ") + ")";
   }
   os << ";";
   return os.str();
 }

Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::dumpSchema ( const TableDescriptor * td ) const

Definition at line 5103 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, DEFAULT_MAX_ROLLBACK_EPOCHS, ColumnDescriptor::default_value, TableDescriptor::fragPageSize, g_leaf_count, ColumnDescriptor::getDefaultValueLiteral(), TableDescriptor::hasDeletedCol, TableDescriptor::is_system_table, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, join(), kCHAR, kENCODING_DICT, kENCODING_GEOINT, kTEXT, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::tableId, and to_string().

Referenced by TableArchiver::dumpTable().

                                                              {
   CHECK(!td->is_system_table);
   cat_read_lock read_lock(this);
 
   std::ostringstream os;
   os << "CREATE TABLE @T (";
   // gather column defines
   const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
   std::string comma;
   std::vector<std::string> shared_dicts;
   std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
   for (const auto cd : cds) {
     if (!(cd->isSystemCol || cd->isVirtualCol)) {
       const auto& ti = cd->columnType;
       os << comma << cd->columnName;
       // CHAR is perculiar... better dump it as TEXT(32) like \d does
       if (ti.get_type() == SQLTypes::kCHAR) {
         os << " "
            << "TEXT";
       } else if (ti.get_subtype() == SQLTypes::kCHAR) {
         os << " "
            << "TEXT[]";
       } else {
         os << " " << ti.get_type_name();
       }
       os << (ti.get_notnull() ? " NOT NULL" : "");
       if (cd->default_value.has_value()) {
         os << " DEFAULT " << cd->getDefaultValueLiteral();
       }
       if (ti.is_string() || (ti.is_array() && ti.get_subtype() == kTEXT)) {
         auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
         if (ti.get_compression() == kENCODING_DICT) {
           // if foreign reference, get referenced tab.col
           const auto dict_id = ti.get_comp_param();
           const DictRef dict_ref(currentDB_.dbId, dict_id);
           const auto dict_it = dictDescriptorMapByRef_.find(dict_ref);
           CHECK(dict_it != dictDescriptorMapByRef_.end());
           const auto dict_name = dict_it->second->dictName;
           // when migrating a table, any foreign dict ref will be dropped
           // and the first cd of a dict will become root of the dict
           if (dict_root_cds.end() == dict_root_cds.find(dict_name)) {
             dict_root_cds[dict_name] = cd;
             os << " ENCODING " << ti.get_compression_name() << "(" << (size * 8) << ")";
           } else {
             const auto dict_root_cd = dict_root_cds[dict_name];
             shared_dicts.push_back("SHARED DICTIONARY (" + cd->columnName +
                                    ") REFERENCES @T(" + dict_root_cd->columnName + ")");
             // "... shouldn't specify an encoding, it borrows from the referenced
             // column"
           }
         } else {
           os << " ENCODING NONE";
         }
       } else if (ti.is_date_in_days() ||
                  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
         const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
         os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
       } else if (ti.is_geometry()) {
         if (ti.get_compression() == kENCODING_GEOINT) {
           os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
              << ")";
         } else {
           os << " ENCODING NONE";
         }
       }
       comma = ", ";
     }
   }
   // gather SHARED DICTIONARYs
   if (shared_dicts.size()) {
     os << ", " << boost::algorithm::join(shared_dicts, ", ");
   }
   // gather WITH options ...
   std::vector<std::string> with_options;
   with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
   with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
   with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
   with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
   with_options.emplace_back(td->hasDeletedCol ? "VACUUM='DELAYED'"
                                               : "VACUUM='IMMEDIATE'");
   if (!td->partitions.empty()) {
     with_options.push_back("PARTITIONS='" + td->partitions + "'");
   }
   if (td->nShards > 0) {
     const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
     CHECK(shard_cd);
     os << ", SHARD KEY(" << shard_cd->columnName << ")";
     with_options.push_back(
         "SHARD_COUNT=" +
         std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
   }
   if (td->sortedColumnId > 0) {
     const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
     CHECK(sort_cd);
     with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
   }
   if (td->maxRollbackEpochs != DEFAULT_MAX_ROLLBACK_EPOCHS &&
       td->maxRollbackEpochs != -1) {
     with_options.push_back("MAX_ROLLBACK_EPOCHS=" +
                            std::to_string(td->maxRollbackEpochs));
   }
   os << ") WITH (" + boost::algorithm::join(with_options, ", ") + ");";
   return os.str();
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void Catalog_Namespace::Catalog::eraseDbMetadata ( )

Definition at line 4880 of file Catalog.cpp.

References shared::kCatalogDirectoryName, and run_benchmark_import::tables.

                               {
   const auto tables = getAllTableMetadata();
   for (const auto table : tables) {
     eraseTableMetadata(table);
   }
   // Physically erase database metadata
   boost::filesystem::remove(basePath_ + "/" + shared::kCatalogDirectoryName + "/" +
                             currentDB_.dbName);
   calciteMgr_->updateMetadata(currentDB_.dbName, "");
 }

void Catalog_Namespace::Catalog::eraseDbPhysicalData ( )

Definition at line 4891 of file Catalog.cpp.

References run_benchmark_import::tables.

                                   {
   const auto tables = getAllTableMetadata();
   for (const auto table : tables) {
     eraseTablePhysicalData(table);
   }
 }

void Catalog_Namespace::Catalog::eraseTableMetadata ( const TableDescriptor * td )

protected

Definition at line 4136 of file Catalog.cpp.

References g_serialize_temp_tables, INJECT_TIMER, table_is_temporary(), TableDescriptor::tableId, and TableDescriptor::tableName.

                                                           {
   executeDropTableSqliteQueries(td);
   if (g_serialize_temp_tables && table_is_temporary(td)) {
     dropTableFromJsonUnlocked(td->tableName);
   }
   calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
   {
     INJECT_TIMER(removeTableFromMap_);
     removeTableFromMap(td->tableName, td->tableId);
   }
 }

Here is the call graph for this function:

void Catalog_Namespace::Catalog::eraseTablePhysicalData ( const TableDescriptor * td )

Definition at line 4898 of file Catalog.cpp.

References Data_Namespace::CPU_LEVEL, Data_Namespace::GPU_LEVEL, INJECT_TIMER, TableDescriptor::isView, and TableDescriptor::tableId.

                                                               {
   const int tableId = td->tableId;
   // must destroy fragmenter before deleteChunks is called.
   removeFragmenterForTable(tableId);
 
   ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
   {
     INJECT_TIMER(deleteChunksWithPrefix);
     // assuming deleteChunksWithPrefix is atomic
     dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
     dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
   }
   if (!td->isView) {
     INJECT_TIMER(Remove_Table);
     dataMgr_->removeTableRelatedDS(

Public Member Functions

Static Public Member Functions

Public Attributes

Static Public Attributes

Protected Types

Protected Member Functions

Protected Attributes

Static Protected Attributes

Private Member Functions

Private Attributes

Static Private Attributes

Friends

Detailed Description

Member Typedef Documentation

Constructor & Destructor Documentation

Member Function Documentation