OmniSciDB  6686921089
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Catalog_Namespace::Catalog Class Referencefinal

class for a per-database catalog. also includes metadata for the current database and the current user. More...

#include <Catalog.h>

+ Collaboration diagram for Catalog_Namespace::Catalog:

Public Member Functions

 Catalog (const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
 Constructor - takes basePath to already extant data directory for writing. More...
 
 Catalog ()
 Constructor builds a hollow catalog used during constructor of other catalogs. More...
 
 ~Catalog ()
 Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite. More...
 
void createTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
 
void createShardedTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
int32_t createDashboard (DashboardDescriptor &vd)
 
void replaceDashboard (DashboardDescriptor &vd)
 
std::string createLink (LinkDescriptor &ld, size_t min_length)
 
void dropTable (const TableDescriptor *td)
 
void truncateTable (const TableDescriptor *td)
 
void renameTable (const TableDescriptor *td, const std::string &newTableName)
 
void renameTable (std::vector< std::pair< std::string, std::string >> &names)
 
void renameColumn (const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
 
void addColumn (const TableDescriptor &td, ColumnDescriptor &cd)
 
void dropColumn (const TableDescriptor &td, const ColumnDescriptor &cd)
 
void removeFragmenterForTable (const int table_id) const
 
const std::map< int, const
ColumnDescriptor * > 
getDictionaryToColumnMapping ()
 
const TableDescriptorgetMetadataForTable (const std::string &tableName, const bool populateFragmenter=true) const
 Returns a pointer to a const TableDescriptor struct matching the provided tableName. More...
 
const TableDescriptorgetMetadataForTable (int tableId, bool populateFragmenter=true) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, const std::string &colName) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, int columnId) const
 
const int getColumnIdBySpi (const int tableId, const size_t spi) const
 
const ColumnDescriptorgetMetadataForColumnBySpi (const int tableId, const size_t spi) const
 
const DashboardDescriptorgetMetadataForDashboard (const std::string &userId, const std::string &dashName) const
 
const DashboardDescriptorgetMetadataForDashboard (const int32_t dashboard_id) const
 
void deleteMetadataForDashboards (const std::vector< int32_t > ids, const UserMetadata &user)
 
const LinkDescriptorgetMetadataForLink (const std::string &link) const
 
const LinkDescriptorgetMetadataForLink (int linkId) const
 
const
foreign_storage::ForeignTable
getForeignTable (const std::string &tableName) const
 
const
foreign_storage::ForeignTable
getForeignTable (int table_id) const
 
std::list< const
ColumnDescriptor * > 
getAllColumnMetadataForTable (const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular table specified by table id. More...
 
std::list< const
TableDescriptor * > 
getAllTableMetadata () const
 
std::vector< TableDescriptorgetAllTableMetadataCopy () const
 
std::list< const
DashboardDescriptor * > 
getAllDashboardsMetadata () const
 
std::vector< DashboardDescriptorgetAllDashboardsMetadataCopy () const
 
const DBMetadatagetCurrentDB () const
 
Data_Namespace::DataMgrgetDataMgr () const
 
std::shared_ptr< CalcitegetCalciteMgr () const
 
const std::string & getCatalogBasePath () const
 
const DictDescriptorgetMetadataForDict (int dict_ref, bool loadDict=true) const
 
const std::vector< LeafHostInfo > & getStringDictionaryHosts () const
 
const ColumnDescriptorgetShardColumnMetadataForTable (const TableDescriptor *td) const
 
std::vector< const
TableDescriptor * > 
getPhysicalTablesDescriptors (const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
 
std::vector< std::string > getTableNamesForUser (const UserMetadata &user, const GetTablesType get_tables_type) const
 
std::vector< TableMetadatagetTablesMetadataForUser (const UserMetadata &user_metadata, const GetTablesType get_tables_type, const std::string &filter_table_name) const
 
int32_t getTableEpoch (const int32_t db_id, const int32_t table_id) const
 
void setTableEpoch (const int db_id, const int table_id, const int new_epoch)
 
void setMaxRollbackEpochs (const int32_t table_id, const int32_t max_rollback_epochs)
 
void setMaxRows (const int32_t table_id, const int64_t max_rows)
 
std::vector< TableEpochInfogetTableEpochs (const int32_t db_id, const int32_t table_id) const
 
void setTableEpochs (const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
 
void setTableEpochsLogExceptions (const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
 
int getDatabaseId () const
 
SqliteConnectorgetSqliteConnector ()
 
void roll (const bool forward)
 
DictRef addDictionary (ColumnDescriptor &cd)
 
void delDictionary (const ColumnDescriptor &cd)
 
void getDictionary (const ColumnDescriptor &cd, std::map< int, StringDictionary * > &stringDicts)
 
const bool checkMetadataForDeletedRecs (const TableDescriptor *td, int column_id) const
 
const ColumnDescriptorgetDeletedColumn (const TableDescriptor *td) const
 
const ColumnDescriptorgetDeletedColumnIfRowsDeleted (const TableDescriptor *td) const
 
void setDeletedColumn (const TableDescriptor *td, const ColumnDescriptor *cd)
 
int getLogicalTableId (const int physicalTableId) const
 
void checkpoint (const int logicalTableId) const
 
void checkpointWithAutoRollback (const int logical_table_id) const
 
std::string name () const
 
void eraseDbMetadata ()
 
void eraseDbPhysicalData ()
 
void eraseTablePhysicalData (const TableDescriptor *td)
 
void setForReload (const int32_t tableId)
 
std::vector< std::string > getTableDataDirectories (const TableDescriptor *td) const
 
std::vector< std::string > getTableDictDirectories (const TableDescriptor *td) const
 
std::string getColumnDictDirectory (const ColumnDescriptor *cd) const
 
std::string dumpSchema (const TableDescriptor *td) const
 
std::string dumpCreateTable (const TableDescriptor *td, bool multiline_formatting=true, bool dump_defaults=false) const
 
void createForeignServer (std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
 
const
foreign_storage::ForeignServer
getForeignServer (const std::string &server_name) const
 
const std::unique_ptr< const
foreign_storage::ForeignServer
getForeignServerFromStorage (const std::string &server_name)
 
const std::unique_ptr< const
foreign_storage::ForeignTable
getForeignTableFromStorage (int table_id)
 
void changeForeignServerOwner (const std::string &server_name, const int new_owner_id)
 
void setForeignServerDataWrapper (const std::string &server_name, const std::string &data_wrapper)
 
void setForeignServerOptions (const std::string &server_name, const std::string &options)
 
void renameForeignServer (const std::string &server_name, const std::string &name)
 
void dropForeignServer (const std::string &server_name)
 
void getForeignServersForUser (const rapidjson::Value *filters, const UserMetadata &user, std::vector< const foreign_storage::ForeignServer * > &results)
 
void createDefaultServersIfNotExists ()
 
bool validateNonExistentTableOrView (const std::string &name, const bool if_not_exists)
 
std::vector< const
TableDescriptor * > 
getAllForeignTablesForRefresh () const
 
void updateForeignTableRefreshTimes (const int32_t table_id)
 
void setForeignTableOptions (const std::string &table_name, foreign_storage::OptionsMap &options_map, bool clear_existing_options=true)
 
void updateLeaf (const LeafHostInfo &string_dict_host)
 
void setUncappedTableEpoch (const std::string &table_name)
 
int32_t createCustomExpression (std::unique_ptr< CustomExpression > custom_expression)
 
const CustomExpressiongetCustomExpression (int32_t custom_expression_id) const
 
const std::unique_ptr< const
CustomExpression
getCustomExpressionFromStorage (int32_t custom_expression_id)
 
std::vector< const
CustomExpression * > 
getCustomExpressionsForUser (const UserMetadata &user) const
 
void updateCustomExpression (int32_t custom_expression_id, const std::string &expression_json)
 
void deleteCustomExpressions (const std::vector< int32_t > &custom_expression_ids, bool do_soft_delete)
 
void reassignOwners (const std::set< std::string > &old_owners, const std::string &new_owner)
 

Static Public Member Functions

static void expandGeoColumn (const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
 
static const std::string getForeignTableSchema (bool if_not_exists=false)
 
static const std::string getForeignServerSchema (bool if_not_exists=false)
 
static const std::string getCustomExpressionsSchema (bool if_not_exists=false)
 

Public Attributes

std::mutex sqliteMutex_
 
mapd_shared_mutex sharedMutex_
 
std::atomic< std::thread::id > thread_holding_sqlite_lock
 
std::atomic< std::thread::id > thread_holding_write_lock
 
bool initialized_ = false
 

Static Public Attributes

static thread_local bool thread_holds_read_lock = false
 

Protected Types

using ColumnDescriptorsForRoll = std::vector< std::pair< ColumnDescriptor *, ColumnDescriptor * >>
 

Protected Member Functions

void CheckAndExecuteMigrations ()
 
void CheckAndExecuteMigrationsPostBuildMaps ()
 
void updateDictionaryNames ()
 
void updateTableDescriptorSchema ()
 
void updateFixlenArrayColumns ()
 
void updateGeoColumns ()
 
void updateFrontendViewSchema ()
 
void updateLinkSchema ()
 
void updateFrontendViewAndLinkUsers ()
 
void updateLogicalToPhysicalTableLinkSchema ()
 
void updateLogicalToPhysicalTableMap (const int32_t logical_tb_id)
 
void updateDictionarySchema ()
 
void updatePageSize ()
 
void updateDeletedColumnIndicator ()
 
void updateDefaultColumnValues ()
 
void updateFrontendViewsToDashboards ()
 
void updateCustomExpressionsSchema ()
 
void updateFsiSchemas ()
 
void recordOwnershipOfObjectsInObjectPermissions ()
 
void checkDateInDaysColumnMigration ()
 
void createDashboardSystemRoles ()
 
void buildMaps ()
 
void addTableToMap (const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
 
void addReferenceToForeignDict (ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def, const bool persist_reference)
 
bool setColumnSharedDictionary (ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
void setColumnDictionary (ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
 
void addFrontendViewToMap (DashboardDescriptor &vd)
 
void addFrontendViewToMapNoLock (DashboardDescriptor &vd)
 
void addLinkToMap (LinkDescriptor &ld)
 
void removeTableFromMap (const std::string &tableName, const int tableId, const bool is_on_error=false)
 
void eraseTableMetadata (const TableDescriptor *td)
 
void executeDropTableSqliteQueries (const TableDescriptor *td)
 
void doTruncateTable (const TableDescriptor *td)
 
void renamePhysicalTable (const TableDescriptor *td, const std::string &newTableName)
 
void renamePhysicalTable (std::vector< std::pair< std::string, std::string >> &names, std::vector< int > &tableIds)
 
void instantiateFragmenter (TableDescriptor *td) const
 
void getAllColumnMetadataForTableImpl (const TableDescriptor *td, std::list< const ColumnDescriptor * > &colDescs, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 
std::string calculateSHA1 (const std::string &data)
 
std::string generatePhysicalTableName (const std::string &logicalTableName, const int32_t &shardNumber)
 
std::vector< DBObjectparseDashboardObjects (const std::string &view_meta, const int &user_id)
 
void createOrUpdateDashboardSystemRole (const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
 
const int getColumnIdBySpiUnlocked (const int table_id, const size_t spi) const
 
void serializeTableJsonUnlocked (const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
 
void dropTableFromJsonUnlocked (const std::string &table_name) const
 

Protected Attributes

std::string basePath_
 
TableDescriptorMap tableDescriptorMap_
 
TableDescriptorMapById tableDescriptorMapById_
 
ColumnDescriptorMap columnDescriptorMap_
 
ColumnDescriptorMapById columnDescriptorMapById_
 
DictDescriptorMapById dictDescriptorMapByRef_
 
DashboardDescriptorMap dashboardDescriptorMap_
 
LinkDescriptorMap linkDescriptorMap_
 
LinkDescriptorMapById linkDescriptorMapById_
 
ForeignServerMap foreignServerMap_
 
ForeignServerMapById foreignServerMapById_
 
CustomExpressionMapById custom_expr_map_by_id_
 
SqliteConnector sqliteConnector_
 
const DBMetadata currentDB_
 
std::shared_ptr
< Data_Namespace::DataMgr
dataMgr_
 
const std::vector< LeafHostInfostring_dict_hosts_
 
std::shared_ptr< CalcitecalciteMgr_
 
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
 
int nextTempTableId_
 
int nextTempDictId_
 
ColumnDescriptorsForRoll columnDescriptorsForRoll
 

Static Protected Attributes

static const std::string physicalTableNameTag_
 

Private Member Functions

void gatherAdditionalInfo (std::vector< std::string > &additional_info, std::set< std::string > &shared_dict_column_names, const TableDescriptor *td) const
 
std::string quoteIfRequired (const std::string &column_name) const
 
void adjustAlteredTableFiles (const std::string &temp_data_dir, const std::unordered_map< int, int > &all_column_ids_map) const
 
void renameTableDirectories (const std::string &temp_data_dir, const std::vector< std::string > &target_paths, const std::string &name_prefix) const
 
void buildForeignServerMap ()
 
void addForeignTableDetails ()
 
void setForeignServerProperty (const std::string &server_name, const std::string &property, const std::string &value)
 
void setForeignTableProperty (const foreign_storage::ForeignTable *table, const std::string &property, const std::string &value)
 
void alterPhysicalTableMetadata (const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
 
void alterTableMetadata (const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
 
void setTableFileMgrParams (const int table_id, const File_Namespace::FileMgrParams &file_mgr_params)
 
bool filterTableByTypeAndUser (const TableDescriptor *td, const UserMetadata &user_metadata, const GetTablesType get_tables_type) const
 
TableDescriptorgetMutableMetadataForTableUnlocked (int table_id) const
 
void createForeignServerNoLocks (std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
 
foreign_storage::ForeignTablegetForeignTableUnlocked (const std::string &tableName) const
 
const CataloggetObjForLock ()
 
void removeChunks (const int table_id) const
 
void buildCustomExpressionsMap ()
 
std::unique_ptr< CustomExpressiongetCustomExpressionFromConnector (size_t row)
 
void restoreOldOwners (const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects, int32_t new_owner_id)
 
void restoreOldOwnersInMemory (const std::map< int32_t, std::vector< DBObject >> &old_owner_db_objects, int32_t new_owner_id)
 
void conditionallyInitializeSystemTables ()
 
void createSystemTableServer (const std::string &server_name, const std::string &data_wrapper_type)
 
void createSystemTable (const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name)
 
void setDeletedColumnUnlocked (const TableDescriptor *td, const ColumnDescriptor *cd)
 

Private Attributes

DeletedColumnPerTableMap deletedColumnPerTable_
 

Static Private Attributes

static constexpr const char * CATALOG_SERVER_NAME {"omnisci_catalog_server"}
 
static constexpr std::array
< const char *, 1 > 
INTERNAL_SERVERS {CATALOG_SERVER_NAME}
 

Detailed Description

class for a per-database catalog. also includes metadata for the current database and the current user.

Catalog

Definition at line 111 of file Catalog.h.

Member Typedef Documentation

Definition at line 658 of file Catalog.h.

Constructor & Destructor Documentation

Catalog_Namespace::Catalog::Catalog ( const std::string &  basePath,
const DBMetadata curDB,
std::shared_ptr< Data_Namespace::DataMgr dataMgr,
const std::vector< LeafHostInfo > &  string_dict_hosts,
std::shared_ptr< Calcite calcite,
bool  is_new_db 
)

Constructor - takes basePath to already extant data directory for writing.

Parameters
basePathdirectory path for writing catalog
dbNamename of the database
fragmenterFragmenter object metadata - expects for this directory to already exist
Catalog_Namespace::Catalog::Catalog ( )

Constructor builds a hollow catalog used during constructor of other catalogs.

Definition at line 197 of file Catalog.cpp.

197 {}
Catalog_Namespace::Catalog::~Catalog ( )

Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite.

Definition at line 199 of file Catalog.cpp.

References basePath_, columnDescriptorMap_, currentDB_, Catalog_Namespace::DBMetadata::dbName, g_serialize_temp_tables, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and tableDescriptorMap_.

199  {
201  // must clean up heap-allocated TableDescriptor and ColumnDescriptor structs
202  for (TableDescriptorMap::iterator tableDescIt = tableDescriptorMap_.begin();
203  tableDescIt != tableDescriptorMap_.end();
204  ++tableDescIt) {
205  tableDescIt->second->fragmenter = nullptr;
206  delete tableDescIt->second;
207  }
208 
209  // TableDescriptorMapById points to the same descriptors. No need to delete
210 
211  for (ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.begin();
212  columnDescIt != columnDescriptorMap_.end();
213  ++columnDescIt) {
214  delete columnDescIt->second;
215  }
216 
217  // ColumnDescriptorMapById points to the same descriptors. No need to delete
218 
220  boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
221  }
222 }
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:630
const DBMetadata currentDB_
Definition: Catalog.h:641
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:156
bool g_serialize_temp_tables
Definition: Catalog.cpp:100
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:116
mapd_unique_lock< mapd_shared_mutex > write_lock
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:628

+ Here is the call graph for this function:

Member Function Documentation

void Catalog_Namespace::Catalog::addColumn ( const TableDescriptor td,
ColumnDescriptor cd 
)

Definition at line 1944 of file Catalog.cpp.

References addDictionary(), columnDescriptorMap_, columnDescriptorMapById_, columnDescriptorsForRoll, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::default_value, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), SqliteConnector::getData(), getPhysicalTablesDescriptors(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, kENCODING_DICT, TableDescriptor::nShards, SqliteConnector::query_with_text_params(), TableDescriptor::shard, sqliteConnector_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, TEXT, to_string(), to_upper(), and ColumnDescriptor::virtualExpr.

1944  {
1945  // caller must handle sqlite/chunk transaction TOGETHER
1946  cd.tableId = td.tableId;
1947  if (td.nShards > 0 && td.shard < 0) {
1948  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
1949  auto shard_cd = cd;
1950  addColumn(*shard, shard_cd);
1951  }
1952  }
1954  addDictionary(cd);
1955  }
1956 
1957  using BindType = SqliteConnector::BindType;
1958  std::vector<BindType> types(17, BindType::TEXT);
1959  if (!cd.default_value.has_value()) {
1960  types[16] = BindType::NULL_TYPE;
1961  }
1963  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
1964  "colscale, is_notnull, "
1965  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
1966  "is_deletedcol, default_value) "
1967  "VALUES (?, "
1968  "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
1969  "?, ?, ?, "
1970  "?, "
1971  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1972  std::vector<std::string>{std::to_string(td.tableId),
1973  std::to_string(td.tableId),
1974  cd.columnName,
1983  "",
1986  cd.virtualExpr,
1988  cd.default_value.value_or("NULL")},
1989  types);
1990 
1992  "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
1993  std::vector<std::string>{std::to_string(td.tableId)});
1994 
1996  "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
1997  std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
1998  cd.columnId = sqliteConnector_.getData<int>(0, 0);
1999 
2000  ++tableDescriptorMapById_[td.tableId]->nColumns;
2001  auto ncd = new ColumnDescriptor(cd);
2004  columnDescriptorsForRoll.emplace_back(nullptr, ncd);
2005 }
std::string virtualExpr
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:330
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
HOST DEVICE int get_size() const
Definition: sqltypes.h:339
T getData(const int row, const int col)
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:630
virtual void query_with_text_params(std::string const &query_only)
void addColumn(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:1944
SqliteConnector sqliteConnector_
Definition: Catalog.h:640
HOST DEVICE int get_scale() const
Definition: sqltypes.h:334
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:329
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:631
std::string to_string(char const *&&v)
#define TEXT
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:629
specifies the content in-memory of a row in the column metadata table
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4104
std::string to_upper(const std::string &str)
std::optional< std::string > default_value
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:337
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:331
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:338
SQLTypeInfo columnType
DictRef addDictionary(ColumnDescriptor &cd)
Definition: Catalog.cpp:1848
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:336
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:659
std::string columnName
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39

+ Here is the call graph for this function:

DictRef Catalog_Namespace::Catalog::addDictionary ( ColumnDescriptor cd)

Definition at line 1848 of file Catalog.cpp.

References CHECK, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, DictDescriptor::dictFolderPath, setColumnDictionary(), string_dict_hosts_, tableDescriptorMapById_, and ColumnDescriptor::tableId.

Referenced by addColumn().

1848  {
1849  cat_write_lock write_lock(this);
1850  const auto& td = *tableDescriptorMapById_[cd.tableId];
1851  list<DictDescriptor> dds;
1852  setColumnDictionary(cd, dds, td, true);
1853  auto& dd = dds.back();
1854  CHECK(dd.dictRef.dictId);
1855 
1856  std::unique_ptr<StringDictionaryClient> client;
1857  if (!string_dict_hosts_.empty()) {
1858  client.reset(new StringDictionaryClient(
1859  string_dict_hosts_.front(), DictRef(currentDB_.dbId, -1), true));
1860  }
1861  if (client) {
1862  client->create(dd.dictRef, dd.dictIsTemp);
1863  }
1864 
1865  DictDescriptor* new_dd = new DictDescriptor(dd);
1866  dictDescriptorMapByRef_[dd.dictRef].reset(new_dd);
1867  if (!dd.dictIsTemp) {
1868  boost::filesystem::create_directory(new_dd->dictFolderPath);
1869  }
1870  return dd.dictRef;
1871 }
std::string dictFolderPath
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:644
const DBMetadata currentDB_
Definition: Catalog.h:641
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:632
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:629
struct dict_ref_t DictRef
Definition: DictRef.h:37
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:116
#define CHECK(condition)
Definition: Logger.h:209
Descriptor for a dictionary for a string columne.
mapd_unique_lock< mapd_shared_mutex > write_lock
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:3325

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addForeignTableDetails ( )
private

Definition at line 4319 of file Catalog.cpp.

References CHECK.

Referenced by buildMaps().

4319  {
4321  "SELECT table_id, server_id, options, last_refresh_time, next_refresh_time from "
4322  "omnisci_foreign_tables");
4323  auto num_rows = sqliteConnector_.getNumRows();
4324  for (size_t r = 0; r < num_rows; r++) {
4325  const auto table_id = sqliteConnector_.getData<int32_t>(r, 0);
4326  const auto server_id = sqliteConnector_.getData<int32_t>(r, 1);
4327  const auto& options = sqliteConnector_.getData<std::string>(r, 2);
4328  const auto last_refresh_time = sqliteConnector_.getData<int>(r, 3);
4329  const auto next_refresh_time = sqliteConnector_.getData<int>(r, 4);
4330 
4331  CHECK(tableDescriptorMapById_.find(table_id) != tableDescriptorMapById_.end());
4332  auto foreign_table =
4333  dynamic_cast<foreign_storage::ForeignTable*>(tableDescriptorMapById_[table_id]);
4334  CHECK(foreign_table);
4335  foreign_table->foreign_server = foreignServerMapById_[server_id].get();
4336  CHECK(foreign_table->foreign_server);
4337  foreign_table->populateOptionsMap(options);
4338  foreign_table->last_refresh_time = last_refresh_time;
4339  foreign_table->next_refresh_time = next_refresh_time;
4340  }
4341 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:640
virtual void query(const std::string &queryString)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:629
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:637
#define CHECK(condition)
Definition: Logger.h:209
virtual size_t getNumRows() const

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addFrontendViewToMap ( DashboardDescriptor vd)
protected

Definition at line 1341 of file Catalog.cpp.

References addFrontendViewToMapNoLock().

1341  {
1342  cat_write_lock write_lock(this);
1344 }
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1346
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:116
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addFrontendViewToMapNoLock ( DashboardDescriptor vd)
protected

Definition at line 1346 of file Catalog.cpp.

References dashboardDescriptorMap_, DashboardDescriptor::dashboardName, to_string(), and DashboardDescriptor::userId.

Referenced by addFrontendViewToMap().

1346  {
1347  cat_write_lock write_lock(this);
1349  std::make_shared<DashboardDescriptor>(vd);
1350 }
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:633
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:116
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addLinkToMap ( LinkDescriptor ld)
protected

Definition at line 1428 of file Catalog.cpp.

References currentDB_, Catalog_Namespace::DBMetadata::dbId, LinkDescriptor::link, linkDescriptorMap_, linkDescriptorMapById_, LinkDescriptor::linkId, and to_string().

1428  {
1429  cat_write_lock write_lock(this);
1430  LinkDescriptor* new_ld = new LinkDescriptor();
1431  *new_ld = ld;
1433  linkDescriptorMapById_[ld.linkId] = new_ld;
1434 }
const DBMetadata currentDB_
Definition: Catalog.h:641
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:116
mapd_unique_lock< mapd_shared_mutex > write_lock
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:635
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:634

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addReferenceToForeignDict ( ColumnDescriptor referencing_column,
Parser::SharedDictionaryDef  shared_dict_def,
const bool  persist_reference 
)
protected

Definition at line 3234 of file Catalog.cpp.

References CHECK, CHECK_GE, ColumnDescriptor::columnType, SQLTypeInfo::get_comp_param(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_foreign_col(), and to_string().

3236  {
3237  cat_write_lock write_lock(this);
3238  const auto foreign_ref_col = get_foreign_col(*this, shared_dict_def);
3239  CHECK(foreign_ref_col);
3240  referencing_column.columnType = foreign_ref_col->columnType;
3241  const int dict_id = referencing_column.columnType.get_comp_param();
3242  const DictRef dict_ref(currentDB_.dbId, dict_id);
3243  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
3244  CHECK(dictIt != dictDescriptorMapByRef_.end());
3245  const auto& dd = dictIt->second;
3246  CHECK_GE(dd->refcount, 1);
3247  ++dd->refcount;
3248  if (persist_reference) {
3249  cat_sqlite_lock sqlite_lock(getObjForLock());
3251  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
3252  {std::to_string(dict_id)});
3253  }
3254 }
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:640
const DBMetadata currentDB_
Definition: Catalog.h:641
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:632
#define CHECK_GE(x, y)
Definition: Logger.h:222
const ColumnDescriptor * get_foreign_col(const Catalog &cat, const Parser::SharedDictionaryDef &shared_dict_def)
Definition: Catalog.cpp:3222
std::string to_string(char const *&&v)
const Catalog * getObjForLock()
Definition: Catalog.cpp:224
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:116
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:338
#define CHECK(condition)
Definition: Logger.h:209
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:117

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addTableToMap ( const TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::list< DictDescriptor > &  dicts 
)
protected

Definition at line 1200 of file Catalog.cpp.

References anonymous_namespace{Utm.h}::a, CHECK, columnDescriptorMap_, columnDescriptorMapById_, ColumnDescriptor::columnId, TableDescriptor::columnIdBySpi_, ColumnDescriptor::columnName, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, DictDescriptor::dictFolderPath, dict_ref_t::dictId, TableDescriptor::hasDeletedCol, TableDescriptor::mutex_, setDeletedColumnUnlocked(), gpu_enabled::sort(), string_dict_hosts_, tableDescriptorMap_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, and to_upper().

Referenced by createTable().

1202  {
1203  cat_write_lock write_lock(this);
1204  TableDescriptor* new_td;
1205 
1206  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
1207  if (foreign_table) {
1208  auto new_foreign_table = new foreign_storage::ForeignTable();
1209  *new_foreign_table = *foreign_table;
1210  new_td = new_foreign_table;
1211  } else {
1212  new_td = new TableDescriptor();
1213  *new_td = *td;
1214  }
1215 
1216  new_td->mutex_ = std::make_shared<std::mutex>();
1217  tableDescriptorMap_[to_upper(td->tableName)] = new_td;
1218  tableDescriptorMapById_[td->tableId] = new_td;
1219  for (auto cd : columns) {
1220  ColumnDescriptor* new_cd = new ColumnDescriptor();
1221  *new_cd = cd;
1222  ColumnKey columnKey(new_cd->tableId, to_upper(new_cd->columnName));
1223  columnDescriptorMap_[columnKey] = new_cd;
1224  ColumnIdKey columnIdKey(new_cd->tableId, new_cd->columnId);
1225  columnDescriptorMapById_[columnIdKey] = new_cd;
1226 
1227  // Add deleted column to the map
1228  if (cd.isDeletedCol) {
1229  CHECK(new_td->hasDeletedCol);
1230  setDeletedColumnUnlocked(new_td, new_cd);
1231  }
1232  }
1233 
1234  std::sort(new_td->columnIdBySpi_.begin(),
1235  new_td->columnIdBySpi_.end(),
1236  [](const size_t a, const size_t b) -> bool { return a < b; });
1237 
1238  std::unique_ptr<StringDictionaryClient> client;
1239  DictRef dict_ref(currentDB_.dbId, -1);
1240  if (!string_dict_hosts_.empty()) {
1241  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1242  }
1243  for (auto dd : dicts) {
1244  if (!dd.dictRef.dictId) {
1245  // Dummy entry created for a shard of a logical table, nothing to do.
1246  continue;
1247  }
1248  dict_ref.dictId = dd.dictRef.dictId;
1249  if (client) {
1250  client->create(dict_ref, dd.dictIsTemp);
1251  }
1252  DictDescriptor* new_dd = new DictDescriptor(dd);
1253  dictDescriptorMapByRef_[dict_ref].reset(new_dd);
1254  if (!dd.dictIsTemp) {
1255  boost::filesystem::create_directory(new_dd->dictFolderPath);
1256  }
1257  }
1258 }
std::string dictFolderPath
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
std::string tableName
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:630
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:644
const DBMetadata currentDB_
Definition: Catalog.h:641
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:632
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
std::vector< int > columnIdBySpi_
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:631
std::shared_ptr< std::mutex > mutex_
constexpr double a
Definition: Utm.h:38
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3213
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:629
specifies the content in-memory of a row in the column metadata table
std::string to_upper(const std::string &str)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:116
#define CHECK(condition)
Definition: Logger.h:209
Descriptor for a dictionary for a string columne.
mapd_unique_lock< mapd_shared_mutex > write_lock
std::string columnName
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:628
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::adjustAlteredTableFiles ( const std::string &  temp_data_dir,
const std::unordered_map< int, int > &  all_column_ids_map 
) const
private
void Catalog_Namespace::Catalog::alterPhysicalTableMetadata ( const TableDescriptor td,
const TableDescriptorUpdateParams table_update_params 
)
private

Definition at line 2951 of file Catalog.cpp.

References CHECK, TableDescriptorUpdateParams::max_rollback_epochs, TableDescriptorUpdateParams::max_rows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::tableId, and to_string().

2953  {
2954  // Only called from parent alterTableParamMetadata, expect already to have catalog and
2955  // sqlite write locks
2956 
2957  // Sqlite transaction should have already been begun in parent alterTableCatalogMetadata
2958 
2960  CHECK(mutable_td);
2961  if (td->maxRollbackEpochs != table_update_params.max_rollback_epochs) {
2963  "UPDATE mapd_tables SET max_rollback_epochs = ? WHERE tableid = ?",
2964  std::vector<std::string>{std::to_string(table_update_params.max_rollback_epochs),
2965  std::to_string(td->tableId)});
2966  mutable_td->maxRollbackEpochs = table_update_params.max_rollback_epochs;
2967  }
2968 
2969  if (td->maxRows != table_update_params.max_rows) {
2971  "UPDATE mapd_tables SET max_rows = ? WHERE tableid = ?",
2972  std::vector<std::string>{std::to_string(table_update_params.max_rows),
2973  std::to_string(td->tableId)});
2974  mutable_td->maxRows = table_update_params.max_rows;
2975  }
2976 }
int32_t maxRollbackEpochs
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:640
TableDescriptor * getMutableMetadataForTableUnlocked(int table_id) const
Definition: Catalog.cpp:1529
std::string to_string(char const *&&v)
#define CHECK(condition)
Definition: Logger.h:209

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::alterTableMetadata ( const TableDescriptor td,
const TableDescriptorUpdateParams table_update_params 
)
private

Definition at line 2978 of file Catalog.cpp.

References CHECK, logger::FATAL, i, LOG, TableDescriptor::tableId, and TableDescriptor::tableName.

2979  {
2980  cat_write_lock write_lock(this);
2981  cat_sqlite_lock sqlite_lock(getObjForLock());
2982  sqliteConnector_.query("BEGIN TRANSACTION");
2983  try {
2984  const auto physical_table_it = logicalToPhysicalTableMapById_.find(td->tableId);
2985  if (physical_table_it != logicalToPhysicalTableMapById_.end()) {
2986  const auto physical_tables = physical_table_it->second;
2987  CHECK(!physical_tables.empty());
2988  for (size_t i = 0; i < physical_tables.size(); i++) {
2989  int32_t physical_tb_id = physical_tables[i];
2990  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id, false);
2991  CHECK(phys_td);
2992  alterPhysicalTableMetadata(phys_td, table_update_params);
2993  }
2994  }
2995  alterPhysicalTableMetadata(td, table_update_params);
2996  } catch (std::exception& e) {
2997  sqliteConnector_.query("ROLLBACK TRANSACTION");
2998  LOG(FATAL) << "Table '" << td->tableName << "' catalog update failed";
2999  }
3000  sqliteConnector_.query("END TRANSACTION");
3001 }
std::string tableName
#define LOG(tag)
Definition: Logger.h:203
SqliteConnector sqliteConnector_
Definition: Catalog.h:640
virtual void query(const std::string &queryString)
void alterPhysicalTableMetadata(const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
Definition: Catalog.cpp:2951
const Catalog * getObjForLock()
Definition: Catalog.cpp:224
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:116
#define CHECK(condition)
Definition: Logger.h:209
mapd_unique_lock< mapd_shared_mutex > write_lock
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:117
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:647
void Catalog_Namespace::Catalog::buildCustomExpressionsMap ( )
private

Definition at line 1172 of file Catalog.cpp.

References custom_expr_map_by_id_, getCustomExpressionFromConnector(), SqliteConnector::getNumRows(), SqliteConnector::query(), and sqliteConnector_.

Referenced by buildMaps().

1172  {
1174  "SELECT id, name, expression_json, data_source_type, data_source_id, "
1175  "is_deleted "
1176  "FROM omnisci_custom_expressions");
1177  auto num_rows = sqliteConnector_.getNumRows();
1178  for (size_t row = 0; row < num_rows; row++) {
1179  auto custom_expr = getCustomExpressionFromConnector(row);
1180  custom_expr_map_by_id_[custom_expr->id] = std::move(custom_expr);
1181  }
1182 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:640
virtual void query(const std::string &queryString)
std::unique_ptr< CustomExpression > getCustomExpressionFromConnector(size_t row)
Definition: Catalog.cpp:1184
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:638
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::buildForeignServerMap ( )
private

Definition at line 4301 of file Catalog.cpp.

Referenced by buildMaps().

4301  {
4303  "SELECT id, name, data_wrapper_type, options, owner_user_id, creation_time FROM "
4304  "omnisci_foreign_servers");
4305  auto num_rows = sqliteConnector_.getNumRows();
4306  for (size_t row = 0; row < num_rows; row++) {
4307  auto foreign_server = std::make_shared<foreign_storage::ForeignServer>(
4308  sqliteConnector_.getData<int>(row, 0),
4309  sqliteConnector_.getData<std::string>(row, 1),
4310  sqliteConnector_.getData<std::string>(row, 2),
4311  sqliteConnector_.getData<std::string>(row, 3),
4312  sqliteConnector_.getData<std::int32_t>(row, 4),
4313  sqliteConnector_.getData<std::int32_t>(row, 5));
4314  foreignServerMap_[foreign_server->name] = foreign_server;
4315  foreignServerMapById_[foreign_server->id] = foreign_server;
4316  }
4317 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:640
virtual void query(const std::string &queryString)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:637
ForeignServerMap foreignServerMap_
Definition: Catalog.h:636
virtual size_t getNumRows() const

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::buildMaps ( )
protected

Definition at line 951 of file Catalog.cpp.

References anonymous_namespace{Utm.h}::a, addForeignTableDetails(), buildCustomExpressionsMap(), buildForeignServerMap(), CHECK, ColumnDescriptor::chunks, columnDescriptorMap_, columnDescriptorMapById_, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, createDefaultServersIfNotExists(), currentDB_, dashboardDescriptorMap_, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, ColumnDescriptor::default_value, dictDescriptorMapByRef_, logger::FATAL, StorageType::FOREIGN_TABLE, TableDescriptor::fragmenter, TableDescriptor::fragments, TableDescriptor::fragPageSize, TableDescriptor::fragType, g_base_path, g_enable_fsi, generate_dashboard_system_rolename(), SQLTypeInfo::get_physical_cols(), SqliteConnector::getData(), SqliteConnector::getNumRows(), getObjForLock(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::getUserFromId(), TableDescriptor::hasDeletedCol, SQLTypeInfo::is_geometry(), TableDescriptor::is_system_table, ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, SqliteConnector::isNull(), ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, TableDescriptor::keyMetainfo, LinkDescriptor::link, linkDescriptorMap_, linkDescriptorMapById_, LinkDescriptor::linkId, LOG, logicalToPhysicalTableMapById_, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nColumns, TableDescriptor::nShards, TableDescriptor::partitions, SqliteConnector::query(), SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_compression(), SQLTypeInfo::set_dimension(), SQLTypeInfo::set_notnull(), SQLTypeInfo::set_scale(), SQLTypeInfo::set_size(), SQLTypeInfo::set_subtype(), SQLTypeInfo::set_type(), setDeletedColumnUnlocked(), TableDescriptor::shard, TableDescriptor::shardedColumnId, gpu_enabled::sort(), TableDescriptor::sortedColumnId, sqliteConnector_, TableDescriptor::storageType, tableDescriptorMap_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), to_upper(), LinkDescriptor::updateTime, LinkDescriptor::userId, TableDescriptor::userId, LinkDescriptor::viewMetadata, TableDescriptor::viewSQL, LinkDescriptor::viewState, and ColumnDescriptor::virtualExpr.

951  {
953  cat_sqlite_lock sqlite_lock(getObjForLock());
954 
955  string dictQuery(
956  "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
957  sqliteConnector_.query(dictQuery);
958  size_t numRows = sqliteConnector_.getNumRows();
959  for (size_t r = 0; r < numRows; ++r) {
960  int dictId = sqliteConnector_.getData<int>(r, 0);
961  std::string dictName = sqliteConnector_.getData<string>(r, 1);
962  int dictNBits = sqliteConnector_.getData<int>(r, 2);
963  bool is_shared = sqliteConnector_.getData<bool>(r, 3);
964  int refcount = sqliteConnector_.getData<int>(r, 4);
965  std::string fname = g_base_path + "/mapd_data/DB_" + std::to_string(currentDB_.dbId) +
966  "_DICT_" + std::to_string(dictId);
967  DictRef dict_ref(currentDB_.dbId, dictId);
968  DictDescriptor* dd = new DictDescriptor(
969  dict_ref, dictName, dictNBits, is_shared, refcount, fname, false);
970  dictDescriptorMapByRef_[dict_ref].reset(dd);
971  }
972 
973  string tableQuery(
974  "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
975  "max_chunk_size, frag_page_size, "
976  "max_rows, partitions, shard_column_id, shard, num_shards, key_metainfo, userid, "
977  "sort_column_id, storage_type, max_rollback_epochs, is_system_table "
978  "from mapd_tables");
979  sqliteConnector_.query(tableQuery);
980  numRows = sqliteConnector_.getNumRows();
981  for (size_t r = 0; r < numRows; ++r) {
982  TableDescriptor* td;
983  const auto& storage_type = sqliteConnector_.getData<string>(r, 17);
984  if (!storage_type.empty() && storage_type != StorageType::FOREIGN_TABLE) {
985  const auto table_id = sqliteConnector_.getData<int>(r, 0);
986  const auto& table_name = sqliteConnector_.getData<string>(r, 1);
987  LOG(FATAL) << "Unable to read Catalog metadata: storage type is currently not a "
988  "supported table option (table "
989  << table_name << " [" << table_id << "] in database "
990  << currentDB_.dbName << ").";
991  }
992 
993  if (storage_type == StorageType::FOREIGN_TABLE) {
995  } else {
996  td = new TableDescriptor();
997  }
998 
999  td->storageType = storage_type;
1000  td->tableId = sqliteConnector_.getData<int>(r, 0);
1001  td->tableName = sqliteConnector_.getData<string>(r, 1);
1002  td->nColumns = sqliteConnector_.getData<int>(r, 2);
1003  td->isView = sqliteConnector_.getData<bool>(r, 3);
1004  td->fragments = sqliteConnector_.getData<string>(r, 4);
1005  td->fragType =
1007  td->maxFragRows = sqliteConnector_.getData<int>(r, 6);
1008  td->maxChunkSize = sqliteConnector_.getData<int64_t>(r, 7);
1009  td->fragPageSize = sqliteConnector_.getData<int>(r, 8);
1010  td->maxRows = sqliteConnector_.getData<int64_t>(r, 9);
1011  td->partitions = sqliteConnector_.getData<string>(r, 10);
1012  td->shardedColumnId = sqliteConnector_.getData<int>(r, 11);
1013  td->shard = sqliteConnector_.getData<int>(r, 12);
1014  td->nShards = sqliteConnector_.getData<int>(r, 13);
1015  td->keyMetainfo = sqliteConnector_.getData<string>(r, 14);
1016  td->userId = sqliteConnector_.getData<int>(r, 15);
1017  td->sortedColumnId =
1018  sqliteConnector_.isNull(r, 16) ? 0 : sqliteConnector_.getData<int>(r, 16);
1019  if (!td->isView) {
1020  td->fragmenter = nullptr;
1021  }
1022  td->maxRollbackEpochs = sqliteConnector_.getData<int>(r, 18);
1023  td->is_system_table = sqliteConnector_.getData<bool>(r, 19);
1024  td->hasDeletedCol = false;
1025 
1027  tableDescriptorMapById_[td->tableId] = td;
1028  }
1029 
1030  if (g_enable_fsi) {
1034  }
1035 
1036  string columnQuery(
1037  "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
1038  "is_notnull, compression, comp_param, "
1039  "size, chunks, is_systemcol, is_virtualcol, virtual_expr, is_deletedcol, "
1040  "default_value from "
1041  "mapd_columns ORDER BY tableid, "
1042  "columnid");
1043  sqliteConnector_.query(columnQuery);
1044  numRows = sqliteConnector_.getNumRows();
1045  int32_t skip_physical_cols = 0;
1046  for (size_t r = 0; r < numRows; ++r) {
1047  ColumnDescriptor* cd = new ColumnDescriptor();
1048  cd->tableId = sqliteConnector_.getData<int>(r, 0);
1049  cd->columnId = sqliteConnector_.getData<int>(r, 1);
1050  cd->columnName = sqliteConnector_.getData<string>(r, 2);
1054  cd->columnType.set_scale(sqliteConnector_.getData<int>(r, 6));
1058  cd->columnType.set_size(sqliteConnector_.getData<int>(r, 10));
1059  cd->chunks = sqliteConnector_.getData<string>(r, 11);
1060  cd->isSystemCol = sqliteConnector_.getData<bool>(r, 12);
1061  cd->isVirtualCol = sqliteConnector_.getData<bool>(r, 13);
1062  cd->virtualExpr = sqliteConnector_.getData<string>(r, 14);
1063  cd->isDeletedCol = sqliteConnector_.getData<bool>(r, 15);
1064  if (sqliteConnector_.isNull(r, 16)) {
1065  cd->default_value = std::nullopt;
1066  } else {
1067  cd->default_value = std::make_optional(sqliteConnector_.getData<string>(r, 16));
1068  }
1069  cd->isGeoPhyCol = skip_physical_cols > 0;
1070  ColumnKey columnKey(cd->tableId, to_upper(cd->columnName));
1071  columnDescriptorMap_[columnKey] = cd;
1072  ColumnIdKey columnIdKey(cd->tableId, cd->columnId);
1073  columnDescriptorMapById_[columnIdKey] = cd;
1074 
1075  if (skip_physical_cols <= 0) {
1076  skip_physical_cols = cd->columnType.get_physical_cols();
1077  }
1078 
1079  auto td_itr = tableDescriptorMapById_.find(cd->tableId);
1080  CHECK(td_itr != tableDescriptorMapById_.end());
1081 
1082  if (cd->isDeletedCol) {
1083  td_itr->second->hasDeletedCol = true;
1084  setDeletedColumnUnlocked(td_itr->second, cd);
1085  } else if (cd->columnType.is_geometry() || skip_physical_cols-- <= 0) {
1086  tableDescriptorMapById_[cd->tableId]->columnIdBySpi_.push_back(cd->columnId);
1087  }
1088  }
1089  // sort columnIdBySpi_ based on columnId
1090  for (auto& tit : tableDescriptorMapById_) {
1091  std::sort(tit.second->columnIdBySpi_.begin(),
1092  tit.second->columnIdBySpi_.end(),
1093  [](const size_t a, const size_t b) -> bool { return a < b; });
1094  }
1095 
1096  string viewQuery("SELECT tableid, sql FROM mapd_views");
1097  sqliteConnector_.query(viewQuery);
1098  numRows = sqliteConnector_.getNumRows();
1099  for (size_t r = 0; r < numRows; ++r) {
1100  int32_t tableId = sqliteConnector_.getData<int>(r, 0);
1101  TableDescriptor* td = tableDescriptorMapById_[tableId];
1102  td->viewSQL = sqliteConnector_.getData<string>(r, 1);
1103  td->fragmenter = nullptr;
1104  }
1105 
1106  string frontendViewQuery(
1107  "SELECT id, state, name, image_hash, strftime('%Y-%m-%dT%H:%M:%SZ', update_time), "
1108  "userid, "
1109  "metadata "
1110  "FROM mapd_dashboards");
1111  sqliteConnector_.query(frontendViewQuery);
1112  numRows = sqliteConnector_.getNumRows();
1113  for (size_t r = 0; r < numRows; ++r) {
1114  std::shared_ptr<DashboardDescriptor> vd = std::make_shared<DashboardDescriptor>();
1115  vd->dashboardId = sqliteConnector_.getData<int>(r, 0);
1116  vd->dashboardState = sqliteConnector_.getData<string>(r, 1);
1117  vd->dashboardName = sqliteConnector_.getData<string>(r, 2);
1118  vd->imageHash = sqliteConnector_.getData<string>(r, 3);
1119  vd->updateTime = sqliteConnector_.getData<string>(r, 4);
1120  vd->userId = sqliteConnector_.getData<int>(r, 5);
1121  vd->dashboardMetadata = sqliteConnector_.getData<string>(r, 6);
1122  vd->user = getUserFromId(vd->userId);
1123  vd->dashboardSystemRoleName = generate_dashboard_system_rolename(
1125  dashboardDescriptorMap_[std::to_string(vd->userId) + ":" + vd->dashboardName] = vd;
1126  }
1127 
1128  string linkQuery(
1129  "SELECT linkid, userid, link, view_state, strftime('%Y-%m-%dT%H:%M:%SZ', "
1130  "update_time), view_metadata "
1131  "FROM mapd_links");
1132  sqliteConnector_.query(linkQuery);
1133  numRows = sqliteConnector_.getNumRows();
1134  for (size_t r = 0; r < numRows; ++r) {
1135  LinkDescriptor* ld = new LinkDescriptor();
1136  ld->linkId = sqliteConnector_.getData<int>(r, 0);
1137  ld->userId = sqliteConnector_.getData<int>(r, 1);
1138  ld->link = sqliteConnector_.getData<string>(r, 2);
1139  ld->viewState = sqliteConnector_.getData<string>(r, 3);
1140  ld->updateTime = sqliteConnector_.getData<string>(r, 4);
1141  ld->viewMetadata = sqliteConnector_.getData<string>(r, 5);
1143  linkDescriptorMapById_[ld->linkId] = ld;
1144  }
1145 
1146  /* rebuild map linking logical tables to corresponding physical ones */
1147  string logicalToPhysicalTableMapQuery(
1148  "SELECT logical_table_id, physical_table_id "
1149  "FROM mapd_logical_to_physical");
1150  sqliteConnector_.query(logicalToPhysicalTableMapQuery);
1151  numRows = sqliteConnector_.getNumRows();
1152  for (size_t r = 0; r < numRows; ++r) {
1153  int32_t logical_tb_id = sqliteConnector_.getData<int>(r, 0);
1154  int32_t physical_tb_id = sqliteConnector_.getData<int>(r, 1);
1155  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
1156  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
1157  /* add new entity to the map logicalToPhysicalTableMapById_ */
1158  std::vector<int32_t> physicalTables;
1159  physicalTables.push_back(physical_tb_id);
1160  const auto it_ok =
1161  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
1162  CHECK(it_ok.second);
1163  } else {
1164  /* update map logicalToPhysicalTableMapById_ */
1165  physicalTableIt->second.push_back(physical_tb_id);
1166  }
1167  }
1168 
1170 }
int32_t maxRollbackEpochs
std::string virtualExpr
void set_compression(EncodingType c)
Definition: sqltypes.h:429
void set_size(int s)
Definition: sqltypes.h:427
std::string partitions
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
T getData(const int row, const int col)
SQLTypes
Definition: sqltypes.h:38
std::string tableName
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:630
#define LOG(tag)
Definition: Logger.h:203
SqliteConnector sqliteConnector_
Definition: Catalog.h:640
const DBMetadata currentDB_
Definition: Catalog.h:641
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:632
std::string storageType
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:420
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
virtual void query(const std::string &queryString)
std::string fragments
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:631
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:633
std::string to_string(char const *&&v)
std::string chunks
constexpr double a
Definition: Utm.h:38
EncodingType
Definition: sqltypes.h:233
int get_physical_cols() const
Definition: sqltypes.h:350
std::string getUserFromId(const int32_t id)
Definition: Catalog.cpp:941
std::string g_base_path
Definition: SysCatalog.cpp:63
void set_scale(int s)
Definition: sqltypes.h:424
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3213
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:629
specifies the content in-memory of a row in the column metadata table
std::string keyMetainfo
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
std::string to_upper(const std::string &str)
void set_comp_param(int p)
Definition: sqltypes.h:430
std::optional< std::string > default_value
bool isNull(const int row, const int col) const
void set_dimension(int d)
Definition: sqltypes.h:421
Fragmenter_Namespace::FragmenterType fragType
const Catalog * getObjForLock()
Definition: Catalog.cpp:224
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:116
void set_notnull(bool n)
Definition: sqltypes.h:426
#define CHECK(condition)
Definition: Logger.h:209
bool is_geometry() const
Definition: sqltypes.h:521
Descriptor for a dictionary for a string columne.
std::string viewSQL
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:117
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:635
static constexpr char const * FOREIGN_TABLE
bool g_enable_fsi
Definition: Catalog.cpp:93
std::string columnName
virtual size_t getNumRows() const
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:647
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:628
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:634
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)
void createDefaultServersIfNotExists()
Definition: Catalog.cpp:4364
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39
HOST DEVICE void set_type(SQLTypes t)
Definition: sqltypes.h:419

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::calculateSHA1 ( const std::string &  data)
protected

Definition at line 4036 of file Catalog.cpp.

References i.

4036  {
4037  boost::uuids::detail::sha1 sha1;
4038  unsigned int digest[5];
4039  sha1.process_bytes(data.c_str(), data.length());
4040  sha1.get_digest(digest);
4041  std::stringstream ss;
4042  for (size_t i = 0; i < 5; i++) {
4043  ss << std::hex << digest[i];
4044  }
4045  return ss.str();
4046 }
void Catalog_Namespace::Catalog::changeForeignServerOwner ( const std::string &  server_name,
const int  new_owner_id 
)

Change the owner of a Foreign Server to a new owner.

Parameters
server_name- Name of the foreign server whose owner to change
new_owner_id- New owner's user id

Definition at line 2674 of file Catalog.cpp.

References CHECK, to_string(), and foreign_storage::ForeignServer::user_id.

2675  {
2676  cat_write_lock write_lock(this);
2677  foreign_storage::ForeignServer* foreign_server =
2678  foreignServerMap_.find(server_name)->second.get();
2679  CHECK(foreign_server);
2680  setForeignServerProperty(server_name, "owner_user_id", std::to_string(new_owner_id));
2681  // update in-memory server
2682  foreign_server->user_id = new_owner_id;
2683 }
std::string to_string(char const *&&v)
ForeignServerMap foreignServerMap_
Definition: Catalog.h:636
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:116
#define CHECK(condition)
Definition: Logger.h:209
mapd_unique_lock< mapd_shared_mutex > write_lock
void setForeignServerProperty(const std::string &server_name, const std::string &property, const std::string &value)
Definition: Catalog.cpp:4343

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::CheckAndExecuteMigrations ( )
protected

Definition at line 914 of file Catalog.cpp.

References g_enable_fsi, recordOwnershipOfObjectsInObjectPermissions(), updateCustomExpressionsSchema(), updateDefaultColumnValues(), updateDeletedColumnIndicator(), updateDictionaryNames(), updateDictionarySchema(), updateFixlenArrayColumns(), updateFrontendViewAndLinkUsers(), updateFrontendViewSchema(), updateFrontendViewsToDashboards(), updateFsiSchemas(), updateGeoColumns(), updateLinkSchema(), updateLogicalToPhysicalTableLinkSchema(), updatePageSize(), and updateTableDescriptorSchema().

914  {
924  updatePageSize();
928  if (g_enable_fsi) {
930  }
933 }
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:435
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:704
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:123
void updateCustomExpressionsSchema()
Definition: Catalog.cpp:670
void updateLogicalToPhysicalTableLinkSchema()
Definition: Catalog.cpp:596
bool g_enable_fsi
Definition: Catalog.cpp:93

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::CheckAndExecuteMigrationsPostBuildMaps ( )
protected

Definition at line 935 of file Catalog.cpp.

References checkDateInDaysColumnMigration(), and createDashboardSystemRoles().

935  {
938 }
void checkDateInDaysColumnMigration()
Definition: Catalog.cpp:832

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::checkDateInDaysColumnMigration ( )
protected

Definition at line 832 of file Catalog.cpp.

References getCurrentDB(), getObjForLock(), migrations::MigrationMgr::migrateDateInDaysMetadata(), sqliteConnector_, and tableDescriptorMapById_.

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

832  {
833  cat_sqlite_lock sqlite_lock(getObjForLock());
836 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:640
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:221
static void migrateDateInDaysMetadata(const Catalog_Namespace::TableDescriptorMapById &table_descriptors_by_id, const int database_id, const Catalog_Namespace::Catalog *cat, SqliteConnector &sqlite)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:629
const Catalog * getObjForLock()
Definition: Catalog.cpp:224
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:117

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const bool Catalog_Namespace::Catalog::checkMetadataForDeletedRecs ( const TableDescriptor td,
int  column_id 
) const

Definition at line 3171 of file Catalog.cpp.

References CHECK, and TableDescriptor::fragmenter.

3172  {
3173  // check if there are rows deleted by examining the deletedColumn metadata
3174  CHECK(td);
3175  auto fragmenter = td->fragmenter;
3176  if (fragmenter) {
3177  return fragmenter->hasDeletedRows(delete_column_id);
3178  } else {
3179  return false;
3180  }
3181 }
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
#define CHECK(condition)
Definition: Logger.h:209
void Catalog_Namespace::Catalog::checkpoint ( const int  logicalTableId) const

Definition at line 4241 of file Catalog.cpp.

Referenced by UpdelRoll::commitUpdate(), TableOptimizer::vacuumDeletedRows(), and TableOptimizer::vacuumFragmentsAboveMinSelectivity().

4241  {
4242  const auto td = getMetadataForTable(logicalTableId);
4243  const auto shards = getPhysicalTablesDescriptors(td);
4244  for (const auto shard : shards) {
4245  getDataMgr().checkpoint(getCurrentDB().dbId, shard->tableId);
4246  }
4247 }
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:222
void checkpoint(const int db_id, const int tb_id)
Definition: DataMgr.cpp:550
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:221
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4104
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::checkpointWithAutoRollback ( const int  logical_table_id) const

Definition at line 4249 of file Catalog.cpp.

Referenced by import_export::Loader::checkpoint(), Parser::LocalConnector::checkpoint(), StorageIOFacility::TransactionParameters::finalizeTransaction(), and TableOptimizer::vacuumFragmentsAboveMinSelectivity().

4249  {
4250  auto table_epochs = getTableEpochs(getDatabaseId(), logical_table_id);
4251  try {
4252  checkpoint(logical_table_id);
4253  } catch (...) {
4254  setTableEpochsLogExceptions(getDatabaseId(), table_epochs);
4255  throw;
4256  }
4257 }
int getDatabaseId() const
Definition: Catalog.h:275
void checkpoint(const int logicalTableId) const
Definition: Catalog.cpp:4241
void setTableEpochsLogExceptions(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3153
std::vector< TableEpochInfo > getTableEpochs(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:3089

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::conditionallyInitializeSystemTables ( )
private

Definition at line 5345 of file Catalog.cpp.

References Catalog_Namespace::DASHBOARDS_SYS_TABLE_NAME, Catalog_Namespace::DATABASES_SYS_TABLE_NAME, g_enable_system_tables, INFORMATION_SCHEMA_DB, foreign_storage::DataWrapperType::INTERNAL_CATALOG, kARRAY, kBIGINT, kBOOLEAN, kENCODING_DICT, kINT, kTEXT, kTIMESTAMP, setup::name, Catalog_Namespace::PERMISSIONS_SYS_TABLE_NAME, Catalog_Namespace::ROLE_ASSIGNMENTS_SYS_TABLE_NAME, Catalog_Namespace::ROLES_SYS_TABLE_NAME, Catalog_Namespace::TABLES_SYS_TABLE_NAME, and Catalog_Namespace::USERS_SYS_TABLE_NAME.

5345  {
5352  {{"user_id", {kINT}},
5353  {"user_name", {kTEXT}},
5354  {"is_super_user", {kBOOLEAN}},
5355  {"default_db_id", {kINT}},
5356  {"can_login", {kBOOLEAN}}});
5357  }
5358 
5363  {{"database_id", {kINT}}, {"database_name", {kTEXT}}, {"owner_id", {kINT}}});
5364  }
5365 
5370  {{"role_name", {kTEXT}},
5371  {"is_user_role", {kBOOLEAN}},
5372  {"database_id", {kINT}},
5373  {"object_name", {kTEXT}},
5374  {"object_id", {kINT}},
5375  {"object_owner_id", {kINT}},
5376  {"object_permission_type", {kTEXT}},
5377  {"object_permissions", {kARRAY, 0, 0, false, kENCODING_DICT, 0, kTEXT}}});
5378  }
5379 
5382  ROLES_SYS_TABLE_NAME, CATALOG_SERVER_NAME, {{"role_name", {kTEXT}}});
5383  }
5384 
5388  {{"database_id", {kINT}},
5389  {"table_id", {kINT}},
5390  {"table_name", {kTEXT}},
5391  {"owner_id", {kINT}},
5392  {"column_count", {kINT}},
5393  {"is_view", {kBOOLEAN}},
5394  {"view_sql", {kTEXT}},
5395  {"max_fragment_size", {kINT}},
5396  {"max_chunk_size", {kBIGINT}},
5397  {"fragment_page_size", {kINT}},
5398  {"max_rows", {kBIGINT}},
5399  {"max_rollback_epochs", {kINT}},
5400  {"shard_count", {kINT}}});
5401  }
5402 
5406  {{"database_id", {kINT}},
5407  {"dashboard_id", {kINT}},
5408  {"dashboard_name", {kTEXT}},
5409  {"owner_id", {kINT}},
5410  {"last_updated_at", {kTIMESTAMP}}});
5411  }
5412 
5416  {{"role_name", {kTEXT}}, {"user_name", {kTEXT}}});
5417  }
5418  }
5419 }
static constexpr const char * DASHBOARDS_SYS_TABLE_NAME
Definition: Catalog.h:99
void createSystemTable(const std::string &table_name, const std::string &server_name, const std::vector< std::pair< std::string, SQLTypeInfo >> &column_type_by_name)
Definition: Catalog.cpp:5432
void createSystemTableServer(const std::string &server_name, const std::string &data_wrapper_type)
Definition: Catalog.cpp:5421
static constexpr char const * INTERNAL_CATALOG
std::string name() const
Definition: Catalog.h:291
static constexpr const char * ROLES_SYS_TABLE_NAME
Definition: Catalog.h:102
bool g_enable_system_tables
Definition: SysCatalog.cpp:65
static constexpr const char * ROLE_ASSIGNMENTS_SYS_TABLE_NAME
Definition: Catalog.h:103
static constexpr const char * USERS_SYS_TABLE_NAME
Definition: Catalog.h:97
static constexpr const char * DATABASES_SYS_TABLE_NAME
Definition: Catalog.h:100
static constexpr const char * TABLES_SYS_TABLE_NAME
Definition: Catalog.h:98
Definition: sqltypes.h:52
static constexpr const char * PERMISSIONS_SYS_TABLE_NAME
Definition: Catalog.h:101
static constexpr const char * CATALOG_SERVER_NAME
Definition: Catalog.h:730
Definition: sqltypes.h:45
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
const std::string INFORMATION_SCHEMA_DB
Definition: SysCatalog.h:65
int32_t Catalog_Namespace::Catalog::createCustomExpression ( std::unique_ptr< CustomExpression custom_expression)

Creates a new custom expression.

Parameters
custom_expression- unique pointer to struct containing custom expression details.
Returns
id of created custom expression

Definition at line 4917 of file Catalog.cpp.

References CHECK, CHECK_EQ, CHECK_GT, and to_string().

4918  {
4919  cat_write_lock write_lock(this);
4920  cat_sqlite_lock sqlite_lock(getObjForLock());
4921  sqliteConnector_.query("BEGIN TRANSACTION");
4922  int32_t custom_expression_id{-1};
4923  try {
4924  auto data_source_type_str =
4925  CustomExpression::dataSourceTypeToString(custom_expression->data_source_type);
4926  auto data_source_id_str = std::to_string(custom_expression->data_source_id);
4927  std::string custom_expr_select_query{
4928  "SELECT id FROM omnisci_custom_expressions WHERE name = ? and data_source_type = "
4929  "? and data_source_id = ? and is_deleted = ?"};
4930  std::vector<std::string> custom_expr_select_params{custom_expression->name,
4931  data_source_type_str,
4932  data_source_id_str,
4933  std::to_string(false)};
4934  sqliteConnector_.query_with_text_params(custom_expr_select_query,
4935  custom_expr_select_params);
4936  if (sqliteConnector_.getNumRows() > 0) {
4937  throw std::runtime_error{
4938  "A custom expression with the given "
4939  "name and data source already exists."};
4940  }
4942  "INSERT INTO omnisci_custom_expressions(name, expression_json, "
4943  "data_source_type, data_source_id, is_deleted) VALUES (?,?,?,?,?)",
4944  std::vector<std::string>{custom_expression->name,
4945  custom_expression->expression_json,
4946  data_source_type_str,
4947  data_source_id_str,
4948  std::to_string(false)});
4949  sqliteConnector_.query_with_text_params(custom_expr_select_query,
4950  custom_expr_select_params);
4951  CHECK_EQ(sqliteConnector_.getNumRows(), static_cast<size_t>(1));
4952  custom_expression->id = sqliteConnector_.getData<int32_t>(0, 0);
4953  custom_expression_id = custom_expression->id;
4954  CHECK(custom_expr_map_by_id_.find(custom_expression->id) ==
4955  custom_expr_map_by_id_.end());
4956  custom_expr_map_by_id_[custom_expression->id] = std::move(custom_expression);
4957  } catch (std::exception& e) {
4958  sqliteConnector_.query("ROLLBACK TRANSACTION");
4959  throw;
4960  }
4961  sqliteConnector_.query("END TRANSACTION");
4962  CHECK_GT(custom_expression_id, 0);
4963  return custom_expression_id;
4964 }
#define CHECK_EQ(x, y)
Definition: Logger.h:217
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:640
virtual void query(const std::string &queryString)
#define CHECK_GT(x, y)
Definition: Logger.h:221
std::string to_string(char const *&&v)
static std::string dataSourceTypeToString(DataSourceType type_enum)
const Catalog * getObjForLock()
Definition: Catalog.cpp:224
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:116
#define CHECK(condition)
Definition: Logger.h:209
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:638
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:117
virtual size_t getNumRows() const

+ Here is the call graph for this function:

int32_t Catalog_Namespace::Catalog::createDashboard ( DashboardDescriptor vd)

Definition at line 3899 of file Catalog.cpp.

References DashboardDescriptor::dashboardId, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, DashboardDescriptor::dashboardSystemRoleName, generate_dashboard_system_rolename(), DashboardDescriptor::imageHash, to_string(), Catalog_Namespace::sqlite_lock< T >::unlock(), Catalog_Namespace::write_lock< T >::unlock(), DashboardDescriptor::updateTime, and DashboardDescriptor::userId.

3899  {
3900  cat_write_lock write_lock(this);
3901  cat_sqlite_lock sqlite_lock(getObjForLock());
3902  sqliteConnector_.query("BEGIN TRANSACTION");
3903  try {
3904  // TODO(andrew): this should be an upsert
3906  "SELECT id FROM mapd_dashboards WHERE name = ? and userid = ?",
3907  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
3908  if (sqliteConnector_.getNumRows() > 0) {
3910  "UPDATE mapd_dashboards SET state = ?, image_hash = ?, metadata = ?, "
3911  "update_time = "
3912  "datetime('now') where name = ? "
3913  "and userid = ?",
3914  std::vector<std::string>{vd.dashboardState,
3915  vd.imageHash,
3916  vd.dashboardMetadata,
3917  vd.dashboardName,
3918  std::to_string(vd.userId)});
3919  } else {
3921  "INSERT INTO mapd_dashboards (name, state, image_hash, metadata, "
3922  "update_time, "
3923  "userid) "
3924  "VALUES "
3925  "(?,?,?,?, "
3926  "datetime('now'), ?)",
3927  std::vector<std::string>{vd.dashboardName,
3928  vd.dashboardState,
3929  vd.imageHash,
3930  vd.dashboardMetadata,
3931  std::to_string(vd.userId)});
3932  }
3933  } catch (std::exception& e) {
3934  sqliteConnector_.query("ROLLBACK TRANSACTION");
3935  throw;
3936  }
3937  sqliteConnector_.query("END TRANSACTION");
3938 
3939  // now get the auto generated dashboardId
3940  try {
3942  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_dashboards "
3943  "WHERE name = ? and userid = ?",
3944  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
3945  vd.dashboardId = sqliteConnector_.getData<int>(0, 0);
3946  vd.updateTime = sqliteConnector_.getData<std::string>(0, 1);
3947  } catch (std::exception& e) {
3948  throw;
3949  }
3953  sqlite_lock.unlock();
3954  write_lock.unlock();
3955  // NOTE(wamsi): Transactionally unsafe
3958  return vd.dashboardId;
3959 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:640
const DBMetadata currentDB_
Definition: Catalog.h:641
virtual void query(const std::string &queryString)
void addFrontendViewToMap(DashboardDescriptor &vd)
Definition: Catalog.cpp:1341
std::string to_string(char const *&&v)
std::string dashboardSystemRoleName
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1382
const Catalog * getObjForLock()
Definition: Catalog.cpp:224
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:116
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:117
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createDashboardSystemRoles ( )
protected

Definition at line 838 of file Catalog.cpp.

References createOrUpdateDashboardSystemRole(), currentDB_, Catalog_Namespace::DBMetadata::dbId, logger::ERROR, generate_dashboard_system_rolename(), SqliteConnector::getData(), Catalog_Namespace::SysCatalog::getGranteesOfSharedDashboards(), SqliteConnector::getNumRows(), getObjForLock(), Catalog_Namespace::SysCatalog::getRoleGrantee(), Catalog_Namespace::SysCatalog::grantRoleBatch(), i, logger::INFO, Catalog_Namespace::SysCatalog::instance(), LOG, MAPD_VERSION, SqliteConnector::query(), SqliteConnector::query_with_text_params(), run_benchmark_import::result, sqliteConnector_, and to_string().

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

838  {
839  std::unordered_map<std::string, std::pair<int, std::string>> dashboards;
840  std::vector<std::string> dashboard_ids;
841  static const std::string migration_name{"dashboard_roles_migration"};
842  {
843  cat_sqlite_lock sqlite_lock(getObjForLock());
844  sqliteConnector_.query("BEGIN TRANSACTION");
845  try {
846  // migration_history should be present in all catalogs by now
847  // if not then would be created before this migration
849  "select * from mapd_version_history where migration_history = '" +
850  migration_name + "'");
851  if (sqliteConnector_.getNumRows() != 0) {
852  // no need for further execution
853  sqliteConnector_.query("END TRANSACTION");
854  return;
855  }
856  LOG(INFO) << "Performing dashboard internal roles Migration.";
857  sqliteConnector_.query("select id, userid, metadata from mapd_dashboards");
858  for (size_t i = 0; i < sqliteConnector_.getNumRows(); ++i) {
861  sqliteConnector_.getData<string>(i, 0)))) {
862  // Successfully created roles during previous migration/crash
863  // No need to include them
864  continue;
865  }
866  dashboards[sqliteConnector_.getData<string>(i, 0)] = std::make_pair(
867  sqliteConnector_.getData<int>(i, 1), sqliteConnector_.getData<string>(i, 2));
868  dashboard_ids.push_back(sqliteConnector_.getData<string>(i, 0));
869  }
870  } catch (const std::exception& e) {
871  sqliteConnector_.query("ROLLBACK TRANSACTION");
872  throw;
873  }
874  sqliteConnector_.query("END TRANSACTION");
875  }
876  // All current grantees with shared dashboards.
877  const auto active_grantees =
879 
880  try {
881  // NOTE(wamsi): Transactionally unsafe
882  for (auto dash : dashboards) {
883  createOrUpdateDashboardSystemRole(dash.second.second,
884  dash.second.first,
886  std::to_string(currentDB_.dbId), dash.first));
887  auto result = active_grantees.find(dash.first);
888  if (result != active_grantees.end()) {
891  dash.first)},
892  result->second);
893  }
894  }
895  cat_sqlite_lock sqlite_lock(getObjForLock());
896  // check if this has already been completed
898  "select * from mapd_version_history where migration_history = '" +
899  migration_name + "'");
900  if (sqliteConnector_.getNumRows() != 0) {
901  return;
902  }
904  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
905  std::vector<std::string>{std::to_string(MAPD_VERSION), migration_name});
906  } catch (const std::exception& e) {
907  LOG(ERROR) << "Failed to create dashboard system roles during migration: "
908  << e.what();
909  throw;
910  }
911  LOG(INFO) << "Successfully created dashboard system roles during migration.";
912 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
#define LOG(tag)
Definition: Logger.h:203
SqliteConnector sqliteConnector_
Definition: Catalog.h:640
const DBMetadata currentDB_
Definition: Catalog.h:641
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
void grantRoleBatch(const std::vector< std::string > &roles, const std::vector< std::string > &grantees)
static SysCatalog & instance()
Definition: SysCatalog.h:325
static const int32_t MAPD_VERSION
Definition: release.h:33
Role * getRoleGrantee(const std::string &name) const
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1382
const Catalog * getObjForLock()
Definition: Catalog.cpp:224
std::unordered_map< std::string, std::vector< std::string > > getGranteesOfSharedDashboards(const std::vector< std::string > &dashboard_ids)
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:117
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createDefaultServersIfNotExists ( )

Creates default local file servers (if they don't already exist).

Definition at line 4364 of file Catalog.cpp.

References foreign_storage::DataWrapperType::CSV, foreign_storage::AbstractFileStorageDataWrapper::LOCAL_FILE_STORAGE_TYPE, OMNISCI_ROOT_USER_ID, foreign_storage::DataWrapperType::PARQUET, foreign_storage::DataWrapperType::REGEX_PARSER, and foreign_storage::AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY.

Referenced by buildMaps().

4364  {
4368 
4369  auto local_csv_server = std::make_unique<foreign_storage::ForeignServer>(
4370  "omnisci_local_csv",
4372  options,
4374  local_csv_server->validate();
4375  createForeignServerNoLocks(std::move(local_csv_server), true);
4376 
4377  auto local_parquet_server = std::make_unique<foreign_storage::ForeignServer>(
4378  "omnisci_local_parquet",
4380  options,
4382  local_parquet_server->validate();
4383  createForeignServerNoLocks(std::move(local_parquet_server), true);
4384 
4385  auto local_regex_parser_server = std::make_unique<foreign_storage::ForeignServer>(
4386  "omnisci_local_regex_parser",
4388  options,
4390  local_regex_parser_server->validate();
4391  createForeignServerNoLocks(std::move(local_regex_parser_server), true);
4392 }
static constexpr char const * REGEX_PARSER
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2585
const int OMNISCI_ROOT_USER_ID
Definition: SysCatalog.h:61
static constexpr char const * CSV
std::map< std::string, std::string, std::less<>> OptionsMap
static constexpr char const * PARQUET

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createForeignServer ( std::unique_ptr< foreign_storage::ForeignServer foreign_server,
bool  if_not_exists 
)

Creates a new foreign server DB object.

Parameters
foreign_server- unique pointer to struct containing foreign server details
if_not_exists- flag indicating whether or not an attempt to create a new foreign server should occur if a server with the same name already exists. An exception is thrown if this flag is set to "false" and an attempt is made to create a pre-existing foreign server

Definition at line 2577 of file Catalog.cpp.

2579  {
2580  cat_write_lock write_lock(this);
2581  cat_sqlite_lock sqlite_lock(getObjForLock());
2582  createForeignServerNoLocks(std::move(foreign_server), if_not_exists);
2583 }
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2585
const Catalog * getObjForLock()
Definition: Catalog.cpp:224
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:116
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:117
void Catalog_Namespace::Catalog::createForeignServerNoLocks ( std::unique_ptr< foreign_storage::ForeignServer foreign_server,
bool  if_not_exists 
)
private

Same as createForeignServer() but without acquiring locks. This should only be called from within a function/code block that already acquires appropriate locks.

Definition at line 2585 of file Catalog.cpp.

References CHECK, CHECK_EQ, and to_string().

2587  {
2589  "SELECT name from omnisci_foreign_servers where name = ?",
2590  std::vector<std::string>{foreign_server->name});
2591 
2592  if (sqliteConnector_.getNumRows() == 0) {
2593  foreign_server->creation_time = std::time(nullptr);
2595  "INSERT INTO omnisci_foreign_servers (name, data_wrapper_type, owner_user_id, "
2596  "creation_time, "
2597  "options) "
2598  "VALUES (?, ?, ?, ?, ?)",
2599  std::vector<std::string>{foreign_server->name,
2600  foreign_server->data_wrapper_type,
2601  std::to_string(foreign_server->user_id),
2602  std::to_string(foreign_server->creation_time),
2603  foreign_server->getOptionsAsJsonString()});
2605  "SELECT id from omnisci_foreign_servers where name = ?",
2606  std::vector<std::string>{foreign_server->name});
2607  CHECK_EQ(sqliteConnector_.getNumRows(), size_t(1));
2608  foreign_server->id = sqliteConnector_.getData<int32_t>(0, 0);
2609  std::shared_ptr<foreign_storage::ForeignServer> foreign_server_shared =
2610  std::move(foreign_server);
2611  CHECK(foreignServerMap_.find(foreign_server_shared->name) == foreignServerMap_.end())
2612  << "Attempting to insert a foreign server into foreign server map that already "
2613  "exists.";
2614  foreignServerMap_[foreign_server_shared->name] = foreign_server_shared;
2615  foreignServerMapById_[foreign_server_shared->id] = foreign_server_shared;
2616  } else if (!if_not_exists) {
2617  throw std::runtime_error{"A foreign server with name \"" + foreign_server->name +
2618  "\" already exists."};
2619  }
2620 }
#define CHECK_EQ(x, y)
Definition: Logger.h:217
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:640
std::string to_string(char const *&&v)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:637
ForeignServerMap foreignServerMap_
Definition: Catalog.h:636
#define CHECK(condition)
Definition: Logger.h:209
virtual size_t getNumRows() const

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::createLink ( LinkDescriptor ld,
size_t  min_length 
)

Definition at line 4048 of file Catalog.cpp.

References LinkDescriptor::link, LinkDescriptor::linkId, to_string(), LinkDescriptor::updateTime, LinkDescriptor::userId, LinkDescriptor::viewMetadata, and LinkDescriptor::viewState.

4048  {
4049  cat_write_lock write_lock(this);
4050  cat_sqlite_lock sqlite_lock(getObjForLock());
4051  sqliteConnector_.query("BEGIN TRANSACTION");
4052  try {
4054  .substr(0, 8);
4056  "SELECT linkid FROM mapd_links WHERE link = ? and userid = ?",
4057  std::vector<std::string>{ld.link, std::to_string(ld.userId)});
4058  if (sqliteConnector_.getNumRows() > 0) {
4060  "UPDATE mapd_links SET update_time = datetime('now') WHERE userid = ? AND "
4061  "link = ?",
4062  std::vector<std::string>{std::to_string(ld.userId), ld.link});
4063  } else {
4065  "INSERT INTO mapd_links (userid, link, view_state, view_metadata, "
4066  "update_time) VALUES (?,?,?,?, datetime('now'))",
4067  std::vector<std::string>{
4068  std::to_string(ld.userId), ld.link, ld.viewState, ld.viewMetadata});
4069  }
4070  // now get the auto generated dashid
4072  "SELECT linkid, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_links "
4073  "WHERE link = ?",
4074  ld.link);
4075  ld.linkId = sqliteConnector_.getData<int>(0, 0);
4076  ld.updateTime = sqliteConnector_.getData<std::string>(0, 1);
4077  } catch (std::exception& e) {
4078  sqliteConnector_.query("ROLLBACK TRANSACTION");
4079  throw;
4080  }
4081  sqliteConnector_.query("END TRANSACTION");
4082  addLinkToMap(ld);
4083  return ld.link;
4084 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:640
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
std::string calculateSHA1(const std::string &data)
Definition: Catalog.cpp:4036
void addLinkToMap(LinkDescriptor &ld)
Definition: Catalog.cpp:1428
const Catalog * getObjForLock()
Definition: Catalog.cpp:224
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:116
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:117
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createOrUpdateDashboardSystemRole ( const std::string &  view_meta,
const int32_t &  user_id,
const std::string &  dash_role_name 
)
protected

Definition at line 1382 of file Catalog.cpp.

References Catalog_Namespace::SysCatalog::createRole(), Grantee::findDbObject(), Grantee::getDbObjects(), Catalog_Namespace::SysCatalog::getRoleGrantee(), Catalog_Namespace::SysCatalog::grantDBObjectPrivilegesBatch(), Catalog_Namespace::SysCatalog::instance(), parseDashboardObjects(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivileges(), TableDBObjectType, and ViewDBObjectType.

Referenced by createDashboardSystemRoles().

1384  {
1385  auto objects = parseDashboardObjects(view_meta, user_id);
1386  Role* rl = SysCatalog::instance().getRoleGrantee(dash_role_name);
1387  if (!rl) {
1388  // Dashboard role does not exist
1389  // create role and grant privileges
1390  // NOTE(wamsi): Transactionally unsafe
1392  dash_role_name, /*user_private_role=*/false, /*is_temporary=*/false);
1393  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1394  } else {
1395  // Dashboard system role already exists
1396  // Add/remove privileges on objects
1397  std::set<DBObjectKey> revoke_keys;
1398  auto ex_objects = rl->getDbObjects(true);
1399  for (auto key : *ex_objects | boost::adaptors::map_keys) {
1400  if (key.permissionType != TableDBObjectType &&
1401  key.permissionType != ViewDBObjectType) {
1402  continue;
1403  }
1404  bool found = false;
1405  for (auto obj : objects) {
1406  found = key == obj.getObjectKey() ? true : false;
1407  if (found) {
1408  break;
1409  }
1410  }
1411  if (!found) {
1412  revoke_keys.insert(key);
1413  }
1414  }
1415  for (auto& key : revoke_keys) {
1416  // revoke privs on object since the object is no
1417  // longer used by the dashboard as source
1418  // NOTE(wamsi): Transactionally unsafe
1420  dash_role_name, *rl->findDbObject(key, true), *this);
1421  }
1422  // Update privileges on remaining objects
1423  // NOTE(wamsi): Transactionally unsafe
1424  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1425  }
1426 }
void revokeDBObjectPrivileges(const std::string &grantee, const DBObject &object, const Catalog_Namespace::Catalog &catalog)
void createRole(const std::string &roleName, const bool user_private_role, const bool is_temporary=false)
Definition: Grantee.h:81
DBObject * findDbObject(const DBObjectKey &objectKey, bool only_direct) const
Definition: Grantee.cpp:75
static SysCatalog & instance()
Definition: SysCatalog.h:325
Role * getRoleGrantee(const std::string &name) const
void grantDBObjectPrivilegesBatch(const std::vector< std::string > &grantees, const std::vector< DBObject > &objects, const Catalog_Namespace::Catalog &catalog)
const DBObjectMap * getDbObjects(bool only_direct) const
Definition: Grantee.h:56
std::vector< DBObject > parseDashboardObjects(const std::string &view_meta, const int &user_id)
Definition: Catalog.cpp:1352

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createShardedTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs 
)

Definition at line 3366 of file Catalog.cpp.

References CHECK, i, TableDescriptor::nShards, TableDescriptor::shard, table_is_temporary(), TableDescriptor::tableId, and TableDescriptor::tableName.

3369  {
3370  /* create logical table */
3371  TableDescriptor* tdl = &td;
3372  createTable(*tdl, cols, shared_dict_defs, true); // create logical table
3373  int32_t logical_tb_id = tdl->tableId;
3374  std::string logical_table_name = tdl->tableName;
3375 
3376  /* create physical tables and link them to the logical table */
3377  std::vector<int32_t> physicalTables;
3378  for (int32_t i = 1; i <= td.nShards; i++) {
3379  TableDescriptor* tdp = &td;
3380  tdp->tableName = generatePhysicalTableName(logical_table_name, i);
3381  tdp->shard = i - 1;
3382  createTable(*tdp, cols, shared_dict_defs, false); // create physical table
3383  int32_t physical_tb_id = tdp->tableId;
3384 
3385  /* add physical table to the vector of physical tables */
3386  physicalTables.push_back(physical_tb_id);
3387  }
3388 
3389  if (!physicalTables.empty()) {
3390  cat_write_lock write_lock(this);
3391  /* add logical to physical tables correspondence to the map */
3392  const auto it_ok =
3393  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
3394  CHECK(it_ok.second);
3395  /* update sqlite mapd_logical_to_physical in sqlite database */
3396  if (!table_is_temporary(&td)) {
3397  updateLogicalToPhysicalTableMap(logical_tb_id);
3398  }
3399  }
3400 }
std::string tableName
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:2243
std::string generatePhysicalTableName(const std::string &logicalTableName, const int32_t &shardNumber)
Definition: Catalog.cpp:4294
bool table_is_temporary(const TableDescriptor *const td)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:116
#define CHECK(condition)
Definition: Logger.h:209
mapd_unique_lock< mapd_shared_mutex > write_lock
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:647
void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id)
Definition: Catalog.cpp:610

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createSystemTable ( const std::string &  table_name,
const std::string &  server_name,
const std::vector< std::pair< std::string, SQLTypeInfo >> &  column_type_by_name 
)
private

Definition at line 5432 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, DEFAULT_FRAGMENT_ROWS, DEFAULT_MAX_CHUNK_SIZE, DEFAULT_MAX_ROWS, DEFAULT_PAGE_SIZE, foreign_storage::ForeignTable::foreign_server, StorageType::FOREIGN_TABLE, TableDescriptor::fragmenter, TableDescriptor::fragments, TableDescriptor::fragPageSize, TableDescriptor::fragType, TableDescriptor::hasDeletedCol, Fragmenter_Namespace::INSERT_ORDER, SQLTypeInfo::is_array(), SQLTypeInfo::is_string(), SQLTypeInfo::is_string_array(), TableDescriptor::is_system_table, ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, kENCODING_DICT, TableDescriptor::keyMetainfo, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRows, TableDescriptor::nColumns, OMNISCI_ROOT_USER_ID, TableDescriptor::partitions, SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_compression(), SQLTypeInfo::set_fixed_size(), SQLTypeInfo::set_notnull(), SQLTypeInfo::set_size(), SQLTypeInfo::set_subtype(), SQLTypeInfo::set_type(), TableDescriptor::storageType, TableDescriptor::tableName, and TableDescriptor::userId.

5435  {
5436  foreign_storage::ForeignTable foreign_table;
5437  foreign_table.tableName = table_name;
5438  foreign_table.nColumns = column_type_by_name.size();
5439  foreign_table.isView = false;
5440  foreign_table.is_system_table = true;
5441  foreign_table.fragmenter = nullptr;
5443  foreign_table.maxFragRows = DEFAULT_FRAGMENT_ROWS;
5444  foreign_table.maxChunkSize = DEFAULT_MAX_CHUNK_SIZE;
5445  foreign_table.fragPageSize = DEFAULT_PAGE_SIZE;
5446  foreign_table.maxRows = DEFAULT_MAX_ROWS;
5447  foreign_table.userId = OMNISCI_ROOT_USER_ID;
5448  foreign_table.storageType = StorageType::FOREIGN_TABLE;
5449  foreign_table.hasDeletedCol = false;
5450  foreign_table.keyMetainfo = "[]";
5451  foreign_table.fragments = "";
5452  foreign_table.partitions = "";
5453  foreign_table.foreign_server = getForeignServer(server_name);
5454  CHECK(foreign_table.foreign_server);
5455 
5456  list<ColumnDescriptor> columns;
5457  for (const auto& [column_name, column_type] : column_type_by_name) {
5458  columns.emplace_back();
5459  auto& cd = columns.back();
5460  cd.columnName = column_name;
5461  cd.columnType.set_type(column_type.get_type());
5462  cd.columnType.set_subtype(column_type.get_subtype());
5463  cd.columnType.set_notnull(false);
5464  if (cd.columnType.is_string() || cd.columnType.is_string_array()) {
5465  cd.columnType.set_compression(kENCODING_DICT);
5466  cd.columnType.set_comp_param(32);
5467  }
5468  if (cd.columnType.is_array()) {
5469  cd.columnType.set_size(-1);
5470  } else {
5471  cd.columnType.set_fixed_size();
5472  }
5473  cd.isSystemCol = false;
5474  cd.isVirtualCol = false;
5475  }
5476  createTable(foreign_table, columns, {}, true);
5477 }
std::string partitions
std::string tableName
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:2243
const foreign_storage::ForeignServer * getForeignServer(const std::string &server_name) const
Definition: Catalog.cpp:2622
std::string storageType
#define DEFAULT_MAX_CHUNK_SIZE
std::string fragments
#define DEFAULT_MAX_ROWS
std::string keyMetainfo
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
#define DEFAULT_PAGE_SIZE
#define DEFAULT_FRAGMENT_ROWS
const int OMNISCI_ROOT_USER_ID
Definition: SysCatalog.h:61
Fragmenter_Namespace::FragmenterType fragType
const ForeignServer * foreign_server
Definition: ForeignTable.h:54
#define CHECK(condition)
Definition: Logger.h:209
static constexpr char const * FOREIGN_TABLE

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createSystemTableServer ( const std::string &  server_name,
const std::string &  data_wrapper_type 
)
private

Definition at line 5421 of file Catalog.cpp.

References OMNISCI_ROOT_USER_ID.

5422  {
5423  auto server =
5424  std::make_unique<foreign_storage::ForeignServer>(server_name,
5425  data_wrapper_type,
5428  server->validate();
5429  createForeignServer(std::move(server), true);
5430 }
void createForeignServer(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2577
const int OMNISCI_ROOT_USER_ID
Definition: SysCatalog.h:61
std::map< std::string, std::string, std::less<>> OptionsMap
void Catalog_Namespace::Catalog::createTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs,
bool  isLogicalTable 
)

Definition at line 2243 of file Catalog.cpp.

References addTableToMap(), calciteMgr_, CHECK, ColumnDescriptor::columnId, TableDescriptor::columnIdBySpi_, ColumnDescriptor::columnName, ColumnDescriptor::columnType, currentDB_, dataMgr_, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, ColumnDescriptor::default_value, dict_ref_t::dictId, Data_Namespace::DISK_LEVEL, expandGeoColumn(), StorageType::FOREIGN_TABLE, TableDescriptor::fragPageSize, TableDescriptor::fragType, g_enable_fsi, g_serialize_temp_tables, g_test_against_columnId_gap, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_next_refresh_time(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), getCurrentDB(), SqliteConnector::getData(), getObjForLock(), TableDescriptor::hasDeletedCol, SQLTypeInfo::is_array(), TableDescriptor::is_system_table, ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, kBIGINT, kBOOLEAN, kENCODING_DICT, TableDescriptor::keyMetainfo, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nColumns, foreign_storage::ForeignTable::next_refresh_time, nextTempDictId_, nextTempTableId_, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, SqliteConnector::query(), SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), serializeTableJsonUnlocked(), SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_size(), setColumnDictionary(), setColumnSharedDictionary(), TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, sqliteConnector_, TableDescriptor::storageType, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, TEXT, to_string(), TableDescriptor::userId, TableDescriptor::viewSQL, and ColumnDescriptor::virtualExpr.

2247  {
2248  cat_write_lock write_lock(this);
2249  list<ColumnDescriptor> cds = cols;
2250  list<DictDescriptor> dds;
2251  std::set<std::string> toplevel_column_names;
2252  list<ColumnDescriptor> columns;
2253 
2254  if (!td.storageType.empty() &&
2257  throw std::runtime_error("Only temporary tables can be backed by foreign storage.");
2258  }
2259  dataMgr_->getForeignStorageInterface()->prepareTable(getCurrentDB().dbId, td, cds);
2260  }
2261 
2262  for (auto cd : cds) {
2263  if (cd.columnName == "rowid") {
2264  throw std::runtime_error(
2265  "Cannot create column with name rowid. rowid is a system defined column.");
2266  }
2267  columns.push_back(cd);
2268  toplevel_column_names.insert(cd.columnName);
2269  if (cd.columnType.is_geometry()) {
2270  expandGeoColumn(cd, columns);
2271  }
2272  }
2273  cds.clear();
2274 
2275  ColumnDescriptor cd;
2276  // add row_id column -- Must be last column in the table
2277  cd.columnName = "rowid";
2278  cd.isSystemCol = true;
2279  cd.columnType = SQLTypeInfo(kBIGINT, true);
2280 #ifdef MATERIALIZED_ROWID
2281  cd.isVirtualCol = false;
2282 #else
2283  cd.isVirtualCol = true;
2284  cd.virtualExpr = "MAPD_FRAG_ID * MAPD_ROWS_PER_FRAG + MAPD_FRAG_ROW_ID";
2285 #endif
2286  columns.push_back(cd);
2287  toplevel_column_names.insert(cd.columnName);
2288 
2289  if (td.hasDeletedCol) {
2290  ColumnDescriptor cd_del;
2291  cd_del.columnName = "$deleted$";
2292  cd_del.isSystemCol = true;
2293  cd_del.isVirtualCol = false;
2294  cd_del.columnType = SQLTypeInfo(kBOOLEAN, true);
2295  cd_del.isDeletedCol = true;
2296 
2297  columns.push_back(cd_del);
2298  }
2299 
2300  td.nColumns = columns.size();
2301  cat_sqlite_lock sqlite_lock(getObjForLock());
2302  sqliteConnector_.query("BEGIN TRANSACTION");
2304  try {
2306  R"(INSERT INTO mapd_tables (name, userid, ncolumns, isview, fragments, frag_type, max_frag_rows, max_chunk_size, frag_page_size, max_rows, partitions, shard_column_id, shard, num_shards, sort_column_id, storage_type, max_rollback_epochs, is_system_table, key_metainfo) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?))",
2307  std::vector<std::string>{td.tableName,
2308  std::to_string(td.userId),
2310  std::to_string(td.isView),
2311  "",
2316  std::to_string(td.maxRows),
2317  td.partitions,
2319  std::to_string(td.shard),
2320  std::to_string(td.nShards),
2322  td.storageType,
2325  td.keyMetainfo});
2326 
2327  // now get the auto generated tableid
2329  "SELECT tableid FROM mapd_tables WHERE name = ?", td.tableName);
2330  td.tableId = sqliteConnector_.getData<int>(0, 0);
2331  int colId = 1;
2332  for (auto cd : columns) {
2334  const bool is_foreign_col =
2335  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2336  if (!is_foreign_col) {
2337  setColumnDictionary(cd, dds, td, isLogicalTable);
2338  }
2339  }
2340 
2341  if (toplevel_column_names.count(cd.columnName)) {
2342  // make up colId gap for sanity test (begin with 1 bc much code depends on it!)
2343  if (colId > 1) {
2344  colId += g_test_against_columnId_gap;
2345  }
2346  if (!cd.isGeoPhyCol) {
2347  td.columnIdBySpi_.push_back(colId);
2348  }
2349  }
2350 
2351  using BindType = SqliteConnector::BindType;
2352  std::vector<BindType> types(17, BindType::TEXT);
2353  if (!cd.default_value.has_value()) {
2354  types[16] = BindType::NULL_TYPE;
2355  }
2357  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, "
2358  "coldim, colscale, is_notnull, "
2359  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
2360  "virtual_expr, is_deletedcol, default_value) "
2361  "VALUES (?, ?, ?, ?, ?, "
2362  "?, "
2363  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2364  std::vector<std::string>{std::to_string(td.tableId),
2365  std::to_string(colId),
2366  cd.columnName,
2375  "",
2378  cd.virtualExpr,
2380  cd.default_value.value_or("NULL")},
2381  types);
2382  cd.tableId = td.tableId;
2383  cd.columnId = colId++;
2384  cds.push_back(cd);
2385  }
2386  if (td.isView) {
2388  "INSERT INTO mapd_views (tableid, sql) VALUES (?,?)",
2389  std::vector<std::string>{std::to_string(td.tableId), td.viewSQL});
2390  }
2392  auto& foreign_table = dynamic_cast<foreign_storage::ForeignTable&>(td);
2393  foreign_table.next_refresh_time = get_next_refresh_time(foreign_table);
2395  "INSERT INTO omnisci_foreign_tables (table_id, server_id, options, "
2396  "last_refresh_time, next_refresh_time) VALUES (?, ?, ?, ?, ?)",
2397  std::vector<std::string>{std::to_string(foreign_table.tableId),
2398  std::to_string(foreign_table.foreign_server->id),
2399  foreign_table.getOptionsAsJsonString(),
2400  std::to_string(foreign_table.last_refresh_time),
2401  std::to_string(foreign_table.next_refresh_time)});
2402  }
2403  } catch (std::exception& e) {
2404  sqliteConnector_.query("ROLLBACK TRANSACTION");
2405  throw;
2406  }
2407  } else { // Temporary table
2408  td.tableId = nextTempTableId_++;
2409  int colId = 1;
2410  for (auto cd : columns) {
2412  const bool is_foreign_col =
2413  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2414 
2415  if (!is_foreign_col) {
2416  // Create a new temporary dictionary
2417  std::string fileName("");
2418  std::string folderPath("");
2420  nextTempDictId_++;
2421  DictDescriptor dd(dict_ref,
2422  fileName,
2424  false,
2425  1,
2426  folderPath,
2427  true); // Is dictName (2nd argument) used?
2428  dds.push_back(dd);
2429  if (!cd.columnType.is_array()) {
2431  }
2432  cd.columnType.set_comp_param(dict_ref.dictId);
2433  }
2434  }
2435  if (toplevel_column_names.count(cd.columnName)) {
2436  // make up colId gap for sanity test (begin with 1 bc much code depends on it!)
2437  if (colId > 1) {
2438  colId += g_test_against_columnId_gap;
2439  }
2440  if (!cd.isGeoPhyCol) {
2441  td.columnIdBySpi_.push_back(colId);
2442  }
2443  }
2444  cd.tableId = td.tableId;
2445  cd.columnId = colId++;
2446  cds.push_back(cd);
2447  }
2448 
2450  serializeTableJsonUnlocked(&td, cds);
2451  }
2452  }
2453 
2454  try {
2455  auto cache = dataMgr_->getPersistentStorageMgr()->getDiskCache();
2456  if (cache) {
2457  CHECK(!cache->hasCachedMetadataForKeyPrefix({getCurrentDB().dbId, td.tableId}))
2458  << "Disk cache at " + cache->getCacheDirectory()
2459  << " contains preexisting data for new table. Please "
2460  "delete or clear cache before continuing";
2461  }
2462 
2463  addTableToMap(&td, cds, dds);
2464  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
2465  if (!td.storageType.empty() && td.storageType != StorageType::FOREIGN_TABLE) {
2466  dataMgr_->getForeignStorageInterface()->registerTable(this, td, cds);
2467  }
2468  } catch (std::exception& e) {
2469  sqliteConnector_.query("ROLLBACK TRANSACTION");
2470  removeTableFromMap(td.tableName, td.tableId, true);
2471  throw;
2472  }
2473  sqliteConnector_.query("END TRANSACTION");
2474 
2476  write_lock.unlock();
2477  sqlite_lock.unlock();
2479  true); // cause instantiateFragmenter() to be called
2480  }
2481 }
int32_t maxRollbackEpochs
void serializeTableJsonUnlocked(const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
Definition: Catalog.cpp:2483
int64_t get_next_refresh_time(const foreign_storage::ForeignTable &foreign_table)
Definition: Catalog.cpp:2231
std::string virtualExpr
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:330
void set_size(int s)
Definition: sqltypes.h:427
std::string partitions
HOST DEVICE int get_size() const
Definition: sqltypes.h:339
T getData(const int row, const int col)
std::string tableName
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:640
const DBMetadata currentDB_
Definition: Catalog.h:641
HOST DEVICE int get_scale() const
Definition: sqltypes.h:334
std::string storageType
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:642
virtual void query(const std::string &queryString)
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:329
std::vector< int > columnIdBySpi_
std::string to_string(char const *&&v)
void addTableToMap(const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
Definition: Catalog.cpp:1200
#define TEXT
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:221
specifies the content in-memory of a row in the column metadata table
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:2100
std::string keyMetainfo
int g_test_against_columnId_gap
Definition: Catalog.cpp:92
bool g_serialize_temp_tables
Definition: Catalog.cpp:100
void set_comp_param(int p)
Definition: sqltypes.h:430
std::optional< std::string > default_value
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:645
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:337
Fragmenter_Namespace::FragmenterType fragType
Data_Namespace::MemoryLevel persistenceLevel
const Catalog * getObjForLock()
Definition: Catalog.cpp:224
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:331
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1260
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:116
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:338
bool setColumnSharedDictionary(ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:3256
#define CHECK(condition)
Definition: Logger.h:209
Descriptor for a dictionary for a string columne.
std::string viewSQL
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:336
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:117
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:3325
static constexpr char const * FOREIGN_TABLE
bool g_enable_fsi
Definition: Catalog.cpp:93
std::string columnName
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
bool is_array() const
Definition: sqltypes.h:517

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::delDictionary ( const ColumnDescriptor cd)

Definition at line 1873 of file Catalog.cpp.

References CHECK, CHECK_GT, ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, g_base_path, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SqliteConnector::getData(), getMetadataForTable(), getObjForLock(), SQLTypeInfo::is_string(), SQLTypeInfo::is_string_array(), kENCODING_DICT, SqliteConnector::query_with_text_param(), File_Namespace::renameForDelete(), sqliteConnector_, string_dict_hosts_, ColumnDescriptor::tableId, to_string(), and VLOG.

Referenced by roll().

1873  {
1874  cat_write_lock write_lock(this);
1875  cat_sqlite_lock sqlite_lock(getObjForLock());
1876  if (!(cd.columnType.is_string() || cd.columnType.is_string_array())) {
1877  return;
1878  }
1879  if (!(cd.columnType.get_compression() == kENCODING_DICT)) {
1880  return;
1881  }
1882  const auto dictId = cd.columnType.get_comp_param();
1883  CHECK_GT(dictId, 0);
1884  // decrement and zero check dict ref count
1885  const auto td = getMetadataForTable(cd.tableId, false);
1886  CHECK(td);
1888  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
1889  std::to_string(dictId));
1891  "SELECT refcount FROM mapd_dictionaries WHERE dictid = ?", std::to_string(dictId));
1892  const auto refcount = sqliteConnector_.getData<int>(0, 0);
1893  VLOG(3) << "Dictionary " << dictId << "from dropped table has reference count "
1894  << refcount;
1895  if (refcount > 0) {
1896  return;
1897  }
1898  const DictRef dictRef(currentDB_.dbId, dictId);
1899  sqliteConnector_.query_with_text_param("DELETE FROM mapd_dictionaries WHERE dictid = ?",
1900  std::to_string(dictId));
1901  File_Namespace::renameForDelete(g_base_path + "/mapd_data/DB_" +
1902  std::to_string(currentDB_.dbId) + "_DICT_" +
1903  std::to_string(dictId));
1904 
1905  std::unique_ptr<StringDictionaryClient> client;
1906  if (!string_dict_hosts_.empty()) {
1907  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dictRef, true));
1908  }
1909  if (client) {
1910  client->drop(dictRef);
1911  }
1912 
1913  dictDescriptorMapByRef_.erase(dictRef);
1914 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:640
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:644
const DBMetadata currentDB_
Definition: Catalog.h:641
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:632
#define CHECK_GT(x, y)
Definition: Logger.h:221
std::string to_string(char const *&&v)
std::string g_base_path
Definition: SysCatalog.cpp:63
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:337
const Catalog * getObjForLock()
Definition: Catalog.cpp:224
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:116
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:338
#define CHECK(condition)
Definition: Logger.h:209
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
bool is_string() const
Definition: sqltypes.h:509
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:218
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:117
bool is_string_array() const
Definition: sqltypes.h:510
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
#define VLOG(n)
Definition: Logger.h:303

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::deleteCustomExpressions ( const std::vector< int32_t > &  custom_expression_ids,
bool  do_soft_delete 
)

Deletes custom expressions with the given ids.

Parameters
custom_expression_ids- ids of custom expressions to delete
do_soft_delete- flag indicating whether or not to do a soft delete

Definition at line 5042 of file Catalog.cpp.

References CHECK, CHECK_EQ, join(), and to_string().

5043  {
5044  cat_write_lock write_lock(this);
5045  cat_sqlite_lock sqlite_lock(getObjForLock());
5046 
5047  std::vector<int32_t> invalid_ids;
5048  for (const auto id : custom_expression_ids) {
5049  if (custom_expr_map_by_id_.find(id) == custom_expr_map_by_id_.end()) {
5050  invalid_ids.emplace_back(id);
5051  }
5052  }
5053  if (!invalid_ids.empty()) {
5054  throw std::runtime_error{"Custom expressions with ids: " + join(invalid_ids, ",") +
5055  " do not exist."};
5056  }
5057  sqliteConnector_.query("BEGIN TRANSACTION");
5058  try {
5059  for (const auto id : custom_expression_ids) {
5061  "SELECT id FROM omnisci_custom_expressions WHERE id = ?",
5062  std::vector<std::string>{std::to_string(id)});
5063  CHECK_EQ(sqliteConnector_.getNumRows(), static_cast<size_t>(1));
5064  if (do_soft_delete) {
5066  "UPDATE omnisci_custom_expressions SET is_deleted = ? WHERE id = ?",
5067  std::vector<std::string>{std::to_string(true), std::to_string(id)});
5068  } else {
5070  "DELETE FROM omnisci_custom_expressions WHERE id = ?",
5071  std::vector<std::string>{std::to_string(id)});
5072  }
5073  }
5074 
5075  for (const auto id : custom_expression_ids) {
5076  if (do_soft_delete) {
5077  auto it = custom_expr_map_by_id_.find(id);
5078  CHECK(it != custom_expr_map_by_id_.end());
5079  it->second->is_deleted = true;
5080  } else {
5081  custom_expr_map_by_id_.erase(id);
5082  }
5083  }
5084  } catch (std::exception& e) {
5085  sqliteConnector_.query("ROLLBACK TRANSACTION");
5086  throw;
5087  }
5088  sqliteConnector_.query("END TRANSACTION");
5089 }
#define CHECK_EQ(x, y)
Definition: Logger.h:217
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:640
std::string join(T const &container, std::string const &delim)
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
const Catalog * getObjForLock()
Definition: Catalog.cpp:224
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:116
#define CHECK(condition)
Definition: Logger.h:209
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:638
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:117
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::deleteMetadataForDashboards ( const std::vector< int32_t >  ids,
const UserMetadata user 
)

Definition at line 1633 of file Catalog.cpp.

References DashboardDBObjectType, dashboardDescriptorMap_, AccessPrivileges::DELETE_DASHBOARD, getMetadataForDashboard(), getObjForLock(), Catalog_Namespace::SysCatalog::instance(), SqliteConnector::query(), SqliteConnector::query_with_text_params(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivilegesFromAllBatch(), sqliteConnector_, and to_string().

1634  {
1635  std::stringstream invalid_ids, restricted_ids;
1636 
1637  for (int32_t dashboard_id : dashboard_ids) {
1638  if (!getMetadataForDashboard(dashboard_id)) {
1639  invalid_ids << (!invalid_ids.str().empty() ? ", " : "") << dashboard_id;
1640  continue;
1641  }
1642  DBObject object(dashboard_id, DashboardDBObjectType);
1643  object.loadKey(*this);
1644  object.setPrivileges(AccessPrivileges::DELETE_DASHBOARD);
1645  std::vector<DBObject> privs = {object};
1646  if (!SysCatalog::instance().checkPrivileges(user, privs)) {
1647  restricted_ids << (!restricted_ids.str().empty() ? ", " : "") << dashboard_id;
1648  }
1649  }
1650 
1651  if (invalid_ids.str().size() > 0 || restricted_ids.str().size() > 0) {
1652  std::stringstream error_message;
1653  error_message << "Delete dashboard(s) failed with error(s):";
1654  if (invalid_ids.str().size() > 0) {
1655  error_message << "\nDashboard id: " << invalid_ids.str()
1656  << " - Dashboard id does not exist";
1657  }
1658  if (restricted_ids.str().size() > 0) {
1659  error_message
1660  << "\nDashboard id: " << restricted_ids.str()
1661  << " - User should be either owner of dashboard or super user to delete it";
1662  }
1663  throw std::runtime_error(error_message.str());
1664  }
1665  std::vector<DBObject> dash_objs;
1666 
1667  for (int32_t dashboard_id : dashboard_ids) {
1668  dash_objs.emplace_back(dashboard_id, DashboardDBObjectType);
1669  }
1670  // BE-5245: Transactionally unsafe (like other combined Catalog/Syscatalog operations)
1672  {
1673  cat_write_lock write_lock(this);
1674  cat_sqlite_lock sqlite_lock(getObjForLock());
1675 
1676  sqliteConnector_.query("BEGIN TRANSACTION");
1677  try {
1678  for (int32_t dashboard_id : dashboard_ids) {
1679  auto dash = getMetadataForDashboard(dashboard_id);
1680  // Dash should still exist if revokeDBObjectPrivileges passed but throw and
1681  // rollback if already deleted
1682  if (!dash) {
1683  throw std::runtime_error(
1684  std::string("Delete dashboard(s) failed with error(s):\nDashboard id: ") +
1685  std::to_string(dashboard_id) + " - Dashboard id does not exist ");
1686  }
1687  std::string user_id = std::to_string(dash->userId);
1688  std::string dash_name = dash->dashboardName;
1689  auto viewDescIt = dashboardDescriptorMap_.find(user_id + ":" + dash_name);
1690  dashboardDescriptorMap_.erase(viewDescIt);
1692  "DELETE FROM mapd_dashboards WHERE name = ? and userid = ?",
1693  std::vector<std::string>{dash_name, user_id});
1694  }
1695  } catch (std::exception& e) {
1696  sqliteConnector_.query("ROLLBACK TRANSACTION");
1697  throw;
1698  }
1699  sqliteConnector_.query("END TRANSACTION");
1700  }
1701 }
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:640
virtual void query(const std::string &queryString)
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:633
std::string to_string(char const *&&v)
static SysCatalog & instance()
Definition: SysCatalog.h:325
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const
const Catalog * getObjForLock()
Definition: Catalog.cpp:224
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:116
static const AccessPrivileges DELETE_DASHBOARD
Definition: DBObject.h:175
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:117
void revokeDBObjectPrivilegesFromAllBatch(std::vector< DBObject > &objects, Catalog *catalog)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::doTruncateTable ( const TableDescriptor td)
protected

Definition at line 3410 of file Catalog.cpp.

References CHECK, CHECK_GE, ColumnDescriptor::columnType, Data_Namespace::CPU_LEVEL, dict_ref_t::dictId, DictDescriptor::dictIsTemp, DictDescriptor::dictRef, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), Data_Namespace::GPU_LEVEL, kENCODING_DICT, File_Namespace::renameForDelete(), ColumnDescriptor::tableId, and TableDescriptor::tableId.

3410  {
3411  // must destroy fragmenter before deleteChunks is called.
3413 
3414  const int tableId = td->tableId;
3415  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
3416  // assuming deleteChunksWithPrefix is atomic
3417  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
3418  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
3419 
3420  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
3421 
3422  cat_write_lock write_lock(this);
3423  std::unique_ptr<StringDictionaryClient> client;
3424  if (SysCatalog::instance().isAggregator()) {
3425  CHECK(!string_dict_hosts_.empty());
3426  DictRef dict_ref(currentDB_.dbId, -1);
3427  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
3428  }
3429  // clean up any dictionaries
3430  // delete all column descriptors for the table
3431  for (const auto& columnDescriptor : columnDescriptorMapById_) {
3432  auto cd = columnDescriptor.second;
3433  if (cd->tableId != td->tableId) {
3434  continue;
3435  }
3436  const int dict_id = cd->columnType.get_comp_param();
3437  // Dummy dictionaries created for a shard of a logical table have the id set to zero.
3438  if (cd->columnType.get_compression() == kENCODING_DICT && dict_id) {
3439  const DictRef dict_ref(currentDB_.dbId, dict_id);
3440  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
3441  CHECK(dictIt != dictDescriptorMapByRef_.end());
3442  const auto& dd = dictIt->second;
3443  CHECK_GE(dd->refcount, 1);
3444  // if this is the only table using this dict reset the dict
3445  if (dd->refcount == 1) {
3446  // close the dictionary
3447  dd->stringDict.reset();
3448  File_Namespace::renameForDelete(dd->dictFolderPath);
3449  if (client) {
3450  client->drop(dd->dictRef);
3451  }
3452  if (!dd->dictIsTemp) {
3453  boost::filesystem::create_directory(dd->dictFolderPath);
3454  }
3455  }
3456 
3457  DictDescriptor* new_dd = new DictDescriptor(dd->dictRef,
3458  dd->dictName,
3459  dd->dictNBits,
3460  dd->dictIsShared,
3461  dd->refcount,
3462  dd->dictFolderPath,
3463  dd->dictIsTemp);
3464  dictDescriptorMapByRef_.erase(dictIt);
3465  // now create new Dict -- need to figure out what to do here for temp tables
3466  if (client) {
3467  client->create(new_dd->dictRef, new_dd->dictIsTemp);
3468  }
3469  dictDescriptorMapByRef_[new_dd->dictRef].reset(new_dd);
3471  }
3472  }
3473 }
std::vector< int > ChunkKey
Definition: types.h:37
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:644
const DBMetadata currentDB_
Definition: Catalog.h:641
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:632
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:642
#define CHECK_GE(x, y)
Definition: Logger.h:222
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:631
static SysCatalog & instance()
Definition: SysCatalog.h:325
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1537
int32_t dictId
Definition: DictRef.h:10
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:116
void removeFragmenterForTable(const int table_id) const
Definition: Catalog.cpp:3475
#define CHECK(condition)
Definition: Logger.h:209
Descriptor for a dictionary for a string columne.
mapd_unique_lock< mapd_shared_mutex > write_lock
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:218

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropColumn ( const TableDescriptor td,
const ColumnDescriptor cd 
)

Definition at line 2007 of file Catalog.cpp.

References CHECK, columnDescriptorMap_, columnDescriptorMapById_, columnDescriptorsForRoll, ColumnDescriptor::columnId, ColumnDescriptor::columnName, getMetadataForColumn(), getObjForLock(), getPhysicalTablesDescriptors(), TableDescriptor::nShards, SqliteConnector::query_with_text_params(), TableDescriptor::shard, sqliteConnector_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), and to_upper().

2007  {
2008  {
2009  cat_write_lock write_lock(this);
2010  cat_sqlite_lock sqlite_lock(getObjForLock());
2011  // caller must handle sqlite/chunk transaction TOGETHER
2013  "DELETE FROM mapd_columns where tableid = ? and columnid = ?",
2014  std::vector<std::string>{std::to_string(td.tableId),
2015  std::to_string(cd.columnId)});
2016 
2018  "UPDATE mapd_tables SET ncolumns = ncolumns - 1 WHERE tableid = ?",
2019  std::vector<std::string>{std::to_string(td.tableId)});
2020 
2021  ColumnDescriptorMap::iterator columnDescIt =
2023  CHECK(columnDescIt != columnDescriptorMap_.end());
2024 
2025  columnDescriptorsForRoll.emplace_back(columnDescIt->second, nullptr);
2026 
2027  columnDescriptorMap_.erase(columnDescIt);
2029  --tableDescriptorMapById_[td.tableId]->nColumns;
2030  }
2031 
2032  // for each shard
2033  if (td.nShards > 0 && td.shard < 0) {
2034  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
2035  const auto shard_cd = getMetadataForColumn(shard->tableId, cd.columnId);
2036  CHECK(shard_cd);
2037  dropColumn(*shard, *shard_cd);
2038  }
2039  }
2040 }
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:630
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:640
void dropColumn(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:2007
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:631
std::string to_string(char const *&&v)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:629
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4104
std::string to_upper(const std::string &str)
const Catalog * getObjForLock()
Definition: Catalog.cpp:224
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:116
#define CHECK(condition)
Definition: Logger.h:209
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:117
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:659
std::string columnName
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropForeignServer ( const std::string &  server_name)

Drops/deletes a foreign server DB object.

Parameters
server_name- Name of foreign server that will be deleted

Definition at line 2738 of file Catalog.cpp.

References CHECK_EQ, and to_string().

2738  {
2739  cat_write_lock write_lock(this);
2740  cat_sqlite_lock sqlite_lock(getObjForLock());
2741 
2743  "SELECT id from omnisci_foreign_servers where name = ?",
2744  std::vector<std::string>{server_name});
2745  auto num_rows = sqliteConnector_.getNumRows();
2746  if (num_rows > 0) {
2747  CHECK_EQ(size_t(1), num_rows);
2748  auto server_id = sqliteConnector_.getData<int32_t>(0, 0);
2750  "SELECT table_id from omnisci_foreign_tables where server_id = ?",
2751  std::to_string(server_id));
2752  if (sqliteConnector_.getNumRows() > 0) {
2753  throw std::runtime_error{"Foreign server \"" + server_name +
2754  "\" is referenced "
2755  "by existing foreign tables and cannot be dropped."};
2756  }
2757  sqliteConnector_.query("BEGIN TRANSACTION");
2758  try {
2760  "DELETE FROM omnisci_foreign_servers WHERE name = ?",
2761  std::vector<std::string>{server_name});
2762  } catch (const std::exception& e) {
2763  sqliteConnector_.query("ROLLBACK TRANSACTION");
2764  throw;
2765  }
2766  sqliteConnector_.query("END TRANSACTION");
2767  foreignServerMap_.erase(server_name);
2768  foreignServerMapById_.erase(server_id);
2769  }
2770 }
#define CHECK_EQ(x, y)
Definition: Logger.h:217
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:640
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:637
ForeignServerMap foreignServerMap_
Definition: Catalog.h:636
const Catalog * getObjForLock()
Definition: Catalog.cpp:224
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:116
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:117
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropTable ( const TableDescriptor td)

Definition at line 3497 of file Catalog.cpp.

References CHECK, i, TableDescriptor::isView, TableDBObjectType, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and ViewDBObjectType.

3497  {
3500  std::vector<const TableDescriptor*> tables_to_drop;
3501  {
3502  cat_read_lock read_lock(this);
3503  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
3504  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
3505  // remove all corresponding physical tables if this is a logical table
3506  const auto physicalTables = physicalTableIt->second;
3507  CHECK(!physicalTables.empty());
3508  for (size_t i = 0; i < physicalTables.size(); i++) {
3509  int32_t physical_tb_id = physicalTables[i];
3510  const TableDescriptor* phys_td =
3511  getMutableMetadataForTableUnlocked(physical_tb_id);
3512  CHECK(phys_td);
3513  tables_to_drop.emplace_back(phys_td);
3514  }
3515  }
3516  tables_to_drop.emplace_back(td);
3517  }
3518 
3519  for (auto table : tables_to_drop) {
3520  eraseTablePhysicalData(table);
3521  }
3522 
3523  {
3524  cat_write_lock write_lock(this);
3525  cat_sqlite_lock sqlite_lock(getObjForLock());
3526  sqliteConnector_.query("BEGIN TRANSACTION");
3527  try {
3528  // remove corresponding record from the logicalToPhysicalTableMap in sqlite database
3530  "DELETE FROM mapd_logical_to_physical WHERE logical_table_id = ?",
3531  std::to_string(td->tableId));
3533  for (auto table : tables_to_drop) {
3534  eraseTableMetadata(table);
3535  }
3536  } catch (std::exception& e) {
3537  sqliteConnector_.query("ROLLBACK TRANSACTION");
3538  throw;
3539  }
3540  sqliteConnector_.query("END TRANSACTION");
3541  }
3542 }
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:4276
std::string tableName
SqliteConnector sqliteConnector_
Definition: Catalog.h:640
void revokeDBObjectPrivilegesFromAll(DBObject object, Catalog *catalog)
virtual void query(const std::string &queryString)
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:115
TableDescriptor * getMutableMetadataForTableUnlocked(int table_id) const
Definition: Catalog.cpp:1529
std::string to_string(char const *&&v)
static SysCatalog & instance()
Definition: SysCatalog.h:325
const Catalog * getObjForLock()
Definition: Catalog.cpp:224
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:116
void eraseTableMetadata(const TableDescriptor *td)
Definition: Catalog.cpp:3544
mapd_shared_lock< mapd_shared_mutex > read_lock
#define CHECK(condition)
Definition: Logger.h:209
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:117
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:647

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropTableFromJsonUnlocked ( const std::string &  table_name) const
protected

Definition at line 2545 of file Catalog.cpp.

References CHECK, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and VLOG.

2545  {
2546  // relies on the catalog write lock
2547  using namespace rapidjson;
2548 
2549  VLOG(1) << "Dropping temporary table " << table_name << " to JSON for Calcite.";
2550 
2551  const auto db_name = currentDB_.dbName;
2552  const auto file_path = table_json_filepath(basePath_, db_name);
2553 
2554  CHECK(boost::filesystem::exists(file_path));
2555  Document d;
2556 
2557  std::ifstream reader(file_path.string());
2558  CHECK(reader.is_open());
2559  IStreamWrapper json_read_wrapper(reader);
2560  d.ParseStream(json_read_wrapper);
2561 
2562  CHECK(d.IsObject());
2563  auto table_name_ref = StringRef(table_name.c_str());
2564  CHECK(d.HasMember(table_name_ref));
2565  CHECK(d.RemoveMember(table_name_ref));
2566 
2567  // Overwrite the existing file
2568  std::ofstream writer(file_path.string(), std::ios::trunc | std::ios::out);
2569  CHECK(writer.is_open());
2570  OStreamWrapper json_wrapper(writer);
2571 
2572  Writer<OStreamWrapper> json_writer(json_wrapper);
2573  d.Accept(json_writer);
2574  writer.close();
2575 }
const DBMetadata currentDB_
Definition: Catalog.h:641
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:156
#define CHECK(condition)
Definition: Logger.h:209
#define VLOG(n)
Definition: Logger.h:303

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::dumpCreateTable ( const TableDescriptor td,
bool  multiline_formatting = true,
bool  dump_defaults = false 
) const

Definition at line 4577 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, Data_Namespace::CPU_LEVEL, DEFAULT_FRAGMENT_ROWS, DEFAULT_MAX_CHUNK_SIZE, DEFAULT_MAX_ROLLBACK_EPOCHS, DEFAULT_MAX_ROWS, DEFAULT_PAGE_SIZE, ColumnDescriptor::default_value, TableDescriptor::fragPageSize, g_leaf_count, ColumnDescriptor::getDefaultValueLiteral(), TableDescriptor::hasDeletedCol, TableDescriptor::is_system_table, ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, join(), kCHAR, kENCODING_DICT, kENCODING_GEOINT, kTEXT, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and TableDescriptor::viewSQL.

4579  {
4580  cat_read_lock read_lock(this);
4581 
4582  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
4583  std::ostringstream os;
4584 
4585  if (foreign_table && !td->is_system_table) {
4586  os << "CREATE FOREIGN TABLE " << td->tableName << " (";
4587  } else if (!td->isView) {
4588  os << "CREATE ";
4590  os << "TEMPORARY ";
4591  }
4592  os << "TABLE " + td->tableName + " (";
4593  } else {
4594  os << "CREATE VIEW " + td->tableName + " AS " << td->viewSQL;
4595  return os.str();
4596  }
4597  // scan column defines
4598  std::vector<std::string> additional_info;
4599  std::set<std::string> shared_dict_column_names;
4600 
4601  gatherAdditionalInfo(additional_info, shared_dict_column_names, td);
4602 
4603  // gather column defines
4604  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
4605  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
4606  bool first = true;
4607  for (const auto cd : cds) {
4608  if (!(cd->isSystemCol || cd->isVirtualCol)) {
4609  const auto& ti = cd->columnType;
4610  if (!first) {
4611  os << ",";
4612  if (!multiline_formatting) {
4613  os << " ";
4614  }
4615  } else {
4616  first = false;
4617  }
4618  if (multiline_formatting) {
4619  os << "\n ";
4620  }
4621  // column name
4622  os << quoteIfRequired(cd->columnName);
4623  // CHAR is perculiar... better dump it as TEXT(32) like \d does
4624  if (ti.get_type() == SQLTypes::kCHAR) {
4625  os << " "
4626  << "TEXT";
4627  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
4628  os << " "
4629  << "TEXT[]";
4630  } else {
4631  os << " " << ti.get_type_name();
4632  }
4633  os << (ti.get_notnull() ? " NOT NULL" : "");
4634  if (cd->default_value.has_value()) {
4635  os << " DEFAULT " << cd->getDefaultValueLiteral();
4636  }
4637  if (shared_dict_column_names.find(cd->columnName) ==
4638  shared_dict_column_names.end()) {
4639  // avoids "Column ... shouldn't specify an encoding, it borrows it
4640  // from the referenced column"
4641  if (ti.is_string() || (ti.is_array() && ti.get_subtype() == kTEXT)) {
4642  auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
4643  if (ti.get_compression() == kENCODING_DICT) {
4644  os << " ENCODING " << ti.get_compression_name() << "(" << (size * 8) << ")";
4645  } else {
4646  os << " ENCODING NONE";
4647  }
4648  } else if (ti.is_date_in_days() ||
4649  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
4650  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
4651  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
4652  } else if (ti.is_geometry()) {
4653  if (ti.get_compression() == kENCODING_GEOINT) {
4654  os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
4655  << ")";
4656  } else {
4657  os << " ENCODING NONE";
4658  }
4659  }
4660  }
4661  }
4662  }
4663  // gather SHARED DICTIONARYs
4664  if (additional_info.size()) {
4665  std::string comma;
4666  if (!multiline_formatting) {
4667  comma = ", ";
4668  } else {
4669  comma = ",\n ";
4670  }
4671  os << comma;
4672  os << boost::algorithm::join(additional_info, comma);
4673  }
4674  os << ")";
4675 
4676  std::vector<std::string> with_options;
4677  if (foreign_table && !td->is_system_table) {
4678  if (multiline_formatting) {
4679  os << "\n";
4680  } else {
4681  os << " ";
4682  }
4683  os << "SERVER " << foreign_table->foreign_server->name;
4684 
4685  // gather WITH options ...
4686  for (const auto& [option, value] : foreign_table->options) {
4687  with_options.emplace_back(option + "='" + value + "'");
4688  }
4689  }
4690 
4691  if (dump_defaults || td->maxFragRows != DEFAULT_FRAGMENT_ROWS) {
4692  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
4693  }
4694  if (!foreign_table && (dump_defaults || td->maxChunkSize != DEFAULT_MAX_CHUNK_SIZE)) {
4695  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
4696  }
4697  if (!foreign_table && (dump_defaults || td->fragPageSize != DEFAULT_PAGE_SIZE)) {
4698  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
4699  }
4700  if (!foreign_table && (dump_defaults || td->maxRows != DEFAULT_MAX_ROWS)) {
4701  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
4702  }
4703  if ((dump_defaults || td->maxRollbackEpochs != DEFAULT_MAX_ROLLBACK_EPOCHS) &&
4704  td->maxRollbackEpochs != -1) {
4705  with_options.push_back("MAX_ROLLBACK_EPOCHS=" +
4707  }
4708  if (!foreign_table && (dump_defaults || !td->hasDeletedCol)) {
4709  with_options.emplace_back(td->hasDeletedCol ? "VACUUM='DELAYED'"
4710  : "VACUUM='IMMEDIATE'");
4711  }
4712  if (!foreign_table && !td->partitions.empty()) {
4713  with_options.push_back("PARTITIONS='" + td->partitions + "'");
4714  }
4715  if (!foreign_table && td->nShards > 0) {
4716  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
4717  CHECK(shard_cd);
4718  with_options.push_back(
4719  "SHARD_COUNT=" +
4720  std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
4721  }
4722  if (!foreign_table && td->sortedColumnId > 0) {
4723  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
4724  CHECK(sort_cd);
4725  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
4726  }
4727 
4728  if (!with_options.empty()) {
4729  if (!multiline_formatting) {
4730  os << " ";
4731  } else {
4732  os << "\n";
4733  }
4734  os << "WITH (" + boost::algorithm::join(with_options, ", ") + ")";
4735  }
4736  os << ";";
4737  return os.str();
4738 }
int32_t maxRollbackEpochs
std::string partitions
std::string tableName
std::string join(T const &container, std::string const &delim)
#define DEFAULT_MAX_CHUNK_SIZE
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:115
std::string to_string(char const *&&v)
std::string quoteIfRequired(const std::string &column_name) const
Definition: Catalog.cpp:4840
#define DEFAULT_MAX_ROWS
void gatherAdditionalInfo(std::vector< std::string > &additional_info, std::set< std::string > &shared_dict_column_names, const TableDescriptor *td) const
Definition: Catalog.cpp:4851
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
#define DEFAULT_MAX_ROLLBACK_EPOCHS
specifies the content in-memory of a row in the table metadata table
#define DEFAULT_PAGE_SIZE
Definition: sqltypes.h:52
#define DEFAULT_FRAGMENT_ROWS
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1794
Data_Namespace::MemoryLevel persistenceLevel
Definition: sqltypes.h:41
mapd_shared_lock< mapd_shared_mutex > read_lock
#define CHECK(condition)
Definition: Logger.h:209
std::string viewSQL
size_t g_leaf_count
Definition: ParserNode.cpp:78

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::dumpSchema ( const TableDescriptor td) const

Definition at line 4450 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, DEFAULT_MAX_ROLLBACK_EPOCHS, ColumnDescriptor::default_value, TableDescriptor::fragPageSize, g_leaf_count, ColumnDescriptor::getDefaultValueLiteral(), TableDescriptor::hasDeletedCol, TableDescriptor::is_system_table, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, join(), kCHAR, kENCODING_DICT, kENCODING_GEOINT, kTEXT, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::tableId, and to_string().

Referenced by TableArchiver::dumpTable().

4450  {
4451  CHECK(!td->is_system_table);
4452  cat_read_lock read_lock(this);
4453 
4454  std::ostringstream os;
4455  os << "CREATE TABLE @T (";
4456  // gather column defines
4457  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
4458  std::string comma;
4459  std::vector<std::string> shared_dicts;
4460  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
4461  for (const auto cd : cds) {
4462  if (!(cd->isSystemCol || cd->isVirtualCol)) {
4463  const auto& ti = cd->columnType;
4464  os << comma << cd->columnName;
4465  // CHAR is perculiar... better dump it as TEXT(32) like \d does
4466  if (ti.get_type() == SQLTypes::kCHAR) {
4467  os << " "
4468  << "TEXT";
4469  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
4470  os << " "
4471  << "TEXT[]";
4472  } else {
4473  os << " " << ti.get_type_name();
4474  }
4475  os << (ti.get_notnull() ? " NOT NULL" : "");
4476  if (cd->default_value.has_value()) {
4477  os << " DEFAULT " << cd->getDefaultValueLiteral();
4478  }
4479  if (ti.is_string() || (ti.is_array() && ti.get_subtype() == kTEXT)) {
4480  auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
4481  if (ti.get_compression() == kENCODING_DICT) {
4482  // if foreign reference, get referenced tab.col
4483  const auto dict_id = ti.get_comp_param();
4484  const DictRef dict_ref(currentDB_.dbId, dict_id);
4485  const auto dict_it = dictDescriptorMapByRef_.find(dict_ref);
4486  CHECK(dict_it != dictDescriptorMapByRef_.end());
4487  const auto dict_name = dict_it->second->dictName;
4488  // when migrating a table, any foreign dict ref will be dropped
4489  // and the first cd of a dict will become root of the dict
4490  if (dict_root_cds.end() == dict_root_cds.find(dict_name)) {
4491  dict_root_cds[dict_name] = cd;
4492  os << " ENCODING " << ti.get_compression_name() << "(" << (size * 8) << ")";
4493  } else {
4494  const auto dict_root_cd = dict_root_cds[dict_name];
4495  shared_dicts.push_back("SHARED DICTIONARY (" + cd->columnName +
4496  ") REFERENCES @T(" + dict_root_cd->columnName + ")");
4497  // "... shouldn't specify an encoding, it borrows from the referenced
4498  // column"
4499  }
4500  } else {
4501  os << " ENCODING NONE";
4502  }
4503  } else if (ti.is_date_in_days() ||
4504  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
4505  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
4506  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
4507  } else if (ti.is_geometry()) {
4508  if (ti.get_compression() == kENCODING_GEOINT) {
4509  os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
4510  << ")";
4511  } else {
4512  os << " ENCODING NONE";
4513  }
4514  }
4515  comma = ", ";
4516  }
4517  }
4518  // gather SHARED DICTIONARYs
4519  if (shared_dicts.size()) {
4520  os << ", " << boost::algorithm::join(shared_dicts, ", ");
4521  }
4522  // gather WITH options ...
4523  std::vector<std::string> with_options;
4524  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
4525  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
4526  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
4527  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
4528  with_options.emplace_back(td->hasDeletedCol ? "VACUUM='DELAYED'"
4529  : "VACUUM='IMMEDIATE'");
4530  if (!td->partitions.empty()) {
4531  with_options.push_back("PARTITIONS='" + td->partitions + "'");
4532  }
4533  if (td->nShards > 0) {
4534  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
4535  CHECK(shard_cd);
4536  os << ", SHARD KEY(" << shard_cd->columnName << ")";
4537  with_options.push_back(
4538  "SHARD_COUNT=" +
4539  std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
4540  }
4541  if (td->sortedColumnId > 0) {
4542  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
4543  CHECK(sort_cd);
4544  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
4545  }
4547  td->maxRollbackEpochs != -1) {
4548  with_options.push_back("MAX_ROLLBACK_EPOCHS=" +
4550  }
4551  os << ") WITH (" + boost::algorithm::join(with_options, ", ") + ");";
4552  return os.str();
4553 }
int32_t maxRollbackEpochs
std::string partitions
const DBMetadata currentDB_
Definition: Catalog.h:641
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:632
std::string join(T const &container, std::string const &delim)
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:115
std::string to_string(char const *&&v)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
#define DEFAULT_MAX_ROLLBACK_EPOCHS
specifies the content in-memory of a row in the table metadata table
Definition: sqltypes.h:52
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1794
Definition: sqltypes.h:41