OmniSciDB  2e3a973ef4
Catalog_Namespace::Catalog Class Referencefinal

class for a per-database catalog. also includes metadata for the current database and the current user. More...

#include <Catalog.h>

+ Collaboration diagram for Catalog_Namespace::Catalog:

Public Member Functions

 Catalog (const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
 Constructor - takes basePath to already extant data directory for writing. More...
 
 ~Catalog ()
 Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite. More...
 
void createTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
 
void createShardedTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
int32_t createDashboard (DashboardDescriptor &vd)
 
void replaceDashboard (DashboardDescriptor &vd)
 
std::string createLink (LinkDescriptor &ld, size_t min_length)
 
void dropTable (const TableDescriptor *td)
 
void truncateTable (const TableDescriptor *td)
 
void renameTable (const TableDescriptor *td, const std::string &newTableName)
 
void renameColumn (const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
 
void addColumn (const TableDescriptor &td, ColumnDescriptor &cd)
 
void dropColumn (const TableDescriptor &td, const ColumnDescriptor &cd)
 
void removeChunks (const int table_id)
 
void removeFragmenterForTable (const int table_id)
 
const std::map< int, const ColumnDescriptor * > getDictionaryToColumnMapping ()
 
const TableDescriptorgetMetadataForTable (const std::string &tableName, const bool populateFragmenter=true) const
 Returns a pointer to a const TableDescriptor struct matching the provided tableName. More...
 
const TableDescriptorgetMetadataForTableImpl (int tableId, const bool populateFragmenter) const
 
const TableDescriptorgetMetadataForTable (int tableId, bool populateFragmenter=true) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, const std::string &colName) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, int columnId) const
 
const ColumnDescriptorgetMetadataForColumnUnlocked (int tableId, int columnId) const
 
const int getColumnIdBySpi (const int tableId, const size_t spi) const
 
const ColumnDescriptorgetMetadataForColumnBySpi (const int tableId, const size_t spi) const
 
const DashboardDescriptorgetMetadataForDashboard (const std::string &userId, const std::string &dashName) const
 
const DashboardDescriptorgetMetadataForDashboard (const int32_t dashboard_id) const
 
void deleteMetadataForDashboards (const std::vector< int32_t > ids, const UserMetadata &user)
 
const LinkDescriptorgetMetadataForLink (const std::string &link) const
 
const LinkDescriptorgetMetadataForLink (int linkId) const
 
const foreign_storage::ForeignTablegetForeignTableUnlocked (int tableId) const
 
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable (const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular table specified by table id. More...
 
std::list< const ColumnDescriptor * > getAllColumnMetadataForTableUnlocked (const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 
std::list< const TableDescriptor * > getAllTableMetadata () const
 
std::list< const DashboardDescriptor * > getAllDashboardsMetadata () const
 
const DBMetadatagetCurrentDB () const
 
Data_Namespace::DataMgrgetDataMgr () const
 
std::shared_ptr< CalcitegetCalciteMgr () const
 
const std::string & getBasePath () const
 
const DictDescriptorgetMetadataForDict (int dict_ref, bool loadDict=true) const
 
const DictDescriptorgetMetadataForDictUnlocked (int dict_ref, bool loadDict) const
 
const std::vector< LeafHostInfo > & getStringDictionaryHosts () const
 
const ColumnDescriptorgetShardColumnMetadataForTable (const TableDescriptor *td) const
 
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors (const TableDescriptor *logicalTableDesc) const
 
std::vector< std::string > getTableNamesForUser (const UserMetadata &user, const GetTablesType get_tables_type) const
 
int32_t getTableEpoch (const int32_t db_id, const int32_t table_id) const
 
void setTableEpoch (const int db_id, const int table_id, const int new_epoch)
 
std::vector< TableEpochInfogetTableEpochs (const int32_t db_id, const int32_t table_id) const
 
void setTableEpochs (const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs)
 
int getDatabaseId () const
 
SqliteConnectorgetSqliteConnector ()
 
void roll (const bool forward)
 
DictRef addDictionary (ColumnDescriptor &cd)
 
void delDictionary (const ColumnDescriptor &cd)
 
void getDictionary (const ColumnDescriptor &cd, std::map< int, StringDictionary *> &stringDicts)
 
const bool checkMetadataForDeletedRecs (const TableDescriptor *td, int column_id) const
 
const ColumnDescriptorgetDeletedColumn (const TableDescriptor *td) const
 
const ColumnDescriptorgetDeletedColumnIfRowsDeleted (const TableDescriptor *td) const
 
void setDeletedColumn (const TableDescriptor *td, const ColumnDescriptor *cd)
 
void setDeletedColumnUnlocked (const TableDescriptor *td, const ColumnDescriptor *cd)
 
int getLogicalTableId (const int physicalTableId) const
 
void checkpoint (const int logicalTableId) const
 
std::string name () const
 
void eraseDBData ()
 
void eraseTablePhysicalData (const TableDescriptor *td)
 
void vacuumDeletedRows (const TableDescriptor *td) const
 
void vacuumDeletedRows (const int logicalTableId) const
 
void setForReload (const int32_t tableId)
 
std::vector< std::string > getTableDataDirectories (const TableDescriptor *td) const
 
std::vector< std::string > getTableDictDirectories (const TableDescriptor *td) const
 
std::string getColumnDictDirectory (const ColumnDescriptor *cd) const
 
std::string dumpSchema (const TableDescriptor *td) const
 
std::string dumpCreateTable (const TableDescriptor *td, bool multiline_formatting=true, bool dump_defaults=false) const
 
void createForeignServer (std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
 
const foreign_storage::ForeignServergetForeignServer (const std::string &server_name) const
 
const std::unique_ptr< const foreign_storage::ForeignServergetForeignServerFromStorage (const std::string &server_name)
 
void changeForeignServerOwner (const std::string &server_name, const int new_owner_id)
 
void setForeignServerDataWrapper (const std::string &server_name, const std::string &data_wrapper)
 
void setForeignServerOptions (const std::string &server_name, const std::string &options)
 
void renameForeignServer (const std::string &server_name, const std::string &name)
 
void dropForeignServer (const std::string &server_name)
 
void getForeignServersForUser (const rapidjson::Value *filters, const UserMetadata &user, std::vector< const foreign_storage::ForeignServer *> &results)
 
void createDefaultServersIfNotExists ()
 
bool validateNonExistentTableOrView (const std::string &name, const bool if_not_exists)
 
std::vector< const TableDescriptor * > getAllForeignTablesForRefresh () const
 
void updateForeignTableRefreshTimes (const int32_t table_id)
 

Static Public Member Functions

static void expandGeoColumn (const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
 
static void set (const std::string &dbName, std::shared_ptr< Catalog > cat)
 
static std::shared_ptr< Catalogget (const std::string &dbName)
 
static std::shared_ptr< Catalogget (const int32_t db_id)
 
static std::shared_ptr< CatalogcheckedGet (const int32_t db_id)
 
static std::shared_ptr< Catalogget (const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
 
static void remove (const std::string &dbName)
 
static const std::string getForeignTableSchema (bool if_not_exists=false)
 
static const std::string getForeignServerSchema (bool if_not_exists=false)
 

Public Attributes

std::mutex sqliteMutex_
 
mapd_shared_mutex sharedMutex_
 
std::atomic< std::thread::id > thread_holding_sqlite_lock
 
std::atomic< std::thread::id > thread_holding_write_lock
 

Static Public Attributes

static thread_local bool thread_holds_read_lock = false
 

Protected Types

using ColumnDescriptorsForRoll = std::vector< std::pair< ColumnDescriptor *, ColumnDescriptor * > >
 

Protected Member Functions

void CheckAndExecuteMigrations ()
 
void CheckAndExecuteMigrationsPostBuildMaps ()
 
void updateDictionaryNames ()
 
void updateTableDescriptorSchema ()
 
void updateFixlenArrayColumns ()
 
void updateGeoColumns ()
 
void updateFrontendViewSchema ()
 
void updateLinkSchema ()
 
void updateFrontendViewAndLinkUsers ()
 
void updateLogicalToPhysicalTableLinkSchema ()
 
void updateLogicalToPhysicalTableMap (const int32_t logical_tb_id)
 
void updateDictionarySchema ()
 
void updatePageSize ()
 
void updateDeletedColumnIndicator ()
 
void updateFrontendViewsToDashboards ()
 
void createFsiSchemasAndDefaultServers ()
 
void dropFsiSchemasAndTables ()
 
void recordOwnershipOfObjectsInObjectPermissions ()
 
void checkDateInDaysColumnMigration ()
 
void createDashboardSystemRoles ()
 
void buildMaps ()
 
void addTableToMap (const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
 
void addReferenceToForeignDict (ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def, const bool persist_reference)
 
bool setColumnSharedDictionary (ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
void setColumnDictionary (ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
 
void addFrontendViewToMap (DashboardDescriptor &vd)
 
void addFrontendViewToMapNoLock (DashboardDescriptor &vd)
 
void addLinkToMap (LinkDescriptor &ld)
 
void removeTableFromMap (const std::string &tableName, const int tableId, const bool is_on_error=false)
 
void doDropTable (const TableDescriptor *td)
 
void executeDropTableSqliteQueries (const TableDescriptor *td)
 
void doTruncateTable (const TableDescriptor *td)
 
void renamePhysicalTable (const TableDescriptor *td, const std::string &newTableName)
 
void instantiateFragmenter (TableDescriptor *td) const
 
void getAllColumnMetadataForTableImpl (const TableDescriptor *td, std::list< const ColumnDescriptor *> &colDescs, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 
std::string calculateSHA1 (const std::string &data)
 
std::string generatePhysicalTableName (const std::string &logicalTableName, const int32_t &shardNumber)
 
std::vector< DBObjectparseDashboardObjects (const std::string &view_meta, const int &user_id)
 
void createOrUpdateDashboardSystemRole (const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
 
const int getColumnIdBySpiUnlocked (const int table_id, const size_t spi) const
 
void serializeTableJsonUnlocked (const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
 
void dropTableFromJsonUnlocked (const std::string &table_name) const
 

Protected Attributes

std::string basePath_
 
TableDescriptorMap tableDescriptorMap_
 
TableDescriptorMapById tableDescriptorMapById_
 
ColumnDescriptorMap columnDescriptorMap_
 
ColumnDescriptorMapById columnDescriptorMapById_
 
DictDescriptorMapById dictDescriptorMapByRef_
 
DashboardDescriptorMap dashboardDescriptorMap_
 
LinkDescriptorMap linkDescriptorMap_
 
LinkDescriptorMapById linkDescriptorMapById_
 
ForeignServerMap foreignServerMap_
 
ForeignServerMapById foreignServerMapById_
 
SqliteConnector sqliteConnector_
 
DBMetadata currentDB_
 
std::shared_ptr< Data_Namespace::DataMgrdataMgr_
 
const std::vector< LeafHostInfostring_dict_hosts_
 
std::shared_ptr< CalcitecalciteMgr_
 
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
 
int nextTempTableId_
 
int nextTempDictId_
 
ColumnDescriptorsForRoll columnDescriptorsForRoll
 

Static Protected Attributes

static const std::string physicalTableNameTag_
 

Private Member Functions

void adjustAlteredTableFiles (const std::string &temp_data_dir, const std::unordered_map< int, int > &all_column_ids_map) const
 
void renameTableDirectories (const std::string &temp_data_dir, const std::vector< std::string > &target_paths, const std::string &name_prefix) const
 
void buildForeignServerMap ()
 
void addForeignTableDetails ()
 
void setForeignServerProperty (const std::string &server_name, const std::string &property, const std::string &value)
 
void createForeignServerNoLocks (std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
 

Private Attributes

DeletedColumnPerTableMap deletedColumnPerTable_
 

Static Private Attributes

static std::map< std::string, std::shared_ptr< Catalog > > mapd_cat_map_
 

Detailed Description

class for a per-database catalog. also includes metadata for the current database and the current user.

Catalog

Definition at line 97 of file Catalog.h.

Member Typedef Documentation

◆ ColumnDescriptorsForRoll

Definition at line 527 of file Catalog.h.

Constructor & Destructor Documentation

◆ Catalog()

Catalog_Namespace::Catalog::Catalog ( const std::string &  basePath,
const DBMetadata curDB,
std::shared_ptr< Data_Namespace::DataMgr dataMgr,
const std::vector< LeafHostInfo > &  string_dict_hosts,
std::shared_ptr< Calcite calcite,
bool  is_new_db 
)

Constructor - takes basePath to already extant data directory for writing.

Parameters
basePathdirectory path for writing catalog
dbNamename of the database
fragmenterFragmenter object metadata - expects for this directory to already exist

Definition at line 161 of file Catalog.cpp.

References basePath_, buildMaps(), CheckAndExecuteMigrations(), CheckAndExecuteMigrationsPostBuildMaps(), currentDB_, Catalog_Namespace::DBMetadata::dbName, g_serialize_temp_tables, and Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath().

167  : basePath_(basePath)
168  , sqliteConnector_(curDB.dbName, basePath + "/mapd_catalogs/")
169  , currentDB_(curDB)
170  , dataMgr_(dataMgr)
171  , string_dict_hosts_(string_dict_hosts)
172  , calciteMgr_(calcite)
175  , sqliteMutex_()
176  , sharedMutex_()
179  if (!is_new_db) {
181  }
182  buildMaps();
183  if (!is_new_db) {
185  }
187  boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
188  }
189 }
const int MAPD_TEMP_TABLE_START_ID
Definition: Catalog.cpp:101
mapd_shared_mutex sharedMutex_
Definition: Catalog.h:556
SqliteConnector sqliteConnector_
Definition: Catalog.h:509
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:513
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:511
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:153
const int MAPD_TEMP_DICT_START_ID
Definition: Catalog.cpp:103
std::atomic< std::thread::id > thread_holding_sqlite_lock
Definition: Catalog.h:557
bool g_serialize_temp_tables
Definition: Catalog.cpp:96
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:514
std::atomic< std::thread::id > thread_holding_write_lock
Definition: Catalog.h:558
void CheckAndExecuteMigrationsPostBuildMaps()
Definition: Catalog.cpp:912
+ Here is the call graph for this function:

◆ ~Catalog()

Catalog_Namespace::Catalog::~Catalog ( )

Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite.

Definition at line 191 of file Catalog.cpp.

References basePath_, columnDescriptorMap_, currentDB_, Catalog_Namespace::DBMetadata::dbName, g_serialize_temp_tables, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and tableDescriptorMap_.

191  {
193  // must clean up heap-allocated TableDescriptor and ColumnDescriptor structs
194  for (TableDescriptorMap::iterator tableDescIt = tableDescriptorMap_.begin();
195  tableDescIt != tableDescriptorMap_.end();
196  ++tableDescIt) {
197  tableDescIt->second->fragmenter = nullptr;
198  delete tableDescIt->second;
199  }
200 
201  // TableDescriptorMapById points to the same descriptors. No need to delete
202 
203  for (ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.begin();
204  columnDescIt != columnDescriptorMap_.end();
205  ++columnDescIt) {
206  delete columnDescIt->second;
207  }
208 
209  // ColumnDescriptorMapById points to the same descriptors. No need to delete
210 
212  boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
213  }
214 }
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:500
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:153
bool g_serialize_temp_tables
Definition: Catalog.cpp:96
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
mapd_unique_lock< mapd_shared_mutex > write_lock
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:498
+ Here is the call graph for this function:

Member Function Documentation

◆ addColumn()

void Catalog_Namespace::Catalog::addColumn ( const TableDescriptor td,
ColumnDescriptor cd 
)

Definition at line 1853 of file Catalog.cpp.

References addDictionary(), columnDescriptorMap_, columnDescriptorMapById_, columnDescriptorsForRoll, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), SqliteConnector::getData(), getPhysicalTablesDescriptors(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, kENCODING_DICT, TableDescriptor::nShards, SqliteConnector::query_with_text_params(), TableDescriptor::shard, sqliteConnector_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), to_upper(), and ColumnDescriptor::virtualExpr.

1853  {
1854  cat_write_lock write_lock(this);
1855  // caller must handle sqlite/chunk transaction TOGETHER
1856  cd.tableId = td.tableId;
1857  if (td.nShards > 0 && td.shard < 0) {
1858  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
1859  auto shard_cd = cd;
1860  addColumn(*shard, shard_cd);
1861  }
1862  }
1864  addDictionary(cd);
1865  }
1866 
1868  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
1869  "colscale, is_notnull, "
1870  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
1871  "is_deletedcol) "
1872  "VALUES (?, "
1873  "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
1874  "?, ?, ?, "
1875  "?, "
1876  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1877  std::vector<std::string>{std::to_string(td.tableId),
1878  std::to_string(td.tableId),
1879  cd.columnName,
1888  "",
1891  cd.virtualExpr,
1893 
1895  "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
1896  std::vector<std::string>{std::to_string(td.tableId)});
1897 
1899  "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
1900  std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
1901  cd.columnId = sqliteConnector_.getData<int>(0, 0);
1902 
1903  ++tableDescriptorMapById_[td.tableId]->nColumns;
1904  auto ncd = new ColumnDescriptor(cd);
1907  columnDescriptorsForRoll.emplace_back(nullptr, ncd);
1908 }
std::string virtualExpr
std::tuple< int, std::string > ColumnKey
Definition: Types.h:36
T getData(const int row, const int col)
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:500
virtual void query_with_text_params(std::string const &query_only)
void addColumn(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:1853
SqliteConnector sqliteConnector_
Definition: Catalog.h:509
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:268
HOST DEVICE int get_size() const
Definition: sqltypes.h:269
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logicalTableDesc) const
Definition: Catalog.cpp:3589
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:267
HOST DEVICE int get_scale() const
Definition: sqltypes.h:264
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:501
std::string to_string(char const *&&v)
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:266
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:260
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:499
specifies the content in-memory of a row in the column metadata table
std::string to_upper(const std::string &str)
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:261
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:259
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
DictRef addDictionary(ColumnDescriptor &cd)
Definition: Catalog.cpp:1758
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:528
std::string columnName
std::tuple< int, int > ColumnIdKey
Definition: Types.h:38
+ Here is the call graph for this function:

◆ addDictionary()

DictRef Catalog_Namespace::Catalog::addDictionary ( ColumnDescriptor cd)

Definition at line 1758 of file Catalog.cpp.

References CHECK, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, DictDescriptor::dictFolderPath, setColumnDictionary(), string_dict_hosts_, tableDescriptorMapById_, and ColumnDescriptor::tableId.

Referenced by addColumn().

1758  {
1759  const auto& td = *tableDescriptorMapById_[cd.tableId];
1760  list<DictDescriptor> dds;
1761  setColumnDictionary(cd, dds, td, true);
1762  auto& dd = dds.back();
1763  CHECK(dd.dictRef.dictId);
1764 
1765  std::unique_ptr<StringDictionaryClient> client;
1766  if (!string_dict_hosts_.empty()) {
1767  client.reset(new StringDictionaryClient(
1768  string_dict_hosts_.front(), DictRef(currentDB_.dbId, -1), true));
1769  }
1770  if (client) {
1771  client->create(dd.dictRef, dd.dictIsTemp);
1772  }
1773 
1774  DictDescriptor* new_dd = new DictDescriptor(dd);
1775  dictDescriptorMapByRef_[dd.dictRef].reset(new_dd);
1776  if (!dd.dictIsTemp) {
1777  boost::filesystem::create_directory(new_dd->dictFolderPath);
1778  }
1779  return dd.dictRef;
1780 }
std::string dictFolderPath
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:513
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:502
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:499
struct dict_ref_t DictRef
Definition: DictRef.h:37
#define CHECK(condition)
Definition: Logger.h:197
Descriptor for a dictionary for a string columne.
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:3007
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ addForeignTableDetails()

void Catalog_Namespace::Catalog::addForeignTableDetails ( )
private

Definition at line 3861 of file Catalog.cpp.

References CHECK, foreignServerMapById_, SqliteConnector::getData(), SqliteConnector::getNumRows(), num_rows, SqliteConnector::query(), sqliteConnector_, and tableDescriptorMapById_.

Referenced by buildMaps().

3861  {
3863  "SELECT table_id, server_id, options, last_refresh_time, next_refresh_time from "
3864  "omnisci_foreign_tables");
3866  for (size_t r = 0; r < num_rows; r++) {
3867  const auto table_id = sqliteConnector_.getData<int32_t>(r, 0);
3868  const auto server_id = sqliteConnector_.getData<int32_t>(r, 1);
3869  const auto& options = sqliteConnector_.getData<std::string>(r, 2);
3870  const auto last_refresh_time = sqliteConnector_.getData<int>(r, 3);
3871  const auto next_refresh_time = sqliteConnector_.getData<int>(r, 4);
3872 
3873  CHECK(tableDescriptorMapById_.find(table_id) != tableDescriptorMapById_.end());
3874  auto foreign_table =
3875  dynamic_cast<foreign_storage::ForeignTable*>(tableDescriptorMapById_[table_id]);
3876  CHECK(foreign_table);
3877  foreign_table->foreign_server = foreignServerMapById_[server_id].get();
3878  CHECK(foreign_table->foreign_server);
3879  foreign_table->populateOptionsMap(options);
3880  foreign_table->last_refresh_time = last_refresh_time;
3881  foreign_table->next_refresh_time = next_refresh_time;
3882  }
3883 }
T getData(const int row, const int col)
const int8_t const int64_t * num_rows
SqliteConnector sqliteConnector_
Definition: Catalog.h:509
virtual void query(const std::string &queryString)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:499
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:507
#define CHECK(condition)
Definition: Logger.h:197
virtual size_t getNumRows() const
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ addFrontendViewToMap()

void Catalog_Namespace::Catalog::addFrontendViewToMap ( DashboardDescriptor vd)
protected

Definition at line 1280 of file Catalog.cpp.

References addFrontendViewToMapNoLock().

Referenced by createDashboard().

1280  {
1281  cat_write_lock write_lock(this);
1283 }
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1285
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
mapd_unique_lock< mapd_shared_mutex > write_lock
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ addFrontendViewToMapNoLock()

void Catalog_Namespace::Catalog::addFrontendViewToMapNoLock ( DashboardDescriptor vd)
protected

Definition at line 1285 of file Catalog.cpp.

References dashboardDescriptorMap_, DashboardDescriptor::dashboardName, to_string(), and DashboardDescriptor::userId.

Referenced by addFrontendViewToMap(), and replaceDashboard().

1285  {
1286  cat_write_lock write_lock(this);
1288  std::make_shared<DashboardDescriptor>(vd);
1289 }
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:503
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
mapd_unique_lock< mapd_shared_mutex > write_lock
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ addLinkToMap()

void Catalog_Namespace::Catalog::addLinkToMap ( LinkDescriptor ld)
protected

Definition at line 1362 of file Catalog.cpp.

References currentDB_, Catalog_Namespace::DBMetadata::dbId, LinkDescriptor::link, linkDescriptorMap_, linkDescriptorMapById_, LinkDescriptor::linkId, and to_string().

Referenced by createLink().

1362  {
1363  cat_write_lock write_lock(this);
1364  LinkDescriptor* new_ld = new LinkDescriptor();
1365  *new_ld = ld;
1367  linkDescriptorMapById_[ld.linkId] = new_ld;
1368 }
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
mapd_unique_lock< mapd_shared_mutex > write_lock
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:505
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:504
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ addReferenceToForeignDict()

void Catalog_Namespace::Catalog::addReferenceToForeignDict ( ColumnDescriptor referencing_column,
Parser::SharedDictionaryDef  shared_dict_def,
const bool  persist_reference 
)
protected

Definition at line 2916 of file Catalog.cpp.

References CHECK, CHECK_GE, ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, SQLTypeInfo::get_comp_param(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_foreign_col(), SqliteConnector::query_with_text_params(), sqliteConnector_, and to_string().

Referenced by setColumnSharedDictionary().

2918  {
2919  cat_write_lock write_lock(this);
2920  const auto foreign_ref_col = get_foreign_col(*this, shared_dict_def);
2921  CHECK(foreign_ref_col);
2922  referencing_column.columnType = foreign_ref_col->columnType;
2923  const int dict_id = referencing_column.columnType.get_comp_param();
2924  const DictRef dict_ref(currentDB_.dbId, dict_id);
2925  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
2926  CHECK(dictIt != dictDescriptorMapByRef_.end());
2927  const auto& dd = dictIt->second;
2928  CHECK_GE(dd->refcount, 1);
2929  ++dd->refcount;
2930  if (persist_reference) {
2931  cat_sqlite_lock sqlite_lock(this);
2933  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
2934  {std::to_string(dict_id)});
2935  }
2936 }
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:509
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:502
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:268
#define CHECK_GE(x, y)
Definition: Logger.h:210
const ColumnDescriptor * get_foreign_col(const Catalog &cat, const Parser::SharedDictionaryDef &shared_dict_def)
Definition: Catalog.cpp:2904
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ addTableToMap()

void Catalog_Namespace::Catalog::addTableToMap ( const TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::list< DictDescriptor > &  dicts 
)
protected

Definition at line 1139 of file Catalog.cpp.

References CHECK, columnDescriptorMap_, columnDescriptorMapById_, ColumnDescriptor::columnId, TableDescriptor::columnIdBySpi_, ColumnDescriptor::columnName, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, DictDescriptor::dictFolderPath, dict_ref_t::dictId, TableDescriptor::hasDeletedCol, TableDescriptor::mutex_, setDeletedColumnUnlocked(), string_dict_hosts_, tableDescriptorMap_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, and to_upper().

Referenced by createTable().

1141  {
1142  cat_write_lock write_lock(this);
1143  TableDescriptor* new_td;
1144 
1145  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
1146  if (foreign_table) {
1147  auto new_foreign_table = new foreign_storage::ForeignTable();
1148  *new_foreign_table = *foreign_table;
1149  new_td = new_foreign_table;
1150  } else {
1151  new_td = new TableDescriptor();
1152  *new_td = *td;
1153  }
1154 
1155  new_td->mutex_ = std::make_shared<std::mutex>();
1156  tableDescriptorMap_[to_upper(td->tableName)] = new_td;
1157  tableDescriptorMapById_[td->tableId] = new_td;
1158  for (auto cd : columns) {
1159  ColumnDescriptor* new_cd = new ColumnDescriptor();
1160  *new_cd = cd;
1161  ColumnKey columnKey(new_cd->tableId, to_upper(new_cd->columnName));
1162  columnDescriptorMap_[columnKey] = new_cd;
1163  ColumnIdKey columnIdKey(new_cd->tableId, new_cd->columnId);
1164  columnDescriptorMapById_[columnIdKey] = new_cd;
1165 
1166  // Add deleted column to the map
1167  if (cd.isDeletedCol) {
1168  CHECK(new_td->hasDeletedCol);
1169  setDeletedColumnUnlocked(new_td, new_cd);
1170  }
1171  }
1172 
1173  std::sort(new_td->columnIdBySpi_.begin(),
1174  new_td->columnIdBySpi_.end(),
1175  [](const size_t a, const size_t b) -> bool { return a < b; });
1176 
1177  std::unique_ptr<StringDictionaryClient> client;
1178  DictRef dict_ref(currentDB_.dbId, -1);
1179  if (!string_dict_hosts_.empty()) {
1180  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1181  }
1182  for (auto dd : dicts) {
1183  if (!dd.dictRef.dictId) {
1184  // Dummy entry created for a shard of a logical table, nothing to do.
1185  continue;
1186  }
1187  dict_ref.dictId = dd.dictRef.dictId;
1188  if (client) {
1189  client->create(dict_ref, dd.dictIsTemp);
1190  }
1191  DictDescriptor* new_dd = new DictDescriptor(dd);
1192  dictDescriptorMapByRef_[dict_ref].reset(new_dd);
1193  if (!dd.dictIsTemp) {
1194  boost::filesystem::create_directory(new_dd->dictFolderPath);
1195  }
1196  }
1197 }
std::string dictFolderPath
std::tuple< int, std::string > ColumnKey
Definition: Types.h:36
std::string tableName
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:500
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:513
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:502
std::vector< int > columnIdBySpi_
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:501
std::shared_ptr< std::mutex > mutex_
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:2895
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:499
specifies the content in-memory of a row in the column metadata table
std::string to_upper(const std::string &str)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
#define CHECK(condition)
Definition: Logger.h:197
Descriptor for a dictionary for a string columne.
mapd_unique_lock< mapd_shared_mutex > write_lock
specifies the content in-memory of a row in the table metadata table
std::string columnName
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:498
std::tuple< int, int > ColumnIdKey
Definition: Types.h:38
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ adjustAlteredTableFiles()

void Catalog_Namespace::Catalog::adjustAlteredTableFiles ( const std::string &  temp_data_dir,
const std::unordered_map< int, int > &  all_column_ids_map 
) const
private

◆ buildForeignServerMap()

void Catalog_Namespace::Catalog::buildForeignServerMap ( )
private

Definition at line 3843 of file Catalog.cpp.

References foreignServerMap_, foreignServerMapById_, SqliteConnector::getData(), SqliteConnector::getNumRows(), num_rows, SqliteConnector::query(), and sqliteConnector_.

Referenced by buildMaps().

3843  {
3845  "SELECT id, name, data_wrapper_type, options, owner_user_id, creation_time FROM "
3846  "omnisci_foreign_servers");
3848  for (size_t row = 0; row < num_rows; row++) {
3849  auto foreign_server = std::make_shared<foreign_storage::ForeignServer>(
3850  sqliteConnector_.getData<int>(row, 0),
3851  sqliteConnector_.getData<std::string>(row, 1),
3852  sqliteConnector_.getData<std::string>(row, 2),
3853  sqliteConnector_.getData<std::string>(row, 3),
3854  sqliteConnector_.getData<std::int32_t>(row, 4),
3855  sqliteConnector_.getData<std::int32_t>(row, 5));
3856  foreignServerMap_[foreign_server->name] = foreign_server;
3857  foreignServerMapById_[foreign_server->id] = foreign_server;
3858  }
3859 }
T getData(const int row, const int col)
const int8_t const int64_t * num_rows
SqliteConnector sqliteConnector_
Definition: Catalog.h:509
virtual void query(const std::string &queryString)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:507
ForeignServerMap foreignServerMap_
Definition: Catalog.h:506
virtual size_t getNumRows() const
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ buildMaps()

void Catalog_Namespace::Catalog::buildMaps ( )
protected

Definition at line 928 of file Catalog.cpp.

References addForeignTableDetails(), basePath_, buildForeignServerMap(), CHECK, ColumnDescriptor::chunks, columnDescriptorMap_, columnDescriptorMapById_, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, currentDB_, dashboardDescriptorMap_, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, dictDescriptorMapByRef_, logger::FATAL, StorageType::FOREIGN_TABLE, TableDescriptor::fragmenter, TableDescriptor::fragments, TableDescriptor::fragPageSize, TableDescriptor::fragType, g_enable_fsi, generate_dashboard_system_rolename(), SQLTypeInfo::get_physical_cols(), SqliteConnector::getData(), SqliteConnector::getNumRows(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::getUserFromId(), TableDescriptor::hasDeletedCol, SQLTypeInfo::is_geometry(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, SqliteConnector::isNull(), ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, TableDescriptor::keyMetainfo, LinkDescriptor::link, linkDescriptorMap_, linkDescriptorMapById_, LinkDescriptor::linkId, LOG, logicalToPhysicalTableMapById_, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRows, TableDescriptor::nColumns, TableDescriptor::nShards, TableDescriptor::partitions, SqliteConnector::query(), SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_compression(), SQLTypeInfo::set_dimension(), SQLTypeInfo::set_notnull(), SQLTypeInfo::set_scale(), SQLTypeInfo::set_size(), SQLTypeInfo::set_subtype(), SQLTypeInfo::set_type(), setDeletedColumnUnlocked(), TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, sqliteConnector_, TableDescriptor::storageType, tableDescriptorMap_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), to_upper(), LinkDescriptor::updateTime, LinkDescriptor::userId, TableDescriptor::userId, LinkDescriptor::viewMetadata, TableDescriptor::viewSQL, LinkDescriptor::viewState, and ColumnDescriptor::virtualExpr.

Referenced by Catalog().

928  {
930  cat_sqlite_lock sqlite_lock(this);
931 
932  string dictQuery(
933  "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
934  sqliteConnector_.query(dictQuery);
935  size_t numRows = sqliteConnector_.getNumRows();
936  for (size_t r = 0; r < numRows; ++r) {
937  int dictId = sqliteConnector_.getData<int>(r, 0);
938  std::string dictName = sqliteConnector_.getData<string>(r, 1);
939  int dictNBits = sqliteConnector_.getData<int>(r, 2);
940  bool is_shared = sqliteConnector_.getData<bool>(r, 3);
941  int refcount = sqliteConnector_.getData<int>(r, 4);
942  std::string fname = basePath_ + "/mapd_data/DB_" + std::to_string(currentDB_.dbId) +
943  "_DICT_" + std::to_string(dictId);
944  DictRef dict_ref(currentDB_.dbId, dictId);
945  DictDescriptor* dd = new DictDescriptor(
946  dict_ref, dictName, dictNBits, is_shared, refcount, fname, false);
947  dictDescriptorMapByRef_[dict_ref].reset(dd);
948  }
949 
950  string tableQuery(
951  "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
952  "max_chunk_size, frag_page_size, "
953  "max_rows, partitions, shard_column_id, shard, num_shards, key_metainfo, userid, "
954  "sort_column_id, storage_type "
955  "from mapd_tables");
956  sqliteConnector_.query(tableQuery);
957  numRows = sqliteConnector_.getNumRows();
958  for (size_t r = 0; r < numRows; ++r) {
959  TableDescriptor* td;
960  const auto& storage_type = sqliteConnector_.getData<string>(r, 17);
961  if (!storage_type.empty() &&
962  (!g_enable_fsi || storage_type != StorageType::FOREIGN_TABLE)) {
963  const auto table_id = sqliteConnector_.getData<int>(r, 0);
964  const auto& table_name = sqliteConnector_.getData<string>(r, 1);
965  LOG(FATAL) << "Unable to read Catalog metadata: storage type is currently not a "
966  "supported table option (table "
967  << table_name << " [" << table_id << "] in database "
968  << currentDB_.dbName << ").";
969  }
970 
971  if (storage_type == StorageType::FOREIGN_TABLE) {
973  } else {
974  td = new TableDescriptor();
975  }
976 
977  td->storageType = storage_type;
978  td->tableId = sqliteConnector_.getData<int>(r, 0);
979  td->tableName = sqliteConnector_.getData<string>(r, 1);
980  td->nColumns = sqliteConnector_.getData<int>(r, 2);
981  td->isView = sqliteConnector_.getData<bool>(r, 3);
982  td->fragments = sqliteConnector_.getData<string>(r, 4);
983  td->fragType =
985  td->maxFragRows = sqliteConnector_.getData<int>(r, 6);
986  td->maxChunkSize = sqliteConnector_.getData<int>(r, 7);
987  td->fragPageSize = sqliteConnector_.getData<int>(r, 8);
988  td->maxRows = sqliteConnector_.getData<int64_t>(r, 9);
989  td->partitions = sqliteConnector_.getData<string>(r, 10);
990  td->shardedColumnId = sqliteConnector_.getData<int>(r, 11);
991  td->shard = sqliteConnector_.getData<int>(r, 12);
992  td->nShards = sqliteConnector_.getData<int>(r, 13);
993  td->keyMetainfo = sqliteConnector_.getData<string>(r, 14);
994  td->userId = sqliteConnector_.getData<int>(r, 15);
995  td->sortedColumnId =
996  sqliteConnector_.isNull(r, 16) ? 0 : sqliteConnector_.getData<int>(r, 16);
997  if (!td->isView) {
998  td->fragmenter = nullptr;
999  }
1000  td->hasDeletedCol = false;
1001 
1003  tableDescriptorMapById_[td->tableId] = td;
1004  }
1005 
1006  if (g_enable_fsi) {
1009  }
1010 
1011  string columnQuery(
1012  "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
1013  "is_notnull, compression, comp_param, "
1014  "size, chunks, is_systemcol, is_virtualcol, virtual_expr, is_deletedcol from "
1015  "mapd_columns ORDER BY tableid, "
1016  "columnid");
1017  sqliteConnector_.query(columnQuery);
1018  numRows = sqliteConnector_.getNumRows();
1019  int32_t skip_physical_cols = 0;
1020  for (size_t r = 0; r < numRows; ++r) {
1021  ColumnDescriptor* cd = new ColumnDescriptor();
1022  cd->tableId = sqliteConnector_.getData<int>(r, 0);
1023  cd->columnId = sqliteConnector_.getData<int>(r, 1);
1024  cd->columnName = sqliteConnector_.getData<string>(r, 2);
1028  cd->columnType.set_scale(sqliteConnector_.getData<int>(r, 6));
1032  cd->columnType.set_size(sqliteConnector_.getData<int>(r, 10));
1033  cd->chunks = sqliteConnector_.getData<string>(r, 11);
1034  cd->isSystemCol = sqliteConnector_.getData<bool>(r, 12);
1035  cd->isVirtualCol = sqliteConnector_.getData<bool>(r, 13);
1036  cd->virtualExpr = sqliteConnector_.getData<string>(r, 14);
1037  cd->isDeletedCol = sqliteConnector_.getData<bool>(r, 15);
1038  cd->isGeoPhyCol = skip_physical_cols > 0;
1039  ColumnKey columnKey(cd->tableId, to_upper(cd->columnName));
1040  columnDescriptorMap_[columnKey] = cd;
1041  ColumnIdKey columnIdKey(cd->tableId, cd->columnId);
1042  columnDescriptorMapById_[columnIdKey] = cd;
1043 
1044  if (skip_physical_cols <= 0) {
1045  skip_physical_cols = cd->columnType.get_physical_cols();
1046  }
1047 
1048  auto td_itr = tableDescriptorMapById_.find(cd->tableId);
1049  CHECK(td_itr != tableDescriptorMapById_.end());
1050 
1051  if (cd->isDeletedCol) {
1052  td_itr->second->hasDeletedCol = true;
1053  setDeletedColumnUnlocked(td_itr->second, cd);
1054  } else if (cd->columnType.is_geometry() || skip_physical_cols-- <= 0) {
1055  tableDescriptorMapById_[cd->tableId]->columnIdBySpi_.push_back(cd->columnId);
1056  }
1057  }
1058  // sort columnIdBySpi_ based on columnId
1059  for (auto& tit : tableDescriptorMapById_) {
1060  std::sort(tit.second->columnIdBySpi_.begin(),
1061  tit.second->columnIdBySpi_.end(),
1062  [](const size_t a, const size_t b) -> bool { return a < b; });
1063  }
1064 
1065  string viewQuery("SELECT tableid, sql FROM mapd_views");
1066  sqliteConnector_.query(viewQuery);
1067  numRows = sqliteConnector_.getNumRows();
1068  for (size_t r = 0; r < numRows; ++r) {
1069  int32_t tableId = sqliteConnector_.getData<int>(r, 0);
1070  TableDescriptor* td = tableDescriptorMapById_[tableId];
1071  td->viewSQL = sqliteConnector_.getData<string>(r, 1);
1072  td->fragmenter = nullptr;
1073  }
1074 
1075  string frontendViewQuery(
1076  "SELECT id, state, name, image_hash, strftime('%Y-%m-%dT%H:%M:%SZ', update_time), "
1077  "userid, "
1078  "metadata "
1079  "FROM mapd_dashboards");
1080  sqliteConnector_.query(frontendViewQuery);
1081  numRows = sqliteConnector_.getNumRows();
1082  for (size_t r = 0; r < numRows; ++r) {
1083  std::shared_ptr<DashboardDescriptor> vd = std::make_shared<DashboardDescriptor>();
1084  vd->dashboardId = sqliteConnector_.getData<int>(r, 0);
1085  vd->dashboardState = sqliteConnector_.getData<string>(r, 1);
1086  vd->dashboardName = sqliteConnector_.getData<string>(r, 2);
1087  vd->imageHash = sqliteConnector_.getData<string>(r, 3);
1088  vd->updateTime = sqliteConnector_.getData<string>(r, 4);
1089  vd->userId = sqliteConnector_.getData<int>(r, 5);
1090  vd->dashboardMetadata = sqliteConnector_.getData<string>(r, 6);
1091  vd->user = getUserFromId(vd->userId);
1092  vd->dashboardSystemRoleName = generate_dashboard_system_rolename(
1094  dashboardDescriptorMap_[std::to_string(vd->userId) + ":" + vd->dashboardName] = vd;
1095  }
1096 
1097  string linkQuery(
1098  "SELECT linkid, userid, link, view_state, strftime('%Y-%m-%dT%H:%M:%SZ', "
1099  "update_time), view_metadata "
1100  "FROM mapd_links");
1101  sqliteConnector_.query(linkQuery);
1102  numRows = sqliteConnector_.getNumRows();
1103  for (size_t r = 0; r < numRows; ++r) {
1104  LinkDescriptor* ld = new LinkDescriptor();
1105  ld->linkId = sqliteConnector_.getData<int>(r, 0);
1106  ld->userId = sqliteConnector_.getData<int>(r, 1);
1107  ld->link = sqliteConnector_.getData<string>(r, 2);
1108  ld->viewState = sqliteConnector_.getData<string>(r, 3);
1109  ld->updateTime = sqliteConnector_.getData<string>(r, 4);
1110  ld->viewMetadata = sqliteConnector_.getData<string>(r, 5);
1112  linkDescriptorMapById_[ld->linkId] = ld;
1113  }
1114 
1115  /* rebuild map linking logical tables to corresponding physical ones */
1116  string logicalToPhysicalTableMapQuery(
1117  "SELECT logical_table_id, physical_table_id "
1118  "FROM mapd_logical_to_physical");
1119  sqliteConnector_.query(logicalToPhysicalTableMapQuery);
1120  numRows = sqliteConnector_.getNumRows();
1121  for (size_t r = 0; r < numRows; ++r) {
1122  int32_t logical_tb_id = sqliteConnector_.getData<int>(r, 0);
1123  int32_t physical_tb_id = sqliteConnector_.getData<int>(r, 1);
1124  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
1125  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
1126  /* add new entity to the map logicalToPhysicalTableMapById_ */
1127  std::vector<int32_t> physicalTables;
1128  physicalTables.push_back(physical_tb_id);
1129  const auto it_ok =
1130  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
1131  CHECK(it_ok.second);
1132  } else {
1133  /* update map logicalToPhysicalTableMapById_ */
1134  physicalTableIt->second.push_back(physical_tb_id);
1135  }
1136  }
1137 }
std::string virtualExpr
void set_compression(EncodingType c)
Definition: sqltypes.h:359
void set_size(int s)
Definition: sqltypes.h:357
std::string partitions
std::tuple< int, std::string > ColumnKey
Definition: Types.h:36
T getData(const int row, const int col)
SQLTypes
Definition: sqltypes.h:40
std::string tableName
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:500
#define LOG(tag)
Definition: Logger.h:188
SqliteConnector sqliteConnector_
Definition: Catalog.h:509
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:502
std::string storageType
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:350
virtual void query(const std::string &queryString)
std::string fragments
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:501
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:503
std::string to_string(char const *&&v)
std::string chunks
EncodingType
Definition: sqltypes.h:156
std::string getUserFromId(const int32_t id)
Definition: Catalog.cpp:918
void set_scale(int s)
Definition: sqltypes.h:354
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:2895
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:499
int get_physical_cols() const
Definition: sqltypes.h:280
specifies the content in-memory of a row in the column metadata table
std::string keyMetainfo
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
std::string to_upper(const std::string &str)
void set_comp_param(int p)
Definition: sqltypes.h:360
bool is_geometry() const
Definition: sqltypes.h:429
void set_dimension(int d)
Definition: sqltypes.h:351
Fragmenter_Namespace::FragmenterType fragType
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
bool isNull(const int row, const int col) const
void set_notnull(bool n)
Definition: sqltypes.h:356
#define CHECK(condition)
Definition: Logger.h:197
Descriptor for a dictionary for a string columne.
virtual size_t getNumRows() const
std::string viewSQL
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
specifies the content in-memory of a row in the table metadata table
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:505
static constexpr char const * FOREIGN_TABLE
bool g_enable_fsi
Definition: Catalog.cpp:91
std::string columnName
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:516
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:498
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:504
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)
std::tuple< int, int > ColumnIdKey
Definition: Types.h:38
HOST DEVICE void set_type(SQLTypes t)
Definition: sqltypes.h:349
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ calculateSHA1()

std::string Catalog_Namespace::Catalog::calculateSHA1 ( const std::string &  data)
protected

Definition at line 3520 of file Catalog.cpp.

Referenced by createLink().

3520  {
3521  boost::uuids::detail::sha1 sha1;
3522  unsigned int digest[5];
3523  sha1.process_bytes(data.c_str(), data.length());
3524  sha1.get_digest(digest);
3525  std::stringstream ss;
3526  for (size_t i = 0; i < 5; i++) {
3527  ss << std::hex << digest[i];
3528  }
3529  return ss.str();
3530 }
+ Here is the caller graph for this function:

◆ changeForeignServerOwner()

void Catalog_Namespace::Catalog::changeForeignServerOwner ( const std::string &  server_name,
const int  new_owner_id 
)

Change the owner of a Foreign Server to a new owner.

Parameters
server_name- Name of the foreign server whose owner to change
new_owner_id- New owner's user id

Definition at line 2525 of file Catalog.cpp.

References CHECK, foreignServerMap_, setForeignServerProperty(), to_string(), and foreign_storage::ForeignServer::user_id.

2526  {
2527  cat_write_lock write_lock(this);
2528  foreign_storage::ForeignServer* foreign_server =
2529  foreignServerMap_.find(server_name)->second.get();
2530  CHECK(foreign_server);
2531  setForeignServerProperty(server_name, "owner_user_id", std::to_string(new_owner_id));
2532  // update in-memory server
2533  foreign_server->user_id = new_owner_id;
2534 }
std::string to_string(char const *&&v)
ForeignServerMap foreignServerMap_
Definition: Catalog.h:506
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock
void setForeignServerProperty(const std::string &server_name, const std::string &property, const std::string &value)
Definition: Catalog.cpp:3885
+ Here is the call graph for this function:

◆ CheckAndExecuteMigrations()

void Catalog_Namespace::Catalog::CheckAndExecuteMigrations ( )
protected

Definition at line 890 of file Catalog.cpp.

References createFsiSchemasAndDefaultServers(), dropFsiSchemasAndTables(), g_enable_fsi, recordOwnershipOfObjectsInObjectPermissions(), updateDeletedColumnIndicator(), updateDictionaryNames(), updateDictionarySchema(), updateFixlenArrayColumns(), updateFrontendViewAndLinkUsers(), updateFrontendViewSchema(), updateFrontendViewsToDashboards(), updateGeoColumns(), updateLinkSchema(), updateLogicalToPhysicalTableLinkSchema(), updatePageSize(), and updateTableDescriptorSchema().

Referenced by Catalog().

890  {
900  updatePageSize();
904 
905  if (g_enable_fsi) {
907  } else {
909  }
910 }
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:408
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:688
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:120
void createFsiSchemasAndDefaultServers()
Definition: Catalog.cpp:609
void updateLogicalToPhysicalTableLinkSchema()
Definition: Catalog.cpp:548
bool g_enable_fsi
Definition: Catalog.cpp:91
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ CheckAndExecuteMigrationsPostBuildMaps()

void Catalog_Namespace::Catalog::CheckAndExecuteMigrationsPostBuildMaps ( )
protected

Definition at line 912 of file Catalog.cpp.

References checkDateInDaysColumnMigration(), and createDashboardSystemRoles().

Referenced by Catalog().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ checkDateInDaysColumnMigration()

void Catalog_Namespace::Catalog::checkDateInDaysColumnMigration ( )
protected

Definition at line 816 of file Catalog.cpp.

References getCurrentDB(), migrations::MigrationMgr::migrateDateInDaysMetadata(), sqliteConnector_, and tableDescriptorMapById_.

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

816  {
817  cat_sqlite_lock sqlite_lock(this);
820 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:509
static void migrateDateInDaysMetadata(const Catalog_Namespace::TableDescriptorMapById &table_descriptors_by_id, const int database_id, const Catalog_Namespace::Catalog *cat, SqliteConnector &sqlite)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:499
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:208
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ checkedGet()

std::shared_ptr< Catalog > Catalog_Namespace::Catalog::checkedGet ( const int32_t  db_id)
static

◆ checkMetadataForDeletedRecs()

const bool Catalog_Namespace::Catalog::checkMetadataForDeletedRecs ( const TableDescriptor td,
int  column_id 
) const

Definition at line 2830 of file Catalog.cpp.

References CHECK, currentDB_, dataMgr_, Catalog_Namespace::DBMetadata::dbId, TableDescriptor::fragmenter, table_is_temporary(), and TableDescriptor::tableId.

Referenced by getDeletedColumnIfRowsDeleted().

2831  {
2832  // check if there are rows deleted by examining the deletedColumn metadata
2833  CHECK(td);
2834 
2835  if (table_is_temporary(td)) {
2836  auto fragmenter = td->fragmenter;
2837  CHECK(fragmenter);
2838  return fragmenter->hasDeletedRows(delete_column_id);
2839  } else {
2840  ChunkKey chunk_key_prefix = {currentDB_.dbId, td->tableId, delete_column_id};
2841  ChunkMetadataVector chunk_metadata_vec;
2842  dataMgr_->getChunkMetadataVecForKeyPrefix(chunk_metadata_vec, chunk_key_prefix);
2843  int64_t chunk_max{0};
2844 
2845  for (auto chunk_metadata : chunk_metadata_vec) {
2846  chunk_max = chunk_metadata.second->chunkStats.max.tinyintval;
2847  // delete has occured
2848  if (chunk_max == 1) {
2849  return true;
2850  }
2851  }
2852  return false;
2853  }
2854 }
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:511
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
bool table_is_temporary(const TableDescriptor *const td)
#define CHECK(condition)
Definition: Logger.h:197
std::vector< int > ChunkKey
Definition: types.h:37
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata > >> ChunkMetadataVector
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ checkpoint()

void Catalog_Namespace::Catalog::checkpoint ( const int  logicalTableId) const

Definition at line 3693 of file Catalog.cpp.

References Data_Namespace::DataMgr::checkpoint(), getCurrentDB(), getDataMgr(), getMetadataForTable(), and getPhysicalTablesDescriptors().

Referenced by import_export::Loader::checkpoint(), and TableOptimizer::vacuumDeletedRows().

3693  {
3694  const auto td = getMetadataForTable(logicalTableId);
3695  const auto shards = getPhysicalTablesDescriptors(td);
3696  for (const auto shard : shards) {
3697  getDataMgr().checkpoint(getCurrentDB().dbId, shard->tableId);
3698  }
3699 }
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:209
void checkpoint(const int db_id, const int tb_id)
Definition: DataMgr.cpp:481
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logicalTableDesc) const
Definition: Catalog.cpp:3589
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:208
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ createDashboard()

int32_t Catalog_Namespace::Catalog::createDashboard ( DashboardDescriptor vd)

Definition at line 3384 of file Catalog.cpp.

References addFrontendViewToMap(), createOrUpdateDashboardSystemRole(), currentDB_, DashboardDescriptor::dashboardId, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, DashboardDescriptor::dashboardSystemRoleName, Catalog_Namespace::DBMetadata::dbId, generate_dashboard_system_rolename(), SqliteConnector::getData(), SqliteConnector::getNumRows(), DashboardDescriptor::imageHash, SqliteConnector::query(), SqliteConnector::query_with_text_params(), sqliteConnector_, to_string(), Catalog_Namespace::sqlite_lock< T >::unlock(), Catalog_Namespace::write_lock< T >::unlock(), DashboardDescriptor::updateTime, and DashboardDescriptor::userId.

3384  {
3385  cat_write_lock write_lock(this);
3386  cat_sqlite_lock sqlite_lock(this);
3387  sqliteConnector_.query("BEGIN TRANSACTION");
3388  try {
3389  // TODO(andrew): this should be an upsert
3391  "SELECT id FROM mapd_dashboards WHERE name = ? and userid = ?",
3392  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
3393  if (sqliteConnector_.getNumRows() > 0) {
3395  "UPDATE mapd_dashboards SET state = ?, image_hash = ?, metadata = ?, "
3396  "update_time = "
3397  "datetime('now') where name = ? "
3398  "and userid = ?",
3399  std::vector<std::string>{vd.dashboardState,
3400  vd.imageHash,
3401  vd.dashboardMetadata,
3402  vd.dashboardName,
3403  std::to_string(vd.userId)});
3404  } else {
3406  "INSERT INTO mapd_dashboards (name, state, image_hash, metadata, "
3407  "update_time, "
3408  "userid) "
3409  "VALUES "
3410  "(?,?,?,?, "
3411  "datetime('now'), ?)",
3412  std::vector<std::string>{vd.dashboardName,
3413  vd.dashboardState,
3414  vd.imageHash,
3415  vd.dashboardMetadata,
3416  std::to_string(vd.userId)});
3417  }
3418  } catch (std::exception& e) {
3419  sqliteConnector_.query("ROLLBACK TRANSACTION");
3420  throw;
3421  }
3422  sqliteConnector_.query("END TRANSACTION");
3423 
3424  // now get the auto generated dashboardId
3425  try {
3427  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_dashboards "
3428  "WHERE name = ? and userid = ?",
3429  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
3430  vd.dashboardId = sqliteConnector_.getData<int>(0, 0);
3431  vd.updateTime = sqliteConnector_.getData<std::string>(0, 1);
3432  } catch (std::exception& e) {
3433  throw;
3434  }
3438  sqlite_lock.unlock();
3439  write_lock.unlock();
3440  // NOTE(wamsi): Transactionally unsafe
3443  return vd.dashboardId;
3444 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:509
virtual void query(const std::string &queryString)
void addFrontendViewToMap(DashboardDescriptor &vd)
Definition: Catalog.cpp:1280
std::string to_string(char const *&&v)
std::string dashboardSystemRoleName
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1321
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
virtual size_t getNumRows() const
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)
+ Here is the call graph for this function:

◆ createDashboardSystemRoles()

void Catalog_Namespace::Catalog::createDashboardSystemRoles ( )
protected

Definition at line 822 of file Catalog.cpp.

References createOrUpdateDashboardSystemRole(), currentDB_, Catalog_Namespace::DBMetadata::dbId, logger::ERROR, generate_dashboard_system_rolename(), SqliteConnector::getData(), Catalog_Namespace::SysCatalog::getGranteesOfSharedDashboards(), SqliteConnector::getNumRows(), Catalog_Namespace::SysCatalog::getRoleGrantee(), Catalog_Namespace::SysCatalog::grantRoleBatch(), logger::INFO, Catalog_Namespace::SysCatalog::instance(), LOG, MAPD_VERSION, SqliteConnector::query(), SqliteConnector::query_with_text_params(), run_benchmark_import::result, sqliteConnector_, and to_string().

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

822  {
823  std::unordered_map<std::string, std::pair<int, std::string>> dashboards;
824  std::vector<std::string> dashboard_ids;
825  static const std::string migration_name{"dashboard_roles_migration"};
826  {
827  cat_sqlite_lock sqlite_lock(this);
828  sqliteConnector_.query("BEGIN TRANSACTION");
829  try {
830  // migration_history should be present in all catalogs by now
831  // if not then would be created before this migration
833  "select * from mapd_version_history where migration_history = '" +
834  migration_name + "'");
835  if (sqliteConnector_.getNumRows() != 0) {
836  // no need for further execution
837  sqliteConnector_.query("END TRANSACTION");
838  return;
839  }
840  LOG(INFO) << "Performing dashboard internal roles Migration.";
841  sqliteConnector_.query("select id, userid, metadata from mapd_dashboards");
842  for (size_t i = 0; i < sqliteConnector_.getNumRows(); ++i) {
845  sqliteConnector_.getData<string>(i, 0)))) {
846  // Successfully created roles during previous migration/crash
847  // No need to include them
848  continue;
849  }
850  dashboards[sqliteConnector_.getData<string>(i, 0)] = std::make_pair(
851  sqliteConnector_.getData<int>(i, 1), sqliteConnector_.getData<string>(i, 2));
852  dashboard_ids.push_back(sqliteConnector_.getData<string>(i, 0));
853  }
854  } catch (const std::exception& e) {
855  sqliteConnector_.query("ROLLBACK TRANSACTION");
856  throw;
857  }
858  sqliteConnector_.query("END TRANSACTION");
859  }
860  // All current grantees with shared dashboards.
861  const auto active_grantees =
863 
864  try {
865  // NOTE(wamsi): Transactionally unsafe
866  for (auto dash : dashboards) {
867  createOrUpdateDashboardSystemRole(dash.second.second,
868  dash.second.first,
870  std::to_string(currentDB_.dbId), dash.first));
871  auto result = active_grantees.find(dash.first);
872  if (result != active_grantees.end()) {
875  dash.first)},
876  result->second);
877  }
878  }
880  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
881  std::vector<std::string>{std::to_string(MAPD_VERSION), migration_name});
882  } catch (const std::exception& e) {
883  LOG(ERROR) << "Failed to create dashboard system roles during migration: "
884  << e.what();
885  throw;
886  }
887  LOG(INFO) << "Successfully created dashboard system roles during migration.";
888 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
#define LOG(tag)
Definition: Logger.h:188
SqliteConnector sqliteConnector_
Definition: Catalog.h:509
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
void grantRoleBatch(const std::vector< std::string > &roles, const std::vector< std::string > &grantees)
static SysCatalog & instance()
Definition: SysCatalog.h:286
static const int32_t MAPD_VERSION
Definition: release.h:33
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1321
std::unordered_map< std::string, std::vector< std::string > > getGranteesOfSharedDashboards(const std::vector< std::string > &dashboard_ids)
virtual size_t getNumRows() const
Role * getRoleGrantee(const std::string &name) const
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ createDefaultServersIfNotExists()

void Catalog_Namespace::Catalog::createDefaultServersIfNotExists ( )

Creates default local file servers (if they don't already exist).

Definition at line 3906 of file Catalog.cpp.

References createForeignServerNoLocks(), foreign_storage::DataWrapperType::CSV, OMNISCI_ROOT_USER_ID, foreign_storage::DataWrapperType::PARQUET, and foreign_storage::ForeignServer::validate().

Referenced by createFsiSchemasAndDefaultServers().

3906  {
3908  std::map<std::string, std::string, std::less<>> options;
3909  options[std::string{ForeignServer::STORAGE_TYPE_KEY}] =
3910  ForeignServer::LOCAL_FILE_STORAGE_TYPE;
3911  options[std::string{ForeignServer::BASE_PATH_KEY}] =
3912  boost::filesystem::path::preferred_separator;
3913 
3914  auto local_csv_server =
3915  std::make_unique<ForeignServer>("omnisci_local_csv",
3917  options,
3919  local_csv_server->validate();
3920  createForeignServerNoLocks(std::move(local_csv_server), true);
3921 
3922  auto local_parquet_server =
3923  std::make_unique<ForeignServer>("omnisci_local_parquet",
3925  options,
3927  local_parquet_server->validate();
3928  createForeignServerNoLocks(std::move(local_parquet_server), true);
3929 }
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2460
const int OMNISCI_ROOT_USER_ID
Definition: SysCatalog.h:58
static constexpr char const * CSV
Definition: ForeignServer.h:35
static constexpr char const * PARQUET
Definition: ForeignServer.h:36
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ createForeignServer()

void Catalog_Namespace::Catalog::createForeignServer ( std::unique_ptr< foreign_storage::ForeignServer foreign_server,
bool  if_not_exists 
)

Creates a new foreign server DB object.

Parameters
foreign_server- unique pointer to struct containing foreign server details
if_not_exists- flag indicating whether or not an attempt to create a new foreign server should occur if a server with the same name already exists. An exception is thrown if this flag is set to "false" and an attempt is made to create a pre-existing foreign server

Definition at line 2452 of file Catalog.cpp.

References createForeignServerNoLocks().

2454  {
2455  cat_write_lock write_lock(this);
2456  cat_sqlite_lock sqlite_lock(this);
2457  createForeignServerNoLocks(std::move(foreign_server), if_not_exists);
2458 }
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2460
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
+ Here is the call graph for this function:

◆ createForeignServerNoLocks()

void Catalog_Namespace::Catalog::createForeignServerNoLocks ( std::unique_ptr< foreign_storage::ForeignServer foreign_server,
bool  if_not_exists 
)
private

Same as createForeignServer() but without acquiring locks. This should only be called from within a function/code block that already acquires appropriate locks.

Definition at line 2460 of file Catalog.cpp.

References CHECK_EQ, foreignServerMap_, foreignServerMapById_, SqliteConnector::getData(), SqliteConnector::getNumRows(), SqliteConnector::query_with_text_params(), sqliteConnector_, and to_string().

Referenced by createDefaultServersIfNotExists(), and createForeignServer().

2462  {
2464  "SELECT name from omnisci_foreign_servers where name = ?",
2465  std::vector<std::string>{foreign_server->name});
2466 
2467  if (sqliteConnector_.getNumRows() == 0) {
2468  foreign_server->creation_time = std::time(nullptr);
2470  "INSERT INTO omnisci_foreign_servers (name, data_wrapper_type, owner_user_id, "
2471  "creation_time, "
2472  "options) "
2473  "VALUES (?, ?, ?, ?, ?)",
2474  std::vector<std::string>{foreign_server->name,
2475  foreign_server->data_wrapper_type,
2476  std::to_string(foreign_server->user_id),
2477  std::to_string(foreign_server->creation_time),
2478  foreign_server->getOptionsAsJsonString()});
2480  "SELECT id from omnisci_foreign_servers where name = ?",
2481  std::vector<std::string>{foreign_server->name});
2482  CHECK_EQ(sqliteConnector_.getNumRows(), size_t(1));
2483  foreign_server->id = sqliteConnector_.getData<int32_t>(0, 0);
2484  } else if (!if_not_exists) {
2485  throw std::runtime_error{"A foreign server with name \"" + foreign_server->name +
2486  "\" already exists."};
2487  }
2488 
2489  std::shared_ptr<foreign_storage::ForeignServer> foreign_server_shared =
2490  std::move(foreign_server);
2491  foreignServerMap_[foreign_server_shared->name] = foreign_server_shared;
2492  foreignServerMapById_[foreign_server_shared->id] = foreign_server_shared;
2493 }
#define CHECK_EQ(x, y)
Definition: Logger.h:205
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:509
std::string to_string(char const *&&v)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:507
ForeignServerMap foreignServerMap_
Definition: Catalog.h:506
virtual size_t getNumRows() const
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ createFsiSchemasAndDefaultServers()

void Catalog_Namespace::Catalog::createFsiSchemasAndDefaultServers ( )
protected

Definition at line 609 of file Catalog.cpp.

References createDefaultServersIfNotExists(), getForeignServerSchema(), getForeignTableSchema(), SqliteConnector::query(), and sqliteConnector_.

Referenced by CheckAndExecuteMigrations().

609  {
610  cat_sqlite_lock sqlite_lock(this);
611  sqliteConnector_.query("BEGIN TRANSACTION");
612  try {
616  } catch (std::exception& e) {
617  sqliteConnector_.query("ROLLBACK TRANSACTION");
618  throw;
619  }
620  sqliteConnector_.query("END TRANSACTION");
621 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:509
virtual void query(const std::string &queryString)
static const std::string getForeignTableSchema(bool if_not_exists=false)
Definition: Catalog.cpp:680
static const std::string getForeignServerSchema(bool if_not_exists=false)
Definition: Catalog.cpp:673
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
void createDefaultServersIfNotExists()
Definition: Catalog.cpp:3906
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ createLink()

std::string Catalog_Namespace::Catalog::createLink ( LinkDescriptor ld,
size_t  min_length 
)

Definition at line 3532 of file Catalog.cpp.

References addLinkToMap(), calculateSHA1(), SqliteConnector::getData(), SqliteConnector::getNumRows(), LinkDescriptor::link, LinkDescriptor::linkId, SqliteConnector::query(), SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), sqliteConnector_, to_string(), LinkDescriptor::updateTime, LinkDescriptor::userId, LinkDescriptor::viewMetadata, and LinkDescriptor::viewState.

3532  {
3533  cat_write_lock write_lock(this);
3534  cat_sqlite_lock sqlite_lock(this);
3535  sqliteConnector_.query("BEGIN TRANSACTION");
3536  try {
3538  .substr(0, 8);
3540  "SELECT linkid FROM mapd_links WHERE link = ? and userid = ?",
3541  std::vector<std::string>{ld.link, std::to_string(ld.userId)});
3542  if (sqliteConnector_.getNumRows() > 0) {
3544  "UPDATE mapd_links SET update_time = datetime('now') WHERE userid = ? AND link "
3545  "= ?",
3546  std::vector<std::string>{std::to_string(ld.userId), ld.link});
3547  } else {
3549  "INSERT INTO mapd_links (userid, link, view_state, view_metadata, update_time) "
3550  "VALUES (?,?,?,?, "
3551  "datetime('now'))",
3552  std::vector<std::string>{
3553  std::to_string(ld.userId), ld.link, ld.viewState, ld.viewMetadata});
3554  }
3555  // now get the auto generated dashid
3557  "SELECT linkid, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_links "
3558  "WHERE link = ?",
3559  ld.link);
3560  ld.linkId = sqliteConnector_.getData<int>(0, 0);
3561  ld.updateTime = sqliteConnector_.getData<std::string>(0, 1);
3562  } catch (std::exception& e) {
3563  sqliteConnector_.query("ROLLBACK TRANSACTION");
3564  throw;
3565  }
3566  sqliteConnector_.query("END TRANSACTION");
3567  addLinkToMap(ld);
3568  return ld.link;
3569 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:509
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
std::string calculateSHA1(const std::string &data)
Definition: Catalog.cpp:3520
void addLinkToMap(LinkDescriptor &ld)
Definition: Catalog.cpp:1362
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
virtual size_t getNumRows() const
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
+ Here is the call graph for this function:

◆ createOrUpdateDashboardSystemRole()

void Catalog_Namespace::Catalog::createOrUpdateDashboardSystemRole ( const std::string &  view_meta,
const int32_t &  user_id,
const std::string &  dash_role_name 
)
protected

Definition at line 1321 of file Catalog.cpp.

References Catalog_Namespace::SysCatalog::createRole(), Grantee::findDbObject(), Grantee::getDbObjects(), Catalog_Namespace::SysCatalog::getRoleGrantee(), Catalog_Namespace::SysCatalog::grantDBObjectPrivilegesBatch(), Catalog_Namespace::SysCatalog::instance(), parseDashboardObjects(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivileges(), TableDBObjectType, and ViewDBObjectType.

Referenced by createDashboard(), createDashboardSystemRoles(), and replaceDashboard().

1323  {
1324  auto objects = parseDashboardObjects(view_meta, user_id);
1325  Role* rl = SysCatalog::instance().getRoleGrantee(dash_role_name);
1326  if (!rl) {
1327  // Dashboard role does not exist
1328  // create role and grant privileges
1329  // NOTE(wamsi): Transactionally unsafe
1330  SysCatalog::instance().createRole(dash_role_name, false);
1331  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1332  } else {
1333  // Dashboard system role already exists
1334  // Add/remove privileges on objects
1335  auto ex_objects = rl->getDbObjects(true);
1336  for (auto key : *ex_objects | boost::adaptors::map_keys) {
1337  if (key.permissionType != TableDBObjectType &&
1338  key.permissionType != ViewDBObjectType) {
1339  continue;
1340  }
1341  bool found = false;
1342  for (auto obj : objects) {
1343  found = key == obj.getObjectKey() ? true : false;
1344  if (found) {
1345  break;
1346  }
1347  }
1348  if (!found) {
1349  // revoke privs on object since the object is no
1350  // longer used by the dashboard as source
1351  // NOTE(wamsi): Transactionally unsafe
1353  dash_role_name, *rl->findDbObject(key, true), *this);
1354  }
1355  }
1356  // Update privileges on remaining objects
1357  // NOTE(wamsi): Transactionally unsafe
1358  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1359  }
1360 }
void revokeDBObjectPrivileges(const std::string &grantee, const DBObject &object, const Catalog_Namespace::Catalog &catalog)
DBObject * findDbObject(const DBObjectKey &objectKey, bool only_direct) const
Definition: Grantee.cpp:72
Definition: Grantee.h:76
void createRole(const std::string &roleName, const bool &userPrivateRole=false)
static SysCatalog & instance()
Definition: SysCatalog.h:286
void grantDBObjectPrivilegesBatch(const std::vector< std::string > &grantees, const std::vector< DBObject > &objects, const Catalog_Namespace::Catalog &catalog)
const DBObjectMap * getDbObjects(bool only_direct) const
Definition: Grantee.h:56
Role * getRoleGrantee(const std::string &name) const
std::vector< DBObject > parseDashboardObjects(const std::string &view_meta, const int &user_id)
Definition: Catalog.cpp:1291
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ createShardedTable()

void Catalog_Namespace::Catalog::createShardedTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs 
)

Definition at line 3048 of file Catalog.cpp.

References CHECK, createTable(), generatePhysicalTableName(), logicalToPhysicalTableMapById_, TableDescriptor::nShards, TableDescriptor::shard, table_is_temporary(), TableDescriptor::tableId, TableDescriptor::tableName, and updateLogicalToPhysicalTableMap().

3051  {
3052  cat_write_lock write_lock(this);
3053 
3054  /* create logical table */
3055  TableDescriptor* tdl = &td;
3056  createTable(*tdl, cols, shared_dict_defs, true); // create logical table
3057  int32_t logical_tb_id = tdl->tableId;
3058 
3059  /* create physical tables and link them to the logical table */
3060  std::vector<int32_t> physicalTables;
3061  for (int32_t i = 1; i <= td.nShards; i++) {
3062  TableDescriptor* tdp = &td;
3064  tdp->shard = i - 1;
3065  createTable(*tdp, cols, shared_dict_defs, false); // create physical table
3066  int32_t physical_tb_id = tdp->tableId;
3067 
3068  /* add physical table to the vector of physical tables */
3069  physicalTables.push_back(physical_tb_id);
3070  }
3071 
3072  if (!physicalTables.empty()) {
3073  /* add logical to physical tables correspondence to the map */
3074  const auto it_ok =
3075  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
3076  CHECK(it_ok.second);
3077  /* update sqlite mapd_logical_to_physical in sqlite database */
3078  if (!table_is_temporary(&td)) {
3079  updateLogicalToPhysicalTableMap(logical_tb_id);
3080  }
3081  }
3082 }
std::string tableName
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:2142
std::string generatePhysicalTableName(const std::string &logicalTableName, const int32_t &shardNumber)
Definition: Catalog.cpp:3742
bool table_is_temporary(const TableDescriptor *const td)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock
specifies the content in-memory of a row in the table metadata table
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:516
void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id)
Definition: Catalog.cpp:562
+ Here is the call graph for this function:

◆ createTable()

void Catalog_Namespace::Catalog::createTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs,
bool  isLogicalTable 
)

Definition at line 2142 of file Catalog.cpp.

References addTableToMap(), calciteMgr_, ColumnDescriptor::columnId, TableDescriptor::columnIdBySpi_, ColumnDescriptor::columnName, ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, dict_ref_t::dictId, Data_Namespace::DISK_LEVEL, expandGeoColumn(), StorageType::FOREIGN_TABLE, TableDescriptor::fragPageSize, TableDescriptor::fragType, g_enable_fsi, g_serialize_temp_tables, g_test_against_columnId_gap, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_next_refresh_time(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), getCurrentDB(), SqliteConnector::getData(), TableDescriptor::hasDeletedCol, SQLTypeInfo::is_array(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, kBIGINT, kBOOLEAN, kENCODING_DICT, TableDescriptor::keyMetainfo, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRows, TableDescriptor::nColumns, foreign_storage::ForeignTable::next_refresh_time, nextTempDictId_, nextTempTableId_, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, ForeignStorageInterface::prepareTable(), SqliteConnector::query(), SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), ForeignStorageInterface::registerTable(), removeTableFromMap(), serializeTableJsonUnlocked(), SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_size(), setColumnDictionary(), setColumnSharedDictionary(), TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, sqliteConnector_, TableDescriptor::storageType, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), TableDescriptor::userId, TableDescriptor::viewSQL, and ColumnDescriptor::virtualExpr.

Referenced by createShardedTable().

2146  {
2147  cat_write_lock write_lock(this);
2148  list<ColumnDescriptor> cds = cols;
2149  list<DictDescriptor> dds;
2150  std::set<std::string> toplevel_column_names;
2151  list<ColumnDescriptor> columns;
2152 
2153  if (!td.storageType.empty() &&
2156  throw std::runtime_error("Only temporary tables can be backed by foreign storage.");
2157  }
2159  }
2160 
2161  for (auto cd : cds) {
2162  if (cd.columnName == "rowid") {
2163  throw std::runtime_error(
2164  "Cannot create column with name rowid. rowid is a system defined column.");
2165  }
2166  columns.push_back(cd);
2167  toplevel_column_names.insert(cd.columnName);
2168  if (cd.columnType.is_geometry()) {
2169  expandGeoColumn(cd, columns);
2170  }
2171  }
2172  cds.clear();
2173 
2174  ColumnDescriptor cd;
2175  // add row_id column -- Must be last column in the table
2176  cd.columnName = "rowid";
2177  cd.isSystemCol = true;
2178  cd.columnType = SQLTypeInfo(kBIGINT, true);
2179 #ifdef MATERIALIZED_ROWID
2180  cd.isVirtualCol = false;
2181 #else
2182  cd.isVirtualCol = true;
2183  cd.virtualExpr = "MAPD_FRAG_ID * MAPD_ROWS_PER_FRAG + MAPD_FRAG_ROW_ID";
2184 #endif
2185  columns.push_back(cd);
2186  toplevel_column_names.insert(cd.columnName);
2187 
2188  if (td.hasDeletedCol) {
2189  ColumnDescriptor cd_del;
2190  cd_del.columnName = "$deleted$";
2191  cd_del.isSystemCol = true;
2192  cd_del.isVirtualCol = false;
2193  cd_del.columnType = SQLTypeInfo(kBOOLEAN, true);
2194  cd_del.isDeletedCol = true;
2195 
2196  columns.push_back(cd_del);
2197  }
2198 
2199  td.nColumns = columns.size();
2200  cat_sqlite_lock sqlite_lock(this);
2201  sqliteConnector_.query("BEGIN TRANSACTION");
2203  try {
2205  R"(INSERT INTO mapd_tables (name, userid, ncolumns, isview, fragments, frag_type, max_frag_rows, max_chunk_size, frag_page_size, max_rows, partitions, shard_column_id, shard, num_shards, sort_column_id, storage_type, key_metainfo) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?))",
2206  std::vector<std::string>{td.tableName,
2207  std::to_string(td.userId),
2209  std::to_string(td.isView),
2210  "",
2215  std::to_string(td.maxRows),
2216  td.partitions,
2218  std::to_string(td.shard),
2219  std::to_string(td.nShards),
2221  td.storageType,
2222  td.keyMetainfo});
2223 
2224  // now get the auto generated tableid
2226  "SELECT tableid FROM mapd_tables WHERE name = ?", td.tableName);
2227  td.tableId = sqliteConnector_.getData<int>(0, 0);
2228  int colId = 1;
2229  for (auto cd : columns) {
2231  const bool is_foreign_col =
2232  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2233  if (!is_foreign_col) {
2234  setColumnDictionary(cd, dds, td, isLogicalTable);
2235  }
2236  }
2237 
2238  if (toplevel_column_names.count(cd.columnName)) {
2239  // make up colId gap for sanity test (begin with 1 bc much code depends on it!)
2240  if (colId > 1) {
2241  colId += g_test_against_columnId_gap;
2242  }
2243  if (!cd.isGeoPhyCol) {
2244  td.columnIdBySpi_.push_back(colId);
2245  }
2246  }
2247 
2249  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, "
2250  "coldim, colscale, is_notnull, "
2251  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
2252  "virtual_expr, is_deletedcol) "
2253  "VALUES (?, ?, ?, ?, ?, "
2254  "?, "
2255  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2256  std::vector<std::string>{std::to_string(td.tableId),
2257  std::to_string(colId),
2258  cd.columnName,
2267  "",
2270  cd.virtualExpr,
2272  cd.tableId = td.tableId;
2273  cd.columnId = colId++;
2274  cds.push_back(cd);
2275  }
2276  if (td.isView) {
2278  "INSERT INTO mapd_views (tableid, sql) VALUES (?,?)",
2279  std::vector<std::string>{std::to_string(td.tableId), td.viewSQL});
2280  }
2282  auto& foreign_table = dynamic_cast<foreign_storage::ForeignTable&>(td);
2283  foreign_table.next_refresh_time = get_next_refresh_time(foreign_table);
2285  "INSERT INTO omnisci_foreign_tables (table_id, server_id, options, "
2286  "last_refresh_time, next_refresh_time) VALUES (?, ?, ?, ?, ?)",
2287  std::vector<std::string>{std::to_string(foreign_table.tableId),
2288  std::to_string(foreign_table.foreign_server->id),
2289  foreign_table.getOptionsAsJsonString(),
2290  std::to_string(foreign_table.last_refresh_time),
2291  std::to_string(foreign_table.next_refresh_time)});
2292  }
2293  } catch (std::exception& e) {
2294  sqliteConnector_.query("ROLLBACK TRANSACTION");
2295  throw;
2296  }
2297  } else { // Temporary table
2298  td.tableId = nextTempTableId_++;
2299  int colId = 1;
2300  for (auto cd : columns) {
2302  const bool is_foreign_col =
2303  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2304 
2305  if (!is_foreign_col) {
2306  // Create a new temporary dictionary
2307  std::string fileName("");
2308  std::string folderPath("");
2310  nextTempDictId_++;
2311  DictDescriptor dd(dict_ref,
2312  fileName,
2314  false,
2315  1,
2316  folderPath,
2317  true); // Is dictName (2nd argument) used?
2318  dds.push_back(dd);
2319  if (!cd.columnType.is_array()) {
2321  }
2322  cd.columnType.set_comp_param(dict_ref.dictId);
2323  }
2324  }
2325  if (toplevel_column_names.count(cd.columnName)) {
2326  // make up colId gap for sanity test (begin with 1 bc much code depends on it!)
2327  if (colId > 1) {
2328  colId += g_test_against_columnId_gap;
2329  }
2330  if (!cd.isGeoPhyCol) {
2331  td.columnIdBySpi_.push_back(colId);
2332  }
2333  }
2334  cd.tableId = td.tableId;
2335  cd.columnId = colId++;
2336  cds.push_back(cd);
2337  }
2338 
2340  serializeTableJsonUnlocked(&td, cds);
2341  }
2342  }
2343 
2344  try {
2345  addTableToMap(&td, cds, dds);
2346  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
2347  if (!td.storageType.empty() && td.storageType != StorageType::FOREIGN_TABLE) {
2349  }
2350  } catch (std::exception& e) {
2351  sqliteConnector_.query("ROLLBACK TRANSACTION");
2352  removeTableFromMap(td.tableName, td.tableId, true);
2353  throw;
2354  }
2355  sqliteConnector_.query("END TRANSACTION");
2356 }
void serializeTableJsonUnlocked(const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
Definition: Catalog.cpp:2358
int64_t get_next_refresh_time(const foreign_storage::ForeignTable &foreign_table)
Definition: Catalog.cpp:2130
std::string virtualExpr
void set_size(int s)
Definition: sqltypes.h:357
std::string partitions
bool is_array() const
Definition: sqltypes.h:425
T getData(const int row, const int col)
std::string tableName
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:509
std::string storageType
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:268
virtual void query(const std::string &queryString)
HOST DEVICE int get_size() const
Definition: sqltypes.h:269
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:267
std::vector< int > columnIdBySpi_
HOST DEVICE int get_scale() const
Definition: sqltypes.h:264
std::string to_string(char const *&&v)
void addTableToMap(const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
Definition: Catalog.cpp:1139
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:266
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:260
specifies the content in-memory of a row in the column metadata table
static void registerTable(Catalog_Namespace::Catalog *catalog, const TableDescriptor &td, const std::list< ColumnDescriptor > &cols)
ids are created
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:208
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:1999
std::string keyMetainfo
int g_test_against_columnId_gap
Definition: Catalog.cpp:90
bool g_serialize_temp_tables
Definition: Catalog.cpp:96
void set_comp_param(int p)
Definition: sqltypes.h:360
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:514
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:261
Fragmenter_Namespace::FragmenterType fragType
Data_Namespace::MemoryLevel persistenceLevel
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1199
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
bool setColumnSharedDictionary(ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:2938
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:259
Descriptor for a dictionary for a string columne.
std::string viewSQL
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:3007
static constexpr char const * FOREIGN_TABLE
bool g_enable_fsi
Definition: Catalog.cpp:91
std::string columnName
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
static void prepareTable(const int db_id, TableDescriptor &td, std::list< ColumnDescriptor > &cols)
prepare table options and modify columns
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ delDictionary()

void Catalog_Namespace::Catalog::delDictionary ( const ColumnDescriptor cd)

Definition at line 1782 of file Catalog.cpp.

References basePath_, CHECK, CHECK_GT, ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SqliteConnector::getData(), getMetadataForTable(), SQLTypeInfo::is_string(), SQLTypeInfo::is_string_array(), kENCODING_DICT, SqliteConnector::query_with_text_param(), File_Namespace::renameForDelete(), sqliteConnector_, string_dict_hosts_, ColumnDescriptor::tableId, to_string(), and VLOG.

Referenced by roll().

1782  {
1783  cat_write_lock write_lock(this);
1784  cat_sqlite_lock sqlite_lock(this);
1785  if (!(cd.columnType.is_string() || cd.columnType.is_string_array())) {
1786  return;
1787  }
1788  if (!(cd.columnType.get_compression() == kENCODING_DICT)) {
1789  return;
1790  }
1791  const auto dictId = cd.columnType.get_comp_param();
1792  CHECK_GT(dictId, 0);
1793  // decrement and zero check dict ref count
1794  const auto td = getMetadataForTable(cd.tableId);
1795  CHECK(td);
1797  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
1798  std::to_string(dictId));
1800  "SELECT refcount FROM mapd_dictionaries WHERE dictid = ?", std::to_string(dictId));
1801  const auto refcount = sqliteConnector_.getData<int>(0, 0);
1802  VLOG(3) << "Dictionary " << dictId << "from dropped table has reference count "
1803  << refcount;
1804  if (refcount > 0) {
1805  return;
1806  }
1807  const DictRef dictRef(currentDB_.dbId, dictId);
1808  sqliteConnector_.query_with_text_param("DELETE FROM mapd_dictionaries WHERE dictid = ?",
1809  std::to_string(dictId));
1810  File_Namespace::renameForDelete(basePath_ + "/mapd_data/DB_" +
1811  std::to_string(currentDB_.dbId) + "_DICT_" +
1812  std::to_string(dictId));
1813 
1814  std::unique_ptr<StringDictionaryClient> client;
1815  if (!string_dict_hosts_.empty()) {
1816  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dictRef, true));
1817  }
1818  if (client) {
1819  client->drop(dictRef);
1820  }
1821 
1822  dictDescriptorMapByRef_.erase(dictRef);
1823 }
T getData(const int row, const int col)
bool is_string() const
Definition: sqltypes.h:417
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
SqliteConnector sqliteConnector_
Definition: Catalog.h:509
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:513
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:502
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:268
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:267
#define CHECK_GT(x, y)
Definition: Logger.h:209
std::string to_string(char const *&&v)
bool is_string_array() const
Definition: sqltypes.h:418
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_<EPOCH>_<oldname>.
Definition: File.cpp:187
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
#define VLOG(n)
Definition: Logger.h:291
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ deleteMetadataForDashboards()

void Catalog_Namespace::Catalog::deleteMetadataForDashboards ( const std::vector< int32_t >  ids,
const UserMetadata user 
)

Definition at line 1557 of file Catalog.cpp.

References DashboardDBObjectType, dashboardDescriptorMap_, AccessPrivileges::DELETE_DASHBOARD, getMetadataForDashboard(), Catalog_Namespace::SysCatalog::instance(), SqliteConnector::query(), SqliteConnector::query_with_text_params(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivilegesFromAllBatch(), sqliteConnector_, and to_string().

1558  {
1559  std::stringstream invalid_ids, restricted_ids;
1560 
1561  for (int32_t dashboard_id : dashboard_ids) {
1562  if (!getMetadataForDashboard(dashboard_id)) {
1563  invalid_ids << (!invalid_ids.str().empty() ? ", " : "") << dashboard_id;
1564  continue;
1565  }
1566  DBObject object(dashboard_id, DashboardDBObjectType);
1567  object.loadKey(*this);
1568  object.setPrivileges(AccessPrivileges::DELETE_DASHBOARD);
1569  std::vector<DBObject> privs = {object};
1570  if (!SysCatalog::instance().checkPrivileges(user, privs))
1571  restricted_ids << (!restricted_ids.str().empty() ? ", " : "") << dashboard_id;
1572  }
1573 
1574  if (invalid_ids.str().size() > 0 || restricted_ids.str().size() > 0) {
1575  std::stringstream error_message;
1576  error_message << "Delete dashboard(s) failed with error(s):";
1577  if (invalid_ids.str().size() > 0)
1578  error_message << "\nDashboard id: " << invalid_ids.str()
1579  << " - Dashboard id does not exist";
1580  if (restricted_ids.str().size() > 0)
1581  error_message
1582  << "\nDashboard id: " << restricted_ids.str()
1583  << " - User should be either owner of dashboard or super user to delete it";
1584  throw std::runtime_error(error_message.str());
1585  }
1586  std::vector<DBObject> dash_objs;
1587 
1588  for (int32_t dashboard_id : dashboard_ids) {
1589  dash_objs.push_back(DBObject(dashboard_id, DashboardDBObjectType));
1590  }
1591  // BE-5245: Transactionally unsafe (like other combined Catalog/Syscatalog operations)
1593  {
1594  cat_write_lock write_lock(this);
1595  cat_sqlite_lock sqlite_lock(this);
1596 
1597  sqliteConnector_.query("BEGIN TRANSACTION");
1598  try {
1599  for (int32_t dashboard_id : dashboard_ids) {
1600  auto dash = getMetadataForDashboard(dashboard_id);
1601  // Dash should still exist if revokeDBObjectPrivileges passed but throw and
1602  // rollback if already deleted
1603  if (!dash) {
1604  throw std::runtime_error(
1605  std::string("Delete dashboard(s) failed with error(s):\nDashboard id: ") +
1606  std::to_string(dashboard_id) + " - Dashboard id does not exist ");
1607  }
1608  std::string user_id = std::to_string(dash->userId);
1609  std::string dash_name = dash->dashboardName;
1610  auto viewDescIt = dashboardDescriptorMap_.find(user_id + ":" + dash_name);
1611  dashboardDescriptorMap_.erase(viewDescIt);
1613  "DELETE FROM mapd_dashboards WHERE name = ? and userid = ?",
1614  std::vector<std::string>{dash_name, user_id});
1615  }
1616  } catch (std::exception& e) {
1617  sqliteConnector_.query("ROLLBACK TRANSACTION");
1618  throw;
1619  }
1620  sqliteConnector_.query("END TRANSACTION");
1621  }
1622 }
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:509
virtual void query(const std::string &queryString)
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:503
std::string to_string(char const *&&v)
static SysCatalog & instance()
Definition: SysCatalog.h:286
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const
static const AccessPrivileges DELETE_DASHBOARD
Definition: DBObject.h:174
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
void revokeDBObjectPrivilegesFromAllBatch(std::vector< DBObject > &objects, Catalog *catalog)
+ Here is the call graph for this function:

◆ doDropTable()

void Catalog_Namespace::Catalog::doDropTable ( const TableDescriptor td)
protected

Definition at line 3237 of file Catalog.cpp.

References dropTableFromJsonUnlocked(), eraseTablePhysicalData(), executeDropTableSqliteQueries(), g_serialize_temp_tables, table_is_temporary(), and TableDescriptor::tableName.

Referenced by dropTable().

3237  {
3241  }
3243 }
void executeDropTableSqliteQueries(const TableDescriptor *td)
Definition: Catalog.cpp:3245
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:3713
std::string tableName
bool g_serialize_temp_tables
Definition: Catalog.cpp:96
bool table_is_temporary(const TableDescriptor *const td)
void dropTableFromJsonUnlocked(const std::string &table_name) const
Definition: Catalog.cpp:2420
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ doTruncateTable()

void Catalog_Namespace::Catalog::doTruncateTable ( const TableDescriptor td)
protected

Definition at line 3102 of file Catalog.cpp.

References CHECK, CHECK_GE, columnDescriptorMapById_, Data_Namespace::CPU_LEVEL, currentDB_, dataMgr_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, dict_ref_t::dictId, DictDescriptor::dictIsTemp, DictDescriptor::dictRef, TableDescriptor::fragmenter, getMetadataForDict(), Data_Namespace::GPU_LEVEL, Catalog_Namespace::SysCatalog::instance(), kENCODING_DICT, File_Namespace::renameForDelete(), string_dict_hosts_, tableDescriptorMapById_, and TableDescriptor::tableId.

Referenced by truncateTable().

3102  {
3103  cat_write_lock write_lock(this);
3104 
3105  const int tableId = td->tableId;
3106  // must destroy fragmenter before deleteChunks is called.
3107  if (td->fragmenter != nullptr) {
3108  auto tableDescIt = tableDescriptorMapById_.find(tableId);
3109  CHECK(tableDescIt != tableDescriptorMapById_.end());
3110  tableDescIt->second->fragmenter = nullptr;
3111  CHECK(td->fragmenter == nullptr);
3112  }
3113  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
3114  // assuming deleteChunksWithPrefix is atomic
3115  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
3116  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
3117 
3118  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
3119 
3120  std::unique_ptr<StringDictionaryClient> client;
3121  if (SysCatalog::instance().isAggregator()) {
3122  CHECK(!string_dict_hosts_.empty());
3123  DictRef dict_ref(currentDB_.dbId, -1);
3124  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
3125  }
3126  // clean up any dictionaries
3127  // delete all column descriptors for the table
3128  for (const auto& columnDescriptor : columnDescriptorMapById_) {
3129  auto cd = columnDescriptor.second;
3130  if (cd->tableId != td->tableId) {
3131  continue;
3132  }
3133  const int dict_id = cd->columnType.get_comp_param();
3134  // Dummy dictionaries created for a shard of a logical table have the id set to zero.
3135  if (cd->columnType.get_compression() == kENCODING_DICT && dict_id) {
3136  const DictRef dict_ref(currentDB_.dbId, dict_id);
3137  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
3138  CHECK(dictIt != dictDescriptorMapByRef_.end());
3139  const auto& dd = dictIt->second;
3140  CHECK_GE(dd->refcount, 1);
3141  // if this is the only table using this dict reset the dict
3142  if (dd->refcount == 1) {
3143  // close the dictionary
3144  dd->stringDict.reset();
3145  File_Namespace::renameForDelete(dd->dictFolderPath);
3146  if (client) {
3147  client->drop(dd->dictRef);
3148  }
3149  if (!dd->dictIsTemp) {
3150  boost::filesystem::create_directory(dd->dictFolderPath);
3151  }
3152  }
3153 
3154  DictDescriptor* new_dd = new DictDescriptor(dd->dictRef,
3155  dd->dictName,
3156  dd->dictNBits,
3157  dd->dictIsShared,
3158  dd->refcount,
3159  dd->dictFolderPath,
3160  dd->dictIsTemp);
3161  dictDescriptorMapByRef_.erase(dictIt);
3162  // now create new Dict -- need to figure out what to do here for temp tables
3163  if (client) {
3164  client->create(new_dd->dictRef, new_dd->dictIsTemp);
3165  }
3166  dictDescriptorMapByRef_[new_dd->dictRef].reset(new_dd);
3168  }
3169  }
3170 }
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:513
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:502
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:511
#define CHECK_GE(x, y)
Definition: Logger.h:210
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:501
static SysCatalog & instance()
Definition: SysCatalog.h:286
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:499
int32_t dictId
Definition: DictRef.h:10
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1451
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
#define CHECK(condition)
Definition: Logger.h:197
std::vector< int > ChunkKey
Definition: types.h:37
Descriptor for a dictionary for a string columne.
mapd_unique_lock< mapd_shared_mutex > write_lock
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_<EPOCH>_<oldname>.
Definition: File.cpp:187
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ dropColumn()

void Catalog_Namespace::Catalog::dropColumn ( const TableDescriptor td,
const ColumnDescriptor cd 
)

Definition at line 1910 of file Catalog.cpp.

References CHECK, columnDescriptorMap_, columnDescriptorMapById_, columnDescriptorsForRoll, ColumnDescriptor::columnId, ColumnDescriptor::columnName, getMetadataForColumn(), getPhysicalTablesDescriptors(), TableDescriptor::nShards, SqliteConnector::query_with_text_params(), TableDescriptor::shard, sqliteConnector_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), and to_upper().

1910  {
1911  cat_write_lock write_lock(this);
1912  cat_sqlite_lock sqlite_lock(this);
1913  // caller must handle sqlite/chunk transaction TOGETHER
1915  "DELETE FROM mapd_columns where tableid = ? and columnid = ?",
1916  std::vector<std::string>{std::to_string(td.tableId), std::to_string(cd.columnId)});
1917 
1919  "UPDATE mapd_tables SET ncolumns = ncolumns - 1 WHERE tableid = ?",
1920  std::vector<std::string>{std::to_string(td.tableId)});
1921 
1922  ColumnDescriptorMap::iterator columnDescIt =
1924  CHECK(columnDescIt != columnDescriptorMap_.end());
1925 
1926  columnDescriptorsForRoll.emplace_back(columnDescIt->second, nullptr);
1927 
1928  columnDescriptorMap_.erase(columnDescIt);
1930  --tableDescriptorMapById_[td.tableId]->nColumns;
1931  // for each shard
1932  if (td.nShards > 0 && td.shard < 0) {
1933  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
1934  const auto shard_cd = getMetadataForColumn(shard->tableId, cd.columnId);
1935  CHECK(shard_cd);
1936  dropColumn(*shard, *shard_cd);
1937  }
1938  }
1939 }
std::tuple< int, std::string > ColumnKey
Definition: Types.h:36
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:500
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:509
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
void dropColumn(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:1910
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logicalTableDesc) const
Definition: Catalog.cpp:3589
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:501
std::string to_string(char const *&&v)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:499
std::string to_upper(const std::string &str)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:528
std::string columnName
std::tuple< int, int > ColumnIdKey
Definition: Types.h:38
+ Here is the call graph for this function:

◆ dropForeignServer()

void Catalog_Namespace::Catalog::dropForeignServer ( const std::string &  server_name)

Drops/deletes a foreign server DB object.

Parameters
server_name- Name of foreign server that will be deleted

Definition at line 2589 of file Catalog.cpp.

References CHECK_EQ, foreignServerMap_, foreignServerMapById_, SqliteConnector::getData(), SqliteConnector::getNumRows(), num_rows, SqliteConnector::query(), SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), sqliteConnector_, and to_string().

2589  {
2590  cat_write_lock write_lock(this);
2591  cat_sqlite_lock sqlite_lock(this);
2592 
2594  "SELECT id from omnisci_foreign_servers where name = ?",
2595  std::vector<std::string>{server_name});
2597  if (num_rows > 0) {
2598  CHECK_EQ(size_t(1), num_rows);
2599  auto server_id = sqliteConnector_.getData<int32_t>(0, 0);
2601  "SELECT table_id from omnisci_foreign_tables where server_id = ?",
2602  std::to_string(server_id));
2603  if (sqliteConnector_.getNumRows() > 0) {
2604  throw std::runtime_error{"Foreign server \"" + server_name +
2605  "\" is referenced "
2606  "by existing foreign tables and cannot be dropped."};
2607  }
2608  sqliteConnector_.query("BEGIN TRANSACTION");
2609  try {
2611  "DELETE FROM omnisci_foreign_servers WHERE name = ?",
2612  std::vector<std::string>{server_name});
2613  } catch (const std::exception& e) {
2614  sqliteConnector_.query("ROLLBACK TRANSACTION");
2615  throw;
2616  }
2617  sqliteConnector_.query("END TRANSACTION");
2618  foreignServerMap_.erase(server_name);
2619  foreignServerMapById_.erase(server_id);
2620  }
2621 }
#define CHECK_EQ(x, y)
Definition: Logger.h:205
T getData(const int row, const int col)
const int8_t const int64_t * num_rows
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:509
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:507
ForeignServerMap foreignServerMap_
Definition: Catalog.h:506
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
virtual size_t getNumRows() const
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
+ Here is the call graph for this function:

◆ dropFsiSchemasAndTables()

void Catalog_Namespace::Catalog::dropFsiSchemasAndTables ( )
protected

Definition at line 623 of file Catalog.cpp.

References CHECK_EQ, executeDropTableSqliteQueries(), SqliteConnector::getData(), SqliteConnector::getNumRows(), Catalog_Namespace::SysCatalog::instance(), num_rows, SqliteConnector::query(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivilegesFromAll(), sqliteConnector_, TableDBObjectType, tableDescriptorMap_, tableDescriptorMapById_, TableDescriptor::tableId, and to_upper().

Referenced by CheckAndExecuteMigrations().

623  {
624  std::vector<foreign_storage::ForeignTable> foreign_tables{};
625  {
626  cat_sqlite_lock sqlite_lock(this);
627  sqliteConnector_.query("BEGIN TRANSACTION");
628  try {
630  "SELECT name FROM sqlite_master WHERE type='table' AND "
631  "name IN ('omnisci_foreign_servers', 'omnisci_foreign_tables', "
632  "'omnisci_user_mappings')");
633  if (sqliteConnector_.getNumRows() > 0) {
635  "SELECT tableid, name, isview, storage_type FROM mapd_tables "
636  "WHERE storage_type = 'FOREIGN_TABLE'");
638  for (size_t r = 0; r < num_rows; r++) {
639  foreign_storage::ForeignTable foreign_table{};
640  foreign_table.tableId = sqliteConnector_.getData<int>(r, 0);
641  foreign_table.tableName = sqliteConnector_.getData<std::string>(r, 1);
642  foreign_table.isView = sqliteConnector_.getData<bool>(r, 2);
643  foreign_table.storageType = sqliteConnector_.getData<std::string>(r, 3);
644  foreign_tables.emplace_back(foreign_table);
645  }
646  for (auto& foreign_table : foreign_tables) {
647  tableDescriptorMap_[to_upper(foreign_table.tableName)] = &foreign_table;
648  tableDescriptorMapById_[foreign_table.tableId] = &foreign_table;
649  executeDropTableSqliteQueries(&foreign_table);
650  }
651  sqliteConnector_.query("SELECT COUNT(*) FROM omnisci_foreign_tables");
652  CHECK_EQ(size_t(1), sqliteConnector_.getNumRows());
653  CHECK_EQ(0, sqliteConnector_.getData<int>(0, 0));
654 
655  sqliteConnector_.query("DROP TABLE omnisci_foreign_tables");
656  sqliteConnector_.query("DROP TABLE omnisci_foreign_servers");
657  }
658  } catch (std::exception& e) {
659  sqliteConnector_.query("ROLLBACK TRANSACTION");
660  throw;
661  }
662  sqliteConnector_.query("END TRANSACTION");
663  }
664 
665  for (auto& foreign_table : foreign_tables) {
667  DBObject(foreign_table.tableName, TableDBObjectType), this);
668  tableDescriptorMap_.erase(to_upper(foreign_table.tableName));
669  tableDescriptorMapById_.erase(foreign_table.tableId);
670  }
671 }
#define CHECK_EQ(x, y)
Definition: Logger.h:205
void executeDropTableSqliteQueries(const TableDescriptor *td)
Definition: Catalog.cpp:3245
T getData(const int row, const int col)
const int8_t const int64_t * num_rows
SqliteConnector sqliteConnector_
Definition: Catalog.h:509
void revokeDBObjectPrivilegesFromAll(DBObject object, Catalog *catalog)
virtual void query(const std::string &queryString)
static SysCatalog & instance()
Definition: SysCatalog.h:286
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:499
std::string to_upper(const std::string &str)
virtual size_t getNumRows() const
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:498
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ dropTable()

void Catalog_Namespace::Catalog::dropTable ( const TableDescriptor td)

Definition at line 3204 of file Catalog.cpp.

References CHECK, doDropTable(), getMetadataForTable(), Catalog_Namespace::SysCatalog::instance(), TableDescriptor::isView, logicalToPhysicalTableMapById_, SqliteConnector::query(), SqliteConnector::query_with_text_param(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivilegesFromAll(), sqliteConnector_, TableDBObjectType, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and ViewDBObjectType.

3204  {
3207  cat_write_lock write_lock(this);
3208  cat_sqlite_lock sqlite_lock(this);
3209  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
3210  sqliteConnector_.query("BEGIN TRANSACTION");
3211  try {
3212  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
3213  // remove all corresponding physical tables if this is a logical table
3214  const auto physicalTables = physicalTableIt->second;
3215  CHECK(!physicalTables.empty());
3216  for (size_t i = 0; i < physicalTables.size(); i++) {
3217  int32_t physical_tb_id = physicalTables[i];
3218  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
3219  CHECK(phys_td);
3220  doDropTable(phys_td);
3221  }
3222 
3223  // remove corresponding record from the logicalToPhysicalTableMap in sqlite database
3225  "DELETE FROM mapd_logical_to_physical WHERE logical_table_id = ?",
3226  std::to_string(td->tableId));
3228  }
3229  doDropTable(td);
3230  } catch (std::exception& e) {
3231  sqliteConnector_.query("ROLLBACK TRANSACTION");
3232  throw;
3233  }
3234  sqliteConnector_.query("END TRANSACTION");
3235 }
void doDropTable(const TableDescriptor *td)
Definition: Catalog.cpp:3237
std::string tableName
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
SqliteConnector sqliteConnector_
Definition: Catalog.h:509
void revokeDBObjectPrivilegesFromAll(DBObject object, Catalog *catalog)
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
static SysCatalog & instance()
Definition: SysCatalog.h:286
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock
specifies the content in-memory of a row in the table metadata table
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:516
+ Here is the call graph for this function:

◆ dropTableFromJsonUnlocked()

void Catalog_Namespace::Catalog::dropTableFromJsonUnlocked ( const std::string &  table_name) const
protected

Definition at line 2420 of file Catalog.cpp.

References basePath_, CHECK, currentDB_, Catalog_Namespace::DBMetadata::dbName, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and VLOG.

Referenced by doDropTable().

2420  {
2421  // relies on the catalog write lock
2422  using namespace rapidjson;
2423 
2424  VLOG(1) << "Dropping temporary table " << table_name << " to JSON for Calcite.";
2425 
2426  const auto db_name = currentDB_.dbName;
2427  const auto file_path = table_json_filepath(basePath_, db_name);
2428 
2429  CHECK(boost::filesystem::exists(file_path));
2430  Document d;
2431 
2432  std::ifstream reader(file_path.string());
2433  CHECK(reader.is_open());
2434  IStreamWrapper json_read_wrapper(reader);
2435  d.ParseStream(json_read_wrapper);
2436 
2437  CHECK(d.IsObject());
2438  auto table_name_ref = StringRef(table_name.c_str());
2439  CHECK(d.HasMember(table_name_ref));
2440  CHECK(d.RemoveMember(table_name_ref));
2441 
2442  // Overwrite the existing file
2443  std::ofstream writer(file_path.string(), writer.trunc | writer.out);
2444  CHECK(writer.is_open());
2445  OStreamWrapper json_wrapper(writer);
2446 
2447  Writer<OStreamWrapper> json_writer(json_wrapper);
2448  d.Accept(json_writer);
2449  writer.close();
2450 }
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:153
#define CHECK(condition)
Definition: Logger.h:197
#define VLOG(n)
Definition: Logger.h:291
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ dumpCreateTable()

std::string Catalog_Namespace::Catalog::dumpCreateTable ( const TableDescriptor td,
bool  multiline_formatting = true,
bool  dump_defaults = false 
) const

Definition at line 4135 of file Catalog.cpp.

References CHECK, Catalog_Namespace::contains_spaces(), Catalog_Namespace::contains_sql_reserved_chars(), Data_Namespace::CPU_LEVEL, DEFAULT_FRAGMENT_ROWS, DEFAULT_MAX_CHUNK_SIZE, DEFAULT_MAX_ROWS, DEFAULT_PAGE_SIZE, TableDescriptor::fragPageSize, g_leaf_count, getAllColumnMetadataForTable(), getMetadataForColumn(), TableDescriptor::hasDeletedCol, Catalog_Namespace::is_reserved_sql_keyword(), TableDescriptor::isView, join(), kCHAR, kENCODING_DICT, kENCODING_GEOINT, TableDescriptor::keyMetainfo, kTEXT, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRows, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::unserialize_key_metainfo(), and TableDescriptor::viewSQL.

4137  {
4138  cat_read_lock read_lock(this);
4139 
4140  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
4141  std::ostringstream os;
4142 
4143  if (foreign_table) {
4144  os << "CREATE FOREIGN TABLE " << td->tableName << " (";
4145  } else if (!td->isView) {
4146  os << "CREATE ";
4148  os << "TEMPORARY ";
4149  }
4150  os << "TABLE " + td->tableName + " (";
4151  } else {
4152  os << "CREATE VIEW " + td->tableName + " AS " << td->viewSQL;
4153  return os.str();
4154  }
4155  // scan column defines
4156  std::vector<std::string> shared_dicts;
4157  std::set<std::string> shared_dict_column_names;
4158  unserialize_key_metainfo(shared_dicts, shared_dict_column_names, td->keyMetainfo);
4159  // gather column defines
4160  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
4161  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
4162  bool first = true;
4163  for (const auto cd : cds) {
4164  if (!(cd->isSystemCol || cd->isVirtualCol)) {
4165  const auto& ti = cd->columnType;
4166  if (!first) {
4167  os << ",";
4168  if (!multiline_formatting) {
4169  os << " ";
4170  }
4171  } else {
4172  first = false;
4173  }
4174  if (multiline_formatting) {
4175  os << "\n ";
4176  }
4177  // column name
4178  bool column_name_needs_quotes = is_reserved_sql_keyword(cd->columnName) ||
4179  contains_spaces(cd->columnName) ||
4180  contains_sql_reserved_chars(cd->columnName);
4181  if (!column_name_needs_quotes) {
4182  os << cd->columnName;
4183  } else {
4184  os << "\"" << cd->columnName << "\"";
4185  }
4186  // CHAR is perculiar... better dump it as TEXT(32) like \d does
4187  if (ti.get_type() == SQLTypes::kCHAR) {
4188  os << " "
4189  << "TEXT";
4190  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
4191  os << " "
4192  << "TEXT[]";
4193  } else {
4194  os << " " << ti.get_type_name();
4195  }
4196  os << (ti.get_notnull() ? " NOT NULL" : "");
4197  if (shared_dict_column_names.find(cd->columnName) ==
4198  shared_dict_column_names.end()) {
4199  // avoids "Exception: Column ... shouldn't specify an encoding, it borrows it from
4200  // the referenced column"
4201  if (ti.is_string() || (ti.is_array() && ti.get_subtype() == kTEXT)) {
4202  auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
4203  if (ti.get_compression() == kENCODING_DICT) {
4204  os << " ENCODING " << ti.get_compression_name() << "(" << (size * 8) << ")";
4205  } else {
4206  os << " ENCODING NONE";
4207  }
4208  } else if (ti.is_date_in_days() ||
4209  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
4210  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
4211  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
4212  } else if (ti.is_geometry()) {
4213  if (ti.get_compression() == kENCODING_GEOINT) {
4214  os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
4215  << ")";
4216  } else {
4217  os << " ENCODING NONE";
4218  }
4219  }
4220  }
4221  }
4222  }
4223  // gather SHARED DICTIONARYs
4224  if (shared_dicts.size()) {
4225  std::string comma;
4226  if (!multiline_formatting) {
4227  comma = ", ";
4228  } else {
4229  comma = ",\n ";
4230  }
4231  os << comma;
4232  os << boost::algorithm::join(shared_dicts, comma);
4233  }
4234  os << ")";
4235 
4236  std::vector<std::string> with_options;
4237  if (foreign_table) {
4238  if (multiline_formatting) {
4239  os << "\n";
4240  } else {
4241  os << " ";
4242  }
4243  os << "SERVER " << foreign_table->foreign_server->name;
4244 
4245  // gather WITH options ...
4246  for (const auto& [option, value] : foreign_table->options) {
4247  with_options.emplace_back(option + "='" + value + "'");
4248  }
4249  }
4250 
4251  if (dump_defaults || td->maxFragRows != DEFAULT_FRAGMENT_ROWS) {
4252  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
4253  }
4254  if (!foreign_table && (dump_defaults || td->maxChunkSize != DEFAULT_MAX_CHUNK_SIZE)) {
4255  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
4256  }
4257  if (!foreign_table && (dump_defaults || td->fragPageSize != DEFAULT_PAGE_SIZE)) {
4258  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
4259  }
4260  if (!foreign_table && (dump_defaults || td->maxRows != DEFAULT_MAX_ROWS)) {
4261  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
4262  }
4263  if (!foreign_table && (dump_defaults || !td->hasDeletedCol)) {
4264  with_options.push_back(td->hasDeletedCol ? "VACUUM='DELAYED'" : "VACUUM='IMMEDIATE'");
4265  }
4266  if (!foreign_table && !td->partitions.empty()) {
4267  with_options.push_back("PARTITIONS='" + td->partitions + "'");
4268  }
4269  if (!foreign_table && td->nShards > 0) {
4270  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
4271  CHECK(shard_cd);
4272  with_options.push_back(
4273  "SHARD_COUNT=" +
4274  std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
4275  }
4276  if (!foreign_table && td->sortedColumnId > 0) {
4277  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
4278  CHECK(sort_cd);
4279  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
4280  }
4281 
4282  if (!with_options.empty()) {
4283  if (!multiline_formatting) {
4284  os << " ";
4285  } else {
4286  os << "\n";
4287  }
4288  os << "WITH (" + boost::algorithm::join(with_options, ", ") + ")";
4289  }
4290  os << ";";
4291  return os.str();
4292 }
std::string partitions
std::string tableName
std::string join(T const &container, std::string const &delim)
#define DEFAULT_MAX_CHUNK_SIZE
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:112
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
void unserialize_key_metainfo(std::vector< std::string > &shared_dicts, std::set< std::string > &shared_dict_column_names, const std::string keyMetainfo)
Definition: Catalog.cpp:4085
bool contains_spaces(std::string_view str)
returns true if the string contains one or more spaces
Definition: Catalog.cpp:4116
std::string to_string(char const *&&v)
#define DEFAULT_MAX_ROWS
bool contains_sql_reserved_chars(std::string_view str, std::string_view chars="`~!@#$%^&*()-=+[{]}\;:'<.>/?")
returns true if the string contains one or more OmniSci SQL reserved characters
Definition: Catalog.cpp:4123
std::string keyMetainfo
#define DEFAULT_PAGE_SIZE
Definition: sqltypes.h:54
#define DEFAULT_FRAGMENT_ROWS
Data_Namespace::MemoryLevel persistenceLevel
Definition: sqltypes.h:43
mapd_shared_lock< mapd_shared_mutex > read_lock
#define CHECK(condition)
Definition: Logger.h:197
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1715
std::string viewSQL
size_t g_leaf_count
Definition: ParserNode.cpp:68
bool is_reserved_sql_keyword(std::string_view str)
returns true if the string equals an OmniSci SQL reserved keyword
Definition: Catalog.cpp:4130
+ Here is the call graph for this function:

◆ dumpSchema()

std::string Catalog_Namespace::Catalog::dumpSchema ( const TableDescriptor td) const

Definition at line 3988 of file Catalog.cpp.

References CHECK, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, TableDescriptor::fragPageSize, g_leaf_count, getAllColumnMetadataForTable(), getMetadataForColumn(), TableDescriptor::hasDeletedCol, join(), kCHAR, kENCODING_DICT, kENCODING_GEOINT, kTEXT, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRows, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::tableId, and to_string().

3988  {
3989  cat_read_lock read_lock(this);
3990 
3991  std::ostringstream os;
3992  os << "CREATE TABLE @T (";
3993  // gather column defines
3994  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
3995  std::string comma;
3996  std::vector<std::string> shared_dicts;
3997  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
3998  for (const auto cd : cds) {
3999  if (!(cd->isSystemCol || cd->isVirtualCol)) {
4000  const auto& ti = cd->columnType;
4001  os << comma << cd->columnName;
4002  // CHAR is perculiar... better dump it as TEXT(32) like \d does
4003  if (ti.get_type() == SQLTypes::kCHAR) {
4004  os << " "
4005  << "TEXT";
4006  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
4007  os << " "
4008  << "TEXT[]";
4009  } else {
4010  os << " " << ti.get_type_name();
4011  }
4012  os << (ti.get_notnull() ? " NOT NULL" : "");
4013  if (ti.is_string() || (ti.is_array() && ti.get_subtype() == kTEXT)) {
4014  auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
4015  if (ti.get_compression() == kENCODING_DICT) {
4016  // if foreign reference, get referenced tab.col
4017  const auto dict_id = ti.get_comp_param();
4018  const DictRef dict_ref(currentDB_.dbId, dict_id);
4019  const auto dict_it = dictDescriptorMapByRef_.find(dict_ref);
4020  CHECK(dict_it != dictDescriptorMapByRef_.end());
4021  const auto dict_name = dict_it->second->dictName;
4022  // when migrating a table, any foreign dict ref will be dropped
4023  // and the first cd of a dict will become root of the dict
4024  if (dict_root_cds.end() == dict_root_cds.find(dict_name)) {
4025  dict_root_cds[dict_name] = cd;
4026  os << " ENCODING " << ti.get_compression_name() << "(" << (size * 8) << ")";
4027  } else {
4028  const auto dict_root_cd = dict_root_cds[dict_name];
4029  shared_dicts.push_back("SHARED DICTIONARY (" + cd->columnName +
4030  ") REFERENCES @T(" + dict_root_cd->columnName + ")");
4031  // "... shouldn't specify an encoding, it borrows from the referenced column"
4032  }
4033  } else {
4034  os << " ENCODING NONE";
4035  }
4036  } else if (ti.is_date_in_days() ||
4037  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
4038  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
4039  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
4040  } else if (ti.is_geometry()) {
4041  if (ti.get_compression() == kENCODING_GEOINT) {
4042  os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
4043  << ")";
4044  } else {
4045  os << " ENCODING NONE";
4046  }
4047  }
4048  comma = ", ";
4049  }
4050  }
4051  // gather SHARED DICTIONARYs
4052  if (shared_dicts.size()) {
4053  os << ", " << boost::algorithm::join(shared_dicts, ", ");
4054  }
4055  // gather WITH options ...
4056  std::vector<std::string> with_options;
4057  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
4058  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
4059  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
4060  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
4061  with_options.emplace_back(td->hasDeletedCol ? "VACUUM='DELAYED'"
4062  : "VACUUM='IMMEDIATE'");
4063  if (!td->partitions.empty()) {
4064  with_options.push_back("PARTITIONS='" + td->partitions + "'");
4065  }
4066  if (td->nShards > 0) {
4067  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
4068  CHECK(shard_cd);
4069  os << ", SHARD KEY(" << shard_cd->columnName << ")";
4070  with_options.push_back(
4071  "SHARD_COUNT=" +
4072  std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
4073  }
4074  if (td->sortedColumnId > 0) {
4075  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
4076  CHECK(sort_cd);
4077  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
4078  }
4079  os << ") WITH (" + boost::algorithm::join(with_options, ", ") + ");";
4080  return os.str();
4081 }
std::string partitions
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:502
std::string join(T const &container, std::string const &delim)
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:112
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
std::string to_string(char const *&&v)
Definition: sqltypes.h:54
Definition: sqltypes.h:43
mapd_shared_lock< mapd_shared_mutex > read_lock
#define CHECK(condition)
Definition: Logger.h:197
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1715
size_t g_leaf_count
Definition: ParserNode.cpp:68
+ Here is the call graph for this function:

◆ eraseDBData()

void Catalog_Namespace::Catalog::eraseDBData ( )

Definition at line 3701 of file Catalog.cpp.

References basePath_, calciteMgr_, currentDB_, Catalog_Namespace::DBMetadata::dbName, eraseTablePhysicalData(), getAllTableMetadata(), and run_benchmark_import::tables.

3701  {
3702  cat_write_lock write_lock(this);
3703  // Physically erase all tables and dictionaries from disc and memory
3704  const auto tables = getAllTableMetadata();
3705  for (const auto table : tables) {
3706  eraseTablePhysicalData(table);
3707  }
3708  // Physically erase database metadata
3709  boost::filesystem::remove(basePath_ + "/mapd_catalogs/" + currentDB_.dbName);
3710  calciteMgr_->updateMetadata(currentDB_.dbName, "");
3711 }
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:1741
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:3713
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:514
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
mapd_unique_lock< mapd_shared_mutex > write_lock
+ Here is the call graph for this function:

◆ eraseTablePhysicalData()

void Catalog_Namespace::Catalog::eraseTablePhysicalData ( const TableDescriptor td)

Definition at line 3713 of file Catalog.cpp.

References calciteMgr_, CHECK, Data_Namespace::CPU_LEVEL, currentDB_, dataMgr_, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, TableDescriptor::fragmenter, Data_Namespace::GPU_LEVEL, INJECT_TIMER, TableDescriptor::isView, removeTableFromMap(), tableDescriptorMapById_, TableDescriptor::tableId, and TableDescriptor::tableName.

Referenced by doDropTable(), and eraseDBData().

3713  {
3714  const int tableId = td->tableId;
3715  // must destroy fragmenter before deleteChunks is called.
3716  if (td->fragmenter != nullptr) {
3717  auto tableDescIt = tableDescriptorMapById_.find(tableId);
3718  CHECK(tableDescIt != tableDescriptorMapById_.end());
3719  {
3720  INJECT_TIMER(deleting_fragmenter);
3721  tableDescIt->second->fragmenter = nullptr;
3722  }
3723  }
3724  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
3725  {
3726  INJECT_TIMER(deleteChunksWithPrefix);
3727  // assuming deleteChunksWithPrefix is atomic
3728  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
3729  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
3730  }
3731  if (!td->isView) {
3732  INJECT_TIMER(Remove_Table);
3733  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
3734  }
3735  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
3736  {
3737  INJECT_TIMER(removeTableFromMap_);
3738  removeTableFromMap(td->tableName, tableId);
3739  }
3740 }
std::string tableName
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:511
#define INJECT_TIMER(DESC)
Definition: measure.h:93
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:499
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:514
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1199
#define CHECK(condition)
Definition: Logger.h:197
std::vector< int > ChunkKey
Definition: types.h:37
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ executeDropTableSqliteQueries()

void Catalog_Namespace::Catalog::executeDropTableSqliteQueries ( const TableDescriptor td)
protected

Definition at line 3245 of file Catalog.cpp.

References StorageType::FOREIGN_TABLE, SqliteConnector::getData(), SqliteConnector::getNumRows(), TableDescriptor::isView, kENCODING_DICT, SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), sqliteConnector_, TableDescriptor::storageType, TableDescriptor::tableId, and to_string().

Referenced by doDropTable(), and dropFsiSchemasAndTables().

3245  {
3246  const int tableId = td->tableId;
3247  sqliteConnector_.query_with_text_param("DELETE FROM mapd_tables WHERE tableid = ?",
3248  std::to_string(tableId));
3250  "select comp_param from mapd_columns where compression = ? and tableid = ?",
3251  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
3252  int numRows = sqliteConnector_.getNumRows();
3253  std::vector<int> dict_id_list;
3254  for (int r = 0; r < numRows; ++r) {
3255  dict_id_list.push_back(sqliteConnector_.getData<int>(r, 0));
3256  }
3257  for (auto dict_id : dict_id_list) {
3259  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
3260  std::vector<std::string>{std::to_string(dict_id)});
3261  }
3263  "DELETE FROM mapd_dictionaries WHERE dictid in (select comp_param from "
3264  "mapd_columns where compression = ? "
3265  "and tableid = ?) and refcount = 0",
3266  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
3267  sqliteConnector_.query_with_text_param("DELETE FROM mapd_columns WHERE tableid = ?",
3268  std::to_string(tableId));
3269  if (td->isView) {
3270  sqliteConnector_.query_with_text_param("DELETE FROM mapd_views WHERE tableid = ?",
3271  std::to_string(tableId));
3272  }
3275  "DELETE FROM omnisci_foreign_tables WHERE table_id = ?", std::to_string(tableId));
3276  }
3277 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:509
std::string storageType
std::string to_string(char const *&&v)
virtual size_t getNumRows() const
static constexpr char const * FOREIGN_TABLE
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ expandGeoColumn()

void Catalog_Namespace::Catalog::expandGeoColumn ( const ColumnDescriptor cd,
std::list< ColumnDescriptor > &  columns 
)
static

Definition at line 1999 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, IS_GEO, kARRAY, kDOUBLE, kENCODING_GEOINT, kENCODING_NONE, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kTINYINT, SQLTypeInfo::set_size(), and SQLTypeInfo::set_subtype().

Referenced by createTable().

2000  {
2001  const auto& col_ti = cd.columnType;
2002  if (IS_GEO(col_ti.get_type())) {
2003  switch (col_ti.get_type()) {
2004  case kPOINT: {
2005  ColumnDescriptor physical_cd_coords(true);
2006  physical_cd_coords.columnName = cd.columnName + "_coords";
2007  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2008  // Raw data: compressed/uncompressed coords
2009  coords_ti.set_subtype(kTINYINT);
2010  size_t unit_size;
2011  if (col_ti.get_compression() == kENCODING_GEOINT &&
2012  col_ti.get_comp_param() == 32) {
2013  unit_size = 4 * sizeof(int8_t);
2014  } else {
2015  CHECK(col_ti.get_compression() == kENCODING_NONE);
2016  unit_size = 8 * sizeof(int8_t);
2017  }
2018  coords_ti.set_size(2 * unit_size);
2019  physical_cd_coords.columnType = coords_ti;
2020  columns.push_back(physical_cd_coords);
2021 
2022  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2023 
2024  break;
2025  }
2026  case kLINESTRING: {
2027  ColumnDescriptor physical_cd_coords(true);
2028  physical_cd_coords.columnName = cd.columnName + "_coords";
2029  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2030  // Raw data: compressed/uncompressed coords
2031  coords_ti.set_subtype(kTINYINT);
2032  physical_cd_coords.columnType = coords_ti;
2033  columns.push_back(physical_cd_coords);
2034 
2035  ColumnDescriptor physical_cd_bounds(true);
2036  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2037  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2038  bounds_ti.set_subtype(kDOUBLE);
2039  bounds_ti.set_size(4 * sizeof(double));
2040  physical_cd_bounds.columnType = bounds_ti;
2041  columns.push_back(physical_cd_bounds);
2042 
2043  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2044 
2045  break;
2046  }
2047  case kPOLYGON: {
2048  ColumnDescriptor physical_cd_coords(true);
2049  physical_cd_coords.columnName = cd.columnName + "_coords";
2050  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2051  // Raw data: compressed/uncompressed coords
2052  coords_ti.set_subtype(kTINYINT);
2053  physical_cd_coords.columnType = coords_ti;
2054  columns.push_back(physical_cd_coords);
2055 
2056  ColumnDescriptor physical_cd_ring_sizes(true);
2057  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
2058  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2059  ring_sizes_ti.set_subtype(kINT);
2060  physical_cd_ring_sizes.columnType = ring_sizes_ti;
2061  columns.push_back(physical_cd_ring_sizes);
2062 
2063  ColumnDescriptor physical_cd_bounds(true);
2064  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2065  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2066  bounds_ti.set_subtype(kDOUBLE);
2067  bounds_ti.set_size(4 * sizeof(double));
2068  physical_cd_bounds.columnType = bounds_ti;
2069  columns.push_back(physical_cd_bounds);
2070 
2071  ColumnDescriptor physical_cd_render_group(true);
2072  physical_cd_render_group.columnName = cd.columnName + "_render_group";
2073  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, col_ti.get_notnull());
2074  physical_cd_render_group.columnType = render_group_ti;
2075  columns.push_back(physical_cd_render_group);
2076 
2077  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2078 
2079  break;
2080  }
2081  case kMULTIPOLYGON: {
2082  ColumnDescriptor physical_cd_coords(true);
2083  physical_cd_coords.columnName = cd.columnName + "_coords";
2084  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2085  // Raw data: compressed/uncompressed coords
2086  coords_ti.set_subtype(kTINYINT);
2087  physical_cd_coords.columnType = coords_ti;
2088  columns.push_back(physical_cd_coords);
2089 
2090  ColumnDescriptor physical_cd_ring_sizes(true);
2091  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
2092  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2093  ring_sizes_ti.set_subtype(kINT);
2094  physical_cd_ring_sizes.columnType = ring_sizes_ti;
2095  columns.push_back(physical_cd_ring_sizes);
2096 
2097  ColumnDescriptor physical_cd_poly_rings(true);
2098  physical_cd_poly_rings.columnName = cd.columnName + "_poly_rings";
2099  SQLTypeInfo poly_rings_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2100  poly_rings_ti.set_subtype(kINT);
2101  physical_cd_poly_rings.columnType = poly_rings_ti;
2102  columns.push_back(physical_cd_poly_rings);
2103 
2104  ColumnDescriptor physical_cd_bounds(true);
2105  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2106  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2107  bounds_ti.set_subtype(kDOUBLE);
2108  bounds_ti.set_size(4 * sizeof(double));
2109  physical_cd_bounds.columnType = bounds_ti;
2110  columns.push_back(physical_cd_bounds);
2111 
2112  ColumnDescriptor physical_cd_render_group(true);
2113  physical_cd_render_group.columnName = cd.columnName + "_render_group";
2114  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, col_ti.get_notnull());
2115  physical_cd_render_group.columnType = render_group_ti;
2116  columns.push_back(physical_cd_render_group);
2117 
2118  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2119 
2120  break;
2121  }
2122  default:
2123  throw runtime_error("Unrecognized geometry type.");
2124  break;
2125  }
2126  }
2127 }
void set_size(int s)
Definition: sqltypes.h:357
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:350
specifies the content in-memory of a row in the column metadata table
#define CHECK(condition)
Definition: Logger.h:197
Definition: sqltypes.h:47
SQLTypeInfo columnType
std::string columnName
#define IS_GEO(T)
Definition: sqltypes.h:174
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ generatePhysicalTableName()

std::string Catalog_Namespace::Catalog::generatePhysicalTableName ( const std::string &  logicalTableName,
const int32_t &  shardNumber 
)
protected

Definition at line 3742 of file Catalog.cpp.

References physicalTableNameTag_, and to_string().

Referenced by createShardedTable(), and renameTable().

3743  {
3744  std::string physicalTableName =
3745  logicalTableName + physicalTableNameTag_ + std::to_string(shardNumber);
3746  return (physicalTableName);
3747 }
std::string to_string(char const *&&v)
static const std::string physicalTableNameTag_
Definition: Catalog.h:518
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ get() [1/3]

std::shared_ptr< Catalog > Catalog_Namespace::Catalog::get ( const std::string &  dbName)
static

Definition at line 3753 of file Catalog.cpp.

References mapd_cat_map_.

Referenced by checkedGet(), EmbeddedDatabase::DBEngineImpl::DBEngineImpl(), and main().

3753  {
3754  auto cat_it = mapd_cat_map_.find(dbName);
3755  if (cat_it != mapd_cat_map_.end()) {
3756  return cat_it->second;
3757  }
3758  return nullptr;
3759 }
static std::map< std::string, std::shared_ptr< Catalog > > mapd_cat_map_
Definition: Catalog.h:531
+ Here is the caller graph for this function:

◆ get() [2/3]

std::shared_ptr< Catalog > Catalog_Namespace::Catalog::get ( const int32_t  db_id)
static

Definition at line 3761 of file Catalog.cpp.

References mapd_cat_map_.

3761  {
3762  for (const auto& entry : mapd_cat_map_) {
3763  if (entry.second->currentDB_.dbId == db_id) {
3764  return entry.second;
3765  }
3766  }
3767  return nullptr;
3768 }
static std::map< std::string, std::shared_ptr< Catalog > > mapd_cat_map_
Definition: Catalog.h:531

◆ get() [3/3]

static std::shared_ptr<Catalog> Catalog_Namespace::Catalog::get ( const std::string &  basePath,
const DBMetadata curDB,
std::shared_ptr< Data_Namespace::DataMgr dataMgr,
const std::vector< LeafHostInfo > &  string_dict_hosts,
std::shared_ptr< Calcite calcite,
bool  is_new_db 
)
static

◆ getAllColumnMetadataForTable()

std::list< const ColumnDescriptor * > Catalog_Namespace::Catalog::getAllColumnMetadataForTable ( const int  tableId,
const bool  fetchSystemColumns,
const bool  fetchVirtualColumns,
const bool  fetchPhysicalColumns 
) const

Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular table specified by table id.

Parameters
tableIdtable id we want the column metadata for
Returns
list of pointers to const ColumnDescriptor structs - one for each and every column in the table

Definition at line 1715 of file Catalog.cpp.

References getAllColumnMetadataForTableUnlocked().

Referenced by Analyzer::RangeTableEntry::add_all_column_descs(), Parser::InsertStmt::analyze(), Parser::InsertValuesStmt::analyze(), Fragmenter_Namespace::copyDataOfShard(), Parser::InsertValuesStmt::determineLeafIndex(), dumpCreateTable(), dumpSchema(), Analyzer::RangeTableEntry::expand_star_in_targetlist(), getDictionaryToColumnMapping(), getShardColumnMetadataForTable(), getTableDictDirectories(), TableOptimizer::recomputeMetadata(), and validate_shared_dictionary().

1719  {
1720  cat_read_lock read_lock(this);
1722  tableId, fetchSystemColumns, fetchVirtualColumns, fetchPhysicalColumns);
1723 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:112
std::list< const ColumnDescriptor * > getAllColumnMetadataForTableUnlocked(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Definition: Catalog.cpp:1725
mapd_shared_lock< mapd_shared_mutex > read_lock
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getAllColumnMetadataForTableImpl()

void Catalog_Namespace::Catalog::getAllColumnMetadataForTableImpl ( const TableDescriptor td,
std::list< const ColumnDescriptor *> &  colDescs,
const bool  fetchSystemColumns,
const bool  fetchVirtualColumns,
const bool  fetchPhysicalColumns 
) const
protected

Definition at line 1685 of file Catalog.cpp.

References columnDescriptorMapById_, and TableDescriptor::tableId.

Referenced by getAllColumnMetadataForTableUnlocked(), and instantiateFragmenter().

1690  {
1691  int32_t skip_physical_cols = 0;
1692  for (const auto& columnDescriptor : columnDescriptorMapById_) {
1693  if (!fetchPhysicalColumns && skip_physical_cols > 0) {
1694  --skip_physical_cols;
1695  continue;
1696  }
1697  auto cd = columnDescriptor.second;
1698  if (cd->tableId != td->tableId) {
1699  continue;
1700  }
1701  if (!fetchSystemColumns && cd->isSystemCol) {
1702  continue;
1703  }
1704  if (!fetchVirtualColumns && cd->isVirtualCol) {
1705  continue;
1706  }
1707  if (!fetchPhysicalColumns) {
1708  const auto& col_ti = cd->columnType;
1709  skip_physical_cols = col_ti.get_physical_cols();
1710  }
1711  columnDescriptors.push_back(cd);
1712  }
1713 }
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:501
+ Here is the caller graph for this function:

◆ getAllColumnMetadataForTableUnlocked()

std::list< const ColumnDescriptor * > Catalog_Namespace::Catalog::getAllColumnMetadataForTableUnlocked ( const int  tableId,
const bool  fetchSystemColumns,
const bool  fetchVirtualColumns,
const bool  fetchPhysicalColumns 
) const

Same as above, but without first taking a catalog read lock.

Definition at line 1725 of file Catalog.cpp.

References getAllColumnMetadataForTableImpl(), and getMetadataForTableImpl().

Referenced by getAllColumnMetadataForTable().

1729  {
1730  std::list<const ColumnDescriptor*> columnDescriptors;
1731  const TableDescriptor* td =
1732  getMetadataForTableImpl(tableId, false); // dont instantiate fragmenter
1734  columnDescriptors,
1735  fetchSystemColumns,
1736  fetchVirtualColumns,
1737  fetchPhysicalColumns);
1738  return columnDescriptors;
1739 }
const TableDescriptor * getMetadataForTableImpl(int tableId, const bool populateFragmenter) const
Definition: Catalog.cpp:1428
specifies the content in-memory of a row in the table metadata table
void getAllColumnMetadataForTableImpl(const TableDescriptor *td, std::list< const ColumnDescriptor *> &colDescs, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Definition: Catalog.cpp:1685
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getAllDashboardsMetadata()

list< const DashboardDescriptor * > Catalog_Namespace::Catalog::getAllDashboardsMetadata ( ) const

Definition at line 1750 of file Catalog.cpp.

References dashboardDescriptorMap_.

1750  {
1751  list<const DashboardDescriptor*> view_list;
1752  for (auto p : dashboardDescriptorMap_) {
1753  view_list.push_back(p.second.get());
1754  }
1755  return view_list;
1756 }
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:503

◆ getAllForeignTablesForRefresh()

std::vector< const TableDescriptor * > Catalog_Namespace::Catalog::getAllForeignTablesForRefresh ( ) const

Gets all the foreign tables that are pending refreshes. The list of tables includes tables that are configured for scheduled refreshes with next refresh timestamps that are in the past.

Returns
foreign tables pending refreshes

Definition at line 4305 of file Catalog.cpp.

References CHECK, StorageType::FOREIGN_TABLE, foreign_storage::ForeignTable::REFRESH_TIMING_TYPE_KEY, foreign_storage::ForeignTable::SCHEDULE_REFRESH_TIMING_TYPE, tableDescriptorMapById_, and run_benchmark_import::tables.

4305  {
4306  cat_read_lock read_lock(this);
4307  std::vector<const TableDescriptor*> tables;
4308  for (auto entry : tableDescriptorMapById_) {
4309  auto table_descriptor = entry.second;
4310  if (table_descriptor->storageType == StorageType::FOREIGN_TABLE) {
4311  auto foreign_table = dynamic_cast<foreign_storage::ForeignTable*>(table_descriptor);
4312  CHECK(foreign_table);
4313  auto timing_type_entry = foreign_table->options.find(
4315  CHECK(timing_type_entry != foreign_table->options.end());
4316  int64_t current_time = std::chrono::duration_cast<std::chrono::seconds>(
4317  std::chrono::system_clock::now().time_since_epoch())
4318  .count();
4319  if (timing_type_entry->second ==
4321  foreign_table->next_refresh_time <= current_time) {
4322  tables.emplace_back(foreign_table);
4323  }
4324  }
4325  }
4326  return tables;
4327 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:112
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:499
mapd_shared_lock< mapd_shared_mutex > read_lock
static constexpr const char * REFRESH_TIMING_TYPE_KEY
Definition: ForeignTable.h:29
#define CHECK(condition)
Definition: Logger.h:197
static constexpr char const * FOREIGN_TABLE
static constexpr const char * SCHEDULE_REFRESH_TIMING_TYPE
Definition: ForeignTable.h:35

◆ getAllTableMetadata()

list< const TableDescriptor * > Catalog_Namespace::Catalog::getAllTableMetadata ( ) const

Definition at line 1741 of file Catalog.cpp.

References tableDescriptorMapById_.

Referenced by eraseDBData(), getDictionaryToColumnMapping(), and getTableNamesForUser().

1741  {
1742  cat_read_lock read_lock(this);
1743  list<const TableDescriptor*> table_list;
1744  for (auto p : tableDescriptorMapById_) {
1745  table_list.push_back(p.second);
1746  }
1747  return table_list;
1748 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:112
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:499
mapd_shared_lock< mapd_shared_mutex > read_lock
+ Here is the caller graph for this function:

◆ getBasePath()

const std::string& Catalog_Namespace::Catalog::getBasePath ( ) const
inline

Definition at line 211 of file Catalog.h.

211 { return basePath_; }

◆ getCalciteMgr()

std::shared_ptr<Calcite> Catalog_Namespace::Catalog::getCalciteMgr ( ) const
inline

Definition at line 210 of file Catalog.h.

210 { return calciteMgr_; }
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:514

◆ getColumnDictDirectory()

std::string Catalog_Namespace::Catalog::getColumnDictDirectory ( const ColumnDescriptor cd) const

Definition at line 3956 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnType, currentDB_,