OmniSciDB  bf83d84833
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Catalog_Namespace::Catalog Class Referencefinal

class for a per-database catalog. also includes metadata for the current database and the current user. More...

#include <Catalog.h>

+ Collaboration diagram for Catalog_Namespace::Catalog:

Public Member Functions

 Catalog (const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
 Constructor - takes basePath to already extant data directory for writing. More...
 
 Catalog ()
 Constructor builds a hollow catalog used during constructor of other catalogs. More...
 
 ~Catalog ()
 Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite. More...
 
void createTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
 
void createShardedTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
int32_t createDashboard (DashboardDescriptor &vd)
 
void replaceDashboard (DashboardDescriptor &vd)
 
std::string createLink (LinkDescriptor &ld, size_t min_length)
 
void dropTable (const TableDescriptor *td)
 
void truncateTable (const TableDescriptor *td)
 
void renameTable (const TableDescriptor *td, const std::string &newTableName)
 
void renameColumn (const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
 
void addColumn (const TableDescriptor &td, ColumnDescriptor &cd)
 
void dropColumn (const TableDescriptor &td, const ColumnDescriptor &cd)
 
void removeChunks (const int table_id)
 
void removeFragmenterForTable (const int table_id)
 
const std::map< int, const
ColumnDescriptor * > 
getDictionaryToColumnMapping ()
 
const TableDescriptorgetMetadataForTable (const std::string &tableName, const bool populateFragmenter=true) const
 Returns a pointer to a const TableDescriptor struct matching the provided tableName. More...
 
const TableDescriptorgetMetadataForTableImpl (int tableId, const bool populateFragmenter) const
 
const TableDescriptorgetMetadataForTable (int tableId, bool populateFragmenter=true) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, const std::string &colName) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, int columnId) const
 
const ColumnDescriptorgetMetadataForColumnUnlocked (int tableId, int columnId) const
 
const int getColumnIdBySpi (const int tableId, const size_t spi) const
 
const ColumnDescriptorgetMetadataForColumnBySpi (const int tableId, const size_t spi) const
 
const DashboardDescriptorgetMetadataForDashboard (const std::string &userId, const std::string &dashName) const
 
const DashboardDescriptorgetMetadataForDashboard (const int32_t dashboard_id) const
 
void deleteMetadataForDashboards (const std::vector< int32_t > ids, const UserMetadata &user)
 
const LinkDescriptorgetMetadataForLink (const std::string &link) const
 
const LinkDescriptorgetMetadataForLink (int linkId) const
 
const
foreign_storage::ForeignTable
getForeignTableUnlocked (int tableId) const
 
const
foreign_storage::ForeignTable
getForeignTable (const std::string &tableName) const
 
const
foreign_storage::ForeignTable
getForeignTable (int table_id) const
 
std::list< const
ColumnDescriptor * > 
getAllColumnMetadataForTable (const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular table specified by table id. More...
 
std::list< const
ColumnDescriptor * > 
getAllColumnMetadataForTableUnlocked (const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 
std::list< const
TableDescriptor * > 
getAllTableMetadata () const
 
std::list< const
DashboardDescriptor * > 
getAllDashboardsMetadata () const
 
const DBMetadatagetCurrentDB () const
 
Data_Namespace::DataMgrgetDataMgr () const
 
std::shared_ptr< CalcitegetCalciteMgr () const
 
const std::string & getBasePath () const
 
const DictDescriptorgetMetadataForDict (int dict_ref, bool loadDict=true) const
 
const DictDescriptorgetMetadataForDictUnlocked (int dict_ref, bool loadDict) const
 
const std::vector< LeafHostInfo > & getStringDictionaryHosts () const
 
const ColumnDescriptorgetShardColumnMetadataForTable (const TableDescriptor *td) const
 
std::vector< const
TableDescriptor * > 
getPhysicalTablesDescriptors (const TableDescriptor *logicalTableDesc) const
 
std::vector< std::string > getTableNamesForUser (const UserMetadata &user, const GetTablesType get_tables_type) const
 
std::vector< TableMetadatagetTablesMetadataForUser (const UserMetadata &user_metadata, const GetTablesType get_tables_type, const std::string &filter_table_name) const
 
int32_t getTableEpoch (const int32_t db_id, const int32_t table_id) const
 
void setTableEpoch (const int db_id, const int table_id, const int new_epoch)
 
void setMaxRollbackEpochs (const int32_t table_id, const int32_t max_rollback_epochs)
 
std::vector< TableEpochInfogetTableEpochs (const int32_t db_id, const int32_t table_id) const
 
void setTableEpochs (const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs)
 
void setTableEpochsLogExceptions (const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs)
 
int getDatabaseId () const
 
SqliteConnectorgetSqliteConnector ()
 
void roll (const bool forward)
 
DictRef addDictionary (ColumnDescriptor &cd)
 
void delDictionary (const ColumnDescriptor &cd)
 
void getDictionary (const ColumnDescriptor &cd, std::map< int, StringDictionary * > &stringDicts)
 
const bool checkMetadataForDeletedRecs (const TableDescriptor *td, int column_id) const
 
const ColumnDescriptorgetDeletedColumn (const TableDescriptor *td) const
 
const ColumnDescriptorgetDeletedColumnIfRowsDeleted (const TableDescriptor *td) const
 
void setDeletedColumn (const TableDescriptor *td, const ColumnDescriptor *cd)
 
void setDeletedColumnUnlocked (const TableDescriptor *td, const ColumnDescriptor *cd)
 
int getLogicalTableId (const int physicalTableId) const
 
void checkpoint (const int logicalTableId) const
 
void checkpointWithAutoRollback (const int logical_table_id)
 
std::string name () const
 
void eraseDBData ()
 
void eraseTablePhysicalData (const TableDescriptor *td)
 
void vacuumDeletedRows (const TableDescriptor *td) const
 
void vacuumDeletedRows (const int logicalTableId) const
 
void setForReload (const int32_t tableId)
 
std::vector< std::string > getTableDataDirectories (const TableDescriptor *td) const
 
std::vector< std::string > getTableDictDirectories (const TableDescriptor *td) const
 
std::string getColumnDictDirectory (const ColumnDescriptor *cd) const
 
std::string dumpSchema (const TableDescriptor *td) const
 
std::string dumpCreateTable (const TableDescriptor *td, bool multiline_formatting=true, bool dump_defaults=false) const
 
void createForeignServer (std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
 
const
foreign_storage::ForeignServer
getForeignServer (const std::string &server_name) const
 
const std::unique_ptr< const
foreign_storage::ForeignServer
getForeignServerFromStorage (const std::string &server_name)
 
const std::unique_ptr< const
foreign_storage::ForeignTable
getForeignTableFromStorage (int table_id)
 
void changeForeignServerOwner (const std::string &server_name, const int new_owner_id)
 
void setForeignServerDataWrapper (const std::string &server_name, const std::string &data_wrapper)
 
void setForeignServerOptions (const std::string &server_name, const std::string &options)
 
void renameForeignServer (const std::string &server_name, const std::string &name)
 
void dropForeignServer (const std::string &server_name)
 
void getForeignServersForUser (const rapidjson::Value *filters, const UserMetadata &user, std::vector< const foreign_storage::ForeignServer * > &results)
 
void createDefaultServersIfNotExists ()
 
bool validateNonExistentTableOrView (const std::string &name, const bool if_not_exists)
 
std::vector< const
TableDescriptor * > 
getAllForeignTablesForRefresh () const
 
void updateForeignTableRefreshTimes (const int32_t table_id)
 
void setForeignTableOptions (const std::string &table_name, foreign_storage::OptionsMap &options_map, bool clear_existing_options=true)
 
void updateLeaf (const LeafHostInfo &string_dict_host)
 

Static Public Member Functions

static void expandGeoColumn (const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
 
static const std::string getForeignTableSchema (bool if_not_exists=false)
 
static const std::string getForeignServerSchema (bool if_not_exists=false)
 

Public Attributes

std::mutex sqliteMutex_
 
mapd_shared_mutex sharedMutex_
 
std::atomic< std::thread::id > thread_holding_sqlite_lock
 
std::atomic< std::thread::id > thread_holding_write_lock
 
bool initialized_ = false
 

Static Public Attributes

static thread_local bool thread_holds_read_lock = false
 

Protected Types

using ColumnDescriptorsForRoll = std::vector< std::pair< ColumnDescriptor *, ColumnDescriptor * >>
 

Protected Member Functions

void CheckAndExecuteMigrations ()
 
void CheckAndExecuteMigrationsPostBuildMaps ()
 
void updateDictionaryNames ()
 
void updateTableDescriptorSchema ()
 
void updateFixlenArrayColumns ()
 
void updateGeoColumns ()
 
void updateFrontendViewSchema ()
 
void updateLinkSchema ()
 
void updateFrontendViewAndLinkUsers ()
 
void updateLogicalToPhysicalTableLinkSchema ()
 
void updateLogicalToPhysicalTableMap (const int32_t logical_tb_id)
 
void updateDictionarySchema ()
 
void updatePageSize ()
 
void updateDeletedColumnIndicator ()
 
void updateFrontendViewsToDashboards ()
 
void createFsiSchemas ()
 
void dropFsiSchemasAndTables ()
 
void recordOwnershipOfObjectsInObjectPermissions ()
 
void checkDateInDaysColumnMigration ()
 
void createDashboardSystemRoles ()
 
void buildMaps ()
 
void addTableToMap (const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
 
void addReferenceToForeignDict (ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def, const bool persist_reference)
 
bool setColumnSharedDictionary (ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
void setColumnDictionary (ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
 
void addFrontendViewToMap (DashboardDescriptor &vd)
 
void addFrontendViewToMapNoLock (DashboardDescriptor &vd)
 
void addLinkToMap (LinkDescriptor &ld)
 
void removeTableFromMap (const std::string &tableName, const int tableId, const bool is_on_error=false)
 
void doDropTable (const TableDescriptor *td)
 
void executeDropTableSqliteQueries (const TableDescriptor *td)
 
void doTruncateTable (const TableDescriptor *td)
 
void renamePhysicalTable (const TableDescriptor *td, const std::string &newTableName)
 
void instantiateFragmenter (TableDescriptor *td) const
 
void getAllColumnMetadataForTableImpl (const TableDescriptor *td, std::list< const ColumnDescriptor * > &colDescs, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 
std::string calculateSHA1 (const std::string &data)
 
std::string generatePhysicalTableName (const std::string &logicalTableName, const int32_t &shardNumber)
 
std::vector< DBObjectparseDashboardObjects (const std::string &view_meta, const int &user_id)
 
void createOrUpdateDashboardSystemRole (const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
 
const int getColumnIdBySpiUnlocked (const int table_id, const size_t spi) const
 
void serializeTableJsonUnlocked (const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
 
void dropTableFromJsonUnlocked (const std::string &table_name) const
 

Protected Attributes

std::string basePath_
 
TableDescriptorMap tableDescriptorMap_
 
TableDescriptorMapById tableDescriptorMapById_
 
ColumnDescriptorMap columnDescriptorMap_
 
ColumnDescriptorMapById columnDescriptorMapById_
 
DictDescriptorMapById dictDescriptorMapByRef_
 
DashboardDescriptorMap dashboardDescriptorMap_
 
LinkDescriptorMap linkDescriptorMap_
 
LinkDescriptorMapById linkDescriptorMapById_
 
ForeignServerMap foreignServerMap_
 
ForeignServerMapById foreignServerMapById_
 
SqliteConnector sqliteConnector_
 
DBMetadata currentDB_
 
std::shared_ptr
< Data_Namespace::DataMgr
dataMgr_
 
const std::vector< LeafHostInfostring_dict_hosts_
 
std::shared_ptr< CalcitecalciteMgr_
 
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
 
int nextTempTableId_
 
int nextTempDictId_
 
ColumnDescriptorsForRoll columnDescriptorsForRoll
 

Static Protected Attributes

static const std::string physicalTableNameTag_
 

Private Member Functions

void adjustAlteredTableFiles (const std::string &temp_data_dir, const std::unordered_map< int, int > &all_column_ids_map) const
 
void renameTableDirectories (const std::string &temp_data_dir, const std::vector< std::string > &target_paths, const std::string &name_prefix) const
 
void buildForeignServerMap ()
 
void addForeignTableDetails ()
 
void setForeignServerProperty (const std::string &server_name, const std::string &property, const std::string &value)
 
void setForeignTableProperty (const foreign_storage::ForeignTable *table, const std::string &property, const std::string &value)
 
void alterPhysicalTableMetadata (const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
 
void alterTableMetadata (const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
 
void setTableFileMgrParams (const int table_id, const File_Namespace::FileMgrParams &file_mgr_params)
 
bool filterTableByTypeAndUser (const TableDescriptor *td, const UserMetadata &user_metadata, const GetTablesType get_tables_type) const
 
TableDescriptorgetMutableMetadataForTableUnlocked (int tableId)
 
void createForeignServerNoLocks (std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
 
foreign_storage::ForeignTablegetForeignTableUnlocked (const std::string &tableName) const
 
const CataloggetObjForLock ()
 

Private Attributes

DeletedColumnPerTableMap deletedColumnPerTable_
 

Detailed Description

class for a per-database catalog. also includes metadata for the current database and the current user.

Catalog

Definition at line 101 of file Catalog.h.

Member Typedef Documentation

Definition at line 571 of file Catalog.h.

Constructor & Destructor Documentation

Catalog_Namespace::Catalog::Catalog ( const std::string &  basePath,
const DBMetadata curDB,
std::shared_ptr< Data_Namespace::DataMgr dataMgr,
const std::vector< LeafHostInfo > &  string_dict_hosts,
std::shared_ptr< Calcite calcite,
bool  is_new_db 
)

Constructor - takes basePath to already extant data directory for writing.

Parameters
basePathdirectory path for writing catalog
dbNamename of the database
fragmenterFragmenter object metadata - expects for this directory to already exist
Catalog_Namespace::Catalog::Catalog ( )

Constructor builds a hollow catalog used during constructor of other catalogs.

Definition at line 193 of file Catalog.cpp.

193 {}
Catalog_Namespace::Catalog::~Catalog ( )

Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite.

Definition at line 195 of file Catalog.cpp.

References basePath_, columnDescriptorMap_, currentDB_, Catalog_Namespace::DBMetadata::dbName, g_serialize_temp_tables, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and tableDescriptorMap_.

195  {
197  // must clean up heap-allocated TableDescriptor and ColumnDescriptor structs
198  for (TableDescriptorMap::iterator tableDescIt = tableDescriptorMap_.begin();
199  tableDescIt != tableDescriptorMap_.end();
200  ++tableDescIt) {
201  tableDescIt->second->fragmenter = nullptr;
202  delete tableDescIt->second;
203  }
204 
205  // TableDescriptorMapById points to the same descriptors. No need to delete
206 
207  for (ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.begin();
208  columnDescIt != columnDescriptorMap_.end();
209  ++columnDescIt) {
210  delete columnDescIt->second;
211  }
212 
213  // ColumnDescriptorMapById points to the same descriptors. No need to delete
214 
216  boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
217  }
218 }
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:544
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:153
bool g_serialize_temp_tables
Definition: Catalog.cpp:97
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
mapd_unique_lock< mapd_shared_mutex > write_lock
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:542

+ Here is the call graph for this function:

Member Function Documentation

void Catalog_Namespace::Catalog::addColumn ( const TableDescriptor td,
ColumnDescriptor cd 
)

Definition at line 1854 of file Catalog.cpp.

References addDictionary(), columnDescriptorMap_, columnDescriptorMapById_, columnDescriptorsForRoll, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), SqliteConnector::getData(), getPhysicalTablesDescriptors(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, kENCODING_DICT, TableDescriptor::nShards, SqliteConnector::query_with_text_params(), TableDescriptor::shard, sqliteConnector_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), to_upper(), and ColumnDescriptor::virtualExpr.

1854  {
1855  cat_write_lock write_lock(this);
1856  // caller must handle sqlite/chunk transaction TOGETHER
1857  cd.tableId = td.tableId;
1858  if (td.nShards > 0 && td.shard < 0) {
1859  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
1860  auto shard_cd = cd;
1861  addColumn(*shard, shard_cd);
1862  }
1863  }
1865  addDictionary(cd);
1866  }
1867 
1869  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
1870  "colscale, is_notnull, "
1871  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
1872  "is_deletedcol) "
1873  "VALUES (?, "
1874  "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
1875  "?, ?, ?, "
1876  "?, "
1877  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1878  std::vector<std::string>{std::to_string(td.tableId),
1879  std::to_string(td.tableId),
1880  cd.columnName,
1889  "",
1892  cd.virtualExpr,
1894 
1896  "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
1897  std::vector<std::string>{std::to_string(td.tableId)});
1898 
1900  "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
1901  std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
1902  cd.columnId = sqliteConnector_.getData<int>(0, 0);
1903 
1904  ++tableDescriptorMapById_[td.tableId]->nColumns;
1905  auto ncd = new ColumnDescriptor(cd);
1908  columnDescriptorsForRoll.emplace_back(nullptr, ncd);
1909 }
std::string virtualExpr
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:312
std::tuple< int, std::string > ColumnKey
Definition: Types.h:36
HOST DEVICE int get_size() const
Definition: sqltypes.h:321
T getData(const int row, const int col)
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:544
virtual void query_with_text_params(std::string const &query_only)
void addColumn(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:1854
SqliteConnector sqliteConnector_
Definition: Catalog.h:553
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logicalTableDesc) const
Definition: Catalog.cpp:3786
HOST DEVICE int get_scale() const
Definition: sqltypes.h:316
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:311
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:545
std::string to_string(char const *&&v)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:543
specifies the content in-memory of a row in the column metadata table
std::string to_upper(const std::string &str)
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:319
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:313
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:320
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
DictRef addDictionary(ColumnDescriptor &cd)
Definition: Catalog.cpp:1759
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:318
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:572
std::string columnName
std::tuple< int, int > ColumnIdKey
Definition: Types.h:38

+ Here is the call graph for this function:

DictRef Catalog_Namespace::Catalog::addDictionary ( ColumnDescriptor cd)

Definition at line 1759 of file Catalog.cpp.

References CHECK, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, DictDescriptor::dictFolderPath, setColumnDictionary(), string_dict_hosts_, tableDescriptorMapById_, and ColumnDescriptor::tableId.

Referenced by addColumn().

1759  {
1760  const auto& td = *tableDescriptorMapById_[cd.tableId];
1761  list<DictDescriptor> dds;
1762  setColumnDictionary(cd, dds, td, true);
1763  auto& dd = dds.back();
1764  CHECK(dd.dictRef.dictId);
1765 
1766  std::unique_ptr<StringDictionaryClient> client;
1767  if (!string_dict_hosts_.empty()) {
1768  client.reset(new StringDictionaryClient(
1769  string_dict_hosts_.front(), DictRef(currentDB_.dbId, -1), true));
1770  }
1771  if (client) {
1772  client->create(dd.dictRef, dd.dictIsTemp);
1773  }
1774 
1775  DictDescriptor* new_dd = new DictDescriptor(dd);
1776  dictDescriptorMapByRef_[dd.dictRef].reset(new_dd);
1777  if (!dd.dictIsTemp) {
1778  boost::filesystem::create_directory(new_dd->dictFolderPath);
1779  }
1780  return dd.dictRef;
1781 }
std::string dictFolderPath
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:557
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:546
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:543
struct dict_ref_t DictRef
Definition: DictRef.h:37
#define CHECK(condition)
Definition: Logger.h:197
Descriptor for a dictionary for a string columne.
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:3205

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addForeignTableDetails ( )
private

Definition at line 4052 of file Catalog.cpp.

References CHECK, foreignServerMapById_, SqliteConnector::getData(), SqliteConnector::getNumRows(), SqliteConnector::query(), sqliteConnector_, and tableDescriptorMapById_.

Referenced by buildMaps().

4052  {
4054  "SELECT table_id, server_id, options, last_refresh_time, next_refresh_time from "
4055  "omnisci_foreign_tables");
4056  auto num_rows = sqliteConnector_.getNumRows();
4057  for (size_t r = 0; r < num_rows; r++) {
4058  const auto table_id = sqliteConnector_.getData<int32_t>(r, 0);
4059  const auto server_id = sqliteConnector_.getData<int32_t>(r, 1);
4060  const auto& options = sqliteConnector_.getData<std::string>(r, 2);
4061  const auto last_refresh_time = sqliteConnector_.getData<int>(r, 3);
4062  const auto next_refresh_time = sqliteConnector_.getData<int>(r, 4);
4063 
4064  CHECK(tableDescriptorMapById_.find(table_id) != tableDescriptorMapById_.end());
4065  auto foreign_table =
4066  dynamic_cast<foreign_storage::ForeignTable*>(tableDescriptorMapById_[table_id]);
4067  CHECK(foreign_table);
4068  foreign_table->foreign_server = foreignServerMapById_[server_id].get();
4069  CHECK(foreign_table->foreign_server);
4070  foreign_table->populateOptionsMap(options);
4071  foreign_table->last_refresh_time = last_refresh_time;
4072  foreign_table->next_refresh_time = next_refresh_time;
4073  }
4074 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:553
virtual void query(const std::string &queryString)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:543
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:551
#define CHECK(condition)
Definition: Logger.h:197
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addFrontendViewToMap ( DashboardDescriptor vd)
protected

Definition at line 1253 of file Catalog.cpp.

References addFrontendViewToMapNoLock().

Referenced by createDashboard().

1253  {
1254  cat_write_lock write_lock(this);
1256 }
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1258
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addFrontendViewToMapNoLock ( DashboardDescriptor vd)
protected

Definition at line 1258 of file Catalog.cpp.

References dashboardDescriptorMap_, DashboardDescriptor::dashboardName, to_string(), and DashboardDescriptor::userId.

Referenced by addFrontendViewToMap(), and replaceDashboard().

1258  {
1259  cat_write_lock write_lock(this);
1261  std::make_shared<DashboardDescriptor>(vd);
1262 }
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:547
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addLinkToMap ( LinkDescriptor ld)
protected

Definition at line 1335 of file Catalog.cpp.

References currentDB_, Catalog_Namespace::DBMetadata::dbId, LinkDescriptor::link, linkDescriptorMap_, linkDescriptorMapById_, LinkDescriptor::linkId, and to_string().

Referenced by createLink().

1335  {
1336  cat_write_lock write_lock(this);
1337  LinkDescriptor* new_ld = new LinkDescriptor();
1338  *new_ld = ld;
1340  linkDescriptorMapById_[ld.linkId] = new_ld;
1341 }
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
mapd_unique_lock< mapd_shared_mutex > write_lock
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:549
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:548

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addReferenceToForeignDict ( ColumnDescriptor referencing_column,
Parser::SharedDictionaryDef  shared_dict_def,
const bool  persist_reference 
)
protected

Definition at line 3114 of file Catalog.cpp.

References CHECK, CHECK_GE, ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, SQLTypeInfo::get_comp_param(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_foreign_col(), getObjForLock(), SqliteConnector::query_with_text_params(), sqliteConnector_, and to_string().

Referenced by setColumnSharedDictionary().

3116  {
3117  cat_write_lock write_lock(this);
3118  const auto foreign_ref_col = get_foreign_col(*this, shared_dict_def);
3119  CHECK(foreign_ref_col);
3120  referencing_column.columnType = foreign_ref_col->columnType;
3121  const int dict_id = referencing_column.columnType.get_comp_param();
3122  const DictRef dict_ref(currentDB_.dbId, dict_id);
3123  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
3124  CHECK(dictIt != dictDescriptorMapByRef_.end());
3125  const auto& dd = dictIt->second;
3126  CHECK_GE(dd->refcount, 1);
3127  ++dd->refcount;
3128  if (persist_reference) {
3129  cat_sqlite_lock sqlite_lock(getObjForLock());
3131  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
3132  {std::to_string(dict_id)});
3133  }
3134 }
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:553
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:546
#define CHECK_GE(x, y)
Definition: Logger.h:210
const ColumnDescriptor * get_foreign_col(const Catalog &cat, const Parser::SharedDictionaryDef &shared_dict_def)
Definition: Catalog.cpp:3102
std::string to_string(char const *&&v)
const Catalog * getObjForLock()
Definition: Catalog.cpp:220
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:320
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addTableToMap ( const TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::list< DictDescriptor > &  dicts 
)
protected

Definition at line 1112 of file Catalog.cpp.

References CHECK, columnDescriptorMap_, columnDescriptorMapById_, ColumnDescriptor::columnId, TableDescriptor::columnIdBySpi_, ColumnDescriptor::columnName, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, DictDescriptor::dictFolderPath, dict_ref_t::dictId, TableDescriptor::hasDeletedCol, TableDescriptor::mutex_, setDeletedColumnUnlocked(), gpu_enabled::sort(), string_dict_hosts_, tableDescriptorMap_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, and to_upper().

Referenced by createTable().

1114  {
1115  cat_write_lock write_lock(this);
1116  TableDescriptor* new_td;
1117 
1118  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
1119  if (foreign_table) {
1120  auto new_foreign_table = new foreign_storage::ForeignTable();
1121  *new_foreign_table = *foreign_table;
1122  new_td = new_foreign_table;
1123  } else {
1124  new_td = new TableDescriptor();
1125  *new_td = *td;
1126  }
1127 
1128  new_td->mutex_ = std::make_shared<std::mutex>();
1129  tableDescriptorMap_[to_upper(td->tableName)] = new_td;
1130  tableDescriptorMapById_[td->tableId] = new_td;
1131  for (auto cd : columns) {
1132  ColumnDescriptor* new_cd = new ColumnDescriptor();
1133  *new_cd = cd;
1134  ColumnKey columnKey(new_cd->tableId, to_upper(new_cd->columnName));
1135  columnDescriptorMap_[columnKey] = new_cd;
1136  ColumnIdKey columnIdKey(new_cd->tableId, new_cd->columnId);
1137  columnDescriptorMapById_[columnIdKey] = new_cd;
1138 
1139  // Add deleted column to the map
1140  if (cd.isDeletedCol) {
1141  CHECK(new_td->hasDeletedCol);
1142  setDeletedColumnUnlocked(new_td, new_cd);
1143  }
1144  }
1145 
1146  std::sort(new_td->columnIdBySpi_.begin(),
1147  new_td->columnIdBySpi_.end(),
1148  [](const size_t a, const size_t b) -> bool { return a < b; });
1149 
1150  std::unique_ptr<StringDictionaryClient> client;
1151  DictRef dict_ref(currentDB_.dbId, -1);
1152  if (!string_dict_hosts_.empty()) {
1153  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1154  }
1155  for (auto dd : dicts) {
1156  if (!dd.dictRef.dictId) {
1157  // Dummy entry created for a shard of a logical table, nothing to do.
1158  continue;
1159  }
1160  dict_ref.dictId = dd.dictRef.dictId;
1161  if (client) {
1162  client->create(dict_ref, dd.dictIsTemp);
1163  }
1164  DictDescriptor* new_dd = new DictDescriptor(dd);
1165  dictDescriptorMapByRef_[dict_ref].reset(new_dd);
1166  if (!dd.dictIsTemp) {
1167  boost::filesystem::create_directory(new_dd->dictFolderPath);
1168  }
1169  }
1170 }
std::string dictFolderPath
std::tuple< int, std::string > ColumnKey
Definition: Types.h:36
std::string tableName
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:544
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:557
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:546
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
std::vector< int > columnIdBySpi_
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:545
std::shared_ptr< std::mutex > mutex_
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3093
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:543
specifies the content in-memory of a row in the column metadata table
std::string to_upper(const std::string &str)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
#define CHECK(condition)
Definition: Logger.h:197
Descriptor for a dictionary for a string columne.
mapd_unique_lock< mapd_shared_mutex > write_lock
std::string columnName
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:542
std::tuple< int, int > ColumnIdKey
Definition: Types.h:38

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::adjustAlteredTableFiles ( const std::string &  temp_data_dir,
const std::unordered_map< int, int > &  all_column_ids_map 
) const
private
void Catalog_Namespace::Catalog::alterPhysicalTableMetadata ( const TableDescriptor td,
const TableDescriptorUpdateParams table_update_params 
)
private

Definition at line 2836 of file Catalog.cpp.

References CHECK, getMutableMetadataForTableUnlocked(), TableDescriptor::maxRollbackEpochs, TableDescriptorUpdateParams::maxRollbackEpochs, SqliteConnector::query_with_text_params(), sqliteConnector_, TableDescriptor::tableId, and to_string().

Referenced by alterTableMetadata().

2838  {
2839  // Only called from parent alterTableParamMetadata, expect already to have catalog and
2840  // sqlite write locks
2841 
2842  // Sqlite transaction should have already been begun in parent alterTableCatalogMetadata
2843 
2845  CHECK(mutable_td);
2846  if (td->maxRollbackEpochs != table_update_params.maxRollbackEpochs) {
2848  "UPDATE mapd_tables SET max_rollback_epochs = ? WHERE tableid = ?",
2849  std::vector<std::string>{std::to_string(table_update_params.maxRollbackEpochs),
2850  std::to_string(td->tableId)});
2851  mutable_td->maxRollbackEpochs = table_update_params.maxRollbackEpochs;
2852  }
2853 }
int32_t maxRollbackEpochs
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:553
std::string to_string(char const *&&v)
TableDescriptor * getMutableMetadataForTableUnlocked(int tableId)
Definition: Catalog.cpp:1445
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::alterTableMetadata ( const TableDescriptor td,
const TableDescriptorUpdateParams table_update_params 
)
private

Definition at line 2855 of file Catalog.cpp.

References alterPhysicalTableMetadata(), CHECK, logger::FATAL, getMetadataForTable(), getObjForLock(), LOG, logicalToPhysicalTableMapById_, SqliteConnector::query(), sqliteConnector_, TableDescriptor::tableId, and TableDescriptor::tableName.

Referenced by setMaxRollbackEpochs().

2856  {
2857  cat_write_lock write_lock(this);
2858  cat_sqlite_lock sqlite_lock(getObjForLock());
2859  sqliteConnector_.query("BEGIN TRANSACTION");
2860  try {
2861  const auto physical_table_it = logicalToPhysicalTableMapById_.find(td->tableId);
2862  if (physical_table_it != logicalToPhysicalTableMapById_.end()) {
2863  const auto physical_tables = physical_table_it->second;
2864  CHECK(!physical_tables.empty());
2865  for (size_t i = 0; i < physical_tables.size(); i++) {
2866  int32_t physical_tb_id = physical_tables[i];
2867  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id, false);
2868  CHECK(phys_td);
2869  alterPhysicalTableMetadata(phys_td, table_update_params);
2870  }
2871  }
2872  alterPhysicalTableMetadata(td, table_update_params);
2873  } catch (std::exception& e) {
2874  sqliteConnector_.query("ROLLBACK TRANSACTION");
2875  LOG(FATAL) << "Table '" << td->tableName << "' catalog update failed";
2876  }
2877  sqliteConnector_.query("END TRANSACTION");
2878 }
std::string tableName
#define LOG(tag)
Definition: Logger.h:188
SqliteConnector sqliteConnector_
Definition: Catalog.h:553
virtual void query(const std::string &queryString)
void alterPhysicalTableMetadata(const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
Definition: Catalog.cpp:2836
const Catalog * getObjForLock()
Definition: Catalog.cpp:220
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:560

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::buildForeignServerMap ( )
private

Definition at line 4034 of file Catalog.cpp.

References foreignServerMap_, foreignServerMapById_, SqliteConnector::getData(), SqliteConnector::getNumRows(), SqliteConnector::query(), and sqliteConnector_.

Referenced by buildMaps().

4034  {
4036  "SELECT id, name, data_wrapper_type, options, owner_user_id, creation_time FROM "
4037  "omnisci_foreign_servers");
4038  auto num_rows = sqliteConnector_.getNumRows();
4039  for (size_t row = 0; row < num_rows; row++) {
4040  auto foreign_server = std::make_shared<foreign_storage::ForeignServer>(
4041  sqliteConnector_.getData<int>(row, 0),
4042  sqliteConnector_.getData<std::string>(row, 1),
4043  sqliteConnector_.getData<std::string>(row, 2),
4044  sqliteConnector_.getData<std::string>(row, 3),
4045  sqliteConnector_.getData<std::int32_t>(row, 4),
4046  sqliteConnector_.getData<std::int32_t>(row, 5));
4047  foreignServerMap_[foreign_server->name] = foreign_server;
4048  foreignServerMapById_[foreign_server->id] = foreign_server;
4049  }
4050 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:553
virtual void query(const std::string &queryString)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:551
ForeignServerMap foreignServerMap_
Definition: Catalog.h:550
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::buildMaps ( )
protected

Definition at line 900 of file Catalog.cpp.

References addForeignTableDetails(), basePath_, buildForeignServerMap(), CHECK, ColumnDescriptor::chunks, columnDescriptorMap_, columnDescriptorMapById_, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, createDefaultServersIfNotExists(), currentDB_, dashboardDescriptorMap_, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, dictDescriptorMapByRef_, logger::FATAL, StorageType::FOREIGN_TABLE, TableDescriptor::fragmenter, TableDescriptor::fragments, TableDescriptor::fragPageSize, TableDescriptor::fragType, g_enable_fsi, generate_dashboard_system_rolename(), SQLTypeInfo::get_physical_cols(), SqliteConnector::getData(), SqliteConnector::getNumRows(), getObjForLock(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::getUserFromId(), TableDescriptor::hasDeletedCol, SQLTypeInfo::is_geometry(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, SqliteConnector::isNull(), ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, TableDescriptor::keyMetainfo, LinkDescriptor::link, linkDescriptorMap_, linkDescriptorMapById_, LinkDescriptor::linkId, LOG, logicalToPhysicalTableMapById_, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nColumns, TableDescriptor::nShards, TableDescriptor::partitions, SqliteConnector::query(), SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_compression(), SQLTypeInfo::set_dimension(), SQLTypeInfo::set_notnull(), SQLTypeInfo::set_scale(), SQLTypeInfo::set_size(), SQLTypeInfo::set_subtype(), SQLTypeInfo::set_type(), setDeletedColumnUnlocked(), TableDescriptor::shard, TableDescriptor::shardedColumnId, gpu_enabled::sort(), TableDescriptor::sortedColumnId, sqliteConnector_, TableDescriptor::storageType, tableDescriptorMap_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), to_upper(), LinkDescriptor::updateTime, LinkDescriptor::userId, TableDescriptor::userId, LinkDescriptor::viewMetadata, TableDescriptor::viewSQL, LinkDescriptor::viewState, and ColumnDescriptor::virtualExpr.

900  {
902  cat_sqlite_lock sqlite_lock(getObjForLock());
903 
904  string dictQuery(
905  "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
906  sqliteConnector_.query(dictQuery);
907  size_t numRows = sqliteConnector_.getNumRows();
908  for (size_t r = 0; r < numRows; ++r) {
909  int dictId = sqliteConnector_.getData<int>(r, 0);
910  std::string dictName = sqliteConnector_.getData<string>(r, 1);
911  int dictNBits = sqliteConnector_.getData<int>(r, 2);
912  bool is_shared = sqliteConnector_.getData<bool>(r, 3);
913  int refcount = sqliteConnector_.getData<int>(r, 4);
914  std::string fname = basePath_ + "/mapd_data/DB_" + std::to_string(currentDB_.dbId) +
915  "_DICT_" + std::to_string(dictId);
916  DictRef dict_ref(currentDB_.dbId, dictId);
917  DictDescriptor* dd = new DictDescriptor(
918  dict_ref, dictName, dictNBits, is_shared, refcount, fname, false);
919  dictDescriptorMapByRef_[dict_ref].reset(dd);
920  }
921 
922  string tableQuery(
923  "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
924  "max_chunk_size, frag_page_size, "
925  "max_rows, partitions, shard_column_id, shard, num_shards, key_metainfo, userid, "
926  "sort_column_id, storage_type, max_rollback_epochs "
927  "from mapd_tables");
928  sqliteConnector_.query(tableQuery);
929  numRows = sqliteConnector_.getNumRows();
930  for (size_t r = 0; r < numRows; ++r) {
931  TableDescriptor* td;
932  const auto& storage_type = sqliteConnector_.getData<string>(r, 17);
933  if (!storage_type.empty() && storage_type != StorageType::FOREIGN_TABLE) {
934  const auto table_id = sqliteConnector_.getData<int>(r, 0);
935  const auto& table_name = sqliteConnector_.getData<string>(r, 1);
936  LOG(FATAL) << "Unable to read Catalog metadata: storage type is currently not a "
937  "supported table option (table "
938  << table_name << " [" << table_id << "] in database "
939  << currentDB_.dbName << ").";
940  }
941 
942  if (storage_type == StorageType::FOREIGN_TABLE) {
944  } else {
945  td = new TableDescriptor();
946  }
947 
948  td->storageType = storage_type;
949  td->tableId = sqliteConnector_.getData<int>(r, 0);
950  td->tableName = sqliteConnector_.getData<string>(r, 1);
951  td->nColumns = sqliteConnector_.getData<int>(r, 2);
952  td->isView = sqliteConnector_.getData<bool>(r, 3);
953  td->fragments = sqliteConnector_.getData<string>(r, 4);
954  td->fragType =
956  td->maxFragRows = sqliteConnector_.getData<int>(r, 6);
957  td->maxChunkSize = sqliteConnector_.getData<int64_t>(r, 7);
958  td->fragPageSize = sqliteConnector_.getData<int>(r, 8);
959  td->maxRows = sqliteConnector_.getData<int64_t>(r, 9);
960  td->partitions = sqliteConnector_.getData<string>(r, 10);
961  td->shardedColumnId = sqliteConnector_.getData<int>(r, 11);
962  td->shard = sqliteConnector_.getData<int>(r, 12);
963  td->nShards = sqliteConnector_.getData<int>(r, 13);
964  td->keyMetainfo = sqliteConnector_.getData<string>(r, 14);
965  td->userId = sqliteConnector_.getData<int>(r, 15);
966  td->sortedColumnId =
967  sqliteConnector_.isNull(r, 16) ? 0 : sqliteConnector_.getData<int>(r, 16);
968  if (!td->isView) {
969  td->fragmenter = nullptr;
970  }
971  td->maxRollbackEpochs = sqliteConnector_.getData<int>(r, 18);
972  td->hasDeletedCol = false;
973 
976  }
977 
978  if (g_enable_fsi) {
982  }
983 
984  string columnQuery(
985  "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
986  "is_notnull, compression, comp_param, "
987  "size, chunks, is_systemcol, is_virtualcol, virtual_expr, is_deletedcol from "
988  "mapd_columns ORDER BY tableid, "
989  "columnid");
990  sqliteConnector_.query(columnQuery);
991  numRows = sqliteConnector_.getNumRows();
992  int32_t skip_physical_cols = 0;
993  for (size_t r = 0; r < numRows; ++r) {
995  cd->tableId = sqliteConnector_.getData<int>(r, 0);
996  cd->columnId = sqliteConnector_.getData<int>(r, 1);
997  cd->columnName = sqliteConnector_.getData<string>(r, 2);
1001  cd->columnType.set_scale(sqliteConnector_.getData<int>(r, 6));
1005  cd->columnType.set_size(sqliteConnector_.getData<int>(r, 10));
1006  cd->chunks = sqliteConnector_.getData<string>(r, 11);
1007  cd->isSystemCol = sqliteConnector_.getData<bool>(r, 12);
1008  cd->isVirtualCol = sqliteConnector_.getData<bool>(r, 13);
1009  cd->virtualExpr = sqliteConnector_.getData<string>(r, 14);
1010  cd->isDeletedCol = sqliteConnector_.getData<bool>(r, 15);
1011  cd->isGeoPhyCol = skip_physical_cols > 0;
1012  ColumnKey columnKey(cd->tableId, to_upper(cd->columnName));
1013  columnDescriptorMap_[columnKey] = cd;
1014  ColumnIdKey columnIdKey(cd->tableId, cd->columnId);
1015  columnDescriptorMapById_[columnIdKey] = cd;
1016 
1017  if (skip_physical_cols <= 0) {
1018  skip_physical_cols = cd->columnType.get_physical_cols();
1019  }
1020 
1021  auto td_itr = tableDescriptorMapById_.find(cd->tableId);
1022  CHECK(td_itr != tableDescriptorMapById_.end());
1023 
1024  if (cd->isDeletedCol) {
1025  td_itr->second->hasDeletedCol = true;
1026  setDeletedColumnUnlocked(td_itr->second, cd);
1027  } else if (cd->columnType.is_geometry() || skip_physical_cols-- <= 0) {
1028  tableDescriptorMapById_[cd->tableId]->columnIdBySpi_.push_back(cd->columnId);
1029  }
1030  }
1031  // sort columnIdBySpi_ based on columnId
1032  for (auto& tit : tableDescriptorMapById_) {
1033  std::sort(tit.second->columnIdBySpi_.begin(),
1034  tit.second->columnIdBySpi_.end(),
1035  [](const size_t a, const size_t b) -> bool { return a < b; });
1036  }
1037 
1038  string viewQuery("SELECT tableid, sql FROM mapd_views");
1039  sqliteConnector_.query(viewQuery);
1040  numRows = sqliteConnector_.getNumRows();
1041  for (size_t r = 0; r < numRows; ++r) {
1042  int32_t tableId = sqliteConnector_.getData<int>(r, 0);
1043  TableDescriptor* td = tableDescriptorMapById_[tableId];
1044  td->viewSQL = sqliteConnector_.getData<string>(r, 1);
1045  td->fragmenter = nullptr;
1046  }
1047 
1048  string frontendViewQuery(
1049  "SELECT id, state, name, image_hash, strftime('%Y-%m-%dT%H:%M:%SZ', update_time), "
1050  "userid, "
1051  "metadata "
1052  "FROM mapd_dashboards");
1053  sqliteConnector_.query(frontendViewQuery);
1054  numRows = sqliteConnector_.getNumRows();
1055  for (size_t r = 0; r < numRows; ++r) {
1056  std::shared_ptr<DashboardDescriptor> vd = std::make_shared<DashboardDescriptor>();
1057  vd->dashboardId = sqliteConnector_.getData<int>(r, 0);
1058  vd->dashboardState = sqliteConnector_.getData<string>(r, 1);
1059  vd->dashboardName = sqliteConnector_.getData<string>(r, 2);
1060  vd->imageHash = sqliteConnector_.getData<string>(r, 3);
1061  vd->updateTime = sqliteConnector_.getData<string>(r, 4);
1062  vd->userId = sqliteConnector_.getData<int>(r, 5);
1063  vd->dashboardMetadata = sqliteConnector_.getData<string>(r, 6);
1064  vd->user = getUserFromId(vd->userId);
1065  vd->dashboardSystemRoleName = generate_dashboard_system_rolename(
1067  dashboardDescriptorMap_[std::to_string(vd->userId) + ":" + vd->dashboardName] = vd;
1068  }
1069 
1070  string linkQuery(
1071  "SELECT linkid, userid, link, view_state, strftime('%Y-%m-%dT%H:%M:%SZ', "
1072  "update_time), view_metadata "
1073  "FROM mapd_links");
1074  sqliteConnector_.query(linkQuery);
1075  numRows = sqliteConnector_.getNumRows();
1076  for (size_t r = 0; r < numRows; ++r) {
1077  LinkDescriptor* ld = new LinkDescriptor();
1078  ld->linkId = sqliteConnector_.getData<int>(r, 0);
1079  ld->userId = sqliteConnector_.getData<int>(r, 1);
1080  ld->link = sqliteConnector_.getData<string>(r, 2);
1081  ld->viewState = sqliteConnector_.getData<string>(r, 3);
1082  ld->updateTime = sqliteConnector_.getData<string>(r, 4);
1083  ld->viewMetadata = sqliteConnector_.getData<string>(r, 5);
1085  linkDescriptorMapById_[ld->linkId] = ld;
1086  }
1087 
1088  /* rebuild map linking logical tables to corresponding physical ones */
1089  string logicalToPhysicalTableMapQuery(
1090  "SELECT logical_table_id, physical_table_id "
1091  "FROM mapd_logical_to_physical");
1092  sqliteConnector_.query(logicalToPhysicalTableMapQuery);
1093  numRows = sqliteConnector_.getNumRows();
1094  for (size_t r = 0; r < numRows; ++r) {
1095  int32_t logical_tb_id = sqliteConnector_.getData<int>(r, 0);
1096  int32_t physical_tb_id = sqliteConnector_.getData<int>(r, 1);
1097  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
1098  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
1099  /* add new entity to the map logicalToPhysicalTableMapById_ */
1100  std::vector<int32_t> physicalTables;
1101  physicalTables.push_back(physical_tb_id);
1102  const auto it_ok =
1103  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
1104  CHECK(it_ok.second);
1105  } else {
1106  /* update map logicalToPhysicalTableMapById_ */
1107  physicalTableIt->second.push_back(physical_tb_id);
1108  }
1109  }
1110 }
int32_t maxRollbackEpochs
std::string virtualExpr
void set_compression(EncodingType c)
Definition: sqltypes.h:411
void set_size(int s)
Definition: sqltypes.h:409
std::string partitions
std::tuple< int, std::string > ColumnKey
Definition: Types.h:36
T getData(const int row, const int col)
SQLTypes
Definition: sqltypes.h:37
std::string tableName
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:544
#define LOG(tag)
Definition: Logger.h:188
SqliteConnector sqliteConnector_
Definition: Catalog.h:553
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:546
std::string storageType
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:402
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
virtual void query(const std::string &queryString)
std::string fragments
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:545
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:547
std::string to_string(char const *&&v)
std::string chunks
EncodingType
Definition: sqltypes.h:224
int get_physical_cols() const
Definition: sqltypes.h:332
std::string getUserFromId(const int32_t id)
Definition: Catalog.cpp:890
void set_scale(int s)
Definition: sqltypes.h:406
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3093
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:543
specifies the content in-memory of a row in the column metadata table
std::string keyMetainfo
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
std::string to_upper(const std::string &str)
void set_comp_param(int p)
Definition: sqltypes.h:412
bool isNull(const int row, const int col) const
void set_dimension(int d)
Definition: sqltypes.h:403
Fragmenter_Namespace::FragmenterType fragType
const Catalog * getObjForLock()
Definition: Catalog.cpp:220
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
void set_notnull(bool n)
Definition: sqltypes.h:408
#define CHECK(condition)
Definition: Logger.h:197
bool is_geometry() const
Definition: sqltypes.h:490
Descriptor for a dictionary for a string columne.
std::string viewSQL
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:549
static constexpr char const * FOREIGN_TABLE
bool g_enable_fsi
Definition: Catalog.cpp:91
std::string columnName
virtual size_t getNumRows() const
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:560
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:542
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:548
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)
void createDefaultServersIfNotExists()
Definition: Catalog.cpp:4097
std::tuple< int, int > ColumnIdKey
Definition: Types.h:38
HOST DEVICE void set_type(SQLTypes t)
Definition: sqltypes.h:401

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::calculateSHA1 ( const std::string &  data)
protected

Definition at line 3717 of file Catalog.cpp.

Referenced by createLink().

3717  {
3718  boost::uuids::detail::sha1 sha1;
3719  unsigned int digest[5];
3720  sha1.process_bytes(data.c_str(), data.length());
3721  sha1.get_digest(digest);
3722  std::stringstream ss;
3723  for (size_t i = 0; i < 5; i++) {
3724  ss << std::hex << digest[i];
3725  }
3726  return ss.str();
3727 }

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::changeForeignServerOwner ( const std::string &  server_name,
const int  new_owner_id 
)

Change the owner of a Foreign Server to a new owner.

Parameters
server_name- Name of the foreign server whose owner to change
new_owner_id- New owner's user id

Definition at line 2551 of file Catalog.cpp.

References CHECK, foreignServerMap_, setForeignServerProperty(), to_string(), and foreign_storage::ForeignServer::user_id.

2552  {
2553  cat_write_lock write_lock(this);
2554  foreign_storage::ForeignServer* foreign_server =
2555  foreignServerMap_.find(server_name)->second.get();
2556  CHECK(foreign_server);
2557  setForeignServerProperty(server_name, "owner_user_id", std::to_string(new_owner_id));
2558  // update in-memory server
2559  foreign_server->user_id = new_owner_id;
2560 }
std::string to_string(char const *&&v)
ForeignServerMap foreignServerMap_
Definition: Catalog.h:550
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock
void setForeignServerProperty(const std::string &server_name, const std::string &property, const std::string &value)
Definition: Catalog.cpp:4076

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::CheckAndExecuteMigrations ( )
protected

Definition at line 865 of file Catalog.cpp.

References createFsiSchemas(), g_enable_fsi, recordOwnershipOfObjectsInObjectPermissions(), updateDeletedColumnIndicator(), updateDictionaryNames(), updateDictionarySchema(), updateFixlenArrayColumns(), updateFrontendViewAndLinkUsers(), updateFrontendViewSchema(), updateFrontendViewsToDashboards(), updateGeoColumns(), updateLinkSchema(), updateLogicalToPhysicalTableLinkSchema(), updatePageSize(), and updateTableDescriptorSchema().

865  {
875  updatePageSize();
879  if (g_enable_fsi) {
881  }
882 }
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:426
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:655
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:120
void updateLogicalToPhysicalTableLinkSchema()
Definition: Catalog.cpp:566
bool g_enable_fsi
Definition: Catalog.cpp:91

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::CheckAndExecuteMigrationsPostBuildMaps ( )
protected

Definition at line 884 of file Catalog.cpp.

References checkDateInDaysColumnMigration(), and createDashboardSystemRoles().

884  {
887 }
void checkDateInDaysColumnMigration()
Definition: Catalog.cpp:783

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::checkDateInDaysColumnMigration ( )
protected

Definition at line 783 of file Catalog.cpp.

References getCurrentDB(), getObjForLock(), migrations::MigrationMgr::migrateDateInDaysMetadata(), sqliteConnector_, and tableDescriptorMapById_.

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

783  {
784  cat_sqlite_lock sqlite_lock(getObjForLock());
787 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:553
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:221
static void migrateDateInDaysMetadata(const Catalog_Namespace::TableDescriptorMapById &table_descriptors_by_id, const int database_id, const Catalog_Namespace::Catalog *cat, SqliteConnector &sqlite)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:543
const Catalog * getObjForLock()
Definition: Catalog.cpp:220
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const bool Catalog_Namespace::Catalog::checkMetadataForDeletedRecs ( const TableDescriptor td,
int  column_id 
) const

Definition at line 3028 of file Catalog.cpp.

References CHECK, currentDB_, dataMgr_, Catalog_Namespace::DBMetadata::dbId, TableDescriptor::fragmenter, table_is_temporary(), and TableDescriptor::tableId.

Referenced by getDeletedColumnIfRowsDeleted().

3029  {
3030  // check if there are rows deleted by examining the deletedColumn metadata
3031  CHECK(td);
3032 
3033  if (table_is_temporary(td)) {
3034  auto fragmenter = td->fragmenter;
3035  CHECK(fragmenter);
3036  return fragmenter->hasDeletedRows(delete_column_id);
3037  } else {
3038  ChunkKey chunk_key_prefix = {currentDB_.dbId, td->tableId, delete_column_id};
3039  ChunkMetadataVector chunk_metadata_vec;
3040  dataMgr_->getChunkMetadataVecForKeyPrefix(chunk_metadata_vec, chunk_key_prefix);
3041  int64_t chunk_max{0};
3042 
3043  for (auto chunk_metadata : chunk_metadata_vec) {
3044  chunk_max = chunk_metadata.second->chunkStats.max.tinyintval;
3045  // delete has occured
3046  if (chunk_max == 1) {
3047  return true;
3048  }
3049  }
3050  return false;
3051  }
3052 }
std::vector< int > ChunkKey
Definition: types.h:37
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:555
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
bool table_is_temporary(const TableDescriptor *const td)
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::checkpoint ( const int  logicalTableId) const

Definition at line 3923 of file Catalog.cpp.

References Data_Namespace::DataMgr::checkpoint(), getCurrentDB(), getDataMgr(), getMetadataForTable(), and getPhysicalTablesDescriptors().

Referenced by checkpointWithAutoRollback(), and TableOptimizer::vacuumDeletedRows().

3923  {
3924  const auto td = getMetadataForTable(logicalTableId);
3925  const auto shards = getPhysicalTablesDescriptors(td);
3926  for (const auto shard : shards) {
3927  getDataMgr().checkpoint(getCurrentDB().dbId, shard->tableId);
3928  }
3929 }
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:222
void checkpoint(const int db_id, const int tb_id)
Definition: DataMgr.cpp:489
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logicalTableDesc) const
Definition: Catalog.cpp:3786
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:221
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::checkpointWithAutoRollback ( const int  logical_table_id)

Definition at line 3931 of file Catalog.cpp.

References checkpoint(), getDatabaseId(), getTableEpochs(), and setTableEpochsLogExceptions().

Referenced by import_export::Loader::checkpoint(), and Parser::LocalConnector::checkpoint().

3931  {
3932  auto table_epochs = getTableEpochs(getDatabaseId(), logical_table_id);
3933  try {
3934  checkpoint(logical_table_id);
3935  } catch (...) {
3936  setTableEpochsLogExceptions(getDatabaseId(), table_epochs);
3937  throw;
3938  }
3939 }
int getDatabaseId() const
Definition: Catalog.h:274
void checkpoint(const int logicalTableId) const
Definition: Catalog.cpp:3923
void setTableEpochsLogExceptions(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs)
Definition: Catalog.cpp:3010
std::vector< TableEpochInfo > getTableEpochs(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:2945

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

int32_t Catalog_Namespace::Catalog::createDashboard ( DashboardDescriptor vd)

Definition at line 3581 of file Catalog.cpp.

References addFrontendViewToMap(), createOrUpdateDashboardSystemRole(), currentDB_, DashboardDescriptor::dashboardId, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, DashboardDescriptor::dashboardSystemRoleName, Catalog_Namespace::DBMetadata::dbId, generate_dashboard_system_rolename(), SqliteConnector::getData(), SqliteConnector::getNumRows(), getObjForLock(), DashboardDescriptor::imageHash, SqliteConnector::query(), SqliteConnector::query_with_text_params(), sqliteConnector_, to_string(), Catalog_Namespace::sqlite_lock< T >::unlock(), Catalog_Namespace::write_lock< T >::unlock(), DashboardDescriptor::updateTime, and DashboardDescriptor::userId.

3581  {
3582  cat_write_lock write_lock(this);
3583  cat_sqlite_lock sqlite_lock(getObjForLock());
3584  sqliteConnector_.query("BEGIN TRANSACTION");
3585  try {
3586  // TODO(andrew): this should be an upsert
3588  "SELECT id FROM mapd_dashboards WHERE name = ? and userid = ?",
3589  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
3590  if (sqliteConnector_.getNumRows() > 0) {
3592  "UPDATE mapd_dashboards SET state = ?, image_hash = ?, metadata = ?, "
3593  "update_time = "
3594  "datetime('now') where name = ? "
3595  "and userid = ?",
3596  std::vector<std::string>{vd.dashboardState,
3597  vd.imageHash,
3598  vd.dashboardMetadata,
3599  vd.dashboardName,
3600  std::to_string(vd.userId)});
3601  } else {
3603  "INSERT INTO mapd_dashboards (name, state, image_hash, metadata, "
3604  "update_time, "
3605  "userid) "
3606  "VALUES "
3607  "(?,?,?,?, "
3608  "datetime('now'), ?)",
3609  std::vector<std::string>{vd.dashboardName,
3610  vd.dashboardState,
3611  vd.imageHash,
3612  vd.dashboardMetadata,
3613  std::to_string(vd.userId)});
3614  }
3615  } catch (std::exception& e) {
3616  sqliteConnector_.query("ROLLBACK TRANSACTION");
3617  throw;
3618  }
3619  sqliteConnector_.query("END TRANSACTION");
3620 
3621  // now get the auto generated dashboardId
3622  try {
3624  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_dashboards "
3625  "WHERE name = ? and userid = ?",
3626  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
3627  vd.dashboardId = sqliteConnector_.getData<int>(0, 0);
3628  vd.updateTime = sqliteConnector_.getData<std::string>(0, 1);
3629  } catch (std::exception& e) {
3630  throw;
3631  }
3635  sqlite_lock.unlock();
3636  write_lock.unlock();
3637  // NOTE(wamsi): Transactionally unsafe
3640  return vd.dashboardId;
3641 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:553
virtual void query(const std::string &queryString)
void addFrontendViewToMap(DashboardDescriptor &vd)
Definition: Catalog.cpp:1253
std::string to_string(char const *&&v)
std::string dashboardSystemRoleName
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1294
const Catalog * getObjForLock()
Definition: Catalog.cpp:220
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createDashboardSystemRoles ( )
protected

Definition at line 789 of file Catalog.cpp.

References createOrUpdateDashboardSystemRole(), currentDB_, Catalog_Namespace::DBMetadata::dbId, logger::ERROR, generate_dashboard_system_rolename(), SqliteConnector::getData(), Catalog_Namespace::SysCatalog::getGranteesOfSharedDashboards(), SqliteConnector::getNumRows(), getObjForLock(), Catalog_Namespace::SysCatalog::getRoleGrantee(), Catalog_Namespace::SysCatalog::grantRoleBatch(), logger::INFO, Catalog_Namespace::SysCatalog::instance(), LOG, MAPD_VERSION, SqliteConnector::query(), SqliteConnector::query_with_text_params(), run_benchmark_import::result, sqliteConnector_, and to_string().

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

789  {
790  std::unordered_map<std::string, std::pair<int, std::string>> dashboards;
791  std::vector<std::string> dashboard_ids;
792  static const std::string migration_name{"dashboard_roles_migration"};
793  {
794  cat_sqlite_lock sqlite_lock(getObjForLock());
795  sqliteConnector_.query("BEGIN TRANSACTION");
796  try {
797  // migration_history should be present in all catalogs by now
798  // if not then would be created before this migration
800  "select * from mapd_version_history where migration_history = '" +
801  migration_name + "'");
802  if (sqliteConnector_.getNumRows() != 0) {
803  // no need for further execution
804  sqliteConnector_.query("END TRANSACTION");
805  return;
806  }
807  LOG(INFO) << "Performing dashboard internal roles Migration.";
808  sqliteConnector_.query("select id, userid, metadata from mapd_dashboards");
809  for (size_t i = 0; i < sqliteConnector_.getNumRows(); ++i) {
812  sqliteConnector_.getData<string>(i, 0)))) {
813  // Successfully created roles during previous migration/crash
814  // No need to include them
815  continue;
816  }
817  dashboards[sqliteConnector_.getData<string>(i, 0)] = std::make_pair(
818  sqliteConnector_.getData<int>(i, 1), sqliteConnector_.getData<string>(i, 2));
819  dashboard_ids.push_back(sqliteConnector_.getData<string>(i, 0));
820  }
821  } catch (const std::exception& e) {
822  sqliteConnector_.query("ROLLBACK TRANSACTION");
823  throw;
824  }
825  sqliteConnector_.query("END TRANSACTION");
826  }
827  // All current grantees with shared dashboards.
828  const auto active_grantees =
830 
831  try {
832  // NOTE(wamsi): Transactionally unsafe
833  for (auto dash : dashboards) {
834  createOrUpdateDashboardSystemRole(dash.second.second,
835  dash.second.first,
837  std::to_string(currentDB_.dbId), dash.first));
838  auto result = active_grantees.find(dash.first);
839  if (result != active_grantees.end()) {
842  dash.first)},
843  result->second);
844  }
845  }
846  cat_sqlite_lock sqlite_lock(getObjForLock());
847  // check if this has already been completed
849  "select * from mapd_version_history where migration_history = '" +
850  migration_name + "'");
851  if (sqliteConnector_.getNumRows() != 0) {
852  return;
853  }
855  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
856  std::vector<std::string>{std::to_string(MAPD_VERSION), migration_name});
857  } catch (const std::exception& e) {
858  LOG(ERROR) << "Failed to create dashboard system roles during migration: "
859  << e.what();
860  throw;
861  }
862  LOG(INFO) << "Successfully created dashboard system roles during migration.";
863 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
#define LOG(tag)
Definition: Logger.h:188
SqliteConnector sqliteConnector_
Definition: Catalog.h:553
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
void grantRoleBatch(const std::vector< std::string > &roles, const std::vector< std::string > &grantees)
static SysCatalog & instance()
Definition: SysCatalog.h:288
static const int32_t MAPD_VERSION
Definition: release.h:33
Role * getRoleGrantee(const std::string &name) const
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1294
const Catalog * getObjForLock()
Definition: Catalog.cpp:220
std::unordered_map< std::string, std::vector< std::string > > getGranteesOfSharedDashboards(const std::vector< std::string > &dashboard_ids)
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createDefaultServersIfNotExists ( )

Creates default local file servers (if they don't already exist).

Definition at line 4097 of file Catalog.cpp.

References createForeignServerNoLocks(), foreign_storage::DataWrapperType::CSV, OMNISCI_ROOT_USER_ID, foreign_storage::DataWrapperType::PARQUET, and foreign_storage::ForeignServer::validate().

Referenced by buildMaps().

4097  {
4099  std::map<std::string, std::string, std::less<>> options;
4100  options[std::string{ForeignServer::STORAGE_TYPE_KEY}] =
4101  ForeignServer::LOCAL_FILE_STORAGE_TYPE;
4102 
4103  auto local_csv_server =
4104  std::make_unique<ForeignServer>("omnisci_local_csv",
4106  options,
4108  local_csv_server->validate();
4109  createForeignServerNoLocks(std::move(local_csv_server), true);
4110 
4111  auto local_parquet_server =
4112  std::make_unique<ForeignServer>("omnisci_local_parquet",
4114  options,
4116  local_parquet_server->validate();
4117  createForeignServerNoLocks(std::move(local_parquet_server), true);
4118 }
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2462
const int OMNISCI_ROOT_USER_ID
Definition: SysCatalog.h:60
static constexpr char const * CSV
Definition: ForeignServer.h:37
static constexpr char const * PARQUET
Definition: ForeignServer.h:38

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createForeignServer ( std::unique_ptr< foreign_storage::ForeignServer foreign_server,
bool  if_not_exists 
)

Creates a new foreign server DB object.

Parameters
foreign_server- unique pointer to struct containing foreign server details
if_not_exists- flag indicating whether or not an attempt to create a new foreign server should occur if a server with the same name already exists. An exception is thrown if this flag is set to "false" and an attempt is made to create a pre-existing foreign server

Definition at line 2454 of file Catalog.cpp.

References createForeignServerNoLocks(), and getObjForLock().

2456  {
2457  cat_write_lock write_lock(this);
2458  cat_sqlite_lock sqlite_lock(getObjForLock());
2459  createForeignServerNoLocks(std::move(foreign_server), if_not_exists);
2460 }
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2462
const Catalog * getObjForLock()
Definition: Catalog.cpp:220
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createForeignServerNoLocks ( std::unique_ptr< foreign_storage::ForeignServer foreign_server,
bool  if_not_exists 
)
private

Same as createForeignServer() but without acquiring locks. This should only be called from within a function/code block that already acquires appropriate locks.

Definition at line 2462 of file Catalog.cpp.

References CHECK, CHECK_EQ, foreignServerMap_, foreignServerMapById_, SqliteConnector::getData(), SqliteConnector::getNumRows(), SqliteConnector::query_with_text_params(), sqliteConnector_, and to_string().

Referenced by createDefaultServersIfNotExists(), and createForeignServer().

2464  {
2466  "SELECT name from omnisci_foreign_servers where name = ?",
2467  std::vector<std::string>{foreign_server->name});
2468 
2469  if (sqliteConnector_.getNumRows() == 0) {
2470  foreign_server->creation_time = std::time(nullptr);
2472  "INSERT INTO omnisci_foreign_servers (name, data_wrapper_type, owner_user_id, "
2473  "creation_time, "
2474  "options) "
2475  "VALUES (?, ?, ?, ?, ?)",
2476  std::vector<std::string>{foreign_server->name,
2477  foreign_server->data_wrapper_type,
2478  std::to_string(foreign_server->user_id),
2479  std::to_string(foreign_server->creation_time),
2480  foreign_server->getOptionsAsJsonString()});
2482  "SELECT id from omnisci_foreign_servers where name = ?",
2483  std::vector<std::string>{foreign_server->name});
2484  CHECK_EQ(sqliteConnector_.getNumRows(), size_t(1));
2485  foreign_server->id = sqliteConnector_.getData<int32_t>(0, 0);
2486  std::shared_ptr<foreign_storage::ForeignServer> foreign_server_shared =
2487  std::move(foreign_server);
2488  CHECK(foreignServerMap_.find(foreign_server_shared->name) == foreignServerMap_.end())
2489  << "Attempting to insert a foreign server into foreign server map that already "
2490  "exists.";
2491  foreignServerMap_[foreign_server_shared->name] = foreign_server_shared;
2492  foreignServerMapById_[foreign_server_shared->id] = foreign_server_shared;
2493  } else if (!if_not_exists) {
2494  throw std::runtime_error{"A foreign server with name \"" + foreign_server->name +
2495  "\" already exists."};
2496  }
2497 }
#define CHECK_EQ(x, y)
Definition: Logger.h:205
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:553
std::string to_string(char const *&&v)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:551
ForeignServerMap foreignServerMap_
Definition: Catalog.h:550
#define CHECK(condition)
Definition: Logger.h:197
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createFsiSchemas ( )
protected

Definition at line 627 of file Catalog.cpp.

References getForeignServerSchema(), getForeignTableSchema(), getObjForLock(), SqliteConnector::query(), and sqliteConnector_.

Referenced by CheckAndExecuteMigrations().

627  {
628  cat_sqlite_lock sqlite_lock(getObjForLock());
629  sqliteConnector_.query("BEGIN TRANSACTION");
630  try {
633  } catch (std::exception& e) {
634  sqliteConnector_.query("ROLLBACK TRANSACTION");
635  throw;
636  }
637  sqliteConnector_.query("END TRANSACTION");
638 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:553
virtual void query(const std::string &queryString)
static const std::string getForeignTableSchema(bool if_not_exists=false)
Definition: Catalog.cpp:647
static const std::string getForeignServerSchema(bool if_not_exists=false)
Definition: Catalog.cpp:640
const Catalog * getObjForLock()
Definition: Catalog.cpp:220
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string Catalog_Namespace::Catalog::createLink ( LinkDescriptor ld,
size_t  min_length 
)

Definition at line 3729 of file Catalog.cpp.

References addLinkToMap(), calculateSHA1(), SqliteConnector::getData(), SqliteConnector::getNumRows(), getObjForLock(), LinkDescriptor::link, LinkDescriptor::linkId, SqliteConnector::query(), SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), sqliteConnector_, to_string(), LinkDescriptor::updateTime, LinkDescriptor::userId, LinkDescriptor::viewMetadata, and LinkDescriptor::viewState.

3729  {
3730  cat_write_lock write_lock(this);
3731  cat_sqlite_lock sqlite_lock(getObjForLock());
3732  sqliteConnector_.query("BEGIN TRANSACTION");
3733  try {
3735  .substr(0, 8);
3737  "SELECT linkid FROM mapd_links WHERE link = ? and userid = ?",
3738  std::vector<std::string>{ld.link, std::to_string(ld.userId)});
3739  if (sqliteConnector_.getNumRows() > 0) {
3741  "UPDATE mapd_links SET update_time = datetime('now') WHERE userid = ? AND link "
3742  "= ?",
3743  std::vector<std::string>{std::to_string(ld.userId), ld.link});
3744  } else {
3746  "INSERT INTO mapd_links (userid, link, view_state, view_metadata, update_time) "
3747  "VALUES (?,?,?,?, "
3748  "datetime('now'))",
3749  std::vector<std::string>{
3750  std::to_string(ld.userId), ld.link, ld.viewState, ld.viewMetadata});
3751  }
3752  // now get the auto generated dashid
3754  "SELECT linkid, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_links "
3755  "WHERE link = ?",
3756  ld.link);
3757  ld.linkId = sqliteConnector_.getData<int>(0, 0);
3758  ld.updateTime = sqliteConnector_.getData<std::string>(0, 1);
3759  } catch (std::exception& e) {
3760  sqliteConnector_.query("ROLLBACK TRANSACTION");
3761  throw;
3762  }
3763  sqliteConnector_.query("END TRANSACTION");
3764  addLinkToMap(ld);
3765  return ld.link;
3766 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:553
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
std::string calculateSHA1(const std::string &data)
Definition: Catalog.cpp:3717
void addLinkToMap(LinkDescriptor &ld)
Definition: Catalog.cpp:1335
const Catalog * getObjForLock()
Definition: Catalog.cpp:220
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createOrUpdateDashboardSystemRole ( const std::string &  view_meta,
const int32_t &  user_id,
const std::string &  dash_role_name 
)
protected

Definition at line 1294 of file Catalog.cpp.

References Catalog_Namespace::SysCatalog::createRole(), Grantee::findDbObject(), Grantee::getDbObjects(), Catalog_Namespace::SysCatalog::getRoleGrantee(), Catalog_Namespace::SysCatalog::grantDBObjectPrivilegesBatch(), Catalog_Namespace::SysCatalog::instance(), parseDashboardObjects(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivileges(), TableDBObjectType, and ViewDBObjectType.

Referenced by createDashboard(), createDashboardSystemRoles(), and replaceDashboard().

1296  {
1297  auto objects = parseDashboardObjects(view_meta, user_id);
1298  Role* rl = SysCatalog::instance().getRoleGrantee(dash_role_name);
1299  if (!rl) {
1300  // Dashboard role does not exist
1301  // create role and grant privileges
1302  // NOTE(wamsi): Transactionally unsafe
1303  SysCatalog::instance().createRole(dash_role_name, false);
1304  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1305  } else {
1306  // Dashboard system role already exists
1307  // Add/remove privileges on objects
1308  auto ex_objects = rl->getDbObjects(true);
1309  for (auto key : *ex_objects | boost::adaptors::map_keys) {
1310  if (key.permissionType != TableDBObjectType &&
1311  key.permissionType != ViewDBObjectType) {
1312  continue;
1313  }
1314  bool found = false;
1315  for (auto obj : objects) {
1316  found = key == obj.getObjectKey() ? true : false;
1317  if (found) {
1318  break;
1319  }
1320  }
1321  if (!found) {
1322  // revoke privs on object since the object is no
1323  // longer used by the dashboard as source
1324  // NOTE(wamsi): Transactionally unsafe
1326  dash_role_name, *rl->findDbObject(key, true), *this);
1327  }
1328  }
1329  // Update privileges on remaining objects
1330  // NOTE(wamsi): Transactionally unsafe
1331  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1332  }
1333 }
void revokeDBObjectPrivileges(const std::string &grantee, const DBObject &object, const Catalog_Namespace::Catalog &catalog)
Definition: Grantee.h:76
DBObject * findDbObject(const DBObjectKey &objectKey, bool only_direct) const
Definition: Grantee.cpp:72
void createRole(const std::string &roleName, const bool &userPrivateRole=false)
static SysCatalog & instance()
Definition: SysCatalog.h:288
Role * getRoleGrantee(const std::string &name) const
void grantDBObjectPrivilegesBatch(const std::vector< std::string > &grantees, const std::vector< DBObject > &objects, const Catalog_Namespace::Catalog &catalog)
const DBObjectMap * getDbObjects(bool only_direct) const
Definition: Grantee.h:56
std::vector< DBObject > parseDashboardObjects(const std::string &view_meta, const int &user_id)
Definition: Catalog.cpp:1264

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createShardedTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs 
)

Definition at line 3246 of file Catalog.cpp.

References CHECK, createTable(), generatePhysicalTableName(), logicalToPhysicalTableMapById_, TableDescriptor::nShards, TableDescriptor::shard, table_is_temporary(), TableDescriptor::tableId, TableDescriptor::tableName, and updateLogicalToPhysicalTableMap().

3249  {
3250  cat_write_lock write_lock(this);
3251 
3252  /* create logical table */
3253  TableDescriptor* tdl = &td;
3254  createTable(*tdl, cols, shared_dict_defs, true); // create logical table
3255  int32_t logical_tb_id = tdl->tableId;
3256  std::string logical_table_name = tdl->tableName;
3257 
3258  /* create physical tables and link them to the logical table */
3259  std::vector<int32_t> physicalTables;
3260  for (int32_t i = 1; i <= td.nShards; i++) {
3261  TableDescriptor* tdp = &td;
3262  tdp->tableName = generatePhysicalTableName(logical_table_name, i);
3263  tdp->shard = i - 1;
3264  createTable(*tdp, cols, shared_dict_defs, false); // create physical table
3265  int32_t physical_tb_id = tdp->tableId;
3266 
3267  /* add physical table to the vector of physical tables */
3268  physicalTables.push_back(physical_tb_id);
3269  }
3270 
3271  if (!physicalTables.empty()) {
3272  /* add logical to physical tables correspondence to the map */
3273  const auto it_ok =
3274  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
3275  CHECK(it_ok.second);
3276  /* update sqlite mapd_logical_to_physical in sqlite database */
3277  if (!table_is_temporary(&td)) {
3278  updateLogicalToPhysicalTableMap(logical_tb_id);
3279  }
3280  }
3281 }
std::string tableName
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:2143
std::string generatePhysicalTableName(const std::string &logicalTableName, const int32_t &shardNumber)
Definition: Catalog.cpp:3982
bool table_is_temporary(const TableDescriptor *const td)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:560
void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id)
Definition: Catalog.cpp:580

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs,
bool  isLogicalTable 
)

Definition at line 2143 of file Catalog.cpp.

References addTableToMap(), calciteMgr_, ColumnDescriptor::columnId, TableDescriptor::columnIdBySpi_, ColumnDescriptor::columnName, ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, dict_ref_t::dictId, Data_Namespace::DISK_LEVEL, expandGeoColumn(), StorageType::FOREIGN_TABLE, TableDescriptor::fragPageSize, TableDescriptor::fragType, g_enable_fsi, g_serialize_temp_tables, g_test_against_columnId_gap, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_next_refresh_time(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), getCurrentDB(), SqliteConnector::getData(), getObjForLock(), TableDescriptor::hasDeletedCol, SQLTypeInfo::is_array(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, kBIGINT, kBOOLEAN, kENCODING_DICT, TableDescriptor::keyMetainfo, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nColumns, foreign_storage::ForeignTable::next_refresh_time, nextTempDictId_, nextTempTableId_, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, ForeignStorageInterface::prepareTable(), SqliteConnector::query(), SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), ForeignStorageInterface::registerTable(), removeTableFromMap(), serializeTableJsonUnlocked(), SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_size(), setColumnDictionary(), setColumnSharedDictionary(), TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, sqliteConnector_, TableDescriptor::storageType, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), TableDescriptor::userId, TableDescriptor::viewSQL, and ColumnDescriptor::virtualExpr.

Referenced by createShardedTable().

2147  {
2148  cat_write_lock write_lock(this);
2149  list<ColumnDescriptor> cds = cols;
2150  list<DictDescriptor> dds;
2151  std::set<std::string> toplevel_column_names;
2152  list<ColumnDescriptor> columns;
2153 
2154  if (!td.storageType.empty() &&
2157  throw std::runtime_error("Only temporary tables can be backed by foreign storage.");
2158  }
2160  }
2161 
2162  for (auto cd : cds) {
2163  if (cd.columnName == "rowid") {
2164  throw std::runtime_error(
2165  "Cannot create column with name rowid. rowid is a system defined column.");
2166  }
2167  columns.push_back(cd);
2168  toplevel_column_names.insert(cd.columnName);
2169  if (cd.columnType.is_geometry()) {
2170  expandGeoColumn(cd, columns);
2171  }
2172  }
2173  cds.clear();
2174 
2175  ColumnDescriptor cd;
2176  // add row_id column -- Must be last column in the table
2177  cd.columnName = "rowid";
2178  cd.isSystemCol = true;
2179  cd.columnType = SQLTypeInfo(kBIGINT, true);
2180 #ifdef MATERIALIZED_ROWID
2181  cd.isVirtualCol = false;
2182 #else
2183  cd.isVirtualCol = true;
2184  cd.virtualExpr = "MAPD_FRAG_ID * MAPD_ROWS_PER_FRAG + MAPD_FRAG_ROW_ID";
2185 #endif
2186  columns.push_back(cd);
2187  toplevel_column_names.insert(cd.columnName);
2188 
2189  if (td.hasDeletedCol) {
2190  ColumnDescriptor cd_del;
2191  cd_del.columnName = "$deleted$";
2192  cd_del.isSystemCol = true;
2193  cd_del.isVirtualCol = false;
2194  cd_del.columnType = SQLTypeInfo(kBOOLEAN, true);
2195  cd_del.isDeletedCol = true;
2196 
2197  columns.push_back(cd_del);
2198  }
2199 
2200  td.nColumns = columns.size();
2201  cat_sqlite_lock sqlite_lock(getObjForLock());
2202  sqliteConnector_.query("BEGIN TRANSACTION");
2204  try {
2206  R"(INSERT INTO mapd_tables (name, userid, ncolumns, isview, fragments, frag_type, max_frag_rows, max_chunk_size, frag_page_size, max_rows, partitions, shard_column_id, shard, num_shards, sort_column_id, storage_type, max_rollback_epochs, key_metainfo) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?))",
2207  std::vector<std::string>{td.tableName,
2208  std::to_string(td.userId),
2210  std::to_string(td.isView),
2211  "",
2216  std::to_string(td.maxRows),
2217  td.partitions,
2219  std::to_string(td.shard),
2220  std::to_string(td.nShards),
2222  td.storageType,
2224  td.keyMetainfo});
2225 
2226  // now get the auto generated tableid
2228  "SELECT tableid FROM mapd_tables WHERE name = ?", td.tableName);
2229  td.tableId = sqliteConnector_.getData<int>(0, 0);
2230  int colId = 1;
2231  for (auto cd : columns) {
2233  const bool is_foreign_col =
2234  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2235  if (!is_foreign_col) {
2236  setColumnDictionary(cd, dds, td, isLogicalTable);
2237  }
2238  }
2239 
2240  if (toplevel_column_names.count(cd.columnName)) {
2241  // make up colId gap for sanity test (begin with 1 bc much code depends on it!)
2242  if (colId > 1) {
2243  colId += g_test_against_columnId_gap;
2244  }
2245  if (!cd.isGeoPhyCol) {
2246  td.columnIdBySpi_.push_back(colId);
2247  }
2248  }
2249 
2251  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, "
2252  "coldim, colscale, is_notnull, "
2253  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
2254  "virtual_expr, is_deletedcol) "
2255  "VALUES (?, ?, ?, ?, ?, "
2256  "?, "
2257  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2258  std::vector<std::string>{std::to_string(td.tableId),
2259  std::to_string(colId),
2260  cd.columnName,
2269  "",
2272  cd.virtualExpr,
2274  cd.tableId = td.tableId;
2275  cd.columnId = colId++;
2276  cds.push_back(cd);
2277  }
2278  if (td.isView) {
2280  "INSERT INTO mapd_views (tableid, sql) VALUES (?,?)",
2281  std::vector<std::string>{std::to_string(td.tableId), td.viewSQL});
2282  }
2284  auto& foreign_table = dynamic_cast<foreign_storage::ForeignTable&>(td);
2285  foreign_table.next_refresh_time = get_next_refresh_time(foreign_table);
2287  "INSERT INTO omnisci_foreign_tables (table_id, server_id, options, "
2288  "last_refresh_time, next_refresh_time) VALUES (?, ?, ?, ?, ?)",
2289  std::vector<std::string>{std::to_string(foreign_table.tableId),
2290  std::to_string(foreign_table.foreign_server->id),
2291  foreign_table.getOptionsAsJsonString(),
2292  std::to_string(foreign_table.last_refresh_time),
2293  std::to_string(foreign_table.next_refresh_time)});
2294  }
2295  } catch (std::exception& e) {
2296  sqliteConnector_.query("ROLLBACK TRANSACTION");
2297  throw;
2298  }
2299  } else { // Temporary table
2300  td.tableId = nextTempTableId_++;
2301  int colId = 1;
2302  for (auto cd : columns) {
2304  const bool is_foreign_col =
2305  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2306 
2307  if (!is_foreign_col) {
2308  // Create a new temporary dictionary
2309  std::string fileName("");
2310  std::string folderPath("");
2312  nextTempDictId_++;
2313  DictDescriptor dd(dict_ref,
2314  fileName,
2316  false,
2317  1,
2318  folderPath,
2319  true); // Is dictName (2nd argument) used?
2320  dds.push_back(dd);
2321  if (!cd.columnType.is_array()) {
2323  }
2324  cd.columnType.set_comp_param(dict_ref.dictId);
2325  }
2326  }
2327  if (toplevel_column_names.count(cd.columnName)) {
2328  // make up colId gap for sanity test (begin with 1 bc much code depends on it!)
2329  if (colId > 1) {
2330  colId += g_test_against_columnId_gap;
2331  }
2332  if (!cd.isGeoPhyCol) {
2333  td.columnIdBySpi_.push_back(colId);
2334  }
2335  }
2336  cd.tableId = td.tableId;
2337  cd.columnId = colId++;
2338  cds.push_back(cd);
2339  }
2340 
2342  serializeTableJsonUnlocked(&td, cds);
2343  }
2344  }
2345 
2346  try {
2347  addTableToMap(&td, cds, dds);
2348  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
2349  if (!td.storageType.empty() && td.storageType != StorageType::FOREIGN_TABLE) {
2351  }
2352  } catch (std::exception& e) {
2353  sqliteConnector_.query("ROLLBACK TRANSACTION");
2354  removeTableFromMap(td.tableName, td.tableId, true);
2355  throw;
2356  }
2357  sqliteConnector_.query("END TRANSACTION");
2358 }
int32_t maxRollbackEpochs
void serializeTableJsonUnlocked(const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
Definition: Catalog.cpp:2360
int64_t get_next_refresh_time(const foreign_storage::ForeignTable &foreign_table)
Definition: Catalog.cpp:2131
std::string virtualExpr
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:312
void set_size(int s)
Definition: sqltypes.h:409
std::string partitions
HOST DEVICE int get_size() const
Definition: sqltypes.h:321
T getData(const int row, const int col)
std::string tableName
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:553
HOST DEVICE int get_scale() const
Definition: sqltypes.h:316
std::string storageType
virtual void query(const std::string &queryString)
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:311
std::vector< int > columnIdBySpi_
std::string to_string(char const *&&v)
void addTableToMap(const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
Definition: Catalog.cpp:1112
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:221
specifies the content in-memory of a row in the column metadata table
static void registerTable(Catalog_Namespace::Catalog *catalog, const TableDescriptor &td, const std::list< ColumnDescriptor > &cols)
ids are created
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:2000
std::string keyMetainfo
int g_test_against_columnId_gap
Definition: Catalog.cpp:90
bool g_serialize_temp_tables
Definition: Catalog.cpp:97
void set_comp_param(int p)
Definition: sqltypes.h:412
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:558
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:319
Fragmenter_Namespace::FragmenterType fragType
Data_Namespace::MemoryLevel persistenceLevel
const Catalog * getObjForLock()
Definition: Catalog.cpp:220
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:313
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1172
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:320
bool setColumnSharedDictionary(ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:3136
Descriptor for a dictionary for a string columne.
std::string viewSQL
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:318
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:3205
static constexpr char const * FOREIGN_TABLE
bool g_enable_fsi
Definition: Catalog.cpp:91
std::string columnName
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
bool is_array() const
Definition: sqltypes.h:486
static void prepareTable(const int db_id, TableDescriptor &td, std::list< ColumnDescriptor > &cols)
prepare table options and modify columns

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::delDictionary ( const ColumnDescriptor cd)

Definition at line 1783 of file Catalog.cpp.

References basePath_, CHECK, CHECK_GT, ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SqliteConnector::getData(), getMetadataForTable(), getObjForLock(), SQLTypeInfo::is_string(), SQLTypeInfo::is_string_array(), kENCODING_DICT, SqliteConnector::query_with_text_param(), File_Namespace::renameForDelete(), sqliteConnector_, string_dict_hosts_, ColumnDescriptor::tableId, to_string(), and VLOG.

Referenced by roll().

1783  {
1784  cat_write_lock write_lock(this);
1785  cat_sqlite_lock sqlite_lock(getObjForLock());
1786  if (!(cd.columnType.is_string() || cd.columnType.is_string_array())) {
1787  return;
1788  }
1789  if (!(cd.columnType.get_compression() == kENCODING_DICT)) {
1790  return;
1791  }
1792  const auto dictId = cd.columnType.get_comp_param();
1793  CHECK_GT(dictId, 0);
1794  // decrement and zero check dict ref count
1795  const auto td = getMetadataForTable(cd.tableId);
1796  CHECK(td);
1798  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
1799  std::to_string(dictId));
1801  "SELECT refcount FROM mapd_dictionaries WHERE dictid = ?", std::to_string(dictId));
1802  const auto refcount = sqliteConnector_.getData<int>(0, 0);
1803  VLOG(3) << "Dictionary " << dictId << "from dropped table has reference count "
1804  << refcount;
1805  if (refcount > 0) {
1806  return;
1807  }
1808  const DictRef dictRef(currentDB_.dbId, dictId);
1809  sqliteConnector_.query_with_text_param("DELETE FROM mapd_dictionaries WHERE dictid = ?",
1810  std::to_string(dictId));
1811  File_Namespace::renameForDelete(basePath_ + "/mapd_data/DB_" +
1812  std::to_string(currentDB_.dbId) + "_DICT_" +
1813  std::to_string(dictId));
1814 
1815  std::unique_ptr<StringDictionaryClient> client;
1816  if (!string_dict_hosts_.empty()) {
1817  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dictRef, true));
1818  }
1819  if (client) {
1820  client->drop(dictRef);
1821  }
1822 
1823  dictDescriptorMapByRef_.erase(dictRef);
1824 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:553
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:557
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:546
#define CHECK_GT(x, y)
Definition: Logger.h:209
std::string to_string(char const *&&v)
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:319
const Catalog * getObjForLock()
Definition: Catalog.cpp:220
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:320
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
bool is_string() const
Definition: sqltypes.h:478
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:188
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
bool is_string_array() const
Definition: sqltypes.h:479
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
#define VLOG(n)
Definition: Logger.h:291

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::deleteMetadataForDashboards ( const std::vector< int32_t >  ids,
const UserMetadata user 
)

Definition at line 1553 of file Catalog.cpp.

References DashboardDBObjectType, dashboardDescriptorMap_, AccessPrivileges::DELETE_DASHBOARD, getMetadataForDashboard(), getObjForLock(), Catalog_Namespace::SysCatalog::instance(), SqliteConnector::query(), SqliteConnector::query_with_text_params(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivilegesFromAllBatch(), sqliteConnector_, and to_string().

1554  {
1555  std::stringstream invalid_ids, restricted_ids;
1556 
1557  for (int32_t dashboard_id : dashboard_ids) {
1558  if (!getMetadataForDashboard(dashboard_id)) {
1559  invalid_ids << (!invalid_ids.str().empty() ? ", " : "") << dashboard_id;
1560  continue;
1561  }
1562  DBObject object(dashboard_id, DashboardDBObjectType);
1563  object.loadKey(*this);
1564  object.setPrivileges(AccessPrivileges::DELETE_DASHBOARD);
1565  std::vector<DBObject> privs = {object};
1566  if (!SysCatalog::instance().checkPrivileges(user, privs))
1567  restricted_ids << (!restricted_ids.str().empty() ? ", " : "") << dashboard_id;
1568  }
1569 
1570  if (invalid_ids.str().size() > 0 || restricted_ids.str().size() > 0) {
1571  std::stringstream error_message;
1572  error_message << "Delete dashboard(s) failed with error(s):";
1573  if (invalid_ids.str().size() > 0)
1574  error_message << "\nDashboard id: " << invalid_ids.str()
1575  << " - Dashboard id does not exist";
1576  if (restricted_ids.str().size() > 0)
1577  error_message
1578  << "\nDashboard id: " << restricted_ids.str()
1579  << " - User should be either owner of dashboard or super user to delete it";
1580  throw std::runtime_error(error_message.str());
1581  }
1582  std::vector<DBObject> dash_objs;
1583 
1584  for (int32_t dashboard_id : dashboard_ids) {
1585  dash_objs.push_back(DBObject(dashboard_id, DashboardDBObjectType));
1586  }
1587  // BE-5245: Transactionally unsafe (like other combined Catalog/Syscatalog operations)
1589  {
1590  cat_write_lock write_lock(this);
1591  cat_sqlite_lock sqlite_lock(getObjForLock());
1592 
1593  sqliteConnector_.query("BEGIN TRANSACTION");
1594  try {
1595  for (int32_t dashboard_id : dashboard_ids) {
1596  auto dash = getMetadataForDashboard(dashboard_id);
1597  // Dash should still exist if revokeDBObjectPrivileges passed but throw and
1598  // rollback if already deleted
1599  if (!dash) {
1600  throw std::runtime_error(
1601  std::string("Delete dashboard(s) failed with error(s):\nDashboard id: ") +
1602  std::to_string(dashboard_id) + " - Dashboard id does not exist ");
1603  }
1604  std::string user_id = std::to_string(dash->userId);
1605  std::string dash_name = dash->dashboardName;
1606  auto viewDescIt = dashboardDescriptorMap_.find(user_id + ":" + dash_name);
1607  dashboardDescriptorMap_.erase(viewDescIt);
1609  "DELETE FROM mapd_dashboards WHERE name = ? and userid = ?",
1610  std::vector<std::string>{dash_name, user_id});
1611  }
1612  } catch (std::exception& e) {
1613  sqliteConnector_.query("ROLLBACK TRANSACTION");
1614  throw;
1615  }
1616  sqliteConnector_.query("END TRANSACTION");
1617  }
1618 }
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:553
virtual void query(const std::string &queryString)
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:547
std::string to_string(char const *&&v)
static SysCatalog & instance()
Definition: SysCatalog.h:288
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const
const Catalog * getObjForLock()
Definition: Catalog.cpp:220
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
static const AccessPrivileges DELETE_DASHBOARD
Definition: DBObject.h:174
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
void revokeDBObjectPrivilegesFromAllBatch(std::vector< DBObject > &objects, Catalog *catalog)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::doDropTable ( const TableDescriptor td)
protected

Definition at line 3434 of file Catalog.cpp.

References dropTableFromJsonUnlocked(), eraseTablePhysicalData(), executeDropTableSqliteQueries(), g_serialize_temp_tables, table_is_temporary(), and TableDescriptor::tableName.

Referenced by dropTable().

3434  {
3438  }
3440 }
void executeDropTableSqliteQueries(const TableDescriptor *td)
Definition: Catalog.cpp:3442
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:3953
std::string tableName
void dropTableFromJsonUnlocked(const std::string &table_name) const
Definition: Catalog.cpp:2422
bool g_serialize_temp_tables
Definition: Catalog.cpp:97
bool table_is_temporary(const TableDescriptor *const td)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::doTruncateTable ( const TableDescriptor td)
protected

Definition at line 3301 of file Catalog.cpp.

References CHECK, CHECK_GE, columnDescriptorMapById_, ColumnDescriptor::columnType, Data_Namespace::CPU_LEVEL, currentDB_, dataMgr_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, dict_ref_t::dictId, DictDescriptor::dictIsTemp, DictDescriptor::dictRef, TableDescriptor::fragmenter, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), getMetadataForDict(), Data_Namespace::GPU_LEVEL, Catalog_Namespace::SysCatalog::instance(), kENCODING_DICT, File_Namespace::renameForDelete(), string_dict_hosts_, tableDescriptorMapById_, ColumnDescriptor::tableId, and TableDescriptor::tableId.

Referenced by truncateTable().

3301  {
3302  cat_write_lock write_lock(this);
3303 
3304  const int tableId = td->tableId;
3305  // must destroy fragmenter before deleteChunks is called.
3306  if (td->fragmenter != nullptr) {
3307  auto tableDescIt = tableDescriptorMapById_.find(tableId);
3308  CHECK(tableDescIt != tableDescriptorMapById_.end());
3309  tableDescIt->second->fragmenter = nullptr;
3310  CHECK(td->fragmenter == nullptr);
3311  }
3312  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
3313  // assuming deleteChunksWithPrefix is atomic
3314  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
3315  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
3316 
3317  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
3318 
3319  std::unique_ptr<StringDictionaryClient> client;
3320  if (SysCatalog::instance().isAggregator()) {
3321  CHECK(!string_dict_hosts_.empty());
3322  DictRef dict_ref(currentDB_.dbId, -1);
3323  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
3324  }
3325  // clean up any dictionaries
3326  // delete all column descriptors for the table
3327  for (const auto& columnDescriptor : columnDescriptorMapById_) {
3328  auto cd = columnDescriptor.second;
3329  if (cd->tableId != td->tableId) {
3330  continue;
3331  }
3332  const int dict_id = cd->columnType.get_comp_param();
3333  // Dummy dictionaries created for a shard of a logical table have the id set to zero.
3334  if (cd->columnType.get_compression() == kENCODING_DICT && dict_id) {
3335  const DictRef dict_ref(currentDB_.dbId, dict_id);
3336  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
3337  CHECK(dictIt != dictDescriptorMapByRef_.end());
3338  const auto& dd = dictIt->second;
3339  CHECK_GE(dd->refcount, 1);
3340  // if this is the only table using this dict reset the dict
3341  if (dd->refcount == 1) {
3342  // close the dictionary
3343  dd->stringDict.reset();
3344  File_Namespace::renameForDelete(dd->dictFolderPath);
3345  if (client) {
3346  client->drop(dd->dictRef);
3347  }
3348  if (!dd->dictIsTemp) {
3349  boost::filesystem::create_directory(dd->dictFolderPath);
3350  }
3351  }
3352 
3353  DictDescriptor* new_dd = new DictDescriptor(dd->dictRef,
3354  dd->dictName,
3355  dd->dictNBits,
3356  dd->dictIsShared,
3357  dd->refcount,
3358  dd->dictFolderPath,
3359  dd->dictIsTemp);
3360  dictDescriptorMapByRef_.erase(dictIt);
3361  // now create new Dict -- need to figure out what to do here for temp tables
3362  if (client) {
3363  client->create(new_dd->dictRef, new_dd->dictIsTemp);
3364  }
3365  dictDescriptorMapByRef_[new_dd->dictRef].reset(new_dd);
3367  }
3368  }
3369 }
std::vector< int > ChunkKey
Definition: types.h:37
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:557
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:546
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:555
#define CHECK_GE(x, y)
Definition: Logger.h:210
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:545
static SysCatalog & instance()
Definition: SysCatalog.h:288
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:543
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1439
int32_t dictId
Definition: DictRef.h:10
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
#define CHECK(condition)
Definition: Logger.h:197
Descriptor for a dictionary for a string columne.
mapd_unique_lock< mapd_shared_mutex > write_lock
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:188

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::dropColumn ( const TableDescriptor td,
const ColumnDescriptor cd 
)

Definition at line 1911 of file Catalog.cpp.

References CHECK, columnDescriptorMap_, columnDescriptorMapById_, columnDescriptorsForRoll, ColumnDescriptor::columnId, ColumnDescriptor::columnName, getMetadataForColumn(), getObjForLock(), getPhysicalTablesDescriptors(), TableDescriptor::nShards, SqliteConnector::query_with_text_params(), TableDescriptor::shard, sqliteConnector_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), and to_upper().

1911  {
1912  cat_write_lock write_lock(this);
1913  cat_sqlite_lock sqlite_lock(getObjForLock());
1914  // caller must handle sqlite/chunk transaction TOGETHER
1916  "DELETE FROM mapd_columns where tableid = ? and columnid = ?",
1917  std::vector<std::string>{std::to_string(td.tableId), std::to_string(cd.columnId)});
1918 
1920  "UPDATE mapd_tables SET ncolumns = ncolumns - 1 WHERE tableid = ?",
1921  std::vector<std::string>{std::to_string(td.tableId)});
1922 
1923  ColumnDescriptorMap::iterator columnDescIt =
1925  CHECK(columnDescIt != columnDescriptorMap_.end());
1926 
1927  columnDescriptorsForRoll.emplace_back(columnDescIt->second, nullptr);
1928 
1929  columnDescriptorMap_.erase(columnDescIt);
1931  --tableDescriptorMapById_[td.tableId]->nColumns;
1932  // for each shard
1933  if (td.nShards > 0 && td.shard < 0) {
1934  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
1935  const auto shard_cd = getMetadataForColumn(shard->tableId, cd.columnId);
1936  CHECK(shard_cd);
1937  dropColumn(*shard, *shard_cd);
1938  }
1939  }
1940 }
std::tuple< int, std::string > ColumnKey
Definition: Types.h:36
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:544
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:553
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logicalTableDesc) const
Definition: Catalog.cpp:3786
void dropColumn(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:1911
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:545
std::string to_string(char const *&&v)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:543
std::string to_upper(const std::string &str)
const Catalog * getObjForLock()
Definition: Catalog.cpp:220
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:572
std::string columnName
std::tuple< int, int > ColumnIdKey
Definition: Types.h:38

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropForeignServer ( const std::string &  server_name)

Drops/deletes a foreign server DB object.

Parameters
server_name- Name of foreign server that will be deleted

Definition at line 2615 of file Catalog.cpp.

References CHECK_EQ, foreignServerMap_, foreignServerMapById_, SqliteConnector::getData(), SqliteConnector::getNumRows(), getObjForLock(), SqliteConnector::query(), SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), sqliteConnector_, and to_string().

2615  {
2616  cat_write_lock write_lock(this);
2617  cat_sqlite_lock sqlite_lock(getObjForLock());
2618 
2620  "SELECT id from omnisci_foreign_servers where name = ?",
2621  std::vector<std::string>{server_name});
2622  auto num_rows = sqliteConnector_.getNumRows();
2623  if (num_rows > 0) {
2624  CHECK_EQ(size_t(1), num_rows);
2625  auto server_id = sqliteConnector_.getData<int32_t>(0, 0);
2627  "SELECT table_id from omnisci_foreign_tables where server_id = ?",
2628  std::to_string(server_id));
2629  if (sqliteConnector_.getNumRows() > 0) {
2630  throw std::runtime_error{"Foreign server \"" + server_name +
2631  "\" is referenced "
2632  "by existing foreign tables and cannot be dropped."};
2633  }
2634  sqliteConnector_.query("BEGIN TRANSACTION");
2635  try {
2637  "DELETE FROM omnisci_foreign_servers WHERE name = ?",
2638  std::vector<std::string>{server_name});
2639  } catch (const std::exception& e) {
2640  sqliteConnector_.query("ROLLBACK TRANSACTION");
2641  throw;
2642  }
2643  sqliteConnector_.query("END TRANSACTION");
2644  foreignServerMap_.erase(server_name);
2645  foreignServerMapById_.erase(server_id);
2646  }
2647 }
#define CHECK_EQ(x, y)
Definition: Logger.h:205
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:553
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:551
ForeignServerMap foreignServerMap_
Definition: Catalog.h:550
const Catalog * getObjForLock()
Definition: Catalog.cpp:220
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropFsiSchemasAndTables ( )
protected
void Catalog_Namespace::Catalog::dropTable ( const TableDescriptor td)

Definition at line 3401 of file Catalog.cpp.

References CHECK, doDropTable(), getMetadataForTable(), getObjForLock(), Catalog_Namespace::SysCatalog::instance(), TableDescriptor::isView, logicalToPhysicalTableMapById_, SqliteConnector::query(), SqliteConnector::query_with_text_param(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivilegesFromAll(), sqliteConnector_, TableDBObjectType, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and ViewDBObjectType.

3401  {
3404  cat_write_lock write_lock(this);
3405  cat_sqlite_lock sqlite_lock(getObjForLock());
3406  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
3407  sqliteConnector_.query("BEGIN TRANSACTION");
3408  try {
3409  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
3410  // remove all corresponding physical tables if this is a logical table
3411  const auto physicalTables = physicalTableIt->second;
3412  CHECK(!physicalTables.empty());
3413  for (size_t i = 0; i < physicalTables.size(); i++) {
3414  int32_t physical_tb_id = physicalTables[i];
3415  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
3416  CHECK(phys_td);
3417  doDropTable(phys_td);
3418  }
3419 
3420  // remove corresponding record from the logicalToPhysicalTableMap in sqlite database
3422  "DELETE FROM mapd_logical_to_physical WHERE logical_table_id = ?",
3423  std::to_string(td->tableId));
3425  }
3426  doDropTable(td);
3427  } catch (std::exception& e) {
3428  sqliteConnector_.query("ROLLBACK TRANSACTION");
3429  throw;
3430  }
3431  sqliteConnector_.query("END TRANSACTION");
3432 }
void doDropTable(const TableDescriptor *td)
Definition: Catalog.cpp:3434
std::string tableName
SqliteConnector sqliteConnector_
Definition: Catalog.h:553
void revokeDBObjectPrivilegesFromAll(DBObject object, Catalog *catalog)
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
static SysCatalog & instance()
Definition: SysCatalog.h:288
const Catalog * getObjForLock()
Definition: Catalog.cpp:220
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:114
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:560

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropTableFromJsonUnlocked ( const std::string &  table_name) const
protected

Definition at line 2422 of file Catalog.cpp.

References basePath_, CHECK, currentDB_, Catalog_Namespace::DBMetadata::dbName, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and VLOG.

Referenced by doDropTable().

2422  {
2423  // relies on the catalog write lock
2424  using namespace rapidjson;
2425 
2426  VLOG(1) << "Dropping temporary table " << table_name << " to JSON for Calcite.";
2427 
2428  const auto db_name = currentDB_.dbName;
2429  const auto file_path = table_json_filepath(basePath_, db_name);
2430 
2431  CHECK(boost::filesystem::exists(file_path));
2432  Document d;
2433 
2434  std::ifstream reader(file_path.string());
2435  CHECK(reader.is_open());
2436  IStreamWrapper json_read_wrapper(reader);
2437  d.ParseStream(json_read_wrapper);
2438 
2439  CHECK(d.IsObject());
2440  auto table_name_ref = StringRef(table_name.c_str());
2441  CHECK(d.HasMember(table_name_ref));
2442  CHECK(d.RemoveMember(table_name_ref));
2443 
2444  // Overwrite the existing file
2445  std::ofstream writer(file_path.string(), std::ios::trunc | std::ios::out);
2446  CHECK(writer.is_open());
2447  OStreamWrapper json_wrapper(writer);
2448 
2449  Writer<OStreamWrapper> json_writer(json_wrapper);
2450  d.Accept(json_writer);
2451  writer.close();
2452 }
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:153
#define CHECK(condition)
Definition: Logger.h:197
#define VLOG(n)
Definition: Logger.h:291

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string Catalog_Namespace::Catalog::dumpCreateTable ( const TableDescriptor td,
bool  multiline_formatting = true,
bool  dump_defaults = false 
) const

Definition at line 4328 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, Catalog_Namespace::contains_spaces(), Catalog_Namespace::contains_sql_reserved_chars(), Data_Namespace::CPU_LEVEL, DEFAULT_FRAGMENT_ROWS, DEFAULT_MAX_CHUNK_SIZE, DEFAULT_MAX_ROLLBACK_EPOCHS, DEFAULT_MAX_ROWS, DEFAULT_PAGE_SIZE, TableDescriptor::fragPageSize, g_leaf_count, getAllColumnMetadataForTable(), getMetadataForColumn(), TableDescriptor::hasDeletedCol, Catalog_Namespace::is_reserved_sql_keyword(), ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, join(), kCHAR, kENCODING_DICT, kENCODING_GEOINT, TableDescriptor::keyMetainfo, kTEXT, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::unserialize_key_metainfo(), and TableDescriptor::viewSQL.

4330  {
4331  cat_read_lock read_lock(this);
4332 
4333  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
4334  std::ostringstream os;
4335 
4336  if (foreign_table) {
4337  os << "CREATE FOREIGN TABLE " << td->tableName << " (";
4338  } else if (!td->isView) {
4339  os << "CREATE ";
4341  os << "TEMPORARY ";
4342  }
4343  os << "TABLE " + td->tableName + " (";
4344  } else {
4345  os << "CREATE VIEW " + td->tableName + " AS " << td->viewSQL;
4346  return os.str();
4347  }
4348  // scan column defines
4349  std::vector<std::string> shared_dicts;
4350  std::set<std::string> shared_dict_column_names;
4351  unserialize_key_metainfo(shared_dicts, shared_dict_column_names, td->keyMetainfo);
4352  // gather column defines
4353  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
4354  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
4355  bool first = true;
4356  for (const auto cd : cds) {
4357  if (!(cd->isSystemCol || cd->isVirtualCol)) {
4358  const auto& ti = cd->columnType;
4359  if (!first) {
4360  os << ",";
4361  if (!multiline_formatting) {
4362  os << " ";
4363  }
4364  } else {
4365  first = false;
4366  }
4367  if (multiline_formatting) {
4368  os << "\n ";
4369  }
4370  // column name
4371  bool column_name_needs_quotes = is_reserved_sql_keyword(cd->columnName) ||
4372  contains_spaces(cd->columnName) ||
4373  contains_sql_reserved_chars(cd->columnName);
4374  if (!column_name_needs_quotes) {
4375  os << cd->columnName;
4376  } else {
4377  os << "\"" << cd->columnName << "\"";
4378  }
4379  // CHAR is perculiar... better dump it as TEXT(32) like \d does
4380  if (ti.get_type() == SQLTypes::kCHAR) {
4381  os << " "
4382  << "TEXT";
4383  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
4384  os << " "
4385  << "TEXT[]";
4386  } else {
4387  os << " " << ti.get_type_name();
4388  }
4389  os << (ti.get_notnull() ? " NOT NULL" : "");
4390  if (shared_dict_column_names.find(cd->columnName) ==
4391  shared_dict_column_names.end()) {
4392  // avoids "Exception: Column ... shouldn't specify an encoding, it borrows it from
4393  // the referenced column"
4394  if (ti.is_string() || (ti.is_array() && ti.get_subtype() == kTEXT)) {
4395  auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
4396  if (ti.get_compression() == kENCODING_DICT) {
4397  os << " ENCODING " << ti.get_compression_name() << "(" << (size * 8) << ")";
4398  } else {
4399  os << " ENCODING NONE";
4400  }
4401  } else if (ti.is_date_in_days() ||
4402  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
4403  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
4404  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
4405  } else if (ti.is_geometry()) {
4406  if (ti.get_compression() == kENCODING_GEOINT) {
4407  os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
4408  << ")";
4409  } else {
4410  os << " ENCODING NONE";
4411  }
4412  }
4413  }
4414  }
4415  }
4416  // gather SHARED DICTIONARYs
4417  if (shared_dicts.size()) {
4418  std::string comma;
4419  if (!multiline_formatting) {
4420  comma = ", ";
4421  } else {
4422  comma = ",\n ";
4423  }
4424  os << comma;
4425  os << boost::algorithm::join(shared_dicts, comma);
4426  }
4427  os << ")";
4428 
4429  std::vector<std::string> with_options;
4430  if (foreign_table) {
4431  if (multiline_formatting) {
4432  os << "\n";
4433  } else {
4434  os << " ";
4435  }
4436  os << "SERVER " << foreign_table->foreign_server->name;
4437 
4438  // gather WITH options ...
4439  for (const auto& [option, value] : foreign_table->options) {
4440  with_options.emplace_back(option + "='" + value + "'");
4441  }
4442  }
4443 
4444  if (dump_defaults || td->maxFragRows != DEFAULT_FRAGMENT_ROWS) {
4445  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
4446  }
4447  if (!foreign_table && (dump_defaults || td->maxChunkSize != DEFAULT_MAX_CHUNK_SIZE)) {
4448  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
4449  }
4450  if (!foreign_table && (dump_defaults || td->fragPageSize != DEFAULT_PAGE_SIZE)) {
4451  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
4452  }
4453  if (!foreign_table && (dump_defaults || td->maxRows != DEFAULT_MAX_ROWS)) {
4454  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
4455  }
4456  if (dump_defaults || td->maxRollbackEpochs != DEFAULT_MAX_ROLLBACK_EPOCHS) {
4457  with_options.push_back("MAX_ROLLBACK_EPOCHS=" +
4459  }
4460  if (!foreign_table && (dump_defaults || !td->hasDeletedCol)) {
4461  with_options.push_back(td->hasDeletedCol ? "VACUUM='DELAYED'" : "VACUUM='IMMEDIATE'");
4462  }
4463  if (!foreign_table && !td->partitions.empty()) {
4464  with_options.push_back("PARTITIONS='" + td->partitions + "'");
4465  }
4466  if (!foreign_table && td->nShards > 0) {
4467  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
4468  CHECK(shard_cd);
4469  with_options.push_back(
4470  "SHARD_COUNT=" +
4471  std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
4472  }
4473  if (!foreign_table && td->sortedColumnId > 0) {
4474  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
4475  CHECK(sort_cd);
4476  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
4477  }
4478 
4479  if (!with_options.empty()) {
4480  if (!multiline_formatting) {
4481  os << " ";
4482  } else {
4483  os << "\n";
4484  }
4485  os << "WITH (" + boost::algorithm::join(with_options, ", ") + ")";
4486  }
4487  os << ";";
4488  return os.str();
4489 }
int32_t maxRollbackEpochs
std::string partitions
std::string tableName
std::string join(T const &container, std::string const &delim)
#define DEFAULT_MAX_CHUNK_SIZE
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:112
void unserialize_key_metainfo(std::vector< std::string > &shared_dicts, std::set< std::string > &shared_dict_column_names, const std::string keyMetainfo)
Definition: Catalog.cpp:4278
bool contains_spaces(std::string_view str)
returns true if the string contains one or more spaces
Definition: Catalog.cpp:4309
std::string to_string(char const *&&v)
bool contains_sql_reserved_chars(std::string_view str, std::string_view chars="`~!@#$%^&*()-=+[{]}\\|;:'\",<.>/?")
returns true if the string contains one or more OmniSci SQL reserved characters
Definition: Catalog.cpp:4316
#define DEFAULT_PAGE_SIZE
Definition: FileMgr.h:47
#define DEFAULT_MAX_ROWS
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
#define DEFAULT_MAX_ROLLBACK_EPOCHS
specifies the content in-memory of a row in the table metadata table
std::string keyMetainfo
Definition: sqltypes.h:51
#define DEFAULT_FRAGMENT_ROWS
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1716
Data_Namespace::MemoryLevel persistenceLevel
Definition: sqltypes.h:40
mapd_shared_lock< mapd_shared_mutex > read_lock
#define CHECK(condition)
Definition: Logger.h:197
std::string viewSQL
size_t g_leaf_count
Definition: ParserNode.cpp:74
bool is_reserved_sql_keyword(std::string_view str)
returns true if the string equals an OmniSci SQL reserved keyword
Definition: Catalog.cpp:4323

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::dumpSchema ( const TableDescriptor td) const

Definition at line 4177 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, TableDescriptor::fragPageSize, g_leaf_count, getAllColumnMetadataForTable(), getMetadataForColumn(), TableDescriptor::hasDeletedCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, join(), kCHAR, kENCODING_DICT, kENCODING_GEOINT, kTEXT, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::tableId, and to_string().

Referenced by TableArchiver::dumpTable().

4177  {
4178  cat_read_lock read_lock(this);
4179 
4180  std::ostringstream os;
4181  os << "CREATE TABLE @T (";
4182  // gather column defines
4183  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
4184  std::string comma;
4185  std::vector<std::string> shared_dicts;
4186  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
4187  for (const auto cd : cds) {
4188  if (!(cd->isSystemCol || cd->isVirtualCol)) {
4189  const auto& ti = cd->columnType;
4190  os << comma << cd->columnName;
4191  // CHAR is perculiar... better dump it as TEXT(32) like \d does
4192  if (ti.get_type() == SQLTypes::kCHAR) {
4193  os << " "
4194  << "TEXT";
4195  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
4196  os << " "
4197  << "TEXT[]";
4198  } else {
4199  os << " " << ti.get_type_name();
4200  }
4201  os << (ti.get_notnull() ? " NOT NULL" : "");
4202  if (ti.is_string() || (ti.is_array() && ti.get_subtype() == kTEXT)) {
4203  auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
4204  if (ti.get_compression() == kENCODING_DICT) {
4205  // if foreign reference, get referenced tab.col
4206  const auto dict_id = ti.get_comp_param();
4207  const DictRef dict_ref(currentDB_.dbId, dict_id);
4208  const auto dict_it = dictDescriptorMapByRef_.find(dict_ref);
4209  CHECK(dict_it != dictDescriptorMapByRef_.end());
4210  const auto dict_name = dict_it->second->dictName;
4211  // when migrating a table, any foreign dict ref will be dropped
4212  // and the first cd of a dict will become root of the dict
4213  if (dict_root_cds.end() == dict_root_cds.find(dict_name)) {
4214  dict_root_cds[dict_name] = cd;
4215  os << " ENCODING " << ti.get_compression_name() << "(" << (size * 8) << ")";
4216  } else {
4217  const auto dict_root_cd = dict_root_cds[dict_name];
4218  shared_dicts.push_back("SHARED DICTIONARY (" + cd->columnName +
4219  ") REFERENCES @T(" + dict_root_cd->columnName + ")");
4220  // "... shouldn't specify an encoding, it borrows from the referenced column"
4221  }
4222  } else {
4223  os << " ENCODING NONE";
4224  }
4225  } else if (ti.is_date_in_days() ||
4226  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
4227  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
4228  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
4229  } else if (ti.is_geometry()) {
4230  if (ti.get_compression() == kENCODING_GEOINT) {
4231  os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
4232  << ")";
4233  } else {
4234  os << " ENCODING NONE";
4235  }
4236  }
4237  comma = ", ";
4238  }
4239  }
4240  // gather SHARED DICTIONARYs
4241  if (shared_dicts.size()) {
4242  os << ", " << boost::algorithm::join(shared_dicts, ", ");
4243  }
4244  // gather WITH options ...
4245  std::vector<std::string> with_options;
4246  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
4247  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
4248  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
4249  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
4250  with_options.emplace_back(td->hasDeletedCol ? "VACUUM='DELAYED'"
4251  : "VACUUM='IMMEDIATE'");
4252  if (!td->partitions.empty()) {
4253  with_options.push_back("PARTITIONS='" + td->partitions + "'");
4254  }
4255  if (td->nShards > 0) {
4256  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
4257  CHECK(shard_cd);
4258  os << ", SHARD KEY(" << shard_cd->columnName << ")";
4259  with_options.push_back(
4260  "SHARD_COUNT=" +
4261  std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
4262  }
4263  if (td->sortedColumnId > 0) {
4264  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
4265  CHECK(sort_cd);
4266  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
4267  }
4268  if (td->maxRollbackEpochs >= 0) {
4269  with_options.push_back("MAX_ROLLBACK_EPOCHS=" +
4271  }
4272  os << ") WITH (" + boost::algorithm::join(with_options, ", ") + ");";
4273  return os.str();
4274 }
int32_t maxRollbackEpochs
std::string partitions
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:546
std::string join(T const &container, std::string const &delim)
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:112
std::string to_string(char const *&&v)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
Definition: sqltypes.h:51
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1716
Definition: sqltypes.h:40
mapd_shared_lock< mapd_shared_mutex > read_lock
#define CHECK(condition)
Definition: Logger.h:197
size_t g_leaf_count
Definition: ParserNode.cpp:74

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::eraseDBData ( )

Definition at line 3941 of file Catalog.cpp.

References basePath_, calciteMgr_, currentDB_, Catalog_Namespace::DBMetadata::dbName, eraseTablePhysicalData(), getAllTableMetadata(), and run_benchmark_import::tables.

3941  {
3942  cat_write_lock write_lock(this);
3943  // Physically erase all tables and dictionaries from disc and memory
3944  const auto tables = getAllTableMetadata();
3945  for (const auto table : tables) {
3946  eraseTablePhysicalData(table);
3947  }
3948  // Physically erase database metadata
3949  boost::filesystem::remove(basePath_ + "/mapd_catalogs/" + currentDB_.dbName);
3950  calciteMgr_->updateMetadata(currentDB_.dbName, "");
3951 }
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:3953
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:1742
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:558
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:113
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::eraseTablePhysicalData ( const TableDescriptor td)

Definition at line 3953 of file Catalog.cpp.

References calciteMgr_, CHECK, Data_Namespace::CPU_LEVEL, currentDB_, dataMgr_, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, TableDescriptor::fragmenter, Data_Namespace::GPU_LEVEL, INJECT_TIMER, TableDescriptor::isView, removeTableFromMap(), tableDescriptorMapById_, TableDescriptor::tableId, and TableDescriptor::tableName.

Referenced by doDropTable(), and eraseDBData().

3953  {
3954  const int tableId = td->tableId;
3955  // must destroy fragmenter before deleteChunks is called.
3956  if (td->fragmenter != nullptr) {
3957  auto tableDescIt = tableDescriptorMapById_.find(tableId);
3958  CHECK(tableDescIt != tableDescriptorMapById_.end());
3959  {
3960  INJECT_TIMER(deleting_fragmenter);
3961  tableDescIt->second->fragmenter = nullptr;
3962  }
3963  }
3964  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
3965  {
3966  INJECT_TIMER(deleteChunksWithPrefix);
3967  // assuming deleteChunksWithPrefix is atomic
3968  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
3969  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
3970  }
3971  if (!td->isView) {
3972  INJECT_TIMER(Remove_Table);
3973  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
3974  }
3975  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
3976  {
3977  INJECT_TIMER(removeTableFromMap_);
3978  removeTableFromMap(td->tableName, tableId);
3979  }
3980 }
std::vector< int > ChunkKey
Definition: types.h:37
std::string tableName
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:555
#define INJECT_TIMER(DESC)
Definition: measure.h:93
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:543
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:558
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1172
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::executeDropTableSqliteQueries ( const TableDescriptor td)
protected

Definition at line 3442 of file Catalog.cpp.

References StorageType::FOREIGN_TABLE, SqliteConnector::getData(), SqliteConnector::getNumRows(), TableDescriptor::isView, kENCODING_DICT, SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), sqliteConnector_, TableDescriptor::storageType, TableDescriptor::tableId, and to_string().

Referenced by doDropTable().

3442  {
3443  const int tableId = td->tableId;
3444  sqliteConnector_.query_with_text_param("DELETE FROM mapd_tables WHERE tableid = ?",
3445  std::to_string(tableId));
3447  "select comp_param from mapd_columns where compression = ? and tableid = ?",
3448  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
3449  int numRows = sqliteConnector_.getNumRows();
3450  std::vector<int> dict_id_list;
3451  for (int r = 0; r < numRows; ++r) {
3452  dict_id_list.push_back(sqliteConnector_.getData<int>(r, 0));
3453  }
3454  for (auto dict_id : dict_id_list) {
3456  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
3457  std::vector<std::string>{std::to_string(dict_id)});
3458  }
3460  "DELETE FROM mapd_dictionaries WHERE dictid in (select comp_param from "
3461  "mapd_columns where compression = ? "
3462  "and tableid = ?) and refcount = 0",
3463  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
3464  sqliteConnector_.query_with_text_param("DELETE FROM mapd_columns WHERE tableid = ?",
3465  std::to_string(tableId));
3466  if (td->isView) {
3467  sqliteConnector_.query_with_text_param("DELETE FROM mapd_views WHERE tableid = ?",
3468  std::to_string(tableId));
3469  }
3472  "DELETE FROM omnisci_foreign_tables WHERE table_id = ?", std::to_string(tableId));
3473  }
3474 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:553
std::string storageType
std::string to_string(char const *&&v)
static constexpr char const * FOREIGN_TABLE
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::expandGeoColumn ( const ColumnDescriptor cd,
std::list< ColumnDescriptor > &  columns 
)
static

Definition at line 2000 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, IS_GEO, kARRAY, kDOUBLE, kENCODING_GEOINT, kENCODING_NONE, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kTINYINT, SQLTypeInfo::set_size(), and SQLTypeInfo::set_subtype().

Referenced by createTable().

2001  {
2002  const auto& col_ti = cd.columnType;
2003  if (IS_GEO(col_ti.get_type())) {
2004  switch (col_ti.get_type()) {
2005  case kPOINT: {
2006  ColumnDescriptor physical_cd_coords(true);
2007  physical_cd_coords.columnName = cd.columnName + "_coords";
2008  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2009  // Raw data: compressed/uncompressed coords
2010  coords_ti.set_subtype(kTINYINT);
2011  size_t unit_size;
2012  if (col_ti.get_compression() == kENCODING_GEOINT &&
2013  col_ti.get_comp_param() == 32) {
2014  unit_size = 4 * sizeof(int8_t);
2015  } else {
2016  CHECK(col_ti.get_compression() == kENCODING_NONE);
2017  unit_size = 8 * sizeof(int8_t);
2018  }
2019  coords_ti.set_size(2 * unit_size);
2020  physical_cd_coords.columnType = coords_ti;
2021  columns.push_back(physical_cd_coords);
2022 
2023  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2024 
2025  break;
2026  }
2027  case kLINESTRING: {
2028  ColumnDescriptor physical_cd_coords(true);
2029  physical_cd_coords.columnName = cd.columnName + "_coords";
2030  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2031  // Raw data: compressed/uncompressed coords
2032  coords_ti.set_subtype(kTINYINT);
2033  physical_cd_coords.columnType = coords_ti;
2034  columns.push_back(physical_cd_coords);
2035 
2036  ColumnDescriptor physical_cd_bounds(true);
2037  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2038  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2039  bounds_ti.set_subtype(kDOUBLE);
2040  bounds_ti.set_size(4 * sizeof(double));
2041  physical_cd_bounds.columnType = bounds_ti;
2042  columns.push_back(physical_cd_bounds);
2043 
2044  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2045 
2046  break;
2047  }
2048  case kPOLYGON: {
2049  ColumnDescriptor physical_cd_coords(true);
2050  physical_cd_coords.columnName = cd.columnName + "_coords";
2051  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2052  // Raw data: compressed/uncompressed coords
2053  coords_ti.set_subtype(kTINYINT);
2054  physical_cd_coords.columnType = coords_ti;
2055  columns.push_back(physical_cd_coords);
2056 
2057  ColumnDescriptor physical_cd_ring_sizes(true);
2058  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
2059  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2060  ring_sizes_ti.set_subtype(kINT);
2061  physical_cd_ring_sizes.columnType = ring_sizes_ti;
2062  columns.push_back(physical_cd_ring_sizes);
2063 
2064  ColumnDescriptor physical_cd_bounds(true);
2065  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2066  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2067  bounds_ti.set_subtype(kDOUBLE);
2068  bounds_ti.set_size(4 * sizeof(double));
2069  physical_cd_bounds.columnType = bounds_ti;
2070  columns.push_back(physical_cd_bounds);
2071 
2072  ColumnDescriptor physical_cd_render_group(true);
2073  physical_cd_render_group.columnName = cd.columnName + "_render_group";
2074  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, col_ti.get_notnull());
2075  physical_cd_render_group.columnType = render_group_ti;
2076  columns.push_back(physical_cd_render_group);
2077 
2078  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2079 
2080  break;
2081  }
2082  case kMULTIPOLYGON: {
2083  ColumnDescriptor physical_cd_coords(true);
2084  physical_cd_coords.columnName = cd.columnName + "_coords";
2085  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2086  // Raw data: compressed/uncompressed coords
2087  coords_ti.set_subtype(kTINYINT);
2088  physical_cd_coords.columnType = coords_ti;
2089  columns.push_back(physical_cd_coords);
2090 
2091  ColumnDescriptor physical_cd_ring_sizes(true);
2092  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
2093  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2094  ring_sizes_ti.set_subtype(kINT);
2095  physical_cd_ring_sizes.columnType = ring_sizes_ti;
2096  columns.push_back(physical_cd_ring_sizes);
2097 
2098  ColumnDescriptor physical_cd_poly_rings(true);
2099  physical_cd_poly_rings.columnName = cd.columnName + "_poly_rings";
2100  SQLTypeInfo poly_rings_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2101  poly_rings_ti.set_subtype(kINT);
2102  physical_cd_poly_rings.columnType = poly_rings_ti;
2103  columns.push_back(physical_cd_poly_rings);
2104 
2105  ColumnDescriptor physical_cd_bounds(true);
2106  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2107  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2108  bounds_ti.set_subtype(kDOUBLE);
2109  bounds_ti.set_size(4 * sizeof(double));
2110  physical_cd_bounds.columnType = bounds_ti;
2111  columns.push_back(physical_cd_bounds);
2112 
2113  ColumnDescriptor physical_cd_render_group(true);
2114  physical_cd_render_group.columnName = cd.columnName + "_render_group";
2115  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, col_ti.get_notnull());
2116  physical_cd_render_group.columnType = render_group_ti;
2117  columns.push_back(physical_cd_render_group);
2118 
2119  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2120 
2121  break;
2122  }
2123  default:
2124  throw runtime_error("Unrecognized geometry type.");
2125  break;
2126  }
2127  }
2128 }
void set_size(int s)
Definition: sqltypes.h:409
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:402
specifies the content in-memory of a row in the column metadata table
#define CHECK(condition)
Definition: Logger.h:197
Definition: sqltypes.h:44
SQLTypeInfo columnType
std::string columnName
#define IS_GEO(T)
Definition: sqltypes.h:242

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool Catalog_Namespace::Catalog::filterTableByTypeAndUser ( const TableDescriptor td,
const UserMetadata user_metadata,
const GetTablesType  get_tables_type 
) const
private

Definition at line 3836 of file Catalog.cpp.

References GET_PHYSICAL_TABLES, GET_VIEWS, Catalog_Namespace::SysCatalog::instance(), TableDescriptor::isView, DBObject::loadKey(), TableDescriptor::shard, TableDBObjectType, TableDescriptor::tableName, and ViewDBObjectType.

Referenced by getTableNamesForUser(), and getTablesMetadataForUser().

3838  {
3839  if (td->shard >= 0) {
3840  // skip shards, they're not standalone tables
3841  return false;
3842  }
3843  switch (get_tables_type) {
3844  case GET_PHYSICAL_TABLES: {
3845  if (td->isView) {
3846  return false;
3847  }
3848  break;
3849  }
3850  case GET_VIEWS: {
3851  if (!td->isView) {
3852  return false;
3853  }
3854  break;
3855  }
3856  default:
3857  break;
3858  }
3860  dbObject.loadKey(*this);
3861  std::vector<DBObject> privObjects = {dbObject};
3862  if (!SysCatalog::instance().hasAnyPrivileges(user_metadata, privObjects)) {
3863  // skip table, as there are no privileges to access it
3864  return false;
3865  }
3866  return true;
3867 }
std::string tableName
static SysCatalog & instance()
Definition: SysCatalog.h:288

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string Catalog_Namespace::Catalog::generatePhysicalTableName ( const std::string &  logicalTableName,
const int32_t &  shardNumber 
)
protected

Definition at line 3982 of file Catalog.cpp.

References physicalTableNameTag_, and to_string().

Referenced by createShardedTable(), and renameTable().

3983  {
3984  std::string physicalTableName =
3985  logicalTableName + physicalTableNameTag_ + std::to_string(shardNumber);
3986  return (physicalTableName);
3987 }
std::string to_string(char const *&&v)
static const std::string physicalTableNameTag_
Definition: Catalog.h:562

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::list< const ColumnDescriptor * > Catalog_Namespace::Catalog::getAllColumnMetadataForTable ( const int  tableId,
const bool  fetchSystemColumns,
const bool  fetchVirtualColumns,
const bool  fetchPhysicalColumns 
) const

Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular table specified by table id.

Parameters
tableIdtable id we want the column metadata for
Returns
list of pointers to const ColumnDescriptor structs - one for each and every column in the table

Definition at line 1716 of file Catalog.cpp.

References getAllColumnMetadataForTableUnlocked().

Referenced by Analyzer::RangeTableEntry::add_all_column_descs(), Parser::InsertStmt::analyze(), Parser::InsertValuesStmt::analyze(), Fragmenter_Namespace::copyDataOfShard(), Parser::InsertValuesStmt::determineLeafIndex(), dumpCreateTable(), dumpSchema(), TableArchiver::dumpTable(), Analyzer::RangeTableEntry::expand_star_in_targetlist(), getDictionaryToColumnMapping(), getShardColumnMetadataForTable(), getTableDictDirectories(), TableOptimizer::recomputeMetadata(), TableArchiver::restoreTable(), and validate_shared_dictionary().

1720  {
1721  cat_read_lock read_lock(this);
1723  tableId, fetchSystemColumns, fetchVirtualColumns, fetchPhysicalColumns);
1724 }
std::list< const ColumnDescriptor * > getAllColumnMetadataForTableUnlocked(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Definition: Catalog.cpp:1726
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:112
mapd_shared_lock< mapd_shared_mutex > read_lock

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::getAllColumnMetadataForTableImpl ( const TableDescriptor td,
std::list< const ColumnDescriptor * > &  colDescs,
const bool  fetchSystemColumns,
const bool  fetchVirtualColumns,
const bool  fetchPhysicalColumns 
) const
protected

Definition at line 1686 of file Catalog.cpp.

References columnDescriptorMapById_, and TableDescriptor::tableId.

Referenced by getAllColumnMetadataForTableUnlocked(), and instantiateFragmenter().

1691  {
1692  int32_t skip_physical_cols = 0;
1693  for (const auto& columnDescriptor : columnDescriptorMapById_) {
1694  if (!fetchPhysicalColumns && skip_physical_cols > 0) {
1695  --skip_physical_cols;
1696  continue;
1697  }
1698  auto cd = columnDescriptor.second;
1699  if (cd->tableId != td->tableId) {
1700  continue;
1701  }
1702  if (!fetchSystemColumns && cd->isSystemCol) {
1703  continue;
1704  }
1705  if (!fetchVirtualColumns && cd->isVirtualCol) {
1706  continue;
1707  }
1708  if (!fetchPhysicalColumns) {
1709  const auto& col_ti = cd->columnType;
1710  skip_physical_cols = col_ti.get_physical_cols();
1711  }
1712  columnDescriptors.push_back(cd);
1713  }
1714 }
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:545

+ Here is the caller graph for this function:

std::list< const ColumnDescriptor * > Catalog_Namespace::Catalog::getAllColumnMetadataForTableUnlocked ( const int  tableId,
const bool  fetchSystemColumns,
const bool  fetchVirtualColumns,
const bool  fetchPhysicalColumns 
) const

Same as above, but without first taking a catalog read lock.

Definition at line 1726 of file Catalog.cpp.

References getAllColumnMetadataForTableImpl(), and getMetadataForTableImpl().

Referenced by getAllColumnMetadataForTable().

1730  {
1731  std::list<const ColumnDescriptor*> columnDescriptors;
1732  const TableDescriptor* td =
1733  getMetadataForTableImpl(tableId, false); // dont instantiate fragmenter
1735  columnDescriptors,
1736  fetchSystemColumns,
1737  fetchVirtualColumns,
1738  fetchPhysicalColumns);
1739  return columnDescriptors;
1740 }
void getAllColumnMetadataForTableImpl(const TableDescriptor *td, std::list< const ColumnDescriptor * > &colDescs, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Definition: Catalog.cpp:1686
const TableDescriptor * getMetadataForTableImpl(int tableId, const bool populateFragmenter) const
Definition: Catalog.cpp:1416

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

list< const DashboardDescriptor * > Catalog_Namespace::Catalog::getAllDashboardsMetadata ( ) const

Definition at line 1751 of file Catalog.cpp.

References dashboardDescriptorMap_.

1751  {
1752  list<const DashboardDescriptor*> view_list;
1753  for (auto p : dashboardDescriptorMap_) {
1754  view_list.push_back(p.second.get());
1755  }
1756  return view_list;
1757 }
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:547
std::vector< const TableDescriptor * > Catalog_Namespace::Catalog::getAllForeignTablesForRefresh ( ) const

Gets all the foreign tables that are pending refreshes. The list of tables includes tables that are configured for scheduled refreshes with next refresh timestamps that are in the past.

Returns
foreign tables pending refreshes

Definition at line 4502 of file Catalog.cpp.

References CHECK, StorageType::FOREIGN_TABLE, foreign_storage::ForeignTable::REFRESH_TIMING_TYPE_KEY, foreign_storage::ForeignTable::SCHEDULE_REFRESH_TIMING_TYPE, tableDescriptorMapById_, and run_benchmark_import::tables.

4502  {
4503  cat_read_lock read_lock(this);
4504  std::vector<const TableDescriptor*> tables;
4505  for (auto entry : tableDescriptorMapById_) {
4506  auto table_descriptor = entry.second;
4507  if (table_descriptor->storageType == StorageType::FOREIGN_TABLE) {
4508  auto foreign_table = dynamic_cast<foreign_storage::ForeignTable*>(table_descriptor);
4509  CHECK(foreign_table);
4510  auto timing_type_entry = foreign_table->options.find(
4512  CHECK(timing_type_entry != foreign_table->options.end());
4513  int64_t current_time = std::chrono::duration_cast<std::chrono::seconds>(
4514  std::chrono::system_clock::now().time_since_epoch())
4515  .count();
4516  if (timing_type_entry->second ==
4518  foreign_table->next_refresh_time <= current_time) {
4519  tables.emplace_back(foreign_table);
4520  }
4521  }
4522  }
4523  return tables;
4524 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:112
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:543
mapd_shared_lock< mapd_shared_mutex > read_lock
static constexpr const char * REFRESH_TIMING