OmniSciDB  a667adc9c8
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Catalog_Namespace::Catalog Class Referencefinal

class for a per-database catalog. also includes metadata for the current database and the current user. More...

#include <Catalog.h>

+ Collaboration diagram for Catalog_Namespace::Catalog:

Public Member Functions

 Catalog (const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
 Constructor - takes basePath to already extant data directory for writing. More...
 
 Catalog ()
 Constructor builds a hollow catalog used during constructor of other catalogs. More...
 
 ~Catalog ()
 Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite. More...
 
void createTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
 
void createShardedTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
int32_t createDashboard (DashboardDescriptor &vd)
 
void replaceDashboard (DashboardDescriptor &vd)
 
std::string createLink (LinkDescriptor &ld, size_t min_length)
 
void dropTable (const TableDescriptor *td)
 
void truncateTable (const TableDescriptor *td)
 
void renameTable (const TableDescriptor *td, const std::string &newTableName)
 
void renameTable (std::vector< std::pair< std::string, std::string >> &names)
 
void renameColumn (const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
 
void addColumn (const TableDescriptor &td, ColumnDescriptor &cd)
 
void dropColumn (const TableDescriptor &td, const ColumnDescriptor &cd)
 
void removeFragmenterForTable (const int table_id) const
 
const std::map< int, const
ColumnDescriptor * > 
getDictionaryToColumnMapping ()
 
const TableDescriptorgetMetadataForTable (const std::string &tableName, const bool populateFragmenter=true) const
 Returns a pointer to a const TableDescriptor struct matching the provided tableName. More...
 
const TableDescriptorgetMetadataForTableImpl (int tableId, const bool populateFragmenter) const
 
const TableDescriptorgetMetadataForTable (int tableId, bool populateFragmenter=true) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, const std::string &colName) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, int columnId) const
 
const ColumnDescriptorgetMetadataForColumnUnlocked (int tableId, int columnId) const
 
const int getColumnIdBySpi (const int tableId, const size_t spi) const
 
const ColumnDescriptorgetMetadataForColumnBySpi (const int tableId, const size_t spi) const
 
const DashboardDescriptorgetMetadataForDashboard (const std::string &userId, const std::string &dashName) const
 
const DashboardDescriptorgetMetadataForDashboard (const int32_t dashboard_id) const
 
void deleteMetadataForDashboards (const std::vector< int32_t > ids, const UserMetadata &user)
 
const LinkDescriptorgetMetadataForLink (const std::string &link) const
 
const LinkDescriptorgetMetadataForLink (int linkId) const
 
const
foreign_storage::ForeignTable
getForeignTableUnlocked (int tableId) const
 
const
foreign_storage::ForeignTable
getForeignTable (const std::string &tableName) const
 
const
foreign_storage::ForeignTable
getForeignTable (int table_id) const
 
std::list< const
ColumnDescriptor * > 
getAllColumnMetadataForTable (const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular table specified by table id. More...
 
std::list< const
ColumnDescriptor * > 
getAllColumnMetadataForTableUnlocked (const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 
std::list< const
TableDescriptor * > 
getAllTableMetadata () const
 
std::list< const
DashboardDescriptor * > 
getAllDashboardsMetadata () const
 
const DBMetadatagetCurrentDB () const
 
Data_Namespace::DataMgrgetDataMgr () const
 
std::shared_ptr< CalcitegetCalciteMgr () const
 
const std::string & getCatalogBasePath () const
 
const DictDescriptorgetMetadataForDict (int dict_ref, bool loadDict=true) const
 
const DictDescriptorgetMetadataForDictUnlocked (int dict_ref, bool loadDict) const
 
const std::vector< LeafHostInfo > & getStringDictionaryHosts () const
 
const ColumnDescriptorgetShardColumnMetadataForTable (const TableDescriptor *td) const
 
std::vector< const
TableDescriptor * > 
getPhysicalTablesDescriptors (const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
 
std::vector< std::string > getTableNamesForUser (const UserMetadata &user, const GetTablesType get_tables_type) const
 
std::vector< TableMetadatagetTablesMetadataForUser (const UserMetadata &user_metadata, const GetTablesType get_tables_type, const std::string &filter_table_name) const
 
int32_t getTableEpoch (const int32_t db_id, const int32_t table_id) const
 
void setTableEpoch (const int db_id, const int table_id, const int new_epoch)
 
void setMaxRollbackEpochs (const int32_t table_id, const int32_t max_rollback_epochs)
 
void setMaxRows (const int32_t table_id, const int64_t max_rows)
 
std::vector< TableEpochInfogetTableEpochs (const int32_t db_id, const int32_t table_id) const
 
void setTableEpochs (const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
 
void setTableEpochsLogExceptions (const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
 
int getDatabaseId () const
 
SqliteConnectorgetSqliteConnector ()
 
void roll (const bool forward)
 
DictRef addDictionary (ColumnDescriptor &cd)
 
void delDictionary (const ColumnDescriptor &cd)
 
void getDictionary (const ColumnDescriptor &cd, std::map< int, StringDictionary * > &stringDicts)
 
const bool checkMetadataForDeletedRecs (const TableDescriptor *td, int column_id) const
 
const ColumnDescriptorgetDeletedColumn (const TableDescriptor *td) const
 
const ColumnDescriptorgetDeletedColumnIfRowsDeleted (const TableDescriptor *td) const
 
void setDeletedColumn (const TableDescriptor *td, const ColumnDescriptor *cd)
 
void setDeletedColumnUnlocked (const TableDescriptor *td, const ColumnDescriptor *cd)
 
int getLogicalTableId (const int physicalTableId) const
 
void checkpoint (const int logicalTableId) const
 
void checkpointWithAutoRollback (const int logical_table_id) const
 
std::string name () const
 
void eraseDBData ()
 
void eraseTablePhysicalData (const TableDescriptor *td)
 
void setForReload (const int32_t tableId)
 
std::vector< std::string > getTableDataDirectories (const TableDescriptor *td) const
 
std::vector< std::string > getTableDictDirectories (const TableDescriptor *td) const
 
std::string getColumnDictDirectory (const ColumnDescriptor *cd) const
 
std::string dumpSchema (const TableDescriptor *td) const
 
std::string dumpCreateTable (const TableDescriptor *td, bool multiline_formatting=true, bool dump_defaults=false) const
 
void createForeignServer (std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
 
const
foreign_storage::ForeignServer
getForeignServer (const std::string &server_name) const
 
const std::unique_ptr< const
foreign_storage::ForeignServer
getForeignServerFromStorage (const std::string &server_name)
 
const std::unique_ptr< const
foreign_storage::ForeignTable
getForeignTableFromStorage (int table_id)
 
void changeForeignServerOwner (const std::string &server_name, const int new_owner_id)
 
void setForeignServerDataWrapper (const std::string &server_name, const std::string &data_wrapper)
 
void setForeignServerOptions (const std::string &server_name, const std::string &options)
 
void renameForeignServer (const std::string &server_name, const std::string &name)
 
void dropForeignServer (const std::string &server_name)
 
void getForeignServersForUser (const rapidjson::Value *filters, const UserMetadata &user, std::vector< const foreign_storage::ForeignServer * > &results)
 
void createDefaultServersIfNotExists ()
 
bool validateNonExistentTableOrView (const std::string &name, const bool if_not_exists)
 
std::vector< const
TableDescriptor * > 
getAllForeignTablesForRefresh () const
 
void updateForeignTableRefreshTimes (const int32_t table_id)
 
void setForeignTableOptions (const std::string &table_name, foreign_storage::OptionsMap &options_map, bool clear_existing_options=true)
 
void updateLeaf (const LeafHostInfo &string_dict_host)
 
void setUncappedTableEpoch (const std::string &table_name)
 

Static Public Member Functions

static void expandGeoColumn (const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
 
static const std::string getForeignTableSchema (bool if_not_exists=false)
 
static const std::string getForeignServerSchema (bool if_not_exists=false)
 

Public Attributes

std::mutex sqliteMutex_
 
mapd_shared_mutex sharedMutex_
 
std::atomic< std::thread::id > thread_holding_sqlite_lock
 
std::atomic< std::thread::id > thread_holding_write_lock
 
bool initialized_ = false
 

Static Public Attributes

static thread_local bool thread_holds_read_lock = false
 

Protected Types

using ColumnDescriptorsForRoll = std::vector< std::pair< ColumnDescriptor *, ColumnDescriptor * >>
 

Protected Member Functions

void CheckAndExecuteMigrations ()
 
void CheckAndExecuteMigrationsPostBuildMaps ()
 
void updateDictionaryNames ()
 
void updateTableDescriptorSchema ()
 
void updateFixlenArrayColumns ()
 
void updateGeoColumns ()
 
void updateFrontendViewSchema ()
 
void updateLinkSchema ()
 
void updateFrontendViewAndLinkUsers ()
 
void updateLogicalToPhysicalTableLinkSchema ()
 
void updateLogicalToPhysicalTableMap (const int32_t logical_tb_id)
 
void updateDictionarySchema ()
 
void updatePageSize ()
 
void updateDeletedColumnIndicator ()
 
void updateFrontendViewsToDashboards ()
 
void createFsiSchemas ()
 
void dropFsiSchemasAndTables ()
 
void recordOwnershipOfObjectsInObjectPermissions ()
 
void checkDateInDaysColumnMigration ()
 
void createDashboardSystemRoles ()
 
void buildMaps ()
 
void addTableToMap (const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
 
void addReferenceToForeignDict (ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def, const bool persist_reference)
 
bool setColumnSharedDictionary (ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
void setColumnDictionary (ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
 
void addFrontendViewToMap (DashboardDescriptor &vd)
 
void addFrontendViewToMapNoLock (DashboardDescriptor &vd)
 
void addLinkToMap (LinkDescriptor &ld)
 
void removeTableFromMap (const std::string &tableName, const int tableId, const bool is_on_error=false)
 
void doDropTable (const TableDescriptor *td)
 
void executeDropTableSqliteQueries (const TableDescriptor *td)
 
void doTruncateTable (const TableDescriptor *td)
 
void renamePhysicalTable (const TableDescriptor *td, const std::string &newTableName)
 
void renamePhysicalTable (std::vector< std::pair< std::string, std::string >> &names, std::vector< int > &tableIds)
 
void instantiateFragmenter (TableDescriptor *td) const
 
void getAllColumnMetadataForTableImpl (const TableDescriptor *td, std::list< const ColumnDescriptor * > &colDescs, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 
std::string calculateSHA1 (const std::string &data)
 
std::string generatePhysicalTableName (const std::string &logicalTableName, const int32_t &shardNumber)
 
std::vector< DBObjectparseDashboardObjects (const std::string &view_meta, const int &user_id)
 
void createOrUpdateDashboardSystemRole (const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
 
const int getColumnIdBySpiUnlocked (const int table_id, const size_t spi) const
 
void serializeTableJsonUnlocked (const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
 
void dropTableFromJsonUnlocked (const std::string &table_name) const
 

Protected Attributes

std::string basePath_
 
TableDescriptorMap tableDescriptorMap_
 
TableDescriptorMapById tableDescriptorMapById_
 
ColumnDescriptorMap columnDescriptorMap_
 
ColumnDescriptorMapById columnDescriptorMapById_
 
DictDescriptorMapById dictDescriptorMapByRef_
 
DashboardDescriptorMap dashboardDescriptorMap_
 
LinkDescriptorMap linkDescriptorMap_
 
LinkDescriptorMapById linkDescriptorMapById_
 
ForeignServerMap foreignServerMap_
 
ForeignServerMapById foreignServerMapById_
 
SqliteConnector sqliteConnector_
 
const DBMetadata currentDB_
 
std::shared_ptr
< Data_Namespace::DataMgr
dataMgr_
 
const std::vector< LeafHostInfostring_dict_hosts_
 
std::shared_ptr< CalcitecalciteMgr_
 
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
 
int nextTempTableId_
 
int nextTempDictId_
 
ColumnDescriptorsForRoll columnDescriptorsForRoll
 

Static Protected Attributes

static const std::string physicalTableNameTag_
 

Private Member Functions

void gatherAdditionalInfo (std::vector< std::string > &additional_info, std::set< std::string > &shared_dict_column_names, const TableDescriptor *td) const
 
std::string quoteIfRequired (const std::string &column_name) const
 
void adjustAlteredTableFiles (const std::string &temp_data_dir, const std::unordered_map< int, int > &all_column_ids_map) const
 
void renameTableDirectories (const std::string &temp_data_dir, const std::vector< std::string > &target_paths, const std::string &name_prefix) const
 
void buildForeignServerMap ()
 
void addForeignTableDetails ()
 
void setForeignServerProperty (const std::string &server_name, const std::string &property, const std::string &value)
 
void setForeignTableProperty (const foreign_storage::ForeignTable *table, const std::string &property, const std::string &value)
 
void alterPhysicalTableMetadata (const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
 
void alterTableMetadata (const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
 
void setTableFileMgrParams (const int table_id, const File_Namespace::FileMgrParams &file_mgr_params)
 
bool filterTableByTypeAndUser (const TableDescriptor *td, const UserMetadata &user_metadata, const GetTablesType get_tables_type) const
 
TableDescriptorgetMutableMetadataForTableUnlocked (int tableId)
 
void createForeignServerNoLocks (std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
 
foreign_storage::ForeignTablegetForeignTableUnlocked (const std::string &tableName) const
 
const CataloggetObjForLock ()
 
void removeChunksUnlocked (const int table_id) const
 

Private Attributes

DeletedColumnPerTableMap deletedColumnPerTable_
 

Detailed Description

class for a per-database catalog. also includes metadata for the current database and the current user.

Catalog

Definition at line 101 of file Catalog.h.

Member Typedef Documentation

Definition at line 576 of file Catalog.h.

Constructor & Destructor Documentation

Catalog_Namespace::Catalog::Catalog ( const std::string &  basePath,
const DBMetadata curDB,
std::shared_ptr< Data_Namespace::DataMgr dataMgr,
const std::vector< LeafHostInfo > &  string_dict_hosts,
std::shared_ptr< Calcite calcite,
bool  is_new_db 
)

Constructor - takes basePath to already extant data directory for writing.

Parameters
basePathdirectory path for writing catalog
dbNamename of the database
fragmenterFragmenter object metadata - expects for this directory to already exist
Catalog_Namespace::Catalog::Catalog ( )

Constructor builds a hollow catalog used during constructor of other catalogs.

Definition at line 194 of file Catalog.cpp.

194 {}
Catalog_Namespace::Catalog::~Catalog ( )

Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite.

Definition at line 196 of file Catalog.cpp.

References basePath_, columnDescriptorMap_, currentDB_, Catalog_Namespace::DBMetadata::dbName, g_serialize_temp_tables, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and tableDescriptorMap_.

196  {
198  // must clean up heap-allocated TableDescriptor and ColumnDescriptor structs
199  for (TableDescriptorMap::iterator tableDescIt = tableDescriptorMap_.begin();
200  tableDescIt != tableDescriptorMap_.end();
201  ++tableDescIt) {
202  tableDescIt->second->fragmenter = nullptr;
203  delete tableDescIt->second;
204  }
205 
206  // TableDescriptorMapById points to the same descriptors. No need to delete
207 
208  for (ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.begin();
209  columnDescIt != columnDescriptorMap_.end();
210  ++columnDescIt) {
211  delete columnDescIt->second;
212  }
213 
214  // ColumnDescriptorMapById points to the same descriptors. No need to delete
215 
217  boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
218  }
219 }
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:549
const DBMetadata currentDB_
Definition: Catalog.h:559
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:154
bool g_serialize_temp_tables
Definition: Catalog.cpp:98
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:547

+ Here is the call graph for this function:

Member Function Documentation

void Catalog_Namespace::Catalog::addColumn ( const TableDescriptor td,
ColumnDescriptor cd 
)

Definition at line 1859 of file Catalog.cpp.

References addDictionary(), columnDescriptorMap_, columnDescriptorMapById_, columnDescriptorsForRoll, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), SqliteConnector::getData(), getPhysicalTablesDescriptors(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, kENCODING_DICT, TableDescriptor::nShards, SqliteConnector::query_with_text_params(), TableDescriptor::shard, sqliteConnector_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), to_upper(), and ColumnDescriptor::virtualExpr.

1859  {
1860  cat_write_lock write_lock(this);
1861  // caller must handle sqlite/chunk transaction TOGETHER
1862  cd.tableId = td.tableId;
1863  if (td.nShards > 0 && td.shard < 0) {
1864  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
1865  auto shard_cd = cd;
1866  addColumn(*shard, shard_cd);
1867  }
1868  }
1870  addDictionary(cd);
1871  }
1872 
1874  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
1875  "colscale, is_notnull, "
1876  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
1877  "is_deletedcol) "
1878  "VALUES (?, "
1879  "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
1880  "?, ?, ?, "
1881  "?, "
1882  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1883  std::vector<std::string>{std::to_string(td.tableId),
1884  std::to_string(td.tableId),
1885  cd.columnName,
1894  "",
1897  cd.virtualExpr,
1899 
1901  "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
1902  std::vector<std::string>{std::to_string(td.tableId)});
1903 
1905  "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
1906  std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
1907  cd.columnId = sqliteConnector_.getData<int>(0, 0);
1908 
1909  ++tableDescriptorMapById_[td.tableId]->nColumns;
1910  auto ncd = new ColumnDescriptor(cd);
1913  columnDescriptorsForRoll.emplace_back(nullptr, ncd);
1914 }
std::string virtualExpr
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:315
std::tuple< int, std::string > ColumnKey
Definition: Types.h:36
HOST DEVICE int get_size() const
Definition: sqltypes.h:324
T getData(const int row, const int col)
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:549
virtual void query_with_text_params(std::string const &query_only)
void addColumn(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:1859
SqliteConnector sqliteConnector_
Definition: Catalog.h:558
HOST DEVICE int get_scale() const
Definition: sqltypes.h:319
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:314
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:550
std::string to_string(char const *&&v)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:548
specifies the content in-memory of a row in the column metadata table
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4013
std::string to_upper(const std::string &str)
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:322
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:316
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:323
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
DictRef addDictionary(ColumnDescriptor &cd)
Definition: Catalog.cpp:1764
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:321
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:577
std::string columnName
std::tuple< int, int > ColumnIdKey
Definition: Types.h:38

+ Here is the call graph for this function:

DictRef Catalog_Namespace::Catalog::addDictionary ( ColumnDescriptor cd)

Definition at line 1764 of file Catalog.cpp.

References CHECK, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, DictDescriptor::dictFolderPath, setColumnDictionary(), string_dict_hosts_, tableDescriptorMapById_, and ColumnDescriptor::tableId.

Referenced by addColumn().

1764  {
1765  const auto& td = *tableDescriptorMapById_[cd.tableId];
1766  list<DictDescriptor> dds;
1767  setColumnDictionary(cd, dds, td, true);
1768  auto& dd = dds.back();
1769  CHECK(dd.dictRef.dictId);
1770 
1771  std::unique_ptr<StringDictionaryClient> client;
1772  if (!string_dict_hosts_.empty()) {
1773  client.reset(new StringDictionaryClient(
1774  string_dict_hosts_.front(), DictRef(currentDB_.dbId, -1), true));
1775  }
1776  if (client) {
1777  client->create(dd.dictRef, dd.dictIsTemp);
1778  }
1779 
1780  DictDescriptor* new_dd = new DictDescriptor(dd);
1781  dictDescriptorMapByRef_[dd.dictRef].reset(new_dd);
1782  if (!dd.dictIsTemp) {
1783  boost::filesystem::create_directory(new_dd->dictFolderPath);
1784  }
1785  return dd.dictRef;
1786 }
std::string dictFolderPath
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:562
const DBMetadata currentDB_
Definition: Catalog.h:559
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:551
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:548
struct dict_ref_t DictRef
Definition: DictRef.h:37
#define CHECK(condition)
Definition: Logger.h:197
Descriptor for a dictionary for a string columne.
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:3228

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addForeignTableDetails ( )
private

Definition at line 4236 of file Catalog.cpp.

References CHECK, foreignServerMapById_, SqliteConnector::getData(), SqliteConnector::getNumRows(), SqliteConnector::query(), test_fsi::r, sqliteConnector_, and tableDescriptorMapById_.

Referenced by buildMaps().

4236  {
4238  "SELECT table_id, server_id, options, last_refresh_time, next_refresh_time from "
4239  "omnisci_foreign_tables");
4240  auto num_rows = sqliteConnector_.getNumRows();
4241  for (size_t r = 0; r < num_rows; r++) {
4242  const auto table_id = sqliteConnector_.getData<int32_t>(r, 0);
4243  const auto server_id = sqliteConnector_.getData<int32_t>(r, 1);
4244  const auto& options = sqliteConnector_.getData<std::string>(r, 2);
4245  const auto last_refresh_time = sqliteConnector_.getData<int>(r, 3);
4246  const auto next_refresh_time = sqliteConnector_.getData<int>(r, 4);
4247 
4248  CHECK(tableDescriptorMapById_.find(table_id) != tableDescriptorMapById_.end());
4249  auto foreign_table =
4250  dynamic_cast<foreign_storage::ForeignTable*>(tableDescriptorMapById_[table_id]);
4251  CHECK(foreign_table);
4252  foreign_table->foreign_server = foreignServerMapById_[server_id].get();
4253  CHECK(foreign_table->foreign_server);
4254  foreign_table->populateOptionsMap(options);
4255  foreign_table->last_refresh_time = last_refresh_time;
4256  foreign_table->next_refresh_time = next_refresh_time;
4257  }
4258 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:558
tuple r
Definition: test_fsi.py:16
virtual void query(const std::string &queryString)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:548
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:556
#define CHECK(condition)
Definition: Logger.h:197
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addFrontendViewToMap ( DashboardDescriptor vd)
protected

Definition at line 1254 of file Catalog.cpp.

References addFrontendViewToMapNoLock().

Referenced by createDashboard().

1254  {
1255  cat_write_lock write_lock(this);
1257 }
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1259
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addFrontendViewToMapNoLock ( DashboardDescriptor vd)
protected

Definition at line 1259 of file Catalog.cpp.

References dashboardDescriptorMap_, DashboardDescriptor::dashboardName, to_string(), and DashboardDescriptor::userId.

Referenced by addFrontendViewToMap(), and replaceDashboard().

1259  {
1260  cat_write_lock write_lock(this);
1262  std::make_shared<DashboardDescriptor>(vd);
1263 }
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:552
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addLinkToMap ( LinkDescriptor ld)
protected

Definition at line 1340 of file Catalog.cpp.

References currentDB_, Catalog_Namespace::DBMetadata::dbId, LinkDescriptor::link, linkDescriptorMap_, linkDescriptorMapById_, LinkDescriptor::linkId, and to_string().

Referenced by createLink().

1340  {
1341  cat_write_lock write_lock(this);
1342  LinkDescriptor* new_ld = new LinkDescriptor();
1343  *new_ld = ld;
1345  linkDescriptorMapById_[ld.linkId] = new_ld;
1346 }
const DBMetadata currentDB_
Definition: Catalog.h:559
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:554
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:553

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addReferenceToForeignDict ( ColumnDescriptor referencing_column,
Parser::SharedDictionaryDef  shared_dict_def,
const bool  persist_reference 
)
protected

Definition at line 3137 of file Catalog.cpp.

References CHECK, CHECK_GE, ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, SQLTypeInfo::get_comp_param(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_foreign_col(), getObjForLock(), SqliteConnector::query_with_text_params(), sqliteConnector_, and to_string().

Referenced by setColumnSharedDictionary().

3139  {
3140  cat_write_lock write_lock(this);
3141  const auto foreign_ref_col = get_foreign_col(*this, shared_dict_def);
3142  CHECK(foreign_ref_col);
3143  referencing_column.columnType = foreign_ref_col->columnType;
3144  const int dict_id = referencing_column.columnType.get_comp_param();
3145  const DictRef dict_ref(currentDB_.dbId, dict_id);
3146  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
3147  CHECK(dictIt != dictDescriptorMapByRef_.end());
3148  const auto& dd = dictIt->second;
3149  CHECK_GE(dd->refcount, 1);
3150  ++dd->refcount;
3151  if (persist_reference) {
3152  cat_sqlite_lock sqlite_lock(getObjForLock());
3154  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
3155  {std::to_string(dict_id)});
3156  }
3157 }
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:558
const DBMetadata currentDB_
Definition: Catalog.h:559
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:551
#define CHECK_GE(x, y)
Definition: Logger.h:210
const ColumnDescriptor * get_foreign_col(const Catalog &cat, const Parser::SharedDictionaryDef &shared_dict_def)
Definition: Catalog.cpp:3125
std::string to_string(char const *&&v)
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:323
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addTableToMap ( const TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::list< DictDescriptor > &  dicts 
)
protected

Definition at line 1113 of file Catalog.cpp.

References CHECK, columnDescriptorMap_, columnDescriptorMapById_, ColumnDescriptor::columnId, TableDescriptor::columnIdBySpi_, ColumnDescriptor::columnName, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, DictDescriptor::dictFolderPath, dict_ref_t::dictId, TableDescriptor::hasDeletedCol, TableDescriptor::mutex_, setDeletedColumnUnlocked(), gpu_enabled::sort(), string_dict_hosts_, tableDescriptorMap_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, and to_upper().

Referenced by createTable().

1115  {
1116  cat_write_lock write_lock(this);
1117  TableDescriptor* new_td;
1118 
1119  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
1120  if (foreign_table) {
1121  auto new_foreign_table = new foreign_storage::ForeignTable();
1122  *new_foreign_table = *foreign_table;
1123  new_td = new_foreign_table;
1124  } else {
1125  new_td = new TableDescriptor();
1126  *new_td = *td;
1127  }
1128 
1129  new_td->mutex_ = std::make_shared<std::mutex>();
1130  tableDescriptorMap_[to_upper(td->tableName)] = new_td;
1131  tableDescriptorMapById_[td->tableId] = new_td;
1132  for (auto cd : columns) {
1133  ColumnDescriptor* new_cd = new ColumnDescriptor();
1134  *new_cd = cd;
1135  ColumnKey columnKey(new_cd->tableId, to_upper(new_cd->columnName));
1136  columnDescriptorMap_[columnKey] = new_cd;
1137  ColumnIdKey columnIdKey(new_cd->tableId, new_cd->columnId);
1138  columnDescriptorMapById_[columnIdKey] = new_cd;
1139 
1140  // Add deleted column to the map
1141  if (cd.isDeletedCol) {
1142  CHECK(new_td->hasDeletedCol);
1143  setDeletedColumnUnlocked(new_td, new_cd);
1144  }
1145  }
1146 
1147  std::sort(new_td->columnIdBySpi_.begin(),
1148  new_td->columnIdBySpi_.end(),
1149  [](const size_t a, const size_t b) -> bool { return a < b; });
1150 
1151  std::unique_ptr<StringDictionaryClient> client;
1152  DictRef dict_ref(currentDB_.dbId, -1);
1153  if (!string_dict_hosts_.empty()) {
1154  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1155  }
1156  for (auto dd : dicts) {
1157  if (!dd.dictRef.dictId) {
1158  // Dummy entry created for a shard of a logical table, nothing to do.
1159  continue;
1160  }
1161  dict_ref.dictId = dd.dictRef.dictId;
1162  if (client) {
1163  client->create(dict_ref, dd.dictIsTemp);
1164  }
1165  DictDescriptor* new_dd = new DictDescriptor(dd);
1166  dictDescriptorMapByRef_[dict_ref].reset(new_dd);
1167  if (!dd.dictIsTemp) {
1168  boost::filesystem::create_directory(new_dd->dictFolderPath);
1169  }
1170  }
1171 }
std::string dictFolderPath
std::tuple< int, std::string > ColumnKey
Definition: Types.h:36
std::string tableName
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:549
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:562
const DBMetadata currentDB_
Definition: Catalog.h:559
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:551
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
std::vector< int > columnIdBySpi_
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:550
std::shared_ptr< std::mutex > mutex_
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3116
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:548
specifies the content in-memory of a row in the column metadata table
std::string to_upper(const std::string &str)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:197
Descriptor for a dictionary for a string columne.
mapd_unique_lock< mapd_shared_mutex > write_lock
std::string columnName
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:547
std::tuple< int, int > ColumnIdKey
Definition: Types.h:38

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::adjustAlteredTableFiles ( const std::string &  temp_data_dir,
const std::unordered_map< int, int > &  all_column_ids_map 
) const
private
void Catalog_Namespace::Catalog::alterPhysicalTableMetadata ( const TableDescriptor td,
const TableDescriptorUpdateParams table_update_params 
)
private

Definition at line 2841 of file Catalog.cpp.

References CHECK, getMutableMetadataForTableUnlocked(), TableDescriptorUpdateParams::max_rollback_epochs, TableDescriptorUpdateParams::max_rows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, SqliteConnector::query_with_text_params(), sqliteConnector_, TableDescriptor::tableId, and to_string().

Referenced by alterTableMetadata().

2843  {
2844  // Only called from parent alterTableParamMetadata, expect already to have catalog and
2845  // sqlite write locks
2846 
2847  // Sqlite transaction should have already been begun in parent alterTableCatalogMetadata
2848 
2850  CHECK(mutable_td);
2851  if (td->maxRollbackEpochs != table_update_params.max_rollback_epochs) {
2853  "UPDATE mapd_tables SET max_rollback_epochs = ? WHERE tableid = ?",
2854  std::vector<std::string>{std::to_string(table_update_params.max_rollback_epochs),
2855  std::to_string(td->tableId)});
2856  mutable_td->maxRollbackEpochs = table_update_params.max_rollback_epochs;
2857  }
2858 
2859  if (td->maxRows != table_update_params.max_rows) {
2861  "UPDATE mapd_tables SET max_rows = ? WHERE tableid = ?",
2862  std::vector<std::string>{std::to_string(table_update_params.max_rows),
2863  std::to_string(td->tableId)});
2864  mutable_td->maxRows = table_update_params.max_rows;
2865  }
2866 }
int32_t maxRollbackEpochs
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:558
std::string to_string(char const *&&v)
TableDescriptor * getMutableMetadataForTableUnlocked(int tableId)
Definition: Catalog.cpp:1450
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::alterTableMetadata ( const TableDescriptor td,
const TableDescriptorUpdateParams table_update_params 
)
private

Definition at line 2868 of file Catalog.cpp.

References alterPhysicalTableMetadata(), CHECK, logger::FATAL, getMetadataForTable(), getObjForLock(), i, LOG, logicalToPhysicalTableMapById_, SqliteConnector::query(), sqliteConnector_, TableDescriptor::tableId, and TableDescriptor::tableName.

Referenced by setMaxRollbackEpochs(), setMaxRows(), and setUncappedTableEpoch().

2869  {
2870  cat_write_lock write_lock(this);
2871  cat_sqlite_lock sqlite_lock(getObjForLock());
2872  sqliteConnector_.query("BEGIN TRANSACTION");
2873  try {
2874  const auto physical_table_it = logicalToPhysicalTableMapById_.find(td->tableId);
2875  if (physical_table_it != logicalToPhysicalTableMapById_.end()) {
2876  const auto physical_tables = physical_table_it->second;
2877  CHECK(!physical_tables.empty());
2878  for (size_t i = 0; i < physical_tables.size(); i++) {
2879  int32_t physical_tb_id = physical_tables[i];
2880  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id, false);
2881  CHECK(phys_td);
2882  alterPhysicalTableMetadata(phys_td, table_update_params);
2883  }
2884  }
2885  alterPhysicalTableMetadata(td, table_update_params);
2886  } catch (std::exception& e) {
2887  sqliteConnector_.query("ROLLBACK TRANSACTION");
2888  LOG(FATAL) << "Table '" << td->tableName << "' catalog update failed";
2889  }
2890  sqliteConnector_.query("END TRANSACTION");
2891 }
std::string tableName
#define LOG(tag)
Definition: Logger.h:188
SqliteConnector sqliteConnector_
Definition: Catalog.h:558
virtual void query(const std::string &queryString)
void alterPhysicalTableMetadata(const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
Definition: Catalog.cpp:2841
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:565

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::buildForeignServerMap ( )
private

Definition at line 4218 of file Catalog.cpp.

References foreignServerMap_, foreignServerMapById_, SqliteConnector::getData(), SqliteConnector::getNumRows(), SqliteConnector::query(), and sqliteConnector_.

Referenced by buildMaps().

4218  {
4220  "SELECT id, name, data_wrapper_type, options, owner_user_id, creation_time FROM "
4221  "omnisci_foreign_servers");
4222  auto num_rows = sqliteConnector_.getNumRows();
4223  for (size_t row = 0; row < num_rows; row++) {
4224  auto foreign_server = std::make_shared<foreign_storage::ForeignServer>(
4225  sqliteConnector_.getData<int>(row, 0),
4226  sqliteConnector_.getData<std::string>(row, 1),
4227  sqliteConnector_.getData<std::string>(row, 2),
4228  sqliteConnector_.getData<std::string>(row, 3),
4229  sqliteConnector_.getData<std::int32_t>(row, 4),
4230  sqliteConnector_.getData<std::int32_t>(row, 5));
4231  foreignServerMap_[foreign_server->name] = foreign_server;
4232  foreignServerMapById_[foreign_server->id] = foreign_server;
4233  }
4234 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:558
virtual void query(const std::string &queryString)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:556
ForeignServerMap foreignServerMap_
Definition: Catalog.h:555
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::buildMaps ( )
protected

Definition at line 901 of file Catalog.cpp.

References addForeignTableDetails(), buildForeignServerMap(), CHECK, ColumnDescriptor::chunks, columnDescriptorMap_, columnDescriptorMapById_, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, createDefaultServersIfNotExists(), currentDB_, dashboardDescriptorMap_, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, dictDescriptorMapByRef_, logger::FATAL, StorageType::FOREIGN_TABLE, TableDescriptor::fragmenter, TableDescriptor::fragments, TableDescriptor::fragPageSize, TableDescriptor::fragType, g_base_path, g_enable_fsi, generate_dashboard_system_rolename(), SQLTypeInfo::get_physical_cols(), SqliteConnector::getData(), SqliteConnector::getNumRows(), getObjForLock(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::getUserFromId(), TableDescriptor::hasDeletedCol, SQLTypeInfo::is_geometry(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, SqliteConnector::isNull(), ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, TableDescriptor::keyMetainfo, LinkDescriptor::link, linkDescriptorMap_, linkDescriptorMapById_, LinkDescriptor::linkId, LOG, logicalToPhysicalTableMapById_, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nColumns, TableDescriptor::nShards, TableDescriptor::partitions, SqliteConnector::query(), test_fsi::r, SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_compression(), SQLTypeInfo::set_dimension(), SQLTypeInfo::set_notnull(), SQLTypeInfo::set_scale(), SQLTypeInfo::set_size(), SQLTypeInfo::set_subtype(), SQLTypeInfo::set_type(), setDeletedColumnUnlocked(), TableDescriptor::shard, TableDescriptor::shardedColumnId, gpu_enabled::sort(), TableDescriptor::sortedColumnId, sqliteConnector_, TableDescriptor::storageType, tableDescriptorMap_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), to_upper(), LinkDescriptor::updateTime, LinkDescriptor::userId, TableDescriptor::userId, LinkDescriptor::viewMetadata, TableDescriptor::viewSQL, LinkDescriptor::viewState, and ColumnDescriptor::virtualExpr.

901  {
903  cat_sqlite_lock sqlite_lock(getObjForLock());
904 
905  string dictQuery(
906  "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
907  sqliteConnector_.query(dictQuery);
908  size_t numRows = sqliteConnector_.getNumRows();
909  for (size_t r = 0; r < numRows; ++r) {
910  int dictId = sqliteConnector_.getData<int>(r, 0);
911  std::string dictName = sqliteConnector_.getData<string>(r, 1);
912  int dictNBits = sqliteConnector_.getData<int>(r, 2);
913  bool is_shared = sqliteConnector_.getData<bool>(r, 3);
914  int refcount = sqliteConnector_.getData<int>(r, 4);
915  std::string fname = g_base_path + "/mapd_data/DB_" + std::to_string(currentDB_.dbId) +
916  "_DICT_" + std::to_string(dictId);
917  DictRef dict_ref(currentDB_.dbId, dictId);
918  DictDescriptor* dd = new DictDescriptor(
919  dict_ref, dictName, dictNBits, is_shared, refcount, fname, false);
920  dictDescriptorMapByRef_[dict_ref].reset(dd);
921  }
922 
923  string tableQuery(
924  "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
925  "max_chunk_size, frag_page_size, "
926  "max_rows, partitions, shard_column_id, shard, num_shards, key_metainfo, userid, "
927  "sort_column_id, storage_type, max_rollback_epochs "
928  "from mapd_tables");
929  sqliteConnector_.query(tableQuery);
930  numRows = sqliteConnector_.getNumRows();
931  for (size_t r = 0; r < numRows; ++r) {
932  TableDescriptor* td;
933  const auto& storage_type = sqliteConnector_.getData<string>(r, 17);
934  if (!storage_type.empty() && storage_type != StorageType::FOREIGN_TABLE) {
935  const auto table_id = sqliteConnector_.getData<int>(r, 0);
936  const auto& table_name = sqliteConnector_.getData<string>(r, 1);
937  LOG(FATAL) << "Unable to read Catalog metadata: storage type is currently not a "
938  "supported table option (table "
939  << table_name << " [" << table_id << "] in database "
940  << currentDB_.dbName << ").";
941  }
942 
943  if (storage_type == StorageType::FOREIGN_TABLE) {
945  } else {
946  td = new TableDescriptor();
947  }
948 
949  td->storageType = storage_type;
950  td->tableId = sqliteConnector_.getData<int>(r, 0);
951  td->tableName = sqliteConnector_.getData<string>(r, 1);
952  td->nColumns = sqliteConnector_.getData<int>(r, 2);
953  td->isView = sqliteConnector_.getData<bool>(r, 3);
954  td->fragments = sqliteConnector_.getData<string>(r, 4);
955  td->fragType =
957  td->maxFragRows = sqliteConnector_.getData<int>(r, 6);
958  td->maxChunkSize = sqliteConnector_.getData<int64_t>(r, 7);
959  td->fragPageSize = sqliteConnector_.getData<int>(r, 8);
960  td->maxRows = sqliteConnector_.getData<int64_t>(r, 9);
961  td->partitions = sqliteConnector_.getData<string>(r, 10);
962  td->shardedColumnId = sqliteConnector_.getData<int>(r, 11);
963  td->shard = sqliteConnector_.getData<int>(r, 12);
964  td->nShards = sqliteConnector_.getData<int>(r, 13);
965  td->keyMetainfo = sqliteConnector_.getData<string>(r, 14);
966  td->userId = sqliteConnector_.getData<int>(r, 15);
967  td->sortedColumnId =
968  sqliteConnector_.isNull(r, 16) ? 0 : sqliteConnector_.getData<int>(r, 16);
969  if (!td->isView) {
970  td->fragmenter = nullptr;
971  }
973  td->hasDeletedCol = false;
974 
977  }
978 
979  if (g_enable_fsi) {
983  }
984 
985  string columnQuery(
986  "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
987  "is_notnull, compression, comp_param, "
988  "size, chunks, is_systemcol, is_virtualcol, virtual_expr, is_deletedcol from "
989  "mapd_columns ORDER BY tableid, "
990  "columnid");
991  sqliteConnector_.query(columnQuery);
992  numRows = sqliteConnector_.getNumRows();
993  int32_t skip_physical_cols = 0;
994  for (size_t r = 0; r < numRows; ++r) {
996  cd->tableId = sqliteConnector_.getData<int>(r, 0);
997  cd->columnId = sqliteConnector_.getData<int>(r, 1);
998  cd->columnName = sqliteConnector_.getData<string>(r, 2);
1002  cd->columnType.set_scale(sqliteConnector_.getData<int>(r, 6));
1006  cd->columnType.set_size(sqliteConnector_.getData<int>(r, 10));
1007  cd->chunks = sqliteConnector_.getData<string>(r, 11);
1008  cd->isSystemCol = sqliteConnector_.getData<bool>(r, 12);
1009  cd->isVirtualCol = sqliteConnector_.getData<bool>(r, 13);
1010  cd->virtualExpr = sqliteConnector_.getData<string>(r, 14);
1011  cd->isDeletedCol = sqliteConnector_.getData<bool>(r, 15);
1012  cd->isGeoPhyCol = skip_physical_cols > 0;
1013  ColumnKey columnKey(cd->tableId, to_upper(cd->columnName));
1014  columnDescriptorMap_[columnKey] = cd;
1015  ColumnIdKey columnIdKey(cd->tableId, cd->columnId);
1016  columnDescriptorMapById_[columnIdKey] = cd;
1017 
1018  if (skip_physical_cols <= 0) {
1019  skip_physical_cols = cd->columnType.get_physical_cols();
1020  }
1021 
1022  auto td_itr = tableDescriptorMapById_.find(cd->tableId);
1023  CHECK(td_itr != tableDescriptorMapById_.end());
1024 
1025  if (cd->isDeletedCol) {
1026  td_itr->second->hasDeletedCol = true;
1027  setDeletedColumnUnlocked(td_itr->second, cd);
1028  } else if (cd->columnType.is_geometry() || skip_physical_cols-- <= 0) {
1029  tableDescriptorMapById_[cd->tableId]->columnIdBySpi_.push_back(cd->columnId);
1030  }
1031  }
1032  // sort columnIdBySpi_ based on columnId
1033  for (auto& tit : tableDescriptorMapById_) {
1034  std::sort(tit.second->columnIdBySpi_.begin(),
1035  tit.second->columnIdBySpi_.end(),
1036  [](const size_t a, const size_t b) -> bool { return a < b; });
1037  }
1038 
1039  string viewQuery("SELECT tableid, sql FROM mapd_views");
1040  sqliteConnector_.query(viewQuery);
1041  numRows = sqliteConnector_.getNumRows();
1042  for (size_t r = 0; r < numRows; ++r) {
1043  int32_t tableId = sqliteConnector_.getData<int>(r, 0);
1044  TableDescriptor* td = tableDescriptorMapById_[tableId];
1045  td->viewSQL = sqliteConnector_.getData<string>(r, 1);
1046  td->fragmenter = nullptr;
1047  }
1048 
1049  string frontendViewQuery(
1050  "SELECT id, state, name, image_hash, strftime('%Y-%m-%dT%H:%M:%SZ', update_time), "
1051  "userid, "
1052  "metadata "
1053  "FROM mapd_dashboards");
1054  sqliteConnector_.query(frontendViewQuery);
1055  numRows = sqliteConnector_.getNumRows();
1056  for (size_t r = 0; r < numRows; ++r) {
1057  std::shared_ptr<DashboardDescriptor> vd = std::make_shared<DashboardDescriptor>();
1058  vd->dashboardId = sqliteConnector_.getData<int>(r, 0);
1059  vd->dashboardState = sqliteConnector_.getData<string>(r, 1);
1060  vd->dashboardName = sqliteConnector_.getData<string>(r, 2);
1061  vd->imageHash = sqliteConnector_.getData<string>(r, 3);
1062  vd->updateTime = sqliteConnector_.getData<string>(r, 4);
1063  vd->userId = sqliteConnector_.getData<int>(r, 5);
1064  vd->dashboardMetadata = sqliteConnector_.getData<string>(r, 6);
1065  vd->user = getUserFromId(vd->userId);
1066  vd->dashboardSystemRoleName = generate_dashboard_system_rolename(
1068  dashboardDescriptorMap_[std::to_string(vd->userId) + ":" + vd->dashboardName] = vd;
1069  }
1070 
1071  string linkQuery(
1072  "SELECT linkid, userid, link, view_state, strftime('%Y-%m-%dT%H:%M:%SZ', "
1073  "update_time), view_metadata "
1074  "FROM mapd_links");
1075  sqliteConnector_.query(linkQuery);
1076  numRows = sqliteConnector_.getNumRows();
1077  for (size_t r = 0; r < numRows; ++r) {
1078  LinkDescriptor* ld = new LinkDescriptor();
1079  ld->linkId = sqliteConnector_.getData<int>(r, 0);
1080  ld->userId = sqliteConnector_.getData<int>(r, 1);
1081  ld->link = sqliteConnector_.getData<string>(r, 2);
1082  ld->viewState = sqliteConnector_.getData<string>(r, 3);
1083  ld->updateTime = sqliteConnector_.getData<string>(r, 4);
1084  ld->viewMetadata = sqliteConnector_.getData<string>(r, 5);
1086  linkDescriptorMapById_[ld->linkId] = ld;
1087  }
1088 
1089  /* rebuild map linking logical tables to corresponding physical ones */
1090  string logicalToPhysicalTableMapQuery(
1091  "SELECT logical_table_id, physical_table_id "
1092  "FROM mapd_logical_to_physical");
1093  sqliteConnector_.query(logicalToPhysicalTableMapQuery);
1094  numRows = sqliteConnector_.getNumRows();
1095  for (size_t r = 0; r < numRows; ++r) {
1096  int32_t logical_tb_id = sqliteConnector_.getData<int>(r, 0);
1097  int32_t physical_tb_id = sqliteConnector_.getData<int>(r, 1);
1098  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
1099  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
1100  /* add new entity to the map logicalToPhysicalTableMapById_ */
1101  std::vector<int32_t> physicalTables;
1102  physicalTables.push_back(physical_tb_id);
1103  const auto it_ok =
1104  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
1105  CHECK(it_ok.second);
1106  } else {
1107  /* update map logicalToPhysicalTableMapById_ */
1108  physicalTableIt->second.push_back(physical_tb_id);
1109  }
1110  }
1111 }
int32_t maxRollbackEpochs
std::string virtualExpr
void set_compression(EncodingType c)
Definition: sqltypes.h:414
void set_size(int s)
Definition: sqltypes.h:412
std::string partitions
std::tuple< int, std::string > ColumnKey
Definition: Types.h:36
T getData(const int row, const int col)
SQLTypes
Definition: sqltypes.h:37
std::string tableName
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:549
#define LOG(tag)
Definition: Logger.h:188
SqliteConnector sqliteConnector_
Definition: Catalog.h:558
const DBMetadata currentDB_
Definition: Catalog.h:559
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:551
std::string storageType
tuple r
Definition: test_fsi.py:16
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:405
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
virtual void query(const std::string &queryString)
std::string fragments
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:550
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:552
std::string to_string(char const *&&v)
std::string chunks
EncodingType
Definition: sqltypes.h:227
int get_physical_cols() const
Definition: sqltypes.h:335
std::string getUserFromId(const int32_t id)
Definition: Catalog.cpp:891
std::string g_base_path
Definition: SysCatalog.cpp:62
void set_scale(int s)
Definition: sqltypes.h:409
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3116
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:548
specifies the content in-memory of a row in the column metadata table
std::string keyMetainfo
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
std::string to_upper(const std::string &str)
void set_comp_param(int p)
Definition: sqltypes.h:415
bool isNull(const int row, const int col) const
void set_dimension(int d)
Definition: sqltypes.h:406
Fragmenter_Namespace::FragmenterType fragType
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
void set_notnull(bool n)
Definition: sqltypes.h:411
#define CHECK(condition)
Definition: Logger.h:197
bool is_geometry() const
Definition: sqltypes.h:500
Descriptor for a dictionary for a string columne.
std::string viewSQL
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:554
static constexpr char const * FOREIGN_TABLE
bool g_enable_fsi
Definition: Catalog.cpp:92
std::string columnName
virtual size_t getNumRows() const
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:565
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:547
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:553
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)
void createDefaultServersIfNotExists()
Definition: Catalog.cpp:4281
std::tuple< int, int > ColumnIdKey
Definition: Types.h:38
HOST DEVICE void set_type(SQLTypes t)
Definition: sqltypes.h:404

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::calculateSHA1 ( const std::string &  data)
protected

Definition at line 3945 of file Catalog.cpp.

References i.

Referenced by createLink().

3945  {
3946  boost::uuids::detail::sha1 sha1;
3947  unsigned int digest[5];
3948  sha1.process_bytes(data.c_str(), data.length());
3949  sha1.get_digest(digest);
3950  std::stringstream ss;
3951  for (size_t i = 0; i < 5; i++) {
3952  ss << std::hex << digest[i];
3953  }
3954  return ss.str();
3955 }

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::changeForeignServerOwner ( const std::string &  server_name,
const int  new_owner_id 
)

Change the owner of a Foreign Server to a new owner.

Parameters
server_name- Name of the foreign server whose owner to change
new_owner_id- New owner's user id

Definition at line 2556 of file Catalog.cpp.

References CHECK, foreignServerMap_, setForeignServerProperty(), to_string(), and foreign_storage::ForeignServer::user_id.

2557  {
2558  cat_write_lock write_lock(this);
2559  foreign_storage::ForeignServer* foreign_server =
2560  foreignServerMap_.find(server_name)->second.get();
2561  CHECK(foreign_server);
2562  setForeignServerProperty(server_name, "owner_user_id", std::to_string(new_owner_id));
2563  // update in-memory server
2564  foreign_server->user_id = new_owner_id;
2565 }
std::string to_string(char const *&&v)
ForeignServerMap foreignServerMap_
Definition: Catalog.h:555
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock
void setForeignServerProperty(const std::string &server_name, const std::string &property, const std::string &value)
Definition: Catalog.cpp:4260

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::CheckAndExecuteMigrations ( )
protected

Definition at line 866 of file Catalog.cpp.

References createFsiSchemas(), g_enable_fsi, recordOwnershipOfObjectsInObjectPermissions(), updateDeletedColumnIndicator(), updateDictionaryNames(), updateDictionarySchema(), updateFixlenArrayColumns(), updateFrontendViewAndLinkUsers(), updateFrontendViewSchema(), updateFrontendViewsToDashboards(), updateGeoColumns(), updateLinkSchema(), updateLogicalToPhysicalTableLinkSchema(), updatePageSize(), and updateTableDescriptorSchema().

866  {
876  updatePageSize();
880  if (g_enable_fsi) {
882  }
883 }
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:427
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:656
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:121
void updateLogicalToPhysicalTableLinkSchema()
Definition: Catalog.cpp:567
bool g_enable_fsi
Definition: Catalog.cpp:92

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::CheckAndExecuteMigrationsPostBuildMaps ( )
protected

Definition at line 885 of file Catalog.cpp.

References checkDateInDaysColumnMigration(), and createDashboardSystemRoles().

885  {
888 }
void checkDateInDaysColumnMigration()
Definition: Catalog.cpp:784

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::checkDateInDaysColumnMigration ( )
protected

Definition at line 784 of file Catalog.cpp.

References getCurrentDB(), getObjForLock(), migrations::MigrationMgr::migrateDateInDaysMetadata(), sqliteConnector_, and tableDescriptorMapById_.

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

784  {
785  cat_sqlite_lock sqlite_lock(getObjForLock());
788 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:558
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:221
static void migrateDateInDaysMetadata(const Catalog_Namespace::TableDescriptorMapById &table_descriptors_by_id, const int database_id, const Catalog_Namespace::Catalog *cat, SqliteConnector &sqlite)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:548
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const bool Catalog_Namespace::Catalog::checkMetadataForDeletedRecs ( const TableDescriptor td,
int  column_id 
) const

Definition at line 3074 of file Catalog.cpp.

References CHECK, and TableDescriptor::fragmenter.

Referenced by getDeletedColumnIfRowsDeleted().

3075  {
3076  // check if there are rows deleted by examining the deletedColumn metadata
3077  CHECK(td);
3078  auto fragmenter = td->fragmenter;
3079  if (fragmenter) {
3080  return fragmenter->hasDeletedRows(delete_column_id);
3081  } else {
3082  return false;
3083  }
3084 }
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::checkpoint ( const int  logicalTableId) const

Definition at line 4152 of file Catalog.cpp.

References Data_Namespace::DataMgr::checkpoint(), getCurrentDB(), getDataMgr(), getMetadataForTable(), and getPhysicalTablesDescriptors().

Referenced by checkpointWithAutoRollback(), UpdelRoll::commitUpdate(), TableOptimizer::vacuumDeletedRows(), and TableOptimizer::vacuumFragmentsAboveMinSelectivity().

4152  {
4153  const auto td = getMetadataForTable(logicalTableId);
4154  const auto shards = getPhysicalTablesDescriptors(td);
4155  for (const auto shard : shards) {
4156  getDataMgr().checkpoint(getCurrentDB().dbId, shard->tableId);
4157  }
4158 }
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:222
void checkpoint(const int db_id, const int tb_id)
Definition: DataMgr.cpp:489
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:221
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4013
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::checkpointWithAutoRollback ( const int  logical_table_id) const

Definition at line 4160 of file Catalog.cpp.

References checkpoint(), getDatabaseId(), getTableEpochs(), and setTableEpochsLogExceptions().

Referenced by import_export::Loader::checkpoint(), Parser::LocalConnector::checkpoint(), and StorageIOFacility::TransactionParameters::finalizeTransaction().

4160  {
4161  auto table_epochs = getTableEpochs(getDatabaseId(), logical_table_id);
4162  try {
4163  checkpoint(logical_table_id);
4164  } catch (...) {
4165  setTableEpochsLogExceptions(getDatabaseId(), table_epochs);
4166  throw;
4167  }
4168 }
int getDatabaseId() const
Definition: Catalog.h:276
void checkpoint(const int logicalTableId) const
Definition: Catalog.cpp:4152
void setTableEpochsLogExceptions(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3056
std::vector< TableEpochInfo > getTableEpochs(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:2991

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

int32_t Catalog_Namespace::Catalog::createDashboard ( DashboardDescriptor vd)

Definition at line 3808 of file Catalog.cpp.

References addFrontendViewToMap(), createOrUpdateDashboardSystemRole(), currentDB_, DashboardDescriptor::dashboardId, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, DashboardDescriptor::dashboardSystemRoleName, Catalog_Namespace::DBMetadata::dbId, generate_dashboard_system_rolename(), SqliteConnector::getData(), SqliteConnector::getNumRows(), getObjForLock(), DashboardDescriptor::imageHash, SqliteConnector::query(), SqliteConnector::query_with_text_params(), sqliteConnector_, to_string(), Catalog_Namespace::sqlite_lock< T >::unlock(), Catalog_Namespace::write_lock< T >::unlock(), DashboardDescriptor::updateTime, and DashboardDescriptor::userId.

3808  {
3809  cat_write_lock write_lock(this);
3810  cat_sqlite_lock sqlite_lock(getObjForLock());
3811  sqliteConnector_.query("BEGIN TRANSACTION");
3812  try {
3813  // TODO(andrew): this should be an upsert
3815  "SELECT id FROM mapd_dashboards WHERE name = ? and userid = ?",
3816  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
3817  if (sqliteConnector_.getNumRows() > 0) {
3819  "UPDATE mapd_dashboards SET state = ?, image_hash = ?, metadata = ?, "
3820  "update_time = "
3821  "datetime('now') where name = ? "
3822  "and userid = ?",
3823  std::vector<std::string>{vd.dashboardState,
3824  vd.imageHash,
3825  vd.dashboardMetadata,
3826  vd.dashboardName,
3827  std::to_string(vd.userId)});
3828  } else {
3830  "INSERT INTO mapd_dashboards (name, state, image_hash, metadata, "
3831  "update_time, "
3832  "userid) "
3833  "VALUES "
3834  "(?,?,?,?, "
3835  "datetime('now'), ?)",
3836  std::vector<std::string>{vd.dashboardName,
3837  vd.dashboardState,
3838  vd.imageHash,
3839  vd.dashboardMetadata,
3840  std::to_string(vd.userId)});
3841  }
3842  } catch (std::exception& e) {
3843  sqliteConnector_.query("ROLLBACK TRANSACTION");
3844  throw;
3845  }
3846  sqliteConnector_.query("END TRANSACTION");
3847 
3848  // now get the auto generated dashboardId
3849  try {
3851  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_dashboards "
3852  "WHERE name = ? and userid = ?",
3853  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
3854  vd.dashboardId = sqliteConnector_.getData<int>(0, 0);
3855  vd.updateTime = sqliteConnector_.getData<std::string>(0, 1);
3856  } catch (std::exception& e) {
3857  throw;
3858  }
3862  sqlite_lock.unlock();
3863  write_lock.unlock();
3864  // NOTE(wamsi): Transactionally unsafe
3867  return vd.dashboardId;
3868 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:558
const DBMetadata currentDB_
Definition: Catalog.h:559
virtual void query(const std::string &queryString)
void addFrontendViewToMap(DashboardDescriptor &vd)
Definition: Catalog.cpp:1254
std::string to_string(char const *&&v)
std::string dashboardSystemRoleName
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1295
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createDashboardSystemRoles ( )
protected

Definition at line 790 of file Catalog.cpp.

References createOrUpdateDashboardSystemRole(), currentDB_, Catalog_Namespace::DBMetadata::dbId, logger::ERROR, generate_dashboard_system_rolename(), SqliteConnector::getData(), Catalog_Namespace::SysCatalog::getGranteesOfSharedDashboards(), SqliteConnector::getNumRows(), getObjForLock(), Catalog_Namespace::SysCatalog::getRoleGrantee(), Catalog_Namespace::SysCatalog::grantRoleBatch(), i, logger::INFO, Catalog_Namespace::SysCatalog::instance(), LOG, MAPD_VERSION, SqliteConnector::query(), SqliteConnector::query_with_text_params(), run_benchmark_import::result, sqliteConnector_, and to_string().

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

790  {
791  std::unordered_map<std::string, std::pair<int, std::string>> dashboards;
792  std::vector<std::string> dashboard_ids;
793  static const std::string migration_name{"dashboard_roles_migration"};
794  {
795  cat_sqlite_lock sqlite_lock(getObjForLock());
796  sqliteConnector_.query("BEGIN TRANSACTION");
797  try {
798  // migration_history should be present in all catalogs by now
799  // if not then would be created before this migration
801  "select * from mapd_version_history where migration_history = '" +
802  migration_name + "'");
803  if (sqliteConnector_.getNumRows() != 0) {
804  // no need for further execution
805  sqliteConnector_.query("END TRANSACTION");
806  return;
807  }
808  LOG(INFO) << "Performing dashboard internal roles Migration.";
809  sqliteConnector_.query("select id, userid, metadata from mapd_dashboards");
810  for (size_t i = 0; i < sqliteConnector_.getNumRows(); ++i) {
813  sqliteConnector_.getData<string>(i, 0)))) {
814  // Successfully created roles during previous migration/crash
815  // No need to include them
816  continue;
817  }
818  dashboards[sqliteConnector_.getData<string>(i, 0)] = std::make_pair(
819  sqliteConnector_.getData<int>(i, 1), sqliteConnector_.getData<string>(i, 2));
820  dashboard_ids.push_back(sqliteConnector_.getData<string>(i, 0));
821  }
822  } catch (const std::exception& e) {
823  sqliteConnector_.query("ROLLBACK TRANSACTION");
824  throw;
825  }
826  sqliteConnector_.query("END TRANSACTION");
827  }
828  // All current grantees with shared dashboards.
829  const auto active_grantees =
831 
832  try {
833  // NOTE(wamsi): Transactionally unsafe
834  for (auto dash : dashboards) {
835  createOrUpdateDashboardSystemRole(dash.second.second,
836  dash.second.first,
838  std::to_string(currentDB_.dbId), dash.first));
839  auto result = active_grantees.find(dash.first);
840  if (result != active_grantees.end()) {
843  dash.first)},
844  result->second);
845  }
846  }
847  cat_sqlite_lock sqlite_lock(getObjForLock());
848  // check if this has already been completed
850  "select * from mapd_version_history where migration_history = '" +
851  migration_name + "'");
852  if (sqliteConnector_.getNumRows() != 0) {
853  return;
854  }
856  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
857  std::vector<std::string>{std::to_string(MAPD_VERSION), migration_name});
858  } catch (const std::exception& e) {
859  LOG(ERROR) << "Failed to create dashboard system roles during migration: "
860  << e.what();
861  throw;
862  }
863  LOG(INFO) << "Successfully created dashboard system roles during migration.";
864 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
#define LOG(tag)
Definition: Logger.h:188
SqliteConnector sqliteConnector_
Definition: Catalog.h:558
const DBMetadata currentDB_
Definition: Catalog.h:559
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
void grantRoleBatch(const std::vector< std::string > &roles, const std::vector< std::string > &grantees)
static SysCatalog & instance()
Definition: SysCatalog.h:292
static const int32_t MAPD_VERSION
Definition: release.h:33
Role * getRoleGrantee(const std::string &name) const
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1295
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
std::unordered_map< std::string, std::vector< std::string > > getGranteesOfSharedDashboards(const std::vector< std::string > &dashboard_ids)
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createDefaultServersIfNotExists ( )

Creates default local file servers (if they don't already exist).

Definition at line 4281 of file Catalog.cpp.

References createForeignServerNoLocks(), foreign_storage::DataWrapperType::CSV, foreign_storage::AbstractFileStorageDataWrapper::LOCAL_FILE_STORAGE_TYPE, OMNISCI_ROOT_USER_ID, foreign_storage::DataWrapperType::PARQUET, and foreign_storage::AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY.

Referenced by buildMaps().

4281  {
4285 
4286  auto local_csv_server = std::make_unique<foreign_storage::ForeignServer>(
4287  "omnisci_local_csv",
4289  options,
4291  local_csv_server->validate();
4292  createForeignServerNoLocks(std::move(local_csv_server), true);
4293 
4294  auto local_parquet_server = std::make_unique<foreign_storage::ForeignServer>(
4295  "omnisci_local_parquet",
4297  options,
4299  local_parquet_server->validate();
4300  createForeignServerNoLocks(std::move(local_parquet_server), true);
4301 }
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2467
const int OMNISCI_ROOT_USER_ID
Definition: SysCatalog.h:61
static constexpr char const * CSV
std::map< std::string, std::string, std::less<>> OptionsMap
static constexpr char const * PARQUET

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createForeignServer ( std::unique_ptr< foreign_storage::ForeignServer foreign_server,
bool  if_not_exists 
)

Creates a new foreign server DB object.

Parameters
foreign_server- unique pointer to struct containing foreign server details
if_not_exists- flag indicating whether or not an attempt to create a new foreign server should occur if a server with the same name already exists. An exception is thrown if this flag is set to "false" and an attempt is made to create a pre-existing foreign server

Definition at line 2459 of file Catalog.cpp.

References createForeignServerNoLocks(), and getObjForLock().

2461  {
2462  cat_write_lock write_lock(this);
2463  cat_sqlite_lock sqlite_lock(getObjForLock());
2464  createForeignServerNoLocks(std::move(foreign_server), if_not_exists);
2465 }
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2467
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createForeignServerNoLocks ( std::unique_ptr< foreign_storage::ForeignServer foreign_server,
bool  if_not_exists 
)
private

Same as createForeignServer() but without acquiring locks. This should only be called from within a function/code block that already acquires appropriate locks.

Definition at line 2467 of file Catalog.cpp.

References CHECK, CHECK_EQ, foreignServerMap_, foreignServerMapById_, SqliteConnector::getData(), SqliteConnector::getNumRows(), SqliteConnector::query_with_text_params(), sqliteConnector_, and to_string().

Referenced by createDefaultServersIfNotExists(), and createForeignServer().

2469  {
2471  "SELECT name from omnisci_foreign_servers where name = ?",
2472  std::vector<std::string>{foreign_server->name});
2473 
2474  if (sqliteConnector_.getNumRows() == 0) {
2475  foreign_server->creation_time = std::time(nullptr);
2477  "INSERT INTO omnisci_foreign_servers (name, data_wrapper_type, owner_user_id, "
2478  "creation_time, "
2479  "options) "
2480  "VALUES (?, ?, ?, ?, ?)",
2481  std::vector<std::string>{foreign_server->name,
2482  foreign_server->data_wrapper_type,
2483  std::to_string(foreign_server->user_id),
2484  std::to_string(foreign_server->creation_time),
2485  foreign_server->getOptionsAsJsonString()});
2487  "SELECT id from omnisci_foreign_servers where name = ?",
2488  std::vector<std::string>{foreign_server->name});
2489  CHECK_EQ(sqliteConnector_.getNumRows(), size_t(1));
2490  foreign_server->id = sqliteConnector_.getData<int32_t>(0, 0);
2491  std::shared_ptr<foreign_storage::ForeignServer> foreign_server_shared =
2492  std::move(foreign_server);
2493  CHECK(foreignServerMap_.find(foreign_server_shared->name) == foreignServerMap_.end())
2494  << "Attempting to insert a foreign server into foreign server map that already "
2495  "exists.";
2496  foreignServerMap_[foreign_server_shared->name] = foreign_server_shared;
2497  foreignServerMapById_[foreign_server_shared->id] = foreign_server_shared;
2498  } else if (!if_not_exists) {
2499  throw std::runtime_error{"A foreign server with name \"" + foreign_server->name +
2500  "\" already exists."};
2501  }
2502 }
#define CHECK_EQ(x, y)
Definition: Logger.h:205
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:558
std::string to_string(char const *&&v)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:556
ForeignServerMap foreignServerMap_
Definition: Catalog.h:555
#define CHECK(condition)
Definition: Logger.h:197
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createFsiSchemas ( )
protected

Definition at line 628 of file Catalog.cpp.

References getForeignServerSchema(), getForeignTableSchema(), getObjForLock(), SqliteConnector::query(), and sqliteConnector_.

Referenced by CheckAndExecuteMigrations().

628  {
629  cat_sqlite_lock sqlite_lock(getObjForLock());
630  sqliteConnector_.query("BEGIN TRANSACTION");
631  try {
634  } catch (std::exception& e) {
635  sqliteConnector_.query("ROLLBACK TRANSACTION");
636  throw;
637  }
638  sqliteConnector_.query("END TRANSACTION");
639 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:558
virtual void query(const std::string &queryString)
static const std::string getForeignTableSchema(bool if_not_exists=false)
Definition: Catalog.cpp:648
static const std::string getForeignServerSchema(bool if_not_exists=false)
Definition: Catalog.cpp:641
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string Catalog_Namespace::Catalog::createLink ( LinkDescriptor ld,
size_t  min_length 
)

Definition at line 3957 of file Catalog.cpp.

References addLinkToMap(), calculateSHA1(), SqliteConnector::getData(), SqliteConnector::getNumRows(), getObjForLock(), LinkDescriptor::link, LinkDescriptor::linkId, SqliteConnector::query(), SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), sqliteConnector_, to_string(), LinkDescriptor::updateTime, LinkDescriptor::userId, LinkDescriptor::viewMetadata, and LinkDescriptor::viewState.

3957  {
3958  cat_write_lock write_lock(this);
3959  cat_sqlite_lock sqlite_lock(getObjForLock());
3960  sqliteConnector_.query("BEGIN TRANSACTION");
3961  try {
3963  .substr(0, 8);
3965  "SELECT linkid FROM mapd_links WHERE link = ? and userid = ?",
3966  std::vector<std::string>{ld.link, std::to_string(ld.userId)});
3967  if (sqliteConnector_.getNumRows() > 0) {
3969  "UPDATE mapd_links SET update_time = datetime('now') WHERE userid = ? AND "
3970  "link = ?",
3971  std::vector<std::string>{std::to_string(ld.userId), ld.link});
3972  } else {
3974  "INSERT INTO mapd_links (userid, link, view_state, view_metadata, "
3975  "update_time) VALUES (?,?,?,?, datetime('now'))",
3976  std::vector<std::string>{
3977  std::to_string(ld.userId), ld.link, ld.viewState, ld.viewMetadata});
3978  }
3979  // now get the auto generated dashid
3981  "SELECT linkid, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_links "
3982  "WHERE link = ?",
3983  ld.link);
3984  ld.linkId = sqliteConnector_.getData<int>(0, 0);
3985  ld.updateTime = sqliteConnector_.getData<std::string>(0, 1);
3986  } catch (std::exception& e) {
3987  sqliteConnector_.query("ROLLBACK TRANSACTION");
3988  throw;
3989  }
3990  sqliteConnector_.query("END TRANSACTION");
3991  addLinkToMap(ld);
3992  return ld.link;
3993 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:558
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
std::string calculateSHA1(const std::string &data)
Definition: Catalog.cpp:3945
void addLinkToMap(LinkDescriptor &ld)
Definition: Catalog.cpp:1340
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createOrUpdateDashboardSystemRole ( const std::string &  view_meta,
const int32_t &  user_id,
const std::string &  dash_role_name 
)
protected

Definition at line 1295 of file Catalog.cpp.

References Catalog_Namespace::SysCatalog::createRole(), Grantee::findDbObject(), Grantee::getDbObjects(), Catalog_Namespace::SysCatalog::getRoleGrantee(), Catalog_Namespace::SysCatalog::grantDBObjectPrivilegesBatch(), Catalog_Namespace::SysCatalog::instance(), parseDashboardObjects(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivileges(), TableDBObjectType, and ViewDBObjectType.

Referenced by createDashboard(), createDashboardSystemRoles(), and replaceDashboard().

1297  {
1298  auto objects = parseDashboardObjects(view_meta, user_id);
1299  Role* rl = SysCatalog::instance().getRoleGrantee(dash_role_name);
1300  if (!rl) {
1301  // Dashboard role does not exist
1302  // create role and grant privileges
1303  // NOTE(wamsi): Transactionally unsafe
1304  SysCatalog::instance().createRole(dash_role_name, false);
1305  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1306  } else {
1307  // Dashboard system role already exists
1308  // Add/remove privileges on objects
1309  std::set<DBObjectKey> revoke_keys;
1310  auto ex_objects = rl->getDbObjects(true);
1311  for (auto key : *ex_objects | boost::adaptors::map_keys) {
1312  if (key.permissionType != TableDBObjectType &&
1313  key.permissionType != ViewDBObjectType) {
1314  continue;
1315  }
1316  bool found = false;
1317  for (auto obj : objects) {
1318  found = key == obj.getObjectKey() ? true : false;
1319  if (found) {
1320  break;
1321  }
1322  }
1323  if (!found) {
1324  revoke_keys.insert(key);
1325  }
1326  }
1327  for (auto& key : revoke_keys) {
1328  // revoke privs on object since the object is no
1329  // longer used by the dashboard as source
1330  // NOTE(wamsi): Transactionally unsafe
1332  dash_role_name, *rl->findDbObject(key, true), *this);
1333  }
1334  // Update privileges on remaining objects
1335  // NOTE(wamsi): Transactionally unsafe
1336  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1337  }
1338 }
void revokeDBObjectPrivileges(const std::string &grantee, const DBObject &object, const Catalog_Namespace::Catalog &catalog)
Definition: Grantee.h:76
DBObject * findDbObject(const DBObjectKey &objectKey, bool only_direct) const
Definition: Grantee.cpp:72
void createRole(const std::string &roleName, const bool &userPrivateRole=false)
static SysCatalog & instance()
Definition: SysCatalog.h:292
Role * getRoleGrantee(const std::string &name) const
void grantDBObjectPrivilegesBatch(const std::vector< std::string > &grantees, const std::vector< DBObject > &objects, const Catalog_Namespace::Catalog &catalog)
const DBObjectMap * getDbObjects(bool only_direct) const
Definition: Grantee.h:56
std::vector< DBObject > parseDashboardObjects(const std::string &view_meta, const int &user_id)
Definition: Catalog.cpp:1265

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createShardedTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs 
)

Definition at line 3269 of file Catalog.cpp.

References CHECK, createTable(), generatePhysicalTableName(), i, logicalToPhysicalTableMapById_, TableDescriptor::nShards, TableDescriptor::shard, table_is_temporary(), TableDescriptor::tableId, TableDescriptor::tableName, and updateLogicalToPhysicalTableMap().

3272  {
3273  cat_write_lock write_lock(this);
3274 
3275  /* create logical table */
3276  TableDescriptor* tdl = &td;
3277  createTable(*tdl, cols, shared_dict_defs, true); // create logical table
3278  int32_t logical_tb_id = tdl->tableId;
3279  std::string logical_table_name = tdl->tableName;
3280 
3281  /* create physical tables and link them to the logical table */
3282  std::vector<int32_t> physicalTables;
3283  for (int32_t i = 1; i <= td.nShards; i++) {
3284  TableDescriptor* tdp = &td;
3285  tdp->tableName = generatePhysicalTableName(logical_table_name, i);
3286  tdp->shard = i - 1;
3287  createTable(*tdp, cols, shared_dict_defs, false); // create physical table
3288  int32_t physical_tb_id = tdp->tableId;
3289 
3290  /* add physical table to the vector of physical tables */
3291  physicalTables.push_back(physical_tb_id);
3292  }
3293 
3294  if (!physicalTables.empty()) {
3295  /* add logical to physical tables correspondence to the map */
3296  const auto it_ok =
3297  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
3298  CHECK(it_ok.second);
3299  /* update sqlite mapd_logical_to_physical in sqlite database */
3300  if (!table_is_temporary(&td)) {
3301  updateLogicalToPhysicalTableMap(logical_tb_id);
3302  }
3303  }
3304 }
std::string tableName
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:2148
std::string generatePhysicalTableName(const std::string &logicalTableName, const int32_t &shardNumber)
Definition: Catalog.cpp:4211
bool table_is_temporary(const TableDescriptor *const td)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:565
void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id)
Definition: Catalog.cpp:581

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs,
bool  isLogicalTable 
)

Definition at line 2148 of file Catalog.cpp.

References addTableToMap(), calciteMgr_, ColumnDescriptor::columnId, TableDescriptor::columnIdBySpi_, ColumnDescriptor::columnName, ColumnDescriptor::columnType, currentDB_, dataMgr_, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, dict_ref_t::dictId, Data_Namespace::DISK_LEVEL, expandGeoColumn(), StorageType::FOREIGN_TABLE, TableDescriptor::fragPageSize, TableDescriptor::fragType, g_enable_fsi, g_serialize_temp_tables, g_test_against_columnId_gap, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_next_refresh_time(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), getCurrentDB(), SqliteConnector::getData(), getObjForLock(), TableDescriptor::hasDeletedCol, SQLTypeInfo::is_array(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, kBIGINT, kBOOLEAN, kENCODING_DICT, TableDescriptor::keyMetainfo, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nColumns, foreign_storage::ForeignTable::next_refresh_time, nextTempDictId_, nextTempTableId_, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, SqliteConnector::query(), SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), removeTableFromMap(), serializeTableJsonUnlocked(), SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_size(), setColumnDictionary(), setColumnSharedDictionary(), TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, sqliteConnector_, TableDescriptor::storageType, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), TableDescriptor::userId, TableDescriptor::viewSQL, and ColumnDescriptor::virtualExpr.

Referenced by createShardedTable().

2152  {
2153  cat_write_lock write_lock(this);
2154  list<ColumnDescriptor> cds = cols;
2155  list<DictDescriptor> dds;
2156  std::set<std::string> toplevel_column_names;
2157  list<ColumnDescriptor> columns;
2158 
2159  if (!td.storageType.empty() &&
2162  throw std::runtime_error("Only temporary tables can be backed by foreign storage.");
2163  }
2164  dataMgr_->getForeignStorageInterface()->prepareTable(getCurrentDB().dbId, td, cds);
2165  }
2166 
2167  for (auto cd : cds) {
2168  if (cd.columnName == "rowid") {
2169  throw std::runtime_error(
2170  "Cannot create column with name rowid. rowid is a system defined column.");
2171  }
2172  columns.push_back(cd);
2173  toplevel_column_names.insert(cd.columnName);
2174  if (cd.columnType.is_geometry()) {
2175  expandGeoColumn(cd, columns);
2176  }
2177  }
2178  cds.clear();
2179 
2180  ColumnDescriptor cd;
2181  // add row_id column -- Must be last column in the table
2182  cd.columnName = "rowid";
2183  cd.isSystemCol = true;
2184  cd.columnType = SQLTypeInfo(kBIGINT, true);
2185 #ifdef MATERIALIZED_ROWID
2186  cd.isVirtualCol = false;
2187 #else
2188  cd.isVirtualCol = true;
2189  cd.virtualExpr = "MAPD_FRAG_ID * MAPD_ROWS_PER_FRAG + MAPD_FRAG_ROW_ID";
2190 #endif
2191  columns.push_back(cd);
2192  toplevel_column_names.insert(cd.columnName);
2193 
2194  if (td.hasDeletedCol) {
2195  ColumnDescriptor cd_del;
2196  cd_del.columnName = "$deleted$";
2197  cd_del.isSystemCol = true;
2198  cd_del.isVirtualCol = false;
2199  cd_del.columnType = SQLTypeInfo(kBOOLEAN, true);
2200  cd_del.isDeletedCol = true;
2201 
2202  columns.push_back(cd_del);
2203  }
2204 
2205  td.nColumns = columns.size();
2206  cat_sqlite_lock sqlite_lock(getObjForLock());
2207  sqliteConnector_.query("BEGIN TRANSACTION");
2209  try {
2211  R"(INSERT INTO mapd_tables (name, userid, ncolumns, isview, fragments, frag_type, max_frag_rows, max_chunk_size, frag_page_size, max_rows, partitions, shard_column_id, shard, num_shards, sort_column_id, storage_type, max_rollback_epochs, key_metainfo) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?))",
2212  std::vector<std::string>{td.tableName,
2213  std::to_string(td.userId),
2215  std::to_string(td.isView),
2216  "",
2221  std::to_string(td.maxRows),
2222  td.partitions,
2224  std::to_string(td.shard),
2225  std::to_string(td.nShards),
2227  td.storageType,
2229  td.keyMetainfo});
2230 
2231  // now get the auto generated tableid
2233  "SELECT tableid FROM mapd_tables WHERE name = ?", td.tableName);
2234  td.tableId = sqliteConnector_.getData<int>(0, 0);
2235  int colId = 1;
2236  for (auto cd : columns) {
2238  const bool is_foreign_col =
2239  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2240  if (!is_foreign_col) {
2241  setColumnDictionary(cd, dds, td, isLogicalTable);
2242  }
2243  }
2244 
2245  if (toplevel_column_names.count(cd.columnName)) {
2246  // make up colId gap for sanity test (begin with 1 bc much code depends on it!)
2247  if (colId > 1) {
2248  colId += g_test_against_columnId_gap;
2249  }
2250  if (!cd.isGeoPhyCol) {
2251  td.columnIdBySpi_.push_back(colId);
2252  }
2253  }
2254 
2256  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, "
2257  "coldim, colscale, is_notnull, "
2258  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
2259  "virtual_expr, is_deletedcol) "
2260  "VALUES (?, ?, ?, ?, ?, "
2261  "?, "
2262  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2263  std::vector<std::string>{std::to_string(td.tableId),
2264  std::to_string(colId),
2265  cd.columnName,
2274  "",
2277  cd.virtualExpr,
2279  cd.tableId = td.tableId;
2280  cd.columnId = colId++;
2281  cds.push_back(cd);
2282  }
2283  if (td.isView) {
2285  "INSERT INTO mapd_views (tableid, sql) VALUES (?,?)",
2286  std::vector<std::string>{std::to_string(td.tableId), td.viewSQL});
2287  }
2289  auto& foreign_table = dynamic_cast<foreign_storage::ForeignTable&>(td);
2290  foreign_table.next_refresh_time = get_next_refresh_time(foreign_table);
2292  "INSERT INTO omnisci_foreign_tables (table_id, server_id, options, "
2293  "last_refresh_time, next_refresh_time) VALUES (?, ?, ?, ?, ?)",
2294  std::vector<std::string>{std::to_string(foreign_table.tableId),
2295  std::to_string(foreign_table.foreign_server->id),
2296  foreign_table.getOptionsAsJsonString(),
2297  std::to_string(foreign_table.last_refresh_time),
2298  std::to_string(foreign_table.next_refresh_time)});
2299  }
2300  } catch (std::exception& e) {
2301  sqliteConnector_.query("ROLLBACK TRANSACTION");
2302  throw;
2303  }
2304  } else { // Temporary table
2305  td.tableId = nextTempTableId_++;
2306  int colId = 1;
2307  for (auto cd : columns) {
2309  const bool is_foreign_col =
2310  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2311 
2312  if (!is_foreign_col) {
2313  // Create a new temporary dictionary
2314  std::string fileName("");
2315  std::string folderPath("");
2317  nextTempDictId_++;
2318  DictDescriptor dd(dict_ref,
2319  fileName,
2321  false,
2322  1,
2323  folderPath,
2324  true); // Is dictName (2nd argument) used?
2325  dds.push_back(dd);
2326  if (!cd.columnType.is_array()) {
2328  }
2329  cd.columnType.set_comp_param(dict_ref.dictId);
2330  }
2331  }
2332  if (toplevel_column_names.count(cd.columnName)) {
2333  // make up colId gap for sanity test (begin with 1 bc much code depends on it!)
2334  if (colId > 1) {
2335  colId += g_test_against_columnId_gap;
2336  }
2337  if (!cd.isGeoPhyCol) {
2338  td.columnIdBySpi_.push_back(colId);
2339  }
2340  }
2341  cd.tableId = td.tableId;
2342  cd.columnId = colId++;
2343  cds.push_back(cd);
2344  }
2345 
2347  serializeTableJsonUnlocked(&td, cds);
2348  }
2349  }
2350 
2351  try {
2352  addTableToMap(&td, cds, dds);
2353  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
2354  if (!td.storageType.empty() && td.storageType != StorageType::FOREIGN_TABLE) {
2355  dataMgr_->getForeignStorageInterface()->registerTable(this, td, cds);
2356  }
2357  } catch (std::exception& e) {
2358  sqliteConnector_.query("ROLLBACK TRANSACTION");
2359  removeTableFromMap(td.tableName, td.tableId, true);
2360  throw;
2361  }
2362  sqliteConnector_.query("END TRANSACTION");
2363 }
int32_t maxRollbackEpochs
void serializeTableJsonUnlocked(const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
Definition: Catalog.cpp:2365
int64_t get_next_refresh_time(const foreign_storage::ForeignTable &foreign_table)
Definition: Catalog.cpp:2136
std::string virtualExpr
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:315
void set_size(int s)
Definition: sqltypes.h:412
std::string partitions
HOST DEVICE int get_size() const
Definition: sqltypes.h:324
T getData(const int row, const int col)
std::string tableName
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:558
const DBMetadata currentDB_
Definition: Catalog.h:559
HOST DEVICE int get_scale() const
Definition: sqltypes.h:319
std::string storageType
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:560
virtual void query(const std::string &queryString)
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:314
std::vector< int > columnIdBySpi_
std::string to_string(char const *&&v)
void addTableToMap(const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
Definition: Catalog.cpp:1113
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:221
specifies the content in-memory of a row in the column metadata table
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:2005
std::string keyMetainfo
int g_test_against_columnId_gap
Definition: Catalog.cpp:91
bool g_serialize_temp_tables
Definition: Catalog.cpp:98
void set_comp_param(int p)
Definition: sqltypes.h:415
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:563
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:322
Fragmenter_Namespace::FragmenterType fragType
Data_Namespace::MemoryLevel persistenceLevel
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:316
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1173
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:323
bool setColumnSharedDictionary(ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:3159
Descriptor for a dictionary for a string columne.
std::string viewSQL
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:321
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:3228
static constexpr char const * FOREIGN_TABLE
bool g_enable_fsi
Definition: Catalog.cpp:92
std::string columnName
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
bool is_array() const
Definition: sqltypes.h:496

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::delDictionary ( const ColumnDescriptor cd)

Definition at line 1788 of file Catalog.cpp.

References CHECK, CHECK_GT, ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, g_base_path, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SqliteConnector::getData(), getMetadataForTable(), getObjForLock(), SQLTypeInfo::is_string(), SQLTypeInfo::is_string_array(), kENCODING_DICT, SqliteConnector::query_with_text_param(), File_Namespace::renameForDelete(), sqliteConnector_, string_dict_hosts_, ColumnDescriptor::tableId, to_string(), and VLOG.

Referenced by roll().

1788  {
1789  cat_write_lock write_lock(this);
1790  cat_sqlite_lock sqlite_lock(getObjForLock());
1791  if (!(cd.columnType.is_string() || cd.columnType.is_string_array())) {
1792  return;
1793  }
1794  if (!(cd.columnType.get_compression() == kENCODING_DICT)) {
1795  return;
1796  }
1797  const auto dictId = cd.columnType.get_comp_param();
1798  CHECK_GT(dictId, 0);
1799  // decrement and zero check dict ref count
1800  const auto td = getMetadataForTable(cd.tableId);
1801  CHECK(td);
1803  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
1804  std::to_string(dictId));
1806  "SELECT refcount FROM mapd_dictionaries WHERE dictid = ?", std::to_string(dictId));
1807  const auto refcount = sqliteConnector_.getData<int>(0, 0);
1808  VLOG(3) << "Dictionary " << dictId << "from dropped table has reference count "
1809  << refcount;
1810  if (refcount > 0) {
1811  return;
1812  }
1813  const DictRef dictRef(currentDB_.dbId, dictId);
1814  sqliteConnector_.query_with_text_param("DELETE FROM mapd_dictionaries WHERE dictid = ?",
1815  std::to_string(dictId));
1816  File_Namespace::renameForDelete(g_base_path + "/mapd_data/DB_" +
1817  std::to_string(currentDB_.dbId) + "_DICT_" +
1818  std::to_string(dictId));
1819 
1820  std::unique_ptr<StringDictionaryClient> client;
1821  if (!string_dict_hosts_.empty()) {
1822  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dictRef, true));
1823  }
1824  if (client) {
1825  client->drop(dictRef);
1826  }
1827 
1828  dictDescriptorMapByRef_.erase(dictRef);
1829 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:558
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:562
const DBMetadata currentDB_
Definition: Catalog.h:559
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:551
#define CHECK_GT(x, y)
Definition: Logger.h:209
std::string to_string(char const *&&v)
std::string g_base_path
Definition: SysCatalog.cpp:62
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:322
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:323
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
bool is_string() const
Definition: sqltypes.h:488
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:218
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
bool is_string_array() const
Definition: sqltypes.h:489
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
#define VLOG(n)
Definition: Logger.h:291

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::deleteMetadataForDashboards ( const std::vector< int32_t >  ids,
const UserMetadata user 
)

Definition at line 1558 of file Catalog.cpp.

References DashboardDBObjectType, dashboardDescriptorMap_, AccessPrivileges::DELETE_DASHBOARD, getMetadataForDashboard(), getObjForLock(), Catalog_Namespace::SysCatalog::instance(), SqliteConnector::query(), SqliteConnector::query_with_text_params(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivilegesFromAllBatch(), sqliteConnector_, and to_string().

1559  {
1560  std::stringstream invalid_ids, restricted_ids;
1561 
1562  for (int32_t dashboard_id : dashboard_ids) {
1563  if (!getMetadataForDashboard(dashboard_id)) {
1564  invalid_ids << (!invalid_ids.str().empty() ? ", " : "") << dashboard_id;
1565  continue;
1566  }
1567  DBObject object(dashboard_id, DashboardDBObjectType);
1568  object.loadKey(*this);
1569  object.setPrivileges(AccessPrivileges::DELETE_DASHBOARD);
1570  std::vector<DBObject> privs = {object};
1571  if (!SysCatalog::instance().checkPrivileges(user, privs))
1572  restricted_ids << (!restricted_ids.str().empty() ? ", " : "") << dashboard_id;
1573  }
1574 
1575  if (invalid_ids.str().size() > 0 || restricted_ids.str().size() > 0) {
1576  std::stringstream error_message;
1577  error_message << "Delete dashboard(s) failed with error(s):";
1578  if (invalid_ids.str().size() > 0)
1579  error_message << "\nDashboard id: " << invalid_ids.str()
1580  << " - Dashboard id does not exist";
1581  if (restricted_ids.str().size() > 0)
1582  error_message
1583  << "\nDashboard id: " << restricted_ids.str()
1584  << " - User should be either owner of dashboard or super user to delete it";
1585  throw std::runtime_error(error_message.str());
1586  }
1587  std::vector<DBObject> dash_objs;
1588 
1589  for (int32_t dashboard_id : dashboard_ids) {
1590  dash_objs.push_back(DBObject(dashboard_id, DashboardDBObjectType));
1591  }
1592  // BE-5245: Transactionally unsafe (like other combined Catalog/Syscatalog operations)
1594  {
1595  cat_write_lock write_lock(this);
1596  cat_sqlite_lock sqlite_lock(getObjForLock());
1597 
1598  sqliteConnector_.query("BEGIN TRANSACTION");
1599  try {
1600  for (int32_t dashboard_id : dashboard_ids) {
1601  auto dash = getMetadataForDashboard(dashboard_id);
1602  // Dash should still exist if revokeDBObjectPrivileges passed but throw and
1603  // rollback if already deleted
1604  if (!dash) {
1605  throw std::runtime_error(
1606  std::string("Delete dashboard(s) failed with error(s):\nDashboard id: ") +
1607  std::to_string(dashboard_id) + " - Dashboard id does not exist ");
1608  }
1609  std::string user_id = std::to_string(dash->userId);
1610  std::string dash_name = dash->dashboardName;
1611  auto viewDescIt = dashboardDescriptorMap_.find(user_id + ":" + dash_name);
1612  dashboardDescriptorMap_.erase(viewDescIt);
1614  "DELETE FROM mapd_dashboards WHERE name = ? and userid = ?",
1615  std::vector<std::string>{dash_name, user_id});
1616  }
1617  } catch (std::exception& e) {
1618  sqliteConnector_.query("ROLLBACK TRANSACTION");
1619  throw;
1620  }
1621  sqliteConnector_.query("END TRANSACTION");
1622  }
1623 }
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:558
virtual void query(const std::string &queryString)
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:552
std::string to_string(char const *&&v)
static SysCatalog & instance()
Definition: SysCatalog.h:292
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
static const AccessPrivileges DELETE_DASHBOARD
Definition: DBObject.h:175
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
void revokeDBObjectPrivilegesFromAllBatch(std::vector< DBObject > &objects, Catalog *catalog)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::doDropTable ( const TableDescriptor td)
protected

Definition at line 3457 of file Catalog.cpp.

References dropTableFromJsonUnlocked(), eraseTablePhysicalData(), executeDropTableSqliteQueries(), g_serialize_temp_tables, table_is_temporary(), and TableDescriptor::tableName.

Referenced by dropTable().

3457  {
3461  }
3463 }
void executeDropTableSqliteQueries(const TableDescriptor *td)
Definition: Catalog.cpp:3465
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:4182
std::string tableName
void dropTableFromJsonUnlocked(const std::string &table_name) const
Definition: Catalog.cpp:2427
bool g_serialize_temp_tables
Definition: Catalog.cpp:98
bool table_is_temporary(const TableDescriptor *const td)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::doTruncateTable ( const TableDescriptor td)
protected

Definition at line 3324 of file Catalog.cpp.

References CHECK, CHECK_GE, columnDescriptorMapById_, ColumnDescriptor::columnType, Data_Namespace::CPU_LEVEL, currentDB_, dataMgr_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, dict_ref_t::dictId, DictDescriptor::dictIsTemp, DictDescriptor::dictRef, TableDescriptor::fragmenter, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), getMetadataForDict(), Data_Namespace::GPU_LEVEL, Catalog_Namespace::SysCatalog::instance(), kENCODING_DICT, File_Namespace::renameForDelete(), string_dict_hosts_, tableDescriptorMapById_, ColumnDescriptor::tableId, and TableDescriptor::tableId.

Referenced by truncateTable().

3324  {
3325  cat_write_lock write_lock(this);
3326 
3327  const int tableId = td->tableId;
3328  // must destroy fragmenter before deleteChunks is called.
3329  if (td->fragmenter != nullptr) {
3330  auto tableDescIt = tableDescriptorMapById_.find(tableId);
3331  CHECK(tableDescIt != tableDescriptorMapById_.end());
3332  tableDescIt->second->fragmenter = nullptr;
3333  CHECK(td->fragmenter == nullptr);
3334  }
3335  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
3336  // assuming deleteChunksWithPrefix is atomic
3337  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
3338  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
3339 
3340  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
3341 
3342  std::unique_ptr<StringDictionaryClient> client;
3343  if (SysCatalog::instance().isAggregator()) {
3344  CHECK(!string_dict_hosts_.empty());
3345  DictRef dict_ref(currentDB_.dbId, -1);
3346  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
3347  }
3348  // clean up any dictionaries
3349  // delete all column descriptors for the table
3350  for (const auto& columnDescriptor : columnDescriptorMapById_) {
3351  auto cd = columnDescriptor.second;
3352  if (cd->tableId != td->tableId) {
3353  continue;
3354  }
3355  const int dict_id = cd->columnType.get_comp_param();
3356  // Dummy dictionaries created for a shard of a logical table have the id set to zero.
3357  if (cd->columnType.get_compression() == kENCODING_DICT && dict_id) {
3358  const DictRef dict_ref(currentDB_.dbId, dict_id);
3359  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
3360  CHECK(dictIt != dictDescriptorMapByRef_.end());
3361  const auto& dd = dictIt->second;
3362  CHECK_GE(dd->refcount, 1);
3363  // if this is the only table using this dict reset the dict
3364  if (dd->refcount == 1) {
3365  // close the dictionary
3366  dd->stringDict.reset();
3367  File_Namespace::renameForDelete(dd->dictFolderPath);
3368  if (client) {
3369  client->drop(dd->dictRef);
3370  }
3371  if (!dd->dictIsTemp) {
3372  boost::filesystem::create_directory(dd->dictFolderPath);
3373  }
3374  }
3375 
3376  DictDescriptor* new_dd = new DictDescriptor(dd->dictRef,
3377  dd->dictName,
3378  dd->dictNBits,
3379  dd->dictIsShared,
3380  dd->refcount,
3381  dd->dictFolderPath,
3382  dd->dictIsTemp);
3383  dictDescriptorMapByRef_.erase(dictIt);
3384  // now create new Dict -- need to figure out what to do here for temp tables
3385  if (client) {
3386  client->create(new_dd->dictRef, new_dd->dictIsTemp);
3387  }
3388  dictDescriptorMapByRef_[new_dd->dictRef].reset(new_dd);
3390  }
3391  }
3392 }
std::vector< int > ChunkKey
Definition: types.h:37
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:562
const DBMetadata currentDB_
Definition: Catalog.h:559
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:551
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:560
#define CHECK_GE(x, y)
Definition: Logger.h:210
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:550
static SysCatalog & instance()
Definition: SysCatalog.h:292
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:548
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1444
int32_t dictId
Definition: DictRef.h:10
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:197
Descriptor for a dictionary for a string columne.
mapd_unique_lock< mapd_shared_mutex > write_lock
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:218

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::dropColumn ( const TableDescriptor td,
const ColumnDescriptor cd 
)

Definition at line 1916 of file Catalog.cpp.

References CHECK, columnDescriptorMap_, columnDescriptorMapById_, columnDescriptorsForRoll, ColumnDescriptor::columnId, ColumnDescriptor::columnName, getMetadataForColumn(), getObjForLock(), getPhysicalTablesDescriptors(), TableDescriptor::nShards, SqliteConnector::query_with_text_params(), TableDescriptor::shard, sqliteConnector_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), and to_upper().

1916  {
1917  cat_write_lock write_lock(this);
1918  cat_sqlite_lock sqlite_lock(getObjForLock());
1919  // caller must handle sqlite/chunk transaction TOGETHER
1921  "DELETE FROM mapd_columns where tableid = ? and columnid = ?",
1922  std::vector<std::string>{std::to_string(td.tableId), std::to_string(cd.columnId)});
1923 
1925  "UPDATE mapd_tables SET ncolumns = ncolumns - 1 WHERE tableid = ?",
1926  std::vector<std::string>{std::to_string(td.tableId)});
1927 
1928  ColumnDescriptorMap::iterator columnDescIt =
1930  CHECK(columnDescIt != columnDescriptorMap_.end());
1931 
1932  columnDescriptorsForRoll.emplace_back(columnDescIt->second, nullptr);
1933 
1934  columnDescriptorMap_.erase(columnDescIt);
1936  --tableDescriptorMapById_[td.tableId]->nColumns;
1937  // for each shard
1938  if (td.nShards > 0 && td.shard < 0) {
1939  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
1940  const auto shard_cd = getMetadataForColumn(shard->tableId, cd.columnId);
1941  CHECK(shard_cd);
1942  dropColumn(*shard, *shard_cd);
1943  }
1944  }
1945 }
std::tuple< int, std::string > ColumnKey
Definition: Types.h:36
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:549
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:558
void dropColumn(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:1916
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:550
std::string to_string(char const *&&v)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:548
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4013
std::string to_upper(const std::string &str)
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:577
std::string columnName
std::tuple< int, int > ColumnIdKey
Definition: Types.h:38

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropForeignServer ( const std::string &  server_name)

Drops/deletes a foreign server DB object.

Parameters
server_name- Name of foreign server that will be deleted

Definition at line 2620 of file Catalog.cpp.

References CHECK_EQ, foreignServerMap_, foreignServerMapById_, SqliteConnector::getData(), SqliteConnector::getNumRows(), getObjForLock(), SqliteConnector::query(), SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), sqliteConnector_, and to_string().

2620  {
2621  cat_write_lock write_lock(this);
2622  cat_sqlite_lock sqlite_lock(getObjForLock());
2623 
2625  "SELECT id from omnisci_foreign_servers where name = ?",
2626  std::vector<std::string>{server_name});
2627  auto num_rows = sqliteConnector_.getNumRows();
2628  if (num_rows > 0) {
2629  CHECK_EQ(size_t(1), num_rows);
2630  auto server_id = sqliteConnector_.getData<int32_t>(0, 0);
2632  "SELECT table_id from omnisci_foreign_tables where server_id = ?",
2633  std::to_string(server_id));
2634  if (sqliteConnector_.getNumRows() > 0) {
2635  throw std::runtime_error{"Foreign server \"" + server_name +
2636  "\" is referenced "
2637  "by existing foreign tables and cannot be dropped."};
2638  }
2639  sqliteConnector_.query("BEGIN TRANSACTION");
2640  try {
2642  "DELETE FROM omnisci_foreign_servers WHERE name = ?",
2643  std::vector<std::string>{server_name});
2644  } catch (const std::exception& e) {
2645  sqliteConnector_.query("ROLLBACK TRANSACTION");
2646  throw;
2647  }
2648  sqliteConnector_.query("END TRANSACTION");
2649  foreignServerMap_.erase(server_name);
2650  foreignServerMapById_.erase(server_id);
2651  }
2652 }
#define CHECK_EQ(x, y)
Definition: Logger.h:205
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:558
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:556
ForeignServerMap foreignServerMap_
Definition: Catalog.h:555
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropFsiSchemasAndTables ( )
protected
void Catalog_Namespace::Catalog::dropTable ( const TableDescriptor td)

Definition at line 3424 of file Catalog.cpp.

References CHECK, doDropTable(), getMetadataForTable(), getObjForLock(), i, Catalog_Namespace::SysCatalog::instance(), TableDescriptor::isView, logicalToPhysicalTableMapById_, SqliteConnector::query(), SqliteConnector::query_with_text_param(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivilegesFromAll(), sqliteConnector_, TableDBObjectType, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and ViewDBObjectType.

3424  {
3427  cat_write_lock write_lock(this);
3428  cat_sqlite_lock sqlite_lock(getObjForLock());
3429  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
3430  sqliteConnector_.query("BEGIN TRANSACTION");
3431  try {
3432  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
3433  // remove all corresponding physical tables if this is a logical table
3434  const auto physicalTables = physicalTableIt->second;
3435  CHECK(!physicalTables.empty());
3436  for (size_t i = 0; i < physicalTables.size(); i++) {
3437  int32_t physical_tb_id = physicalTables[i];
3438  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
3439  CHECK(phys_td);
3440  doDropTable(phys_td);
3441  }
3442 
3443  // remove corresponding record from the logicalToPhysicalTableMap in sqlite database
3445  "DELETE FROM mapd_logical_to_physical WHERE logical_table_id = ?",
3446  std::to_string(td->tableId));
3448  }
3449  doDropTable(td);
3450  } catch (std::exception& e) {
3451  sqliteConnector_.query("ROLLBACK TRANSACTION");
3452  throw;
3453  }
3454  sqliteConnector_.query("END TRANSACTION");
3455 }
void doDropTable(const TableDescriptor *td)
Definition: Catalog.cpp:3457
std::string tableName
SqliteConnector sqliteConnector_
Definition: Catalog.h:558
void revokeDBObjectPrivilegesFromAll(DBObject object, Catalog *catalog)
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
static SysCatalog & instance()
Definition: SysCatalog.h:292
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:197
mapd_unique_lock< mapd_shared_mutex > write_lock
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:565

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropTableFromJsonUnlocked ( const std::string &  table_name) const
protected

Definition at line 2427 of file Catalog.cpp.

References basePath_, CHECK, currentDB_, test_fsi::d, Catalog_Namespace::DBMetadata::dbName, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and VLOG.

Referenced by doDropTable().

2427  {
2428  // relies on the catalog write lock
2429  using namespace rapidjson;
2430 
2431  VLOG(1) << "Dropping temporary table " << table_name << " to JSON for Calcite.";
2432 
2433  const auto db_name = currentDB_.dbName;
2434  const auto file_path = table_json_filepath(basePath_, db_name);
2435 
2436  CHECK(boost::filesystem::exists(file_path));
2437  Document d;
2438 
2439  std::ifstream reader(file_path.string());
2440  CHECK(reader.is_open());
2441  IStreamWrapper json_read_wrapper(reader);
2442  d.ParseStream(json_read_wrapper);
2443 
2444  CHECK(d.IsObject());
2445  auto table_name_ref = StringRef(table_name.c_str());
2446  CHECK(d.HasMember(table_name_ref));
2447  CHECK(d.RemoveMember(table_name_ref));
2448 
2449  // Overwrite the existing file
2450  std::ofstream writer(file_path.string(), std::ios::trunc | std::ios::out);
2451  CHECK(writer.is_open());
2452  OStreamWrapper json_wrapper(writer);
2453 
2454  Writer<OStreamWrapper> json_writer(json_wrapper);
2455  d.Accept(json_writer);
2456  writer.close();
2457 }
tuple d
Definition: test_fsi.py:9
const DBMetadata currentDB_
Definition: Catalog.h:559
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:154
#define CHECK(condition)
Definition: Logger.h:197
#define VLOG(n)
Definition: Logger.h:291

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string Catalog_Namespace::Catalog::dumpCreateTable ( const TableDescriptor td,
bool  multiline_formatting = true,
bool  dump_defaults = false 
) const

Definition at line 4483 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, Data_Namespace::CPU_LEVEL, DEFAULT_FRAGMENT_ROWS, DEFAULT_MAX_CHUNK_SIZE, DEFAULT_MAX_ROLLBACK_EPOCHS, DEFAULT_MAX_ROWS, DEFAULT_PAGE_SIZE, TableDescriptor::fragPageSize, g_leaf_count, gatherAdditionalInfo(), getAllColumnMetadataForTable(), getMetadataForColumn(), TableDescriptor::hasDeletedCol, ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, join(), kCHAR, kENCODING_DICT, kENCODING_GEOINT, kTEXT, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, quoteIfRequired(), TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and TableDescriptor::viewSQL.

4485  {
4486  cat_read_lock read_lock(this);
4487 
4488  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
4489  std::ostringstream os;
4490 
4491  if (foreign_table) {
4492  os << "CREATE FOREIGN TABLE " << td->tableName << " (";
4493  } else if (!td->isView) {
4494  os << "CREATE ";
4496  os << "TEMPORARY ";
4497  }
4498  os << "TABLE " + td->tableName + " (";
4499  } else {
4500  os << "CREATE VIEW " + td->tableName + " AS " << td->viewSQL;
4501  return os.str();
4502  }
4503  // scan column defines
4504  std::vector<std::string> additional_info;
4505  std::set<std::string> shared_dict_column_names;
4506 
4507  gatherAdditionalInfo(additional_info, shared_dict_column_names, td);
4508 
4509  // gather column defines
4510  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
4511  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
4512  bool first = true;
4513  for (const auto cd : cds) {
4514  if (!(cd->isSystemCol || cd->isVirtualCol)) {
4515  const auto& ti = cd->columnType;
4516  if (!first) {
4517  os << ",";
4518  if (!multiline_formatting) {
4519  os << " ";
4520  }
4521  } else {
4522  first = false;
4523  }
4524  if (multiline_formatting) {
4525  os << "\n ";
4526  }
4527  // column name
4528  os << quoteIfRequired(cd->columnName);
4529  // CHAR is perculiar... better dump it as TEXT(32) like \d does
4530  if (ti.get_type() == SQLTypes::kCHAR) {
4531  os << " "
4532  << "TEXT";
4533  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
4534  os << " "
4535  << "TEXT[]";
4536  } else {
4537  os << " " << ti.get_type_name();
4538  }
4539  os << (ti.get_notnull() ? " NOT NULL" : "");
4540  if (shared_dict_column_names.find(cd->columnName) ==
4541  shared_dict_column_names.end()) {
4542  // avoids "Exception: Column ... shouldn't specify an encoding, it borrows it
4543  // from the referenced column"
4544  if (ti.is_string() || (ti.is_array() && ti.get_subtype() == kTEXT)) {
4545  auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
4546  if (ti.get_compression() == kENCODING_DICT) {
4547  os << " ENCODING " << ti.get_compression_name() << "(" << (size * 8) << ")";
4548  } else {
4549  os << " ENCODING NONE";
4550  }
4551  } else if (ti.is_date_in_days() ||
4552  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
4553  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
4554  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
4555  } else if (ti.is_geometry()) {
4556  if (ti.get_compression() == kENCODING_GEOINT) {
4557  os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
4558  << ")";
4559  } else {
4560  os << " ENCODING NONE";
4561  }
4562  }
4563  }
4564  }
4565  }
4566  // gather SHARED DICTIONARYs
4567  if (additional_info.size()) {
4568  std::string comma;
4569  if (!multiline_formatting) {
4570  comma = ", ";
4571  } else {
4572  comma = ",\n ";
4573  }
4574  os << comma;
4575  os << boost::algorithm::join(additional_info, comma);
4576  }
4577  os << ")";
4578 
4579  std::vector<std::string> with_options;
4580  if (foreign_table) {
4581  if (multiline_formatting) {
4582  os << "\n";
4583  } else {
4584  os << " ";
4585  }
4586  os << "SERVER " << foreign_table->foreign_server->name;
4587 
4588  // gather WITH options ...
4589  for (const auto& [option, value] : foreign_table->options) {
4590  with_options.emplace_back(option + "='" + value + "'");
4591  }
4592  }
4593 
4594  if (dump_defaults || td->maxFragRows != DEFAULT_FRAGMENT_ROWS) {
4595  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
4596  }
4597  if (!foreign_table && (dump_defaults || td->maxChunkSize != DEFAULT_MAX_CHUNK_SIZE)) {
4598  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
4599  }
4600  if (!foreign_table && (dump_defaults || td->fragPageSize != DEFAULT_PAGE_SIZE)) {
4601  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
4602  }
4603  if (!foreign_table && (dump_defaults || td->maxRows != DEFAULT_MAX_ROWS)) {
4604  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
4605  }
4606  if ((dump_defaults || td->maxRollbackEpochs != DEFAULT_MAX_ROLLBACK_EPOCHS) &&
4607  td->maxRollbackEpochs != -1) {
4608  with_options.push_back("MAX_ROLLBACK_EPOCHS=" +
4610  }
4611  if (!foreign_table && (dump_defaults || !td->hasDeletedCol)) {
4612  with_options.push_back(td->hasDeletedCol ? "VACUUM='DELAYED'" : "VACUUM='IMMEDIATE'");
4613  }
4614  if (!foreign_table && !td->partitions.empty()) {
4615  with_options.push_back("PARTITIONS='" + td->partitions + "'");
4616  }
4617  if (!foreign_table && td->nShards > 0) {
4618  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
4619  CHECK(shard_cd);
4620  with_options.push_back(
4621  "SHARD_COUNT=" +
4622  std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
4623  }
4624  if (!foreign_table && td->sortedColumnId > 0) {
4625  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
4626  CHECK(sort_cd);
4627  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
4628  }
4629 
4630  if (!with_options.empty()) {
4631  if (!multiline_formatting) {
4632  os << " ";
4633  } else {
4634  os << "\n";
4635  }
4636  os << "WITH (" + boost::algorithm::join(with_options, ", ") + ")";
4637  }
4638  os << ";";
4639  return os.str();
4640 }
int32_t maxRollbackEpochs
std::string partitions
std::string tableName
std::string join(T const &container, std::string const &delim)
#define DEFAULT_MAX_CHUNK_SIZE
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:113
std::string to_string(char const *&&v)
std::string quoteIfRequired(const std::string &column_name) const
Definition: Catalog.cpp:4742
#define DEFAULT_MAX_ROWS
void gatherAdditionalInfo(std::vector< std::string > &additional_info, std::set< std::string > &shared_dict_column_names, const TableDescriptor *td) const
Definition: Catalog.cpp:4753
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
#define DEFAULT_MAX_ROLLBACK_EPOCHS
specifies the content in-memory of a row in the table metadata table
#define DEFAULT_PAGE_SIZE
Definition: sqltypes.h:51
#define DEFAULT_FRAGMENT_ROWS
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1721
Data_Namespace::MemoryLevel persistenceLevel
Definition: sqltypes.h:40
mapd_shared_lock< mapd_shared_mutex > read_lock
#define CHECK(condition)
Definition: Logger.h:197
std::string viewSQL
size_t g_leaf_count
Definition: ParserNode.cpp:75

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::dumpSchema ( const TableDescriptor td) const

Definition at line 4360 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, DEFAULT_MAX_ROLLBACK_EPOCHS, dictDescriptorMapByRef_, TableDescriptor::fragPageSize, g_leaf_count, getAllColumnMetadataForTable(), getMetadataForColumn(), TableDescriptor::hasDeletedCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, join(), kCHAR, kENCODING_DICT, kENCODING_GEOINT, kTEXT, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::tableId, and to_string().

Referenced by TableArchiver::dumpTable().

4360  {
4361  cat_read_lock read_lock(this);
4362 
4363  std::ostringstream os;
4364  os << "CREATE TABLE @T (";
4365  // gather column defines
4366  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
4367  std::string comma;
4368  std::vector<std::string> shared_dicts;
4369  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
4370  for (const auto cd : cds) {
4371  if (!(cd->isSystemCol || cd->isVirtualCol)) {
4372  const auto& ti = cd->columnType;
4373  os << comma << cd->columnName;
4374  // CHAR is perculiar... better dump it as TEXT(32) like \d does
4375  if (ti.get_type() == SQLTypes::kCHAR) {
4376  os << " "
4377  << "TEXT";
4378  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
4379  os << " "
4380  << "TEXT[]";
4381  } else {
4382  os << " " << ti.get_type_name();
4383  }
4384  os << (ti.get_notnull() ? " NOT NULL" : "");
4385  if (ti.is_string() || (ti.is_array() && ti.get_subtype() == kTEXT)) {
4386  auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
4387  if (ti.get_compression() == kENCODING_DICT) {
4388  // if foreign reference, get referenced tab.col
4389  const auto dict_id = ti.get_comp_param();
4390  const DictRef dict_ref(currentDB_.dbId, dict_id);
4391  const auto dict_it = dictDescriptorMapByRef_.find(dict_ref);
4392  CHECK(dict_it != dictDescriptorMapByRef_.end());
4393  const auto dict_name = dict_it->second->dictName;
4394  // when migrating a table, any foreign dict ref will be dropped
4395  // and the first cd of a dict will become root of the dict
4396  if (dict_root_cds.end() == dict_root_cds.find(dict_name)) {
4397  dict_root_cds[dict_name] = cd;
4398  os << " ENCODING " << ti.get_compression_name() << "(" << (size * 8) << ")";
4399  } else {
4400  const auto dict_root_cd = dict_root_cds[dict_name];
4401  shared_dicts.push_back("SHARED DICTIONARY (" + cd->columnName +
4402  ") REFERENCES @T(" + dict_root_cd->columnName + ")");
4403  // "... shouldn't specify an encoding, it borrows from the referenced
4404  // column"
4405  }
4406  } else {
4407  os << " ENCODING NONE";
4408  }
4409  } else if (ti.is_date_in_days() ||
4410  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
4411  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
4412  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
4413  } else if (ti.is_geometry()) {
4414  if (ti.get_compression() == kENCODING_GEOINT) {
4415  os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
4416  << ")";
4417  } else {
4418  os << " ENCODING NONE";
4419  }
4420  }
4421  comma = ", ";
4422  }
4423  }
4424  // gather SHARED DICTIONARYs
4425  if (shared_dicts.size()) {
4426  os << ", " << boost::algorithm::join(shared_dicts, ", ");
4427  }
4428  // gather WITH options ...
4429  std::vector<std::string> with_options;
4430  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
4431  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
4432  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
4433  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
4434  with_options.emplace_back(td->hasDeletedCol ? "VACUUM='DELAYED'"
4435  : "VACUUM='IMMEDIATE'");
4436  if (!td->partitions.empty()) {
4437  with_options.push_back("PARTITIONS='" + td->partitions + "'");
4438  }
4439  if (td->nShards > 0) {
4440  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
4441  CHECK(shard_cd);
4442  os << ", SHARD KEY(" << shard_cd->columnName << ")";
4443  with_options.push_back(
4444  "SHARD_COUNT=" +
4445  std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
4446  }
4447  if (td->sortedColumnId > 0) {
4448  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
4449  CHECK(sort_cd);
4450  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
4451  }
4453  td->maxRollbackEpochs != -1) {
4454  with_options.push_back("MAX_ROLLBACK_EPOCHS=" +
4456  }
4457  os << ") WITH (" + boost::algorithm::join(with_options, ", ") + ");";
4458  return os.str();
4459 }
int32_t maxRollbackEpochs
std::string partitions
const DBMetadata currentDB_
Definition: Catalog.h:559
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:551
std::string join(T const &container, std::string const &delim)
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:113
std::string to_string(char const *&&v)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
#define DEFAULT_MAX_ROLLBACK_EPOCHS
specifies the content in-memory of a row in the table metadata table
Definition: sqltypes.h:51
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1721
Definition: sqltypes.h:40
mapd_shared_lock< mapd_shared_mutex > read_lock
#define CHECK(condition)
Definition: Logger.h:197
size_t g_leaf_count
Definition: ParserNode.cpp:75

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::eraseDBData ( )

Definition at line 4170 of file Catalog.cpp.

References basePath_, calciteMgr_, currentDB_, Catalog_Namespace::DBMetadata::dbName, eraseTablePhysicalData(), getAllTableMetadata(), test_readcsv::table, and run_benchmark_import::tables.

4170  {
4171  cat_write_lock write_lock(this);
4172  // Physically erase all tables and dictionaries from disc and memory
4173  const auto tables = getAllTableMetadata();
4174  for (const auto table : tables) {
4176  }
4177  // Physically erase database metadata
4178  boost::filesystem::remove(basePath_ + "/mapd_catalogs/" + currentDB_.dbName);
4179  calciteMgr_->updateMetadata(currentDB_.dbName, "");
4180 }
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:4182
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:1747
const DBMetadata currentDB_
Definition: Catalog.h:559
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:563
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::eraseTablePhysicalData ( const TableDescriptor td)

Definition at line 4182 of file Catalog.cpp.

References calciteMgr_, CHECK, Data_Namespace::CPU_LEVEL, currentDB_, dataMgr_, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, TableDescriptor::fragmenter, Data_Namespace::GPU_LEVEL, INJECT_TIMER, TableDescriptor::isView, removeTableFromMap(), tableDescriptorMapById_, TableDescriptor::tableId, and TableDescriptor::tableName.

Referenced by doDropTable(), and eraseDBData().

4182  {
4183  const int tableId = td->tableId;
4184  // must destroy fragmenter before deleteChunks is called.
4185  if (td->fragmenter != nullptr) {
4186  auto tableDescIt = tableDescriptorMapById_.find(tableId);
4187  CHECK(tableDescIt != tableDescriptorMapById_.end());
4188  {
4189  INJECT_TIMER(deleting_fragmenter);
4190  tableDescIt->second->fragmenter = nullptr;
4191  }
4192  }
4193  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
4194  {
4195  INJECT_TIMER(deleteChunksWithPrefix);
4196  // assuming deleteChunksWithPrefix is atomic
4197  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
4198  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
4199  }
4200  if (!td->isView) {
4201  INJECT_TIMER(Remove_Table);
4202  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
4203  }
4204  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
4205  {
4206  INJECT_TIMER(removeTableFromMap_);
4207  removeTableFromMap(td->tableName, tableId);
4208  }
4209 }
std::vector< int > ChunkKey
Definition: types.h:37
std::string tableName
const DBMetadata currentDB_
Definition: Catalog.h:559
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:560
#define INJECT_TIMER(DESC)
Definition: measure.h:93
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:548
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:563
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1173
#define CHECK(condition)
Definition: Logger.h:197

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::executeDropTableSqliteQueries ( const TableDescriptor td)
protected

Definition at line 3465 of file Catalog.cpp.

References StorageType::FOREIGN_TABLE, SqliteConnector::getData(), SqliteConnector::getNumRows(), TableDescriptor::isView, kENCODING_DICT, SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), test_fsi::r, sqliteConnector_, TableDescriptor::storageType, TableDescriptor::tableId, and to_string().

Referenced by doDropTable().

3465  {
3466  const int tableId = td->tableId;
3467  sqliteConnector_.query_with_text_param("DELETE FROM mapd_tables WHERE tableid = ?",
3468  std::to_string(tableId));
3470  "select comp_param from mapd_columns where compression = ? and tableid = ?",
3471  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
3472  int numRows = sqliteConnector_.getNumRows();
3473  std::vector<int> dict_id_list;
3474  for (int r = 0; r < numRows; ++r) {
3475  dict_id_list.push_back(sqliteConnector_.getData<int>(r, 0));
3476  }
3477  for (auto dict_id : dict_id_list) {
3479  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
3480  std::vector<std::string>{std::to_string(dict_id)});
3481  }
3483  "DELETE FROM mapd_dictionaries WHERE dictid in (select comp_param from "
3484  "mapd_columns where compression = ? "
3485  "and tableid = ?) and refcount = 0",
3486  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
3487  sqliteConnector_.query_with_text_param("DELETE FROM mapd_columns WHERE tableid = ?",
3488  std::to_string(tableId));
3489  if (td->isView) {
3490  sqliteConnector_.query_with_text_param("DELETE FROM mapd_views WHERE tableid = ?",
3491  std::to_string(tableId));
3492  }
3495  "DELETE FROM omnisci_foreign_tables WHERE table_id = ?", std::to_string(tableId));
3496  }
3497 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:558
std::string storageType
tuple r
Definition: test_fsi.py:16
std::string to_string(char const *&&v)
static constexpr char const * FOREIGN_TABLE
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::expandGeoColumn ( const ColumnDescriptor cd,
std::list< ColumnDescriptor > &  columns 
)
static

Definition at line 2005 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, IS_GEO, kARRAY, kDOUBLE, kENCODING_GEOINT, kENCODING_NONE, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kTINYINT, SQLTypeInfo::set_size(), and SQLTypeInfo::set_subtype().

Referenced by createTable().

2006  {
2007  const auto& col_ti = cd.columnType;
2008  if (IS_GEO(col_ti.get_type())) {
2009  switch (col_ti.get_type()) {
2010  case kPOINT: {
2011  ColumnDescriptor physical_cd_coords(true);
2012  physical_cd_coords.columnName = cd.columnName + "_coords";
2013  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2014  // Raw data: compressed/uncompressed coords
2015  coords_ti.set_subtype(kTINYINT);
2016  size_t unit_size;
2017  if (col_ti.get_compression() == kENCODING_GEOINT &&
2018  col_ti.get_comp_param() == 32) {
2019  unit_size = 4 * sizeof(int8_t);
2020  } else {
2021  CHECK(col_ti.get_compression() == kENCODING_NONE);
2022  unit_size = 8 * sizeof(int8_t);
2023  }
2024  coords_ti.set_size(2 * unit_size);
2025  physical_cd_coords.columnType = coords_ti;
2026  columns.push_back(physical_cd_coords);
2027 
2028  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2029 
2030  break;
2031  }
2032  case kLINESTRING: {
2033  ColumnDescriptor physical_cd_coords(true);
2034  physical_cd_coords.columnName = cd.columnName + "_coords";
2035  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2036  // Raw data: compressed/uncompressed coords
2037  coords_ti.set_subtype(kTINYINT);
2038  physical_cd_coords.columnType = coords_ti;
2039  columns.push_back(physical_cd_coords);
2040 
2041  ColumnDescriptor physical_cd_bounds(true);
2042  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2043  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2044  bounds_ti.set_subtype(kDOUBLE);
2045  bounds_ti.set_size(4 * sizeof(double));
2046  physical_cd_bounds.columnType = bounds_ti;
2047  columns.push_back(physical_cd_bounds);
2048 
2049  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2050 
2051  break;
2052  }
2053  case kPOLYGON: {
2054  ColumnDescriptor physical_cd_coords(true);
2055  physical_cd_coords.columnName = cd.columnName + "_coords";
2056  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2057  // Raw data: compressed/uncompressed coords
2058  coords_ti.set_subtype(kTINYINT);
2059  physical_cd_coords.columnType = coords_ti;
2060  columns.push_back(physical_cd_coords);
2061 
2062  ColumnDescriptor physical_cd_ring_sizes(true);
2063  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
2064  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2065  ring_sizes_ti.set_subtype(kINT);
2066  physical_cd_ring_sizes.columnType = ring_sizes_ti;
2067  columns.push_back(physical_cd_ring_sizes);
2068 
2069  ColumnDescriptor physical_cd_bounds(true);
2070  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2071  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2072  bounds_ti.set_subtype(kDOUBLE);
2073  bounds_ti.set_size(4 * sizeof(double));
2074  physical_cd_bounds.columnType = bounds_ti;
2075  columns.push_back(physical_cd_bounds);
2076 
2077  ColumnDescriptor physical_cd_render_group(true);
2078  physical_cd_render_group.columnName = cd.columnName + "_render_group";
2079  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, col_ti.get_notnull());
2080  physical_cd_render_group.columnType = render_group_ti;
2081  columns.push_back(physical_cd_render_group);
2082 
2083  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2084 
2085  break;
2086  }
2087  case kMULTIPOLYGON: {
2088  ColumnDescriptor physical_cd_coords(true);
2089  physical_cd_coords.columnName = cd.columnName + "_coords";
2090  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2091  // Raw data: compressed/uncompressed coords
2092  coords_ti.set_subtype(kTINYINT);
2093  physical_cd_coords.columnType = coords_ti;
2094  columns.push_back(physical_cd_coords);
2095 
2096  ColumnDescriptor physical_cd_ring_sizes(true);
2097  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
2098  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2099  ring_sizes_ti.set_subtype(kINT);
2100  physical_cd_ring_sizes.columnType = ring_sizes_ti;
2101  columns.push_back(physical_cd_ring_sizes);
2102 
2103  ColumnDescriptor physical_cd_poly_rings(true);
2104  physical_cd_poly_rings.columnName = cd.columnName + "_poly_rings";
2105  SQLTypeInfo poly_rings_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2106  poly_rings_ti.set_subtype(kINT);
2107  physical_cd_poly_rings.columnType = poly_rings_ti;
2108  columns.push_back(physical_cd_poly_rings);
2109 
2110  ColumnDescriptor physical_cd_bounds(true);
2111  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2112  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2113  bounds_ti.set_subtype(kDOUBLE);
2114  bounds_ti.set_size(4 * sizeof(double));
2115  physical_cd_bounds.columnType = bounds_ti;
2116  columns.push_back(physical_cd_bounds);
2117 
2118  ColumnDescriptor physical_cd_render_group(true);
2119  physical_cd_render_group.columnName = cd.columnName + "_render_group";
2120  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, col_ti.get_notnull());
2121  physical_cd_render_group.columnType = render_group_ti;
2122  columns.push_back(physical_cd_render_group);
2123 
2124  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2125 
2126  break;
2127  }
2128  default:
2129  throw runtime_error("Unrecognized geometry type.");
2130  break;
2131  }
2132  }
2133 }
void set_size(int s)
Definition: sqltypes.h:412
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:405
specifies the content in-memory of a row in the column metadata table
#define CHECK(condition)
Definition: Logger.h:197
Definition: sqltypes.h:44
SQLTypeInfo columnType
std::string columnName
#define IS_GEO(T)
Definition: sqltypes.h:245

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

bool Catalog_Namespace::Catalog::filterTableByTypeAndUser ( const TableDescriptor td,
const UserMetadata user_metadata,
const GetTablesType  get_tables_type 
) const
private

Definition at line 4065 of file Catalog.cpp.

References GET_PHYSICAL_TABLES, GET_VIEWS, Catalog_Namespace::SysCatalog::instance(), TableDescriptor::isView, DBObject::loadKey(), TableDescriptor::shard, TableDBObjectType, TableDescriptor::tableName, and ViewDBObjectType.

Referenced by getTableNamesForUser(), and getTablesMetadataForUser().

4067  {
4068  if (td->shard >= 0) {
4069  // skip shards, they're not standalone tables
4070  return false;
4071  }
4072  switch (get_tables_type) {
4073  case GET_PHYSICAL_TABLES: {
4074  if (td->isView) {
4075  return false;
4076  }
4077  break;
4078  }
4079  case GET_VIEWS: {
4080  if (!td->isView) {
4081  return false;
4082  }
4083  break;
4084  }
4085  default:
4086  break;
4087  }
4089  dbObject.loadKey(*this);
4090  std::vector<DBObject> privObjects = {dbObject};
4091  if (!SysCatalog::instance().hasAnyPrivileges(user_metadata, privObjects)) {
4092  // skip table, as there are no privileges to access it
4093  return false;
4094  }
4095  return true;
4096 }
std::string tableName
static SysCatalog & instance()
Definition: SysCatalog.h:292