OmniSciDB  8fa3bf436f
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Catalog_Namespace::Catalog Class Referencefinal

class for a per-database catalog. also includes metadata for the current database and the current user. More...

#include <Catalog.h>

+ Collaboration diagram for Catalog_Namespace::Catalog:

Public Member Functions

 Catalog (const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
 Constructor - takes basePath to already extant data directory for writing. More...
 
 Catalog ()
 Constructor builds a hollow catalog used during constructor of other catalogs. More...
 
 ~Catalog ()
 Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite. More...
 
void createTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
 
void createShardedTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
int32_t createDashboard (DashboardDescriptor &vd)
 
void replaceDashboard (DashboardDescriptor &vd)
 
std::string createLink (LinkDescriptor &ld, size_t min_length)
 
void dropTable (const TableDescriptor *td)
 
void truncateTable (const TableDescriptor *td)
 
void renameTable (const TableDescriptor *td, const std::string &newTableName)
 
void renameTable (std::vector< std::pair< std::string, std::string >> &names)
 
void renameColumn (const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
 
void addColumn (const TableDescriptor &td, ColumnDescriptor &cd)
 
void dropColumn (const TableDescriptor &td, const ColumnDescriptor &cd)
 
void removeFragmenterForTable (const int table_id) const
 
const std::map< int, const
ColumnDescriptor * > 
getDictionaryToColumnMapping ()
 
const TableDescriptorgetMetadataForTable (const std::string &tableName, const bool populateFragmenter=true) const
 Returns a pointer to a const TableDescriptor struct matching the provided tableName. More...
 
const TableDescriptorgetMetadataForTableImpl (int tableId, const bool populateFragmenter) const
 
const TableDescriptorgetMetadataForTable (int tableId, bool populateFragmenter=true) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, const std::string &colName) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, int columnId) const
 
const ColumnDescriptorgetMetadataForColumnUnlocked (int tableId, int columnId) const
 
const int getColumnIdBySpi (const int tableId, const size_t spi) const
 
const ColumnDescriptorgetMetadataForColumnBySpi (const int tableId, const size_t spi) const
 
const DashboardDescriptorgetMetadataForDashboard (const std::string &userId, const std::string &dashName) const
 
const DashboardDescriptorgetMetadataForDashboard (const int32_t dashboard_id) const
 
void deleteMetadataForDashboards (const std::vector< int32_t > ids, const UserMetadata &user)
 
const LinkDescriptorgetMetadataForLink (const std::string &link) const
 
const LinkDescriptorgetMetadataForLink (int linkId) const
 
const
foreign_storage::ForeignTable
getForeignTableUnlocked (int tableId) const
 
const
foreign_storage::ForeignTable
getForeignTable (const std::string &tableName) const
 
const
foreign_storage::ForeignTable
getForeignTable (int table_id) const
 
std::list< const
ColumnDescriptor * > 
getAllColumnMetadataForTable (const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular table specified by table id. More...
 
std::list< const
ColumnDescriptor * > 
getAllColumnMetadataForTableUnlocked (const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 
std::list< const
TableDescriptor * > 
getAllTableMetadata () const
 
std::list< const
DashboardDescriptor * > 
getAllDashboardsMetadata () const
 
const DBMetadatagetCurrentDB () const
 
Data_Namespace::DataMgrgetDataMgr () const
 
std::shared_ptr< CalcitegetCalciteMgr () const
 
const std::string & getCatalogBasePath () const
 
const DictDescriptorgetMetadataForDict (int dict_ref, bool loadDict=true) const
 
const DictDescriptorgetMetadataForDictUnlocked (int dict_ref, bool loadDict) const
 
const std::vector< LeafHostInfo > & getStringDictionaryHosts () const
 
const ColumnDescriptorgetShardColumnMetadataForTable (const TableDescriptor *td) const
 
std::vector< const
TableDescriptor * > 
getPhysicalTablesDescriptors (const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
 
std::vector< std::string > getTableNamesForUser (const UserMetadata &user, const GetTablesType get_tables_type) const
 
std::vector< TableMetadatagetTablesMetadataForUser (const UserMetadata &user_metadata, const GetTablesType get_tables_type, const std::string &filter_table_name) const
 
int32_t getTableEpoch (const int32_t db_id, const int32_t table_id) const
 
void setTableEpoch (const int db_id, const int table_id, const int new_epoch)
 
void setMaxRollbackEpochs (const int32_t table_id, const int32_t max_rollback_epochs)
 
void setMaxRows (const int32_t table_id, const int64_t max_rows)
 
std::vector< TableEpochInfogetTableEpochs (const int32_t db_id, const int32_t table_id) const
 
void setTableEpochs (const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
 
void setTableEpochsLogExceptions (const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
 
int getDatabaseId () const
 
SqliteConnectorgetSqliteConnector ()
 
void roll (const bool forward)
 
DictRef addDictionary (ColumnDescriptor &cd)
 
void delDictionary (const ColumnDescriptor &cd)
 
void getDictionary (const ColumnDescriptor &cd, std::map< int, StringDictionary * > &stringDicts)
 
const bool checkMetadataForDeletedRecs (const TableDescriptor *td, int column_id) const
 
const ColumnDescriptorgetDeletedColumn (const TableDescriptor *td) const
 
const ColumnDescriptorgetDeletedColumnIfRowsDeleted (const TableDescriptor *td) const
 
void setDeletedColumn (const TableDescriptor *td, const ColumnDescriptor *cd)
 
void setDeletedColumnUnlocked (const TableDescriptor *td, const ColumnDescriptor *cd)
 
int getLogicalTableId (const int physicalTableId) const
 
void checkpoint (const int logicalTableId) const
 
void checkpointWithAutoRollback (const int logical_table_id) const
 
std::string name () const
 
void eraseDBData ()
 
void eraseTablePhysicalData (const TableDescriptor *td)
 
void setForReload (const int32_t tableId)
 
std::vector< std::string > getTableDataDirectories (const TableDescriptor *td) const
 
std::vector< std::string > getTableDictDirectories (const TableDescriptor *td) const
 
std::string getColumnDictDirectory (const ColumnDescriptor *cd) const
 
std::string dumpSchema (const TableDescriptor *td) const
 
std::string dumpCreateTable (const TableDescriptor *td, bool multiline_formatting=true, bool dump_defaults=false) const
 
void createForeignServer (std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
 
const
foreign_storage::ForeignServer
getForeignServer (const std::string &server_name) const
 
const std::unique_ptr< const
foreign_storage::ForeignServer
getForeignServerFromStorage (const std::string &server_name)
 
const std::unique_ptr< const
foreign_storage::ForeignTable
getForeignTableFromStorage (int table_id)
 
void changeForeignServerOwner (const std::string &server_name, const int new_owner_id)
 
void setForeignServerDataWrapper (const std::string &server_name, const std::string &data_wrapper)
 
void setForeignServerOptions (const std::string &server_name, const std::string &options)
 
void renameForeignServer (const std::string &server_name, const std::string &name)
 
void dropForeignServer (const std::string &server_name)
 
void getForeignServersForUser (const rapidjson::Value *filters, const UserMetadata &user, std::vector< const foreign_storage::ForeignServer * > &results)
 
void createDefaultServersIfNotExists ()
 
bool validateNonExistentTableOrView (const std::string &name, const bool if_not_exists)
 
std::vector< const
TableDescriptor * > 
getAllForeignTablesForRefresh () const
 
void updateForeignTableRefreshTimes (const int32_t table_id)
 
void setForeignTableOptions (const std::string &table_name, foreign_storage::OptionsMap &options_map, bool clear_existing_options=true)
 
void updateLeaf (const LeafHostInfo &string_dict_host)
 
void setUncappedTableEpoch (const std::string &table_name)
 
int32_t createCustomExpression (std::unique_ptr< CustomExpression > custom_expression)
 
const CustomExpressiongetCustomExpression (int32_t custom_expression_id) const
 
const std::unique_ptr< const
CustomExpression
getCustomExpressionFromStorage (int32_t custom_expression_id)
 
std::vector< const
CustomExpression * > 
getCustomExpressionsForUser (const UserMetadata &user) const
 
void updateCustomExpression (int32_t custom_expression_id, const std::string &expression_json)
 
void deleteCustomExpressions (const std::vector< int32_t > &custom_expression_ids, bool do_soft_delete)
 

Static Public Member Functions

static void expandGeoColumn (const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
 
static const std::string getForeignTableSchema (bool if_not_exists=false)
 
static const std::string getForeignServerSchema (bool if_not_exists=false)
 
static const std::string getCustomExpressionsSchema (bool if_not_exists=false)
 

Public Attributes

std::mutex sqliteMutex_
 
mapd_shared_mutex sharedMutex_
 
std::atomic< std::thread::id > thread_holding_sqlite_lock
 
std::atomic< std::thread::id > thread_holding_write_lock
 
bool initialized_ = false
 

Static Public Attributes

static thread_local bool thread_holds_read_lock = false
 

Protected Types

using ColumnDescriptorsForRoll = std::vector< std::pair< ColumnDescriptor *, ColumnDescriptor * >>
 

Protected Member Functions

void CheckAndExecuteMigrations ()
 
void CheckAndExecuteMigrationsPostBuildMaps ()
 
void updateDictionaryNames ()
 
void updateTableDescriptorSchema ()
 
void updateFixlenArrayColumns ()
 
void updateGeoColumns ()
 
void updateFrontendViewSchema ()
 
void updateLinkSchema ()
 
void updateFrontendViewAndLinkUsers ()
 
void updateLogicalToPhysicalTableLinkSchema ()
 
void updateLogicalToPhysicalTableMap (const int32_t logical_tb_id)
 
void updateDictionarySchema ()
 
void updatePageSize ()
 
void updateDeletedColumnIndicator ()
 
void updateFrontendViewsToDashboards ()
 
void updateCustomExpressionsSchema ()
 
void updateFsiSchemas ()
 
void recordOwnershipOfObjectsInObjectPermissions ()
 
void checkDateInDaysColumnMigration ()
 
void createDashboardSystemRoles ()
 
void buildMaps ()
 
void addTableToMap (const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
 
void addReferenceToForeignDict (ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def, const bool persist_reference)
 
bool setColumnSharedDictionary (ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
void setColumnDictionary (ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
 
void addFrontendViewToMap (DashboardDescriptor &vd)
 
void addFrontendViewToMapNoLock (DashboardDescriptor &vd)
 
void addLinkToMap (LinkDescriptor &ld)
 
void removeTableFromMap (const std::string &tableName, const int tableId, const bool is_on_error=false)
 
void doDropTable (const TableDescriptor *td)
 
void executeDropTableSqliteQueries (const TableDescriptor *td)
 
void doTruncateTable (const TableDescriptor *td)
 
void renamePhysicalTable (const TableDescriptor *td, const std::string &newTableName)
 
void renamePhysicalTable (std::vector< std::pair< std::string, std::string >> &names, std::vector< int > &tableIds)
 
void instantiateFragmenter (TableDescriptor *td) const
 
void getAllColumnMetadataForTableImpl (const TableDescriptor *td, std::list< const ColumnDescriptor * > &colDescs, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 
std::string calculateSHA1 (const std::string &data)
 
std::string generatePhysicalTableName (const std::string &logicalTableName, const int32_t &shardNumber)
 
std::vector< DBObjectparseDashboardObjects (const std::string &view_meta, const int &user_id)
 
void createOrUpdateDashboardSystemRole (const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
 
const int getColumnIdBySpiUnlocked (const int table_id, const size_t spi) const
 
void serializeTableJsonUnlocked (const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
 
void dropTableFromJsonUnlocked (const std::string &table_name) const
 

Protected Attributes

std::string basePath_
 
TableDescriptorMap tableDescriptorMap_
 
TableDescriptorMapById tableDescriptorMapById_
 
ColumnDescriptorMap columnDescriptorMap_
 
ColumnDescriptorMapById columnDescriptorMapById_
 
DictDescriptorMapById dictDescriptorMapByRef_
 
DashboardDescriptorMap dashboardDescriptorMap_
 
LinkDescriptorMap linkDescriptorMap_
 
LinkDescriptorMapById linkDescriptorMapById_
 
ForeignServerMap foreignServerMap_
 
ForeignServerMapById foreignServerMapById_
 
CustomExpressionMapById custom_expr_map_by_id_
 
SqliteConnector sqliteConnector_
 
const DBMetadata currentDB_
 
std::shared_ptr
< Data_Namespace::DataMgr
dataMgr_
 
const std::vector< LeafHostInfostring_dict_hosts_
 
std::shared_ptr< CalcitecalciteMgr_
 
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
 
int nextTempTableId_
 
int nextTempDictId_
 
ColumnDescriptorsForRoll columnDescriptorsForRoll
 

Static Protected Attributes

static const std::string physicalTableNameTag_
 

Private Member Functions

void gatherAdditionalInfo (std::vector< std::string > &additional_info, std::set< std::string > &shared_dict_column_names, const TableDescriptor *td) const
 
std::string quoteIfRequired (const std::string &column_name) const
 
void adjustAlteredTableFiles (const std::string &temp_data_dir, const std::unordered_map< int, int > &all_column_ids_map) const
 
void renameTableDirectories (const std::string &temp_data_dir, const std::vector< std::string > &target_paths, const std::string &name_prefix) const
 
void buildForeignServerMap ()
 
void addForeignTableDetails ()
 
void setForeignServerProperty (const std::string &server_name, const std::string &property, const std::string &value)
 
void setForeignTableProperty (const foreign_storage::ForeignTable *table, const std::string &property, const std::string &value)
 
void alterPhysicalTableMetadata (const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
 
void alterTableMetadata (const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
 
void setTableFileMgrParams (const int table_id, const File_Namespace::FileMgrParams &file_mgr_params)
 
bool filterTableByTypeAndUser (const TableDescriptor *td, const UserMetadata &user_metadata, const GetTablesType get_tables_type) const
 
TableDescriptorgetMutableMetadataForTableUnlocked (int tableId)
 
void createForeignServerNoLocks (std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
 
foreign_storage::ForeignTablegetForeignTableUnlocked (const std::string &tableName) const
 
const CataloggetObjForLock ()
 
void removeChunksUnlocked (const int table_id) const
 
void buildCustomExpressionsMap ()
 
std::unique_ptr< CustomExpressiongetCustomExpressionFromConnector (size_t row)
 

Private Attributes

DeletedColumnPerTableMap deletedColumnPerTable_
 

Detailed Description

class for a per-database catalog. also includes metadata for the current database and the current user.

Catalog

Definition at line 102 of file Catalog.h.

Member Typedef Documentation

Definition at line 648 of file Catalog.h.

Constructor & Destructor Documentation

Catalog_Namespace::Catalog::Catalog ( const std::string &  basePath,
const DBMetadata curDB,
std::shared_ptr< Data_Namespace::DataMgr dataMgr,
const std::vector< LeafHostInfo > &  string_dict_hosts,
std::shared_ptr< Calcite calcite,
bool  is_new_db 
)

Constructor - takes basePath to already extant data directory for writing.

Parameters
basePathdirectory path for writing catalog
dbNamename of the database
fragmenterFragmenter object metadata - expects for this directory to already exist
Catalog_Namespace::Catalog::Catalog ( )

Constructor builds a hollow catalog used during constructor of other catalogs.

Definition at line 194 of file Catalog.cpp.

194 {}
Catalog_Namespace::Catalog::~Catalog ( )

Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite.

Definition at line 196 of file Catalog.cpp.

References basePath_, columnDescriptorMap_, currentDB_, Catalog_Namespace::DBMetadata::dbName, g_serialize_temp_tables, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and tableDescriptorMap_.

196  {
198  // must clean up heap-allocated TableDescriptor and ColumnDescriptor structs
199  for (TableDescriptorMap::iterator tableDescIt = tableDescriptorMap_.begin();
200  tableDescIt != tableDescriptorMap_.end();
201  ++tableDescIt) {
202  tableDescIt->second->fragmenter = nullptr;
203  delete tableDescIt->second;
204  }
205 
206  // TableDescriptorMapById points to the same descriptors. No need to delete
207 
208  for (ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.begin();
209  columnDescIt != columnDescriptorMap_.end();
210  ++columnDescIt) {
211  delete columnDescIt->second;
212  }
213 
214  // ColumnDescriptorMapById points to the same descriptors. No need to delete
215 
217  boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
218  }
219 }
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:620
const DBMetadata currentDB_
Definition: Catalog.h:631
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:154
bool g_serialize_temp_tables
Definition: Catalog.cpp:98
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:618

+ Here is the call graph for this function:

Member Function Documentation

void Catalog_Namespace::Catalog::addColumn ( const TableDescriptor td,
ColumnDescriptor cd 
)

Definition at line 1909 of file Catalog.cpp.

References addDictionary(), columnDescriptorMap_, columnDescriptorMapById_, columnDescriptorsForRoll, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), SqliteConnector::getData(), getPhysicalTablesDescriptors(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, kENCODING_DICT, TableDescriptor::nShards, SqliteConnector::query_with_text_params(), TableDescriptor::shard, sqliteConnector_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), to_upper(), and ColumnDescriptor::virtualExpr.

1909  {
1910  cat_write_lock write_lock(this);
1911  // caller must handle sqlite/chunk transaction TOGETHER
1912  cd.tableId = td.tableId;
1913  if (td.nShards > 0 && td.shard < 0) {
1914  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
1915  auto shard_cd = cd;
1916  addColumn(*shard, shard_cd);
1917  }
1918  }
1920  addDictionary(cd);
1921  }
1922 
1924  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
1925  "colscale, is_notnull, "
1926  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
1927  "is_deletedcol) "
1928  "VALUES (?, "
1929  "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
1930  "?, ?, ?, "
1931  "?, "
1932  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1933  std::vector<std::string>{std::to_string(td.tableId),
1934  std::to_string(td.tableId),
1935  cd.columnName,
1944  "",
1947  cd.virtualExpr,
1949 
1951  "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
1952  std::vector<std::string>{std::to_string(td.tableId)});
1953 
1955  "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
1956  std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
1957  cd.columnId = sqliteConnector_.getData<int>(0, 0);
1958 
1959  ++tableDescriptorMapById_[td.tableId]->nColumns;
1960  auto ncd = new ColumnDescriptor(cd);
1963  columnDescriptorsForRoll.emplace_back(nullptr, ncd);
1964 }
std::string virtualExpr
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:315
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
HOST DEVICE int get_size() const
Definition: sqltypes.h:324
T getData(const int row, const int col)
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:620
virtual void query_with_text_params(std::string const &query_only)
void addColumn(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:1909
SqliteConnector sqliteConnector_
Definition: Catalog.h:630
HOST DEVICE int get_scale() const
Definition: sqltypes.h:319
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:314
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:621
std::string to_string(char const *&&v)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:619
specifies the content in-memory of a row in the column metadata table
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4063
std::string to_upper(const std::string &str)
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:322
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:316
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:323
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
DictRef addDictionary(ColumnDescriptor &cd)
Definition: Catalog.cpp:1814
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:321
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:649
std::string columnName
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39

+ Here is the call graph for this function:

DictRef Catalog_Namespace::Catalog::addDictionary ( ColumnDescriptor cd)

Definition at line 1814 of file Catalog.cpp.

References CHECK, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, DictDescriptor::dictFolderPath, setColumnDictionary(), string_dict_hosts_, tableDescriptorMapById_, and ColumnDescriptor::tableId.

Referenced by addColumn().

1814  {
1815  const auto& td = *tableDescriptorMapById_[cd.tableId];
1816  list<DictDescriptor> dds;
1817  setColumnDictionary(cd, dds, td, true);
1818  auto& dd = dds.back();
1819  CHECK(dd.dictRef.dictId);
1820 
1821  std::unique_ptr<StringDictionaryClient> client;
1822  if (!string_dict_hosts_.empty()) {
1823  client.reset(new StringDictionaryClient(
1824  string_dict_hosts_.front(), DictRef(currentDB_.dbId, -1), true));
1825  }
1826  if (client) {
1827  client->create(dd.dictRef, dd.dictIsTemp);
1828  }
1829 
1830  DictDescriptor* new_dd = new DictDescriptor(dd);
1831  dictDescriptorMapByRef_[dd.dictRef].reset(new_dd);
1832  if (!dd.dictIsTemp) {
1833  boost::filesystem::create_directory(new_dd->dictFolderPath);
1834  }
1835  return dd.dictRef;
1836 }
std::string dictFolderPath
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:634
const DBMetadata currentDB_
Definition: Catalog.h:631
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:622
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:619
struct dict_ref_t DictRef
Definition: DictRef.h:37
#define CHECK(condition)
Definition: Logger.h:203
Descriptor for a dictionary for a string columne.
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:3278

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addForeignTableDetails ( )
private

Definition at line 4286 of file Catalog.cpp.

References CHECK, foreignServerMapById_, SqliteConnector::getData(), SqliteConnector::getNumRows(), SqliteConnector::query(), test_fsi::r, sqliteConnector_, and tableDescriptorMapById_.

Referenced by buildMaps().

4286  {
4288  "SELECT table_id, server_id, options, last_refresh_time, next_refresh_time from "
4289  "omnisci_foreign_tables");
4290  auto num_rows = sqliteConnector_.getNumRows();
4291  for (size_t r = 0; r < num_rows; r++) {
4292  const auto table_id = sqliteConnector_.getData<int32_t>(r, 0);
4293  const auto server_id = sqliteConnector_.getData<int32_t>(r, 1);
4294  const auto& options = sqliteConnector_.getData<std::string>(r, 2);
4295  const auto last_refresh_time = sqliteConnector_.getData<int>(r, 3);
4296  const auto next_refresh_time = sqliteConnector_.getData<int>(r, 4);
4297 
4298  CHECK(tableDescriptorMapById_.find(table_id) != tableDescriptorMapById_.end());
4299  auto foreign_table =
4300  dynamic_cast<foreign_storage::ForeignTable*>(tableDescriptorMapById_[table_id]);
4301  CHECK(foreign_table);
4302  foreign_table->foreign_server = foreignServerMapById_[server_id].get();
4303  CHECK(foreign_table->foreign_server);
4304  foreign_table->populateOptionsMap(options);
4305  foreign_table->last_refresh_time = last_refresh_time;
4306  foreign_table->next_refresh_time = next_refresh_time;
4307  }
4308 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:630
tuple r
Definition: test_fsi.py:16
virtual void query(const std::string &queryString)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:619
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:627
#define CHECK(condition)
Definition: Logger.h:203
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addFrontendViewToMap ( DashboardDescriptor vd)
protected

Definition at line 1304 of file Catalog.cpp.

References addFrontendViewToMapNoLock().

Referenced by createDashboard().

1304  {
1305  cat_write_lock write_lock(this);
1307 }
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1309
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addFrontendViewToMapNoLock ( DashboardDescriptor vd)
protected

Definition at line 1309 of file Catalog.cpp.

References dashboardDescriptorMap_, DashboardDescriptor::dashboardName, to_string(), and DashboardDescriptor::userId.

Referenced by addFrontendViewToMap(), and replaceDashboard().

1309  {
1310  cat_write_lock write_lock(this);
1312  std::make_shared<DashboardDescriptor>(vd);
1313 }
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:623
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addLinkToMap ( LinkDescriptor ld)
protected

Definition at line 1390 of file Catalog.cpp.

References currentDB_, Catalog_Namespace::DBMetadata::dbId, LinkDescriptor::link, linkDescriptorMap_, linkDescriptorMapById_, LinkDescriptor::linkId, and to_string().

Referenced by createLink().

1390  {
1391  cat_write_lock write_lock(this);
1392  LinkDescriptor* new_ld = new LinkDescriptor();
1393  *new_ld = ld;
1395  linkDescriptorMapById_[ld.linkId] = new_ld;
1396 }
const DBMetadata currentDB_
Definition: Catalog.h:631
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:625
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:624

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addReferenceToForeignDict ( ColumnDescriptor referencing_column,
Parser::SharedDictionaryDef  shared_dict_def,
const bool  persist_reference 
)
protected

Definition at line 3187 of file Catalog.cpp.

References CHECK, CHECK_GE, ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, SQLTypeInfo::get_comp_param(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_foreign_col(), getObjForLock(), SqliteConnector::query_with_text_params(), sqliteConnector_, and to_string().

Referenced by setColumnSharedDictionary().

3189  {
3190  cat_write_lock write_lock(this);
3191  const auto foreign_ref_col = get_foreign_col(*this, shared_dict_def);
3192  CHECK(foreign_ref_col);
3193  referencing_column.columnType = foreign_ref_col->columnType;
3194  const int dict_id = referencing_column.columnType.get_comp_param();
3195  const DictRef dict_ref(currentDB_.dbId, dict_id);
3196  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
3197  CHECK(dictIt != dictDescriptorMapByRef_.end());
3198  const auto& dd = dictIt->second;
3199  CHECK_GE(dd->refcount, 1);
3200  ++dd->refcount;
3201  if (persist_reference) {
3202  cat_sqlite_lock sqlite_lock(getObjForLock());
3204  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
3205  {std::to_string(dict_id)});
3206  }
3207 }
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:630
const DBMetadata currentDB_
Definition: Catalog.h:631
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:622
#define CHECK_GE(x, y)
Definition: Logger.h:216
const ColumnDescriptor * get_foreign_col(const Catalog &cat, const Parser::SharedDictionaryDef &shared_dict_def)
Definition: Catalog.cpp:3175
std::string to_string(char const *&&v)
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:323
#define CHECK(condition)
Definition: Logger.h:203
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addTableToMap ( const TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::list< DictDescriptor > &  dicts 
)
protected

Definition at line 1163 of file Catalog.cpp.

References CHECK, columnDescriptorMap_, columnDescriptorMapById_, ColumnDescriptor::columnId, TableDescriptor::columnIdBySpi_, ColumnDescriptor::columnName, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, DictDescriptor::dictFolderPath, dict_ref_t::dictId, TableDescriptor::hasDeletedCol, TableDescriptor::mutex_, setDeletedColumnUnlocked(), gpu_enabled::sort(), string_dict_hosts_, tableDescriptorMap_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, and to_upper().

Referenced by createTable().

1165  {
1166  cat_write_lock write_lock(this);
1167  TableDescriptor* new_td;
1168 
1169  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
1170  if (foreign_table) {
1171  auto new_foreign_table = new foreign_storage::ForeignTable();
1172  *new_foreign_table = *foreign_table;
1173  new_td = new_foreign_table;
1174  } else {
1175  new_td = new TableDescriptor();
1176  *new_td = *td;
1177  }
1178 
1179  new_td->mutex_ = std::make_shared<std::mutex>();
1180  tableDescriptorMap_[to_upper(td->tableName)] = new_td;
1181  tableDescriptorMapById_[td->tableId] = new_td;
1182  for (auto cd : columns) {
1183  ColumnDescriptor* new_cd = new ColumnDescriptor();
1184  *new_cd = cd;
1185  ColumnKey columnKey(new_cd->tableId, to_upper(new_cd->columnName));
1186  columnDescriptorMap_[columnKey] = new_cd;
1187  ColumnIdKey columnIdKey(new_cd->tableId, new_cd->columnId);
1188  columnDescriptorMapById_[columnIdKey] = new_cd;
1189 
1190  // Add deleted column to the map
1191  if (cd.isDeletedCol) {
1192  CHECK(new_td->hasDeletedCol);
1193  setDeletedColumnUnlocked(new_td, new_cd);
1194  }
1195  }
1196 
1197  std::sort(new_td->columnIdBySpi_.begin(),
1198  new_td->columnIdBySpi_.end(),
1199  [](const size_t a, const size_t b) -> bool { return a < b; });
1200 
1201  std::unique_ptr<StringDictionaryClient> client;
1202  DictRef dict_ref(currentDB_.dbId, -1);
1203  if (!string_dict_hosts_.empty()) {
1204  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1205  }
1206  for (auto dd : dicts) {
1207  if (!dd.dictRef.dictId) {
1208  // Dummy entry created for a shard of a logical table, nothing to do.
1209  continue;
1210  }
1211  dict_ref.dictId = dd.dictRef.dictId;
1212  if (client) {
1213  client->create(dict_ref, dd.dictIsTemp);
1214  }
1215  DictDescriptor* new_dd = new DictDescriptor(dd);
1216  dictDescriptorMapByRef_[dict_ref].reset(new_dd);
1217  if (!dd.dictIsTemp) {
1218  boost::filesystem::create_directory(new_dd->dictFolderPath);
1219  }
1220  }
1221 }
std::string dictFolderPath
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
std::string tableName
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:620
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:634
const DBMetadata currentDB_
Definition: Catalog.h:631
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:622
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
std::vector< int > columnIdBySpi_
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:621
std::shared_ptr< std::mutex > mutex_
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3166
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:619
specifies the content in-memory of a row in the column metadata table
std::string to_upper(const std::string &str)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:203
Descriptor for a dictionary for a string columne.
mapd_unique_lock< mapd_shared_mutex > write_lock
std::string columnName
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:618
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::adjustAlteredTableFiles ( const std::string &  temp_data_dir,
const std::unordered_map< int, int > &  all_column_ids_map 
) const
private
void Catalog_Namespace::Catalog::alterPhysicalTableMetadata ( const TableDescriptor td,
const TableDescriptorUpdateParams table_update_params 
)
private

Definition at line 2891 of file Catalog.cpp.

References CHECK, getMutableMetadataForTableUnlocked(), TableDescriptorUpdateParams::max_rollback_epochs, TableDescriptorUpdateParams::max_rows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, SqliteConnector::query_with_text_params(), sqliteConnector_, TableDescriptor::tableId, and to_string().

Referenced by alterTableMetadata().

2893  {
2894  // Only called from parent alterTableParamMetadata, expect already to have catalog and
2895  // sqlite write locks
2896 
2897  // Sqlite transaction should have already been begun in parent alterTableCatalogMetadata
2898 
2900  CHECK(mutable_td);
2901  if (td->maxRollbackEpochs != table_update_params.max_rollback_epochs) {
2903  "UPDATE mapd_tables SET max_rollback_epochs = ? WHERE tableid = ?",
2904  std::vector<std::string>{std::to_string(table_update_params.max_rollback_epochs),
2905  std::to_string(td->tableId)});
2906  mutable_td->maxRollbackEpochs = table_update_params.max_rollback_epochs;
2907  }
2908 
2909  if (td->maxRows != table_update_params.max_rows) {
2911  "UPDATE mapd_tables SET max_rows = ? WHERE tableid = ?",
2912  std::vector<std::string>{std::to_string(table_update_params.max_rows),
2913  std::to_string(td->tableId)});
2914  mutable_td->maxRows = table_update_params.max_rows;
2915  }
2916 }
int32_t maxRollbackEpochs
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:630
std::string to_string(char const *&&v)
TableDescriptor * getMutableMetadataForTableUnlocked(int tableId)
Definition: Catalog.cpp:1500
#define CHECK(condition)
Definition: Logger.h:203

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::alterTableMetadata ( const TableDescriptor td,
const TableDescriptorUpdateParams table_update_params 
)
private

Definition at line 2918 of file Catalog.cpp.

References alterPhysicalTableMetadata(), CHECK, logger::FATAL, getMetadataForTable(), getObjForLock(), i, LOG, logicalToPhysicalTableMapById_, SqliteConnector::query(), sqliteConnector_, TableDescriptor::tableId, and TableDescriptor::tableName.

Referenced by setMaxRollbackEpochs(), setMaxRows(), and setUncappedTableEpoch().

2919  {
2920  cat_write_lock write_lock(this);
2921  cat_sqlite_lock sqlite_lock(getObjForLock());
2922  sqliteConnector_.query("BEGIN TRANSACTION");
2923  try {
2924  const auto physical_table_it = logicalToPhysicalTableMapById_.find(td->tableId);
2925  if (physical_table_it != logicalToPhysicalTableMapById_.end()) {
2926  const auto physical_tables = physical_table_it->second;
2927  CHECK(!physical_tables.empty());
2928  for (size_t i = 0; i < physical_tables.size(); i++) {
2929  int32_t physical_tb_id = physical_tables[i];
2930  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id, false);
2931  CHECK(phys_td);
2932  alterPhysicalTableMetadata(phys_td, table_update_params);
2933  }
2934  }
2935  alterPhysicalTableMetadata(td, table_update_params);
2936  } catch (std::exception& e) {
2937  sqliteConnector_.query("ROLLBACK TRANSACTION");
2938  LOG(FATAL) << "Table '" << td->tableName << "' catalog update failed";
2939  }
2940  sqliteConnector_.query("END TRANSACTION");
2941 }
std::string tableName
#define LOG(tag)
Definition: Logger.h:194
SqliteConnector sqliteConnector_
Definition: Catalog.h:630
virtual void query(const std::string &queryString)
void alterPhysicalTableMetadata(const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
Definition: Catalog.cpp:2891
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:203
mapd_unique_lock< mapd_shared_mutex > write_lock
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:637

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::buildCustomExpressionsMap ( )
private

Definition at line 1135 of file Catalog.cpp.

References custom_expr_map_by_id_, getCustomExpressionFromConnector(), SqliteConnector::getNumRows(), SqliteConnector::query(), and sqliteConnector_.

Referenced by buildMaps().

1135  {
1137  "SELECT id, name, expression_json, data_source_type, data_source_id, "
1138  "is_deleted "
1139  "FROM omnisci_custom_expressions");
1140  auto num_rows = sqliteConnector_.getNumRows();
1141  for (size_t row = 0; row < num_rows; row++) {
1142  auto custom_expr = getCustomExpressionFromConnector(row);
1143  custom_expr_map_by_id_[custom_expr->id] = std::move(custom_expr);
1144  }
1145 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:630
virtual void query(const std::string &queryString)
std::unique_ptr< CustomExpression > getCustomExpressionFromConnector(size_t row)
Definition: Catalog.cpp:1147
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:628
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::buildForeignServerMap ( )
private

Definition at line 4268 of file Catalog.cpp.

References foreignServerMap_, foreignServerMapById_, SqliteConnector::getData(), SqliteConnector::getNumRows(), SqliteConnector::query(), and sqliteConnector_.

Referenced by buildMaps().

4268  {
4270  "SELECT id, name, data_wrapper_type, options, owner_user_id, creation_time FROM "
4271  "omnisci_foreign_servers");
4272  auto num_rows = sqliteConnector_.getNumRows();
4273  for (size_t row = 0; row < num_rows; row++) {
4274  auto foreign_server = std::make_shared<foreign_storage::ForeignServer>(
4275  sqliteConnector_.getData<int>(row, 0),
4276  sqliteConnector_.getData<std::string>(row, 1),
4277  sqliteConnector_.getData<std::string>(row, 2),
4278  sqliteConnector_.getData<std::string>(row, 3),
4279  sqliteConnector_.getData<std::int32_t>(row, 4),
4280  sqliteConnector_.getData<std::int32_t>(row, 5));
4281  foreignServerMap_[foreign_server->name] = foreign_server;
4282  foreignServerMapById_[foreign_server->id] = foreign_server;
4283  }
4284 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:630
virtual void query(const std::string &queryString)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:627
ForeignServerMap foreignServerMap_
Definition: Catalog.h:626
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::buildMaps ( )
protected

Definition at line 921 of file Catalog.cpp.

References addForeignTableDetails(), buildCustomExpressionsMap(), buildForeignServerMap(), CHECK, ColumnDescriptor::chunks, columnDescriptorMap_, columnDescriptorMapById_, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, createDefaultServersIfNotExists(), currentDB_, dashboardDescriptorMap_, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, dictDescriptorMapByRef_, logger::FATAL, StorageType::FOREIGN_TABLE, TableDescriptor::fragmenter, TableDescriptor::fragments, TableDescriptor::fragPageSize, TableDescriptor::fragType, g_base_path, g_enable_fsi, generate_dashboard_system_rolename(), SQLTypeInfo::get_physical_cols(), SqliteConnector::getData(), SqliteConnector::getNumRows(), getObjForLock(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::getUserFromId(), TableDescriptor::hasDeletedCol, SQLTypeInfo::is_geometry(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, SqliteConnector::isNull(), ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, TableDescriptor::keyMetainfo, LinkDescriptor::link, linkDescriptorMap_, linkDescriptorMapById_, LinkDescriptor::linkId, LOG, logicalToPhysicalTableMapById_, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nColumns, TableDescriptor::nShards, TableDescriptor::partitions, SqliteConnector::query(), test_fsi::r, SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_compression(), SQLTypeInfo::set_dimension(), SQLTypeInfo::set_notnull(), SQLTypeInfo::set_scale(), SQLTypeInfo::set_size(), SQLTypeInfo::set_subtype(), SQLTypeInfo::set_type(), setDeletedColumnUnlocked(), TableDescriptor::shard, TableDescriptor::shardedColumnId, gpu_enabled::sort(), TableDescriptor::sortedColumnId, sqliteConnector_, TableDescriptor::storageType, tableDescriptorMap_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), to_upper(), LinkDescriptor::updateTime, LinkDescriptor::userId, TableDescriptor::userId, LinkDescriptor::viewMetadata, TableDescriptor::viewSQL, LinkDescriptor::viewState, and ColumnDescriptor::virtualExpr.

921  {
923  cat_sqlite_lock sqlite_lock(getObjForLock());
924 
925  string dictQuery(
926  "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
927  sqliteConnector_.query(dictQuery);
928  size_t numRows = sqliteConnector_.getNumRows();
929  for (size_t r = 0; r < numRows; ++r) {
930  int dictId = sqliteConnector_.getData<int>(r, 0);
931  std::string dictName = sqliteConnector_.getData<string>(r, 1);
932  int dictNBits = sqliteConnector_.getData<int>(r, 2);
933  bool is_shared = sqliteConnector_.getData<bool>(r, 3);
934  int refcount = sqliteConnector_.getData<int>(r, 4);
935  std::string fname = g_base_path + "/mapd_data/DB_" + std::to_string(currentDB_.dbId) +
936  "_DICT_" + std::to_string(dictId);
937  DictRef dict_ref(currentDB_.dbId, dictId);
938  DictDescriptor* dd = new DictDescriptor(
939  dict_ref, dictName, dictNBits, is_shared, refcount, fname, false);
940  dictDescriptorMapByRef_[dict_ref].reset(dd);
941  }
942 
943  string tableQuery(
944  "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
945  "max_chunk_size, frag_page_size, "
946  "max_rows, partitions, shard_column_id, shard, num_shards, key_metainfo, userid, "
947  "sort_column_id, storage_type, max_rollback_epochs "
948  "from mapd_tables");
949  sqliteConnector_.query(tableQuery);
950  numRows = sqliteConnector_.getNumRows();
951  for (size_t r = 0; r < numRows; ++r) {
952  TableDescriptor* td;
953  const auto& storage_type = sqliteConnector_.getData<string>(r, 17);
954  if (!storage_type.empty() && storage_type != StorageType::FOREIGN_TABLE) {
955  const auto table_id = sqliteConnector_.getData<int>(r, 0);
956  const auto& table_name = sqliteConnector_.getData<string>(r, 1);
957  LOG(FATAL) << "Unable to read Catalog metadata: storage type is currently not a "
958  "supported table option (table "
959  << table_name << " [" << table_id << "] in database "
960  << currentDB_.dbName << ").";
961  }
962 
963  if (storage_type == StorageType::FOREIGN_TABLE) {
965  } else {
966  td = new TableDescriptor();
967  }
968 
969  td->storageType = storage_type;
970  td->tableId = sqliteConnector_.getData<int>(r, 0);
971  td->tableName = sqliteConnector_.getData<string>(r, 1);
972  td->nColumns = sqliteConnector_.getData<int>(r, 2);
973  td->isView = sqliteConnector_.getData<bool>(r, 3);
974  td->fragments = sqliteConnector_.getData<string>(r, 4);
975  td->fragType =
977  td->maxFragRows = sqliteConnector_.getData<int>(r, 6);
978  td->maxChunkSize = sqliteConnector_.getData<int64_t>(r, 7);
979  td->fragPageSize = sqliteConnector_.getData<int>(r, 8);
980  td->maxRows = sqliteConnector_.getData<int64_t>(r, 9);
981  td->partitions = sqliteConnector_.getData<string>(r, 10);
982  td->shardedColumnId = sqliteConnector_.getData<int>(r, 11);
983  td->shard = sqliteConnector_.getData<int>(r, 12);
984  td->nShards = sqliteConnector_.getData<int>(r, 13);
985  td->keyMetainfo = sqliteConnector_.getData<string>(r, 14);
986  td->userId = sqliteConnector_.getData<int>(r, 15);
987  td->sortedColumnId =
988  sqliteConnector_.isNull(r, 16) ? 0 : sqliteConnector_.getData<int>(r, 16);
989  if (!td->isView) {
990  td->fragmenter = nullptr;
991  }
993  td->hasDeletedCol = false;
994 
997  }
998 
999  if (g_enable_fsi) {
1003  }
1004 
1005  string columnQuery(
1006  "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
1007  "is_notnull, compression, comp_param, "
1008  "size, chunks, is_systemcol, is_virtualcol, virtual_expr, is_deletedcol from "
1009  "mapd_columns ORDER BY tableid, "
1010  "columnid");
1011  sqliteConnector_.query(columnQuery);
1012  numRows = sqliteConnector_.getNumRows();
1013  int32_t skip_physical_cols = 0;
1014  for (size_t r = 0; r < numRows; ++r) {
1015  ColumnDescriptor* cd = new ColumnDescriptor();
1016  cd->tableId = sqliteConnector_.getData<int>(r, 0);
1017  cd->columnId = sqliteConnector_.getData<int>(r, 1);
1018  cd->columnName = sqliteConnector_.getData<string>(r, 2);
1022  cd->columnType.set_scale(sqliteConnector_.getData<int>(r, 6));
1026  cd->columnType.set_size(sqliteConnector_.getData<int>(r, 10));
1027  cd->chunks = sqliteConnector_.getData<string>(r, 11);
1028  cd->isSystemCol = sqliteConnector_.getData<bool>(r, 12);
1029  cd->isVirtualCol = sqliteConnector_.getData<bool>(r, 13);
1030  cd->virtualExpr = sqliteConnector_.getData<string>(r, 14);
1031  cd->isDeletedCol = sqliteConnector_.getData<bool>(r, 15);
1032  cd->isGeoPhyCol = skip_physical_cols > 0;
1033  ColumnKey columnKey(cd->tableId, to_upper(cd->columnName));
1034  columnDescriptorMap_[columnKey] = cd;
1035  ColumnIdKey columnIdKey(cd->tableId, cd->columnId);
1036  columnDescriptorMapById_[columnIdKey] = cd;
1037 
1038  if (skip_physical_cols <= 0) {
1039  skip_physical_cols = cd->columnType.get_physical_cols();
1040  }
1041 
1042  auto td_itr = tableDescriptorMapById_.find(cd->tableId);
1043  CHECK(td_itr != tableDescriptorMapById_.end());
1044 
1045  if (cd->isDeletedCol) {
1046  td_itr->second->hasDeletedCol = true;
1047  setDeletedColumnUnlocked(td_itr->second, cd);
1048  } else if (cd->columnType.is_geometry() || skip_physical_cols-- <= 0) {
1049  tableDescriptorMapById_[cd->tableId]->columnIdBySpi_.push_back(cd->columnId);
1050  }
1051  }
1052  // sort columnIdBySpi_ based on columnId
1053  for (auto& tit : tableDescriptorMapById_) {
1054  std::sort(tit.second->columnIdBySpi_.begin(),
1055  tit.second->columnIdBySpi_.end(),
1056  [](const size_t a, const size_t b) -> bool { return a < b; });
1057  }
1058 
1059  string viewQuery("SELECT tableid, sql FROM mapd_views");
1060  sqliteConnector_.query(viewQuery);
1061  numRows = sqliteConnector_.getNumRows();
1062  for (size_t r = 0; r < numRows; ++r) {
1063  int32_t tableId = sqliteConnector_.getData<int>(r, 0);
1064  TableDescriptor* td = tableDescriptorMapById_[tableId];
1065  td->viewSQL = sqliteConnector_.getData<string>(r, 1);
1066  td->fragmenter = nullptr;
1067  }
1068 
1069  string frontendViewQuery(
1070  "SELECT id, state, name, image_hash, strftime('%Y-%m-%dT%H:%M:%SZ', update_time), "
1071  "userid, "
1072  "metadata "
1073  "FROM mapd_dashboards");
1074  sqliteConnector_.query(frontendViewQuery);
1075  numRows = sqliteConnector_.getNumRows();
1076  for (size_t r = 0; r < numRows; ++r) {
1077  std::shared_ptr<DashboardDescriptor> vd = std::make_shared<DashboardDescriptor>();
1078  vd->dashboardId = sqliteConnector_.getData<int>(r, 0);
1079  vd->dashboardState = sqliteConnector_.getData<string>(r, 1);
1080  vd->dashboardName = sqliteConnector_.getData<string>(r, 2);
1081  vd->imageHash = sqliteConnector_.getData<string>(r, 3);
1082  vd->updateTime = sqliteConnector_.getData<string>(r, 4);
1083  vd->userId = sqliteConnector_.getData<int>(r, 5);
1084  vd->dashboardMetadata = sqliteConnector_.getData<string>(r, 6);
1085  vd->user = getUserFromId(vd->userId);
1086  vd->dashboardSystemRoleName = generate_dashboard_system_rolename(
1088  dashboardDescriptorMap_[std::to_string(vd->userId) + ":" + vd->dashboardName] = vd;
1089  }
1090 
1091  string linkQuery(
1092  "SELECT linkid, userid, link, view_state, strftime('%Y-%m-%dT%H:%M:%SZ', "
1093  "update_time), view_metadata "
1094  "FROM mapd_links");
1095  sqliteConnector_.query(linkQuery);
1096  numRows = sqliteConnector_.getNumRows();
1097  for (size_t r = 0; r < numRows; ++r) {
1098  LinkDescriptor* ld = new LinkDescriptor();
1099  ld->linkId = sqliteConnector_.getData<int>(r, 0);
1100  ld->userId = sqliteConnector_.getData<int>(r, 1);
1101  ld->link = sqliteConnector_.getData<string>(r, 2);
1102  ld->viewState = sqliteConnector_.getData<string>(r, 3);
1103  ld->updateTime = sqliteConnector_.getData<string>(r, 4);
1104  ld->viewMetadata = sqliteConnector_.getData<string>(r, 5);
1106  linkDescriptorMapById_[ld->linkId] = ld;
1107  }
1108 
1109  /* rebuild map linking logical tables to corresponding physical ones */
1110  string logicalToPhysicalTableMapQuery(
1111  "SELECT logical_table_id, physical_table_id "
1112  "FROM mapd_logical_to_physical");
1113  sqliteConnector_.query(logicalToPhysicalTableMapQuery);
1114  numRows = sqliteConnector_.getNumRows();
1115  for (size_t r = 0; r < numRows; ++r) {
1116  int32_t logical_tb_id = sqliteConnector_.getData<int>(r, 0);
1117  int32_t physical_tb_id = sqliteConnector_.getData<int>(r, 1);
1118  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
1119  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
1120  /* add new entity to the map logicalToPhysicalTableMapById_ */
1121  std::vector<int32_t> physicalTables;
1122  physicalTables.push_back(physical_tb_id);
1123  const auto it_ok =
1124  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
1125  CHECK(it_ok.second);
1126  } else {
1127  /* update map logicalToPhysicalTableMapById_ */
1128  physicalTableIt->second.push_back(physical_tb_id);
1129  }
1130  }
1131 
1133 }
int32_t maxRollbackEpochs
std::string virtualExpr
void set_compression(EncodingType c)
Definition: sqltypes.h:414
void set_size(int s)
Definition: sqltypes.h:412
std::string partitions
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
T getData(const int row, const int col)
SQLTypes
Definition: sqltypes.h:37
std::string tableName
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:620
#define LOG(tag)
Definition: Logger.h:194
SqliteConnector sqliteConnector_
Definition: Catalog.h:630
const DBMetadata currentDB_
Definition: Catalog.h:631
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:622
std::string storageType
tuple r
Definition: test_fsi.py:16
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:405
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
virtual void query(const std::string &queryString)
std::string fragments
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:621
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:623
std::string to_string(char const *&&v)
std::string chunks
EncodingType
Definition: sqltypes.h:227
int get_physical_cols() const
Definition: sqltypes.h:335
std::string getUserFromId(const int32_t id)
Definition: Catalog.cpp:911
std::string g_base_path
Definition: SysCatalog.cpp:62
void set_scale(int s)
Definition: sqltypes.h:409
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3166
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:619
specifies the content in-memory of a row in the column metadata table
std::string keyMetainfo
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
std::string to_upper(const std::string &str)
void set_comp_param(int p)
Definition: sqltypes.h:415
bool isNull(const int row, const int col) const
void set_dimension(int d)
Definition: sqltypes.h:406
Fragmenter_Namespace::FragmenterType fragType
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
void set_notnull(bool n)
Definition: sqltypes.h:411
#define CHECK(condition)
Definition: Logger.h:203
bool is_geometry() const
Definition: sqltypes.h:501
Descriptor for a dictionary for a string columne.
std::string viewSQL
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:625
static constexpr char const * FOREIGN_TABLE
bool g_enable_fsi
Definition: Catalog.cpp:92
std::string columnName
virtual size_t getNumRows() const
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:637
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:618
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:624
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)
void createDefaultServersIfNotExists()
Definition: Catalog.cpp:4331
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39
HOST DEVICE void set_type(SQLTypes t)
Definition: sqltypes.h:404

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::calculateSHA1 ( const std::string &  data)
protected

Definition at line 3995 of file Catalog.cpp.

References i.

Referenced by createLink().

3995  {
3996  boost::uuids::detail::sha1 sha1;
3997  unsigned int digest[5];
3998  sha1.process_bytes(data.c_str(), data.length());
3999  sha1.get_digest(digest);
4000  std::stringstream ss;
4001  for (size_t i = 0; i < 5; i++) {
4002  ss << std::hex << digest[i];
4003  }
4004  return ss.str();
4005 }

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::changeForeignServerOwner ( const std::string &  server_name,
const int  new_owner_id 
)

Change the owner of a Foreign Server to a new owner.

Parameters
server_name- Name of the foreign server whose owner to change
new_owner_id- New owner's user id

Definition at line 2606 of file Catalog.cpp.

References CHECK, foreignServerMap_, setForeignServerProperty(), to_string(), and foreign_storage::ForeignServer::user_id.

2607  {
2608  cat_write_lock write_lock(this);
2609  foreign_storage::ForeignServer* foreign_server =
2610  foreignServerMap_.find(server_name)->second.get();
2611  CHECK(foreign_server);
2612  setForeignServerProperty(server_name, "owner_user_id", std::to_string(new_owner_id));
2613  // update in-memory server
2614  foreign_server->user_id = new_owner_id;
2615 }
std::string to_string(char const *&&v)
ForeignServerMap foreignServerMap_
Definition: Catalog.h:626
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:203
mapd_unique_lock< mapd_shared_mutex > write_lock
void setForeignServerProperty(const std::string &server_name, const std::string &property, const std::string &value)
Definition: Catalog.cpp:4310

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::CheckAndExecuteMigrations ( )
protected

Definition at line 885 of file Catalog.cpp.

References g_enable_fsi, recordOwnershipOfObjectsInObjectPermissions(), updateCustomExpressionsSchema(), updateDeletedColumnIndicator(), updateDictionaryNames(), updateDictionarySchema(), updateFixlenArrayColumns(), updateFrontendViewAndLinkUsers(), updateFrontendViewSchema(), updateFrontendViewsToDashboards(), updateFsiSchemas(), updateGeoColumns(), updateLinkSchema(), updateLogicalToPhysicalTableLinkSchema(), updatePageSize(), and updateTableDescriptorSchema().

885  {
895  updatePageSize();
899  if (g_enable_fsi) {
901  }
903 }
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:427
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:675
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:121
void updateCustomExpressionsSchema()
Definition: Catalog.cpp:641
void updateLogicalToPhysicalTableLinkSchema()
Definition: Catalog.cpp:567
bool g_enable_fsi
Definition: Catalog.cpp:92

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::CheckAndExecuteMigrationsPostBuildMaps ( )
protected

Definition at line 905 of file Catalog.cpp.

References checkDateInDaysColumnMigration(), and createDashboardSystemRoles().

905  {
908 }
void checkDateInDaysColumnMigration()
Definition: Catalog.cpp:803

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::checkDateInDaysColumnMigration ( )
protected

Definition at line 803 of file Catalog.cpp.

References getCurrentDB(), getObjForLock(), migrations::MigrationMgr::migrateDateInDaysMetadata(), sqliteConnector_, and tableDescriptorMapById_.

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

803  {
804  cat_sqlite_lock sqlite_lock(getObjForLock());
807 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:630
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:222
static void migrateDateInDaysMetadata(const Catalog_Namespace::TableDescriptorMapById &table_descriptors_by_id, const int database_id, const Catalog_Namespace::Catalog *cat, SqliteConnector &sqlite)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:619
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const bool Catalog_Namespace::Catalog::checkMetadataForDeletedRecs ( const TableDescriptor td,
int  column_id 
) const

Definition at line 3124 of file Catalog.cpp.

References CHECK, and TableDescriptor::fragmenter.

Referenced by getDeletedColumnIfRowsDeleted().

3125  {
3126  // check if there are rows deleted by examining the deletedColumn metadata
3127  CHECK(td);
3128  auto fragmenter = td->fragmenter;
3129  if (fragmenter) {
3130  return fragmenter->hasDeletedRows(delete_column_id);
3131  } else {
3132  return false;
3133  }
3134 }
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
#define CHECK(condition)
Definition: Logger.h:203

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::checkpoint ( const int  logicalTableId) const

Definition at line 4202 of file Catalog.cpp.

References Data_Namespace::DataMgr::checkpoint(), getCurrentDB(), getDataMgr(), getMetadataForTable(), and getPhysicalTablesDescriptors().

Referenced by checkpointWithAutoRollback(), UpdelRoll::commitUpdate(), TableOptimizer::vacuumDeletedRows(), and TableOptimizer::vacuumFragmentsAboveMinSelectivity().

4202  {
4203  const auto td = getMetadataForTable(logicalTableId);
4204  const auto shards = getPhysicalTablesDescriptors(td);
4205  for (const auto shard : shards) {
4206  getDataMgr().checkpoint(getCurrentDB().dbId, shard->tableId);
4207  }
4208 }
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:223
void checkpoint(const int db_id, const int tb_id)
Definition: DataMgr.cpp:489
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:222
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4063
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::checkpointWithAutoRollback ( const int  logical_table_id) const

Definition at line 4210 of file Catalog.cpp.

References checkpoint(), getDatabaseId(), getTableEpochs(), and setTableEpochsLogExceptions().

Referenced by import_export::Loader::checkpoint(), Parser::LocalConnector::checkpoint(), and StorageIOFacility::TransactionParameters::finalizeTransaction().

4210  {
4211  auto table_epochs = getTableEpochs(getDatabaseId(), logical_table_id);
4212  try {
4213  checkpoint(logical_table_id);
4214  } catch (...) {
4215  setTableEpochsLogExceptions(getDatabaseId(), table_epochs);
4216  throw;
4217  }
4218 }
int getDatabaseId() const
Definition: Catalog.h:277
void checkpoint(const int logicalTableId) const
Definition: Catalog.cpp:4202
void setTableEpochsLogExceptions(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3106
std::vector< TableEpochInfo > getTableEpochs(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:3041

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

int32_t Catalog_Namespace::Catalog::createCustomExpression ( std::unique_ptr< CustomExpression custom_expression)

Creates a new custom expression.

Parameters
custom_expression- unique pointer to struct containing custom expression details.
Returns
id of created custom expression

Definition at line 4869 of file Catalog.cpp.

References CHECK, CHECK_EQ, CHECK_GT, custom_expr_map_by_id_, Catalog_Namespace::CustomExpression::dataSourceTypeToString(), SqliteConnector::getData(), SqliteConnector::getNumRows(), getObjForLock(), SqliteConnector::query(), SqliteConnector::query_with_text_params(), sqliteConnector_, and to_string().

4870  {
4871  cat_write_lock write_lock(this);
4872  cat_sqlite_lock sqlite_lock(getObjForLock());
4873  sqliteConnector_.query("BEGIN TRANSACTION");
4874  int32_t custom_expression_id{-1};
4875  try {
4876  auto data_source_type_str =
4877  CustomExpression::dataSourceTypeToString(custom_expression->data_source_type);
4878  auto data_source_id_str = std::to_string(custom_expression->data_source_id);
4879  std::string custom_expr_select_query{
4880  "SELECT id FROM omnisci_custom_expressions WHERE name = ? and data_source_type = "
4881  "? and data_source_id = ? and is_deleted = ?"};
4882  std::vector<std::string> custom_expr_select_params{custom_expression->name,
4883  data_source_type_str,
4884  data_source_id_str,
4885  std::to_string(false)};
4886  sqliteConnector_.query_with_text_params(custom_expr_select_query,
4887  custom_expr_select_params);
4888  if (sqliteConnector_.getNumRows() > 0) {
4889  throw std::runtime_error{
4890  "A custom expression with the given "
4891  "name and data source already exists."};
4892  }
4894  "INSERT INTO omnisci_custom_expressions(name, expression_json, "
4895  "data_source_type, data_source_id, is_deleted) VALUES (?,?,?,?,?)",
4896  std::vector<std::string>{custom_expression->name,
4897  custom_expression->expression_json,
4898  data_source_type_str,
4899  data_source_id_str,
4900  std::to_string(false)});
4901  sqliteConnector_.query_with_text_params(custom_expr_select_query,
4902  custom_expr_select_params);
4903  CHECK_EQ(sqliteConnector_.getNumRows(), static_cast<size_t>(1));
4904  custom_expression->id = sqliteConnector_.getData<int32_t>(0, 0);
4905  custom_expression_id = custom_expression->id;
4906  CHECK(custom_expr_map_by_id_.find(custom_expression->id) ==
4907  custom_expr_map_by_id_.end());
4908  custom_expr_map_by_id_[custom_expression->id] = std::move(custom_expression);
4909  } catch (std::exception& e) {
4910  sqliteConnector_.query("ROLLBACK TRANSACTION");
4911  throw;
4912  }
4913  sqliteConnector_.query("END TRANSACTION");
4914  CHECK_GT(custom_expression_id, 0);
4915  return custom_expression_id;
4916 }
#define CHECK_EQ(x, y)
Definition: Logger.h:211
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:630
virtual void query(const std::string &queryString)
#define CHECK_GT(x, y)
Definition: Logger.h:215
std::string to_string(char const *&&v)
static std::string dataSourceTypeToString(DataSourceType type_enum)
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:203
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:628
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
virtual size_t getNumRows() const

+ Here is the call graph for this function:

int32_t Catalog_Namespace::Catalog::createDashboard ( DashboardDescriptor vd)

Definition at line 3858 of file Catalog.cpp.

References addFrontendViewToMap(), createOrUpdateDashboardSystemRole(), currentDB_, DashboardDescriptor::dashboardId, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, DashboardDescriptor::dashboardSystemRoleName, Catalog_Namespace::DBMetadata::dbId, generate_dashboard_system_rolename(), SqliteConnector::getData(), SqliteConnector::getNumRows(), getObjForLock(), DashboardDescriptor::imageHash, SqliteConnector::query(), SqliteConnector::query_with_text_params(), sqliteConnector_, to_string(), Catalog_Namespace::sqlite_lock< T >::unlock(), Catalog_Namespace::write_lock< T >::unlock(), DashboardDescriptor::updateTime, and DashboardDescriptor::userId.

3858  {
3859  cat_write_lock write_lock(this);
3860  cat_sqlite_lock sqlite_lock(getObjForLock());
3861  sqliteConnector_.query("BEGIN TRANSACTION");
3862  try {
3863  // TODO(andrew): this should be an upsert
3865  "SELECT id FROM mapd_dashboards WHERE name = ? and userid = ?",
3866  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
3867  if (sqliteConnector_.getNumRows() > 0) {
3869  "UPDATE mapd_dashboards SET state = ?, image_hash = ?, metadata = ?, "
3870  "update_time = "
3871  "datetime('now') where name = ? "
3872  "and userid = ?",
3873  std::vector<std::string>{vd.dashboardState,
3874  vd.imageHash,
3875  vd.dashboardMetadata,
3876  vd.dashboardName,
3877  std::to_string(vd.userId)});
3878  } else {
3880  "INSERT INTO mapd_dashboards (name, state, image_hash, metadata, "
3881  "update_time, "
3882  "userid) "
3883  "VALUES "
3884  "(?,?,?,?, "
3885  "datetime('now'), ?)",
3886  std::vector<std::string>{vd.dashboardName,
3887  vd.dashboardState,
3888  vd.imageHash,
3889  vd.dashboardMetadata,
3890  std::to_string(vd.userId)});
3891  }
3892  } catch (std::exception& e) {
3893  sqliteConnector_.query("ROLLBACK TRANSACTION");
3894  throw;
3895  }
3896  sqliteConnector_.query("END TRANSACTION");
3897 
3898  // now get the auto generated dashboardId
3899  try {
3901  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_dashboards "
3902  "WHERE name = ? and userid = ?",
3903  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
3904  vd.dashboardId = sqliteConnector_.getData<int>(0, 0);
3905  vd.updateTime = sqliteConnector_.getData<std::string>(0, 1);
3906  } catch (std::exception& e) {
3907  throw;
3908  }
3912  sqlite_lock.unlock();
3913  write_lock.unlock();
3914  // NOTE(wamsi): Transactionally unsafe
3917  return vd.dashboardId;
3918 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:630
const DBMetadata currentDB_
Definition: Catalog.h:631
virtual void query(const std::string &queryString)
void addFrontendViewToMap(DashboardDescriptor &vd)
Definition: Catalog.cpp:1304
std::string to_string(char const *&&v)
std::string dashboardSystemRoleName
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1345
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createDashboardSystemRoles ( )
protected

Definition at line 809 of file Catalog.cpp.

References createOrUpdateDashboardSystemRole(), currentDB_, Catalog_Namespace::DBMetadata::dbId, logger::ERROR, generate_dashboard_system_rolename(), SqliteConnector::getData(), Catalog_Namespace::SysCatalog::getGranteesOfSharedDashboards(), SqliteConnector::getNumRows(), getObjForLock(), Catalog_Namespace::SysCatalog::getRoleGrantee(), Catalog_Namespace::SysCatalog::grantRoleBatch(), i, logger::INFO, Catalog_Namespace::SysCatalog::instance(), LOG, MAPD_VERSION, SqliteConnector::query(), SqliteConnector::query_with_text_params(), run_benchmark_import::result, sqliteConnector_, and to_string().

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

809  {
810  std::unordered_map<std::string, std::pair<int, std::string>> dashboards;
811  std::vector<std::string> dashboard_ids;
812  static const std::string migration_name{"dashboard_roles_migration"};
813  {
814  cat_sqlite_lock sqlite_lock(getObjForLock());
815  sqliteConnector_.query("BEGIN TRANSACTION");
816  try {
817  // migration_history should be present in all catalogs by now
818  // if not then would be created before this migration
820  "select * from mapd_version_history where migration_history = '" +
821  migration_name + "'");
822  if (sqliteConnector_.getNumRows() != 0) {
823  // no need for further execution
824  sqliteConnector_.query("END TRANSACTION");
825  return;
826  }
827  LOG(INFO) << "Performing dashboard internal roles Migration.";
828  sqliteConnector_.query("select id, userid, metadata from mapd_dashboards");
829  for (size_t i = 0; i < sqliteConnector_.getNumRows(); ++i) {
832  sqliteConnector_.getData<string>(i, 0)))) {
833  // Successfully created roles during previous migration/crash
834  // No need to include them
835  continue;
836  }
837  dashboards[sqliteConnector_.getData<string>(i, 0)] = std::make_pair(
838  sqliteConnector_.getData<int>(i, 1), sqliteConnector_.getData<string>(i, 2));
839  dashboard_ids.push_back(sqliteConnector_.getData<string>(i, 0));
840  }
841  } catch (const std::exception& e) {
842  sqliteConnector_.query("ROLLBACK TRANSACTION");
843  throw;
844  }
845  sqliteConnector_.query("END TRANSACTION");
846  }
847  // All current grantees with shared dashboards.
848  const auto active_grantees =
850 
851  try {
852  // NOTE(wamsi): Transactionally unsafe
853  for (auto dash : dashboards) {
854  createOrUpdateDashboardSystemRole(dash.second.second,
855  dash.second.first,
857  std::to_string(currentDB_.dbId), dash.first));
858  auto result = active_grantees.find(dash.first);
859  if (result != active_grantees.end()) {
862  dash.first)},
863  result->second);
864  }
865  }
866  cat_sqlite_lock sqlite_lock(getObjForLock());
867  // check if this has already been completed
869  "select * from mapd_version_history where migration_history = '" +
870  migration_name + "'");
871  if (sqliteConnector_.getNumRows() != 0) {
872  return;
873  }
875  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
876  std::vector<std::string>{std::to_string(MAPD_VERSION), migration_name});
877  } catch (const std::exception& e) {
878  LOG(ERROR) << "Failed to create dashboard system roles during migration: "
879  << e.what();
880  throw;
881  }
882  LOG(INFO) << "Successfully created dashboard system roles during migration.";
883 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
#define LOG(tag)
Definition: Logger.h:194
SqliteConnector sqliteConnector_
Definition: Catalog.h:630
const DBMetadata currentDB_
Definition: Catalog.h:631
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
void grantRoleBatch(const std::vector< std::string > &roles, const std::vector< std::string > &grantees)
static SysCatalog & instance()
Definition: SysCatalog.h:292
static const int32_t MAPD_VERSION
Definition: release.h:33
Role * getRoleGrantee(const std::string &name) const
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1345
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
std::unordered_map< std::string, std::vector< std::string > > getGranteesOfSharedDashboards(const std::vector< std::string > &dashboard_ids)
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createDefaultServersIfNotExists ( )

Creates default local file servers (if they don't already exist).

Definition at line 4331 of file Catalog.cpp.

References createForeignServerNoLocks(), foreign_storage::DataWrapperType::CSV, foreign_storage::AbstractFileStorageDataWrapper::LOCAL_FILE_STORAGE_TYPE, OMNISCI_ROOT_USER_ID, foreign_storage::DataWrapperType::PARQUET, and foreign_storage::AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY.

Referenced by buildMaps().

4331  {
4335 
4336  auto local_csv_server = std::make_unique<foreign_storage::ForeignServer>(
4337  "omnisci_local_csv",
4339  options,
4341  local_csv_server->validate();
4342  createForeignServerNoLocks(std::move(local_csv_server), true);
4343 
4344  auto local_parquet_server = std::make_unique<foreign_storage::ForeignServer>(
4345  "omnisci_local_parquet",
4347  options,
4349  local_parquet_server->validate();
4350  createForeignServerNoLocks(std::move(local_parquet_server), true);
4351 }
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2517
const int OMNISCI_ROOT_USER_ID
Definition: SysCatalog.h:61
static constexpr char const * CSV
std::map< std::string, std::string, std::less<>> OptionsMap
static constexpr char const * PARQUET

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createForeignServer ( std::unique_ptr< foreign_storage::ForeignServer foreign_server,
bool  if_not_exists 
)

Creates a new foreign server DB object.

Parameters
foreign_server- unique pointer to struct containing foreign server details
if_not_exists- flag indicating whether or not an attempt to create a new foreign server should occur if a server with the same name already exists. An exception is thrown if this flag is set to "false" and an attempt is made to create a pre-existing foreign server

Definition at line 2509 of file Catalog.cpp.

References createForeignServerNoLocks(), and getObjForLock().

2511  {
2512  cat_write_lock write_lock(this);
2513  cat_sqlite_lock sqlite_lock(getObjForLock());
2514  createForeignServerNoLocks(std::move(foreign_server), if_not_exists);
2515 }
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2517
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createForeignServerNoLocks ( std::unique_ptr< foreign_storage::ForeignServer foreign_server,
bool  if_not_exists 
)
private

Same as createForeignServer() but without acquiring locks. This should only be called from within a function/code block that already acquires appropriate locks.

Definition at line 2517 of file Catalog.cpp.

References CHECK, CHECK_EQ, foreignServerMap_, foreignServerMapById_, SqliteConnector::getData(), SqliteConnector::getNumRows(), SqliteConnector::query_with_text_params(), sqliteConnector_, and to_string().

Referenced by createDefaultServersIfNotExists(), and createForeignServer().

2519  {
2521  "SELECT name from omnisci_foreign_servers where name = ?",
2522  std::vector<std::string>{foreign_server->name});
2523 
2524  if (sqliteConnector_.getNumRows() == 0) {
2525  foreign_server->creation_time = std::time(nullptr);
2527  "INSERT INTO omnisci_foreign_servers (name, data_wrapper_type, owner_user_id, "
2528  "creation_time, "
2529  "options) "
2530  "VALUES (?, ?, ?, ?, ?)",
2531  std::vector<std::string>{foreign_server->name,
2532  foreign_server->data_wrapper_type,
2533  std::to_string(foreign_server->user_id),
2534  std::to_string(foreign_server->creation_time),
2535  foreign_server->getOptionsAsJsonString()});
2537  "SELECT id from omnisci_foreign_servers where name = ?",
2538  std::vector<std::string>{foreign_server->name});
2539  CHECK_EQ(sqliteConnector_.getNumRows(), size_t(1));
2540  foreign_server->id = sqliteConnector_.getData<int32_t>(0, 0);
2541  std::shared_ptr<foreign_storage::ForeignServer> foreign_server_shared =
2542  std::move(foreign_server);
2543  CHECK(foreignServerMap_.find(foreign_server_shared->name) == foreignServerMap_.end())
2544  << "Attempting to insert a foreign server into foreign server map that already "
2545  "exists.";
2546  foreignServerMap_[foreign_server_shared->name] = foreign_server_shared;
2547  foreignServerMapById_[foreign_server_shared->id] = foreign_server_shared;
2548  } else if (!if_not_exists) {
2549  throw std::runtime_error{"A foreign server with name \"" + foreign_server->name +
2550  "\" already exists."};
2551  }
2552 }
#define CHECK_EQ(x, y)
Definition: Logger.h:211
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:630
std::string to_string(char const *&&v)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:627
ForeignServerMap foreignServerMap_
Definition: Catalog.h:626
#define CHECK(condition)
Definition: Logger.h:203
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string Catalog_Namespace::Catalog::createLink ( LinkDescriptor ld,
size_t  min_length 
)

Definition at line 4007 of file Catalog.cpp.

References addLinkToMap(), calculateSHA1(), SqliteConnector::getData(), SqliteConnector::getNumRows(), getObjForLock(), LinkDescriptor::link, LinkDescriptor::linkId, SqliteConnector::query(), SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), sqliteConnector_, to_string(), LinkDescriptor::updateTime, LinkDescriptor::userId, LinkDescriptor::viewMetadata, and LinkDescriptor::viewState.

4007  {
4008  cat_write_lock write_lock(this);
4009  cat_sqlite_lock sqlite_lock(getObjForLock());
4010  sqliteConnector_.query("BEGIN TRANSACTION");
4011  try {
4013  .substr(0, 8);
4015  "SELECT linkid FROM mapd_links WHERE link = ? and userid = ?",
4016  std::vector<std::string>{ld.link, std::to_string(ld.userId)});
4017  if (sqliteConnector_.getNumRows() > 0) {
4019  "UPDATE mapd_links SET update_time = datetime('now') WHERE userid = ? AND "
4020  "link = ?",
4021  std::vector<std::string>{std::to_string(ld.userId), ld.link});
4022  } else {
4024  "INSERT INTO mapd_links (userid, link, view_state, view_metadata, "
4025  "update_time) VALUES (?,?,?,?, datetime('now'))",
4026  std::vector<std::string>{
4027  std::to_string(ld.userId), ld.link, ld.viewState, ld.viewMetadata});
4028  }
4029  // now get the auto generated dashid
4031  "SELECT linkid, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_links "
4032  "WHERE link = ?",
4033  ld.link);
4034  ld.linkId = sqliteConnector_.getData<int>(0, 0);
4035  ld.updateTime = sqliteConnector_.getData<std::string>(0, 1);
4036  } catch (std::exception& e) {
4037  sqliteConnector_.query("ROLLBACK TRANSACTION");
4038  throw;
4039  }
4040  sqliteConnector_.query("END TRANSACTION");
4041  addLinkToMap(ld);
4042  return ld.link;
4043 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:630
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
std::string calculateSHA1(const std::string &data)
Definition: Catalog.cpp:3995
void addLinkToMap(LinkDescriptor &ld)
Definition: Catalog.cpp:1390
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createOrUpdateDashboardSystemRole ( const std::string &  view_meta,
const int32_t &  user_id,
const std::string &  dash_role_name 
)
protected

Definition at line 1345 of file Catalog.cpp.

References Catalog_Namespace::SysCatalog::createRole(), Grantee::findDbObject(), Grantee::getDbObjects(), Catalog_Namespace::SysCatalog::getRoleGrantee(), Catalog_Namespace::SysCatalog::grantDBObjectPrivilegesBatch(), Catalog_Namespace::SysCatalog::instance(), parseDashboardObjects(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivileges(), TableDBObjectType, and ViewDBObjectType.

Referenced by createDashboard(), createDashboardSystemRoles(), and replaceDashboard().

1347  {
1348  auto objects = parseDashboardObjects(view_meta, user_id);
1349  Role* rl = SysCatalog::instance().getRoleGrantee(dash_role_name);
1350  if (!rl) {
1351  // Dashboard role does not exist
1352  // create role and grant privileges
1353  // NOTE(wamsi): Transactionally unsafe
1354  SysCatalog::instance().createRole(dash_role_name, false);
1355  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1356  } else {
1357  // Dashboard system role already exists
1358  // Add/remove privileges on objects
1359  std::set<DBObjectKey> revoke_keys;
1360  auto ex_objects = rl->getDbObjects(true);
1361  for (auto key : *ex_objects | boost::adaptors::map_keys) {
1362  if (key.permissionType != TableDBObjectType &&
1363  key.permissionType != ViewDBObjectType) {
1364  continue;
1365  }
1366  bool found = false;
1367  for (auto obj : objects) {
1368  found = key == obj.getObjectKey() ? true : false;
1369  if (found) {
1370  break;
1371  }
1372  }
1373  if (!found) {
1374  revoke_keys.insert(key);
1375  }
1376  }
1377  for (auto& key : revoke_keys) {
1378  // revoke privs on object since the object is no
1379  // longer used by the dashboard as source
1380  // NOTE(wamsi): Transactionally unsafe
1382  dash_role_name, *rl->findDbObject(key, true), *this);
1383  }
1384  // Update privileges on remaining objects
1385  // NOTE(wamsi): Transactionally unsafe
1386  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1387  }
1388 }
void revokeDBObjectPrivileges(const std::string &grantee, const DBObject &object, const Catalog_Namespace::Catalog &catalog)
Definition: Grantee.h:76
DBObject * findDbObject(const DBObjectKey &objectKey, bool only_direct) const
Definition: Grantee.cpp:72
void createRole(const std::string &roleName, const bool &userPrivateRole=false)
static SysCatalog & instance()
Definition: SysCatalog.h:292
Role * getRoleGrantee(const std::string &name) const
void grantDBObjectPrivilegesBatch(const std::vector< std::string > &grantees, const std::vector< DBObject > &objects, const Catalog_Namespace::Catalog &catalog)
const DBObjectMap * getDbObjects(bool only_direct) const
Definition: Grantee.h:56
std::vector< DBObject > parseDashboardObjects(const std::string &view_meta, const int &user_id)
Definition: Catalog.cpp:1315

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createShardedTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs 
)

Definition at line 3319 of file Catalog.cpp.

References CHECK, createTable(), generatePhysicalTableName(), i, logicalToPhysicalTableMapById_, TableDescriptor::nShards, TableDescriptor::shard, table_is_temporary(), TableDescriptor::tableId, TableDescriptor::tableName, and updateLogicalToPhysicalTableMap().

3322  {
3323  cat_write_lock write_lock(this);
3324 
3325  /* create logical table */
3326  TableDescriptor* tdl = &td;
3327  createTable(*tdl, cols, shared_dict_defs, true); // create logical table
3328  int32_t logical_tb_id = tdl->tableId;
3329  std::string logical_table_name = tdl->tableName;
3330 
3331  /* create physical tables and link them to the logical table */
3332  std::vector<int32_t> physicalTables;
3333  for (int32_t i = 1; i <= td.nShards; i++) {
3334  TableDescriptor* tdp = &td;
3335  tdp->tableName = generatePhysicalTableName(logical_table_name, i);
3336  tdp->shard = i - 1;
3337  createTable(*tdp, cols, shared_dict_defs, false); // create physical table
3338  int32_t physical_tb_id = tdp->tableId;
3339 
3340  /* add physical table to the vector of physical tables */
3341  physicalTables.push_back(physical_tb_id);
3342  }
3343 
3344  if (!physicalTables.empty()) {
3345  /* add logical to physical tables correspondence to the map */
3346  const auto it_ok =
3347  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
3348  CHECK(it_ok.second);
3349  /* update sqlite mapd_logical_to_physical in sqlite database */
3350  if (!table_is_temporary(&td)) {
3351  updateLogicalToPhysicalTableMap(logical_tb_id);
3352  }
3353  }
3354 }
std::string tableName
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:2198
std::string generatePhysicalTableName(const std::string &logicalTableName, const int32_t &shardNumber)
Definition: Catalog.cpp:4261
bool table_is_temporary(const TableDescriptor *const td)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:203
mapd_unique_lock< mapd_shared_mutex > write_lock
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:637
void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id)
Definition: Catalog.cpp:581

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs,
bool  isLogicalTable 
)

Definition at line 2198 of file Catalog.cpp.

References addTableToMap(), calciteMgr_, ColumnDescriptor::columnId, TableDescriptor::columnIdBySpi_, ColumnDescriptor::columnName, ColumnDescriptor::columnType, currentDB_, dataMgr_, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, dict_ref_t::dictId, Data_Namespace::DISK_LEVEL, expandGeoColumn(), StorageType::FOREIGN_TABLE, TableDescriptor::fragPageSize, TableDescriptor::fragType, g_enable_fsi, g_serialize_temp_tables, g_test_against_columnId_gap, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_next_refresh_time(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), getCurrentDB(), SqliteConnector::getData(), getObjForLock(), TableDescriptor::hasDeletedCol, SQLTypeInfo::is_array(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, kBIGINT, kBOOLEAN, kENCODING_DICT, TableDescriptor::keyMetainfo, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nColumns, foreign_storage::ForeignTable::next_refresh_time, nextTempDictId_, nextTempTableId_, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, SqliteConnector::query(), SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), removeTableFromMap(), serializeTableJsonUnlocked(), SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_size(), setColumnDictionary(), setColumnSharedDictionary(), TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, sqliteConnector_, TableDescriptor::storageType, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), TableDescriptor::userId, TableDescriptor::viewSQL, and ColumnDescriptor::virtualExpr.

Referenced by createShardedTable().

2202  {
2203  cat_write_lock write_lock(this);
2204  list<ColumnDescriptor> cds = cols;
2205  list<DictDescriptor> dds;
2206  std::set<std::string> toplevel_column_names;
2207  list<ColumnDescriptor> columns;
2208 
2209  if (!td.storageType.empty() &&
2212  throw std::runtime_error("Only temporary tables can be backed by foreign storage.");
2213  }
2214  dataMgr_->getForeignStorageInterface()->prepareTable(getCurrentDB().dbId, td, cds);
2215  }
2216 
2217  for (auto cd : cds) {
2218  if (cd.columnName == "rowid") {
2219  throw std::runtime_error(
2220  "Cannot create column with name rowid. rowid is a system defined column.");
2221  }
2222  columns.push_back(cd);
2223  toplevel_column_names.insert(cd.columnName);
2224  if (cd.columnType.is_geometry()) {
2225  expandGeoColumn(cd, columns);
2226  }
2227  }
2228  cds.clear();
2229 
2230  ColumnDescriptor cd;
2231  // add row_id column -- Must be last column in the table
2232  cd.columnName = "rowid";
2233  cd.isSystemCol = true;
2234  cd.columnType = SQLTypeInfo(kBIGINT, true);
2235 #ifdef MATERIALIZED_ROWID
2236  cd.isVirtualCol = false;
2237 #else
2238  cd.isVirtualCol = true;
2239  cd.virtualExpr = "MAPD_FRAG_ID * MAPD_ROWS_PER_FRAG + MAPD_FRAG_ROW_ID";
2240 #endif
2241  columns.push_back(cd);
2242  toplevel_column_names.insert(cd.columnName);
2243 
2244  if (td.hasDeletedCol) {
2245  ColumnDescriptor cd_del;
2246  cd_del.columnName = "$deleted$";
2247  cd_del.isSystemCol = true;
2248  cd_del.isVirtualCol = false;
2249  cd_del.columnType = SQLTypeInfo(kBOOLEAN, true);
2250  cd_del.isDeletedCol = true;
2251 
2252  columns.push_back(cd_del);
2253  }
2254 
2255  td.nColumns = columns.size();
2256  cat_sqlite_lock sqlite_lock(getObjForLock());
2257  sqliteConnector_.query("BEGIN TRANSACTION");
2259  try {
2261  R"(INSERT INTO mapd_tables (name, userid, ncolumns, isview, fragments, frag_type, max_frag_rows, max_chunk_size, frag_page_size, max_rows, partitions, shard_column_id, shard, num_shards, sort_column_id, storage_type, max_rollback_epochs, key_metainfo) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?))",
2262  std::vector<std::string>{td.tableName,
2263  std::to_string(td.userId),
2265  std::to_string(td.isView),
2266  "",
2271  std::to_string(td.maxRows),
2272  td.partitions,
2274  std::to_string(td.shard),
2275  std::to_string(td.nShards),
2277  td.storageType,
2279  td.keyMetainfo});
2280 
2281  // now get the auto generated tableid
2283  "SELECT tableid FROM mapd_tables WHERE name = ?", td.tableName);
2284  td.tableId = sqliteConnector_.getData<int>(0, 0);
2285  int colId = 1;
2286  for (auto cd : columns) {
2288  const bool is_foreign_col =
2289  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2290  if (!is_foreign_col) {
2291  setColumnDictionary(cd, dds, td, isLogicalTable);
2292  }
2293  }
2294 
2295  if (toplevel_column_names.count(cd.columnName)) {
2296  // make up colId gap for sanity test (begin with 1 bc much code depends on it!)
2297  if (colId > 1) {
2298  colId += g_test_against_columnId_gap;
2299  }
2300  if (!cd.isGeoPhyCol) {
2301  td.columnIdBySpi_.push_back(colId);
2302  }
2303  }
2304 
2306  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, "
2307  "coldim, colscale, is_notnull, "
2308  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
2309  "virtual_expr, is_deletedcol) "
2310  "VALUES (?, ?, ?, ?, ?, "
2311  "?, "
2312  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2313  std::vector<std::string>{std::to_string(td.tableId),
2314  std::to_string(colId),
2315  cd.columnName,
2324  "",
2327  cd.virtualExpr,
2329  cd.tableId = td.tableId;
2330  cd.columnId = colId++;
2331  cds.push_back(cd);
2332  }
2333  if (td.isView) {
2335  "INSERT INTO mapd_views (tableid, sql) VALUES (?,?)",
2336  std::vector<std::string>{std::to_string(td.tableId), td.viewSQL});
2337  }
2339  auto& foreign_table = dynamic_cast<foreign_storage::ForeignTable&>(td);
2340  foreign_table.next_refresh_time = get_next_refresh_time(foreign_table);
2342  "INSERT INTO omnisci_foreign_tables (table_id, server_id, options, "
2343  "last_refresh_time, next_refresh_time) VALUES (?, ?, ?, ?, ?)",
2344  std::vector<std::string>{std::to_string(foreign_table.tableId),
2345  std::to_string(foreign_table.foreign_server->id),
2346  foreign_table.getOptionsAsJsonString(),
2347  std::to_string(foreign_table.last_refresh_time),
2348  std::to_string(foreign_table.next_refresh_time)});
2349  }
2350  } catch (std::exception& e) {
2351  sqliteConnector_.query("ROLLBACK TRANSACTION");
2352  throw;
2353  }
2354  } else { // Temporary table
2355  td.tableId = nextTempTableId_++;
2356  int colId = 1;
2357  for (auto cd : columns) {
2359  const bool is_foreign_col =
2360  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2361 
2362  if (!is_foreign_col) {
2363  // Create a new temporary dictionary
2364  std::string fileName("");
2365  std::string folderPath("");
2367  nextTempDictId_++;
2368  DictDescriptor dd(dict_ref,
2369  fileName,
2371  false,
2372  1,
2373  folderPath,
2374  true); // Is dictName (2nd argument) used?
2375  dds.push_back(dd);
2376  if (!cd.columnType.is_array()) {
2378  }
2379  cd.columnType.set_comp_param(dict_ref.dictId);
2380  }
2381  }
2382  if (toplevel_column_names.count(cd.columnName)) {
2383  // make up colId gap for sanity test (begin with 1 bc much code depends on it!)
2384  if (colId > 1) {
2385  colId += g_test_against_columnId_gap;
2386  }
2387  if (!cd.isGeoPhyCol) {
2388  td.columnIdBySpi_.push_back(colId);
2389  }
2390  }
2391  cd.tableId = td.tableId;
2392  cd.columnId = colId++;
2393  cds.push_back(cd);
2394  }
2395 
2397  serializeTableJsonUnlocked(&td, cds);
2398  }
2399  }
2400 
2401  try {
2402  addTableToMap(&td, cds, dds);
2403  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
2404  if (!td.storageType.empty() && td.storageType != StorageType::FOREIGN_TABLE) {
2405  dataMgr_->getForeignStorageInterface()->registerTable(this, td, cds);
2406  }
2407  } catch (std::exception& e) {
2408  sqliteConnector_.query("ROLLBACK TRANSACTION");
2409  removeTableFromMap(td.tableName, td.tableId, true);
2410  throw;
2411  }
2412  sqliteConnector_.query("END TRANSACTION");
2413 }
int32_t maxRollbackEpochs
void serializeTableJsonUnlocked(const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
Definition: Catalog.cpp:2415
int64_t get_next_refresh_time(const foreign_storage::ForeignTable &foreign_table)
Definition: Catalog.cpp:2186
std::string virtualExpr
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:315
void set_size(int s)
Definition: sqltypes.h:412
std::string partitions
HOST DEVICE int get_size() const
Definition: sqltypes.h:324
T getData(const int row, const int col)
std::string tableName
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:630
const DBMetadata currentDB_
Definition: Catalog.h:631
HOST DEVICE int get_scale() const
Definition: sqltypes.h:319
std::string storageType
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:632
virtual void query(const std::string &queryString)
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:314
std::vector< int > columnIdBySpi_
std::string to_string(char const *&&v)
void addTableToMap(const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
Definition: Catalog.cpp:1163
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:222
specifies the content in-memory of a row in the column metadata table
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:2055
std::string keyMetainfo
int g_test_against_columnId_gap
Definition: Catalog.cpp:91
bool g_serialize_temp_tables
Definition: Catalog.cpp:98
void set_comp_param(int p)
Definition: sqltypes.h:415
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:635
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:322
Fragmenter_Namespace::FragmenterType fragType
Data_Namespace::MemoryLevel persistenceLevel
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:316
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1223
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:323
bool setColumnSharedDictionary(ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:3209
Descriptor for a dictionary for a string columne.
std::string viewSQL
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:321
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:3278
static constexpr char const * FOREIGN_TABLE
bool g_enable_fsi
Definition: Catalog.cpp:92
std::string columnName
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
bool is_array() const
Definition: sqltypes.h:497

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::delDictionary ( const ColumnDescriptor cd)

Definition at line 1838 of file Catalog.cpp.

References CHECK, CHECK_GT, ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, g_base_path, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SqliteConnector::getData(), getMetadataForTable(), getObjForLock(), SQLTypeInfo::is_string(), SQLTypeInfo::is_string_array(), kENCODING_DICT, SqliteConnector::query_with_text_param(), File_Namespace::renameForDelete(), sqliteConnector_, string_dict_hosts_, ColumnDescriptor::tableId, to_string(), and VLOG.

Referenced by roll().

1838  {
1839  cat_write_lock write_lock(this);
1840  cat_sqlite_lock sqlite_lock(getObjForLock());
1841  if (!(cd.columnType.is_string() || cd.columnType.is_string_array())) {
1842  return;
1843  }
1844  if (!(cd.columnType.get_compression() == kENCODING_DICT)) {
1845  return;
1846  }
1847  const auto dictId = cd.columnType.get_comp_param();
1848  CHECK_GT(dictId, 0);
1849  // decrement and zero check dict ref count
1850  const auto td = getMetadataForTable(cd.tableId);
1851  CHECK(td);
1853  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
1854  std::to_string(dictId));
1856  "SELECT refcount FROM mapd_dictionaries WHERE dictid = ?", std::to_string(dictId));
1857  const auto refcount = sqliteConnector_.getData<int>(0, 0);
1858  VLOG(3) << "Dictionary " << dictId << "from dropped table has reference count "
1859  << refcount;
1860  if (refcount > 0) {
1861  return;
1862  }
1863  const DictRef dictRef(currentDB_.dbId, dictId);
1864  sqliteConnector_.query_with_text_param("DELETE FROM mapd_dictionaries WHERE dictid = ?",
1865  std::to_string(dictId));
1866  File_Namespace::renameForDelete(g_base_path + "/mapd_data/DB_" +
1867  std::to_string(currentDB_.dbId) + "_DICT_" +
1868  std::to_string(dictId));
1869 
1870  std::unique_ptr<StringDictionaryClient> client;
1871  if (!string_dict_hosts_.empty()) {
1872  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dictRef, true));
1873  }
1874  if (client) {
1875  client->drop(dictRef);
1876  }
1877 
1878  dictDescriptorMapByRef_.erase(dictRef);
1879 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:630
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:634
const DBMetadata currentDB_
Definition: Catalog.h:631
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:622
#define CHECK_GT(x, y)
Definition: Logger.h:215
std::string to_string(char const *&&v)
std::string g_base_path
Definition: SysCatalog.cpp:62
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:322
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:323
#define CHECK(condition)
Definition: Logger.h:203
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
bool is_string() const
Definition: sqltypes.h:489
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:218
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
bool is_string_array() const
Definition: sqltypes.h:490
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
#define VLOG(n)
Definition: Logger.h:297

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::deleteCustomExpressions ( const std::vector< int32_t > &  custom_expression_ids,
bool  do_soft_delete 
)

Deletes custom expressions with the given ids.

Parameters
custom_expression_ids- ids of custom expressions to delete
do_soft_delete- flag indicating whether or not to do a soft delete

Definition at line 4994 of file Catalog.cpp.

References CHECK, CHECK_EQ, custom_expr_map_by_id_, SqliteConnector::getNumRows(), getObjForLock(), join(), SqliteConnector::query(), SqliteConnector::query_with_text_params(), sqliteConnector_, and to_string().

4995  {
4996  cat_write_lock write_lock(this);
4997  cat_sqlite_lock sqlite_lock(getObjForLock());
4998 
4999  std::vector<int32_t> invalid_ids;
5000  for (const auto id : custom_expression_ids) {
5001  if (custom_expr_map_by_id_.find(id) == custom_expr_map_by_id_.end()) {
5002  invalid_ids.emplace_back(id);
5003  }
5004  }
5005  if (!invalid_ids.empty()) {
5006  throw std::runtime_error{"Custom expressions with ids: " + join(invalid_ids, ",") +
5007  " do not exist."};
5008  }
5009  sqliteConnector_.query("BEGIN TRANSACTION");
5010  try {
5011  for (const auto id : custom_expression_ids) {
5013  "SELECT id FROM omnisci_custom_expressions WHERE id = ?",
5014  std::vector<std::string>{std::to_string(id)});
5015  CHECK_EQ(sqliteConnector_.getNumRows(), static_cast<size_t>(1));
5016  if (do_soft_delete) {
5018  "UPDATE omnisci_custom_expressions SET is_deleted = ? WHERE id = ?",
5019  std::vector<std::string>{std::to_string(true), std::to_string(id)});
5020  } else {
5022  "DELETE FROM omnisci_custom_expressions WHERE id = ?",
5023  std::vector<std::string>{std::to_string(id)});
5024  }
5025  }
5026 
5027  for (const auto id : custom_expression_ids) {
5028  if (do_soft_delete) {
5029  auto it = custom_expr_map_by_id_.find(id);
5030  CHECK(it != custom_expr_map_by_id_.end());
5031  it->second->is_deleted = true;
5032  } else {
5033  custom_expr_map_by_id_.erase(id);
5034  }
5035  }
5036  } catch (std::exception& e) {
5037  sqliteConnector_.query("ROLLBACK TRANSACTION");
5038  throw;
5039  }
5040  sqliteConnector_.query("END TRANSACTION");
5041 }
#define CHECK_EQ(x, y)
Definition: Logger.h:211
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:630
std::string join(T const &container, std::string const &delim)
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:203
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:628
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::deleteMetadataForDashboards ( const std::vector< int32_t >  ids,
const UserMetadata user 
)

Definition at line 1608 of file Catalog.cpp.

References DashboardDBObjectType, dashboardDescriptorMap_, AccessPrivileges::DELETE_DASHBOARD, getMetadataForDashboard(), getObjForLock(), Catalog_Namespace::SysCatalog::instance(), SqliteConnector::query(), SqliteConnector::query_with_text_params(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivilegesFromAllBatch(), sqliteConnector_, and to_string().

1609  {
1610  std::stringstream invalid_ids, restricted_ids;
1611 
1612  for (int32_t dashboard_id : dashboard_ids) {
1613  if (!getMetadataForDashboard(dashboard_id)) {
1614  invalid_ids << (!invalid_ids.str().empty() ? ", " : "") << dashboard_id;
1615  continue;
1616  }
1617  DBObject object(dashboard_id, DashboardDBObjectType);
1618  object.loadKey(*this);
1619  object.setPrivileges(AccessPrivileges::DELETE_DASHBOARD);
1620  std::vector<DBObject> privs = {object};
1621  if (!SysCatalog::instance().checkPrivileges(user, privs))
1622  restricted_ids << (!restricted_ids.str().empty() ? ", " : "") << dashboard_id;
1623  }
1624 
1625  if (invalid_ids.str().size() > 0 || restricted_ids.str().size() > 0) {
1626  std::stringstream error_message;
1627  error_message << "Delete dashboard(s) failed with error(s):";
1628  if (invalid_ids.str().size() > 0)
1629  error_message << "\nDashboard id: " << invalid_ids.str()
1630  << " - Dashboard id does not exist";
1631  if (restricted_ids.str().size() > 0)
1632  error_message
1633  << "\nDashboard id: " << restricted_ids.str()
1634  << " - User should be either owner of dashboard or super user to delete it";
1635  throw std::runtime_error(error_message.str());
1636  }
1637  std::vector<DBObject> dash_objs;
1638 
1639  for (int32_t dashboard_id : dashboard_ids) {
1640  dash_objs.push_back(DBObject(dashboard_id, DashboardDBObjectType));
1641  }
1642  // BE-5245: Transactionally unsafe (like other combined Catalog/Syscatalog operations)
1644  {
1645  cat_write_lock write_lock(this);
1646  cat_sqlite_lock sqlite_lock(getObjForLock());
1647 
1648  sqliteConnector_.query("BEGIN TRANSACTION");
1649  try {
1650  for (int32_t dashboard_id : dashboard_ids) {
1651  auto dash = getMetadataForDashboard(dashboard_id);
1652  // Dash should still exist if revokeDBObjectPrivileges passed but throw and
1653  // rollback if already deleted
1654  if (!dash) {
1655  throw std::runtime_error(
1656  std::string("Delete dashboard(s) failed with error(s):\nDashboard id: ") +
1657  std::to_string(dashboard_id) + " - Dashboard id does not exist ");
1658  }
1659  std::string user_id = std::to_string(dash->userId);
1660  std::string dash_name = dash->dashboardName;
1661  auto viewDescIt = dashboardDescriptorMap_.find(user_id + ":" + dash_name);
1662  dashboardDescriptorMap_.erase(viewDescIt);
1664  "DELETE FROM mapd_dashboards WHERE name = ? and userid = ?",
1665  std::vector<std::string>{dash_name, user_id});
1666  }
1667  } catch (std::exception& e) {
1668  sqliteConnector_.query("ROLLBACK TRANSACTION");
1669  throw;
1670  }
1671  sqliteConnector_.query("END TRANSACTION");
1672  }
1673 }
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:630
virtual void query(const std::string &queryString)
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:623
std::string to_string(char const *&&v)
static SysCatalog & instance()
Definition: SysCatalog.h:292
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
static const AccessPrivileges DELETE_DASHBOARD
Definition: DBObject.h:175
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
void revokeDBObjectPrivilegesFromAllBatch(std::vector< DBObject > &objects, Catalog *catalog)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::doDropTable ( const TableDescriptor td)
protected

Definition at line 3507 of file Catalog.cpp.

References dropTableFromJsonUnlocked(), eraseTablePhysicalData(), executeDropTableSqliteQueries(), g_serialize_temp_tables, table_is_temporary(), and TableDescriptor::tableName.

Referenced by dropTable().

3507  {
3511  }
3513 }
void executeDropTableSqliteQueries(const TableDescriptor *td)
Definition: Catalog.cpp:3515
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:4232
std::string tableName
void dropTableFromJsonUnlocked(const std::string &table_name) const
Definition: Catalog.cpp:2477
bool g_serialize_temp_tables
Definition: Catalog.cpp:98
bool table_is_temporary(const TableDescriptor *const td)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::doTruncateTable ( const TableDescriptor td)
protected

Definition at line 3374 of file Catalog.cpp.

References CHECK, CHECK_GE, columnDescriptorMapById_, ColumnDescriptor::columnType, Data_Namespace::CPU_LEVEL, currentDB_, dataMgr_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, dict_ref_t::dictId, DictDescriptor::dictIsTemp, DictDescriptor::dictRef, TableDescriptor::fragmenter, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), getMetadataForDict(), Data_Namespace::GPU_LEVEL, Catalog_Namespace::SysCatalog::instance(), kENCODING_DICT, File_Namespace::renameForDelete(), string_dict_hosts_, tableDescriptorMapById_, ColumnDescriptor::tableId, and TableDescriptor::tableId.

Referenced by truncateTable().

3374  {
3375  cat_write_lock write_lock(this);
3376 
3377  const int tableId = td->tableId;
3378  // must destroy fragmenter before deleteChunks is called.
3379  if (td->fragmenter != nullptr) {
3380  auto tableDescIt = tableDescriptorMapById_.find(tableId);
3381  CHECK(tableDescIt != tableDescriptorMapById_.end());
3382  tableDescIt->second->fragmenter = nullptr;
3383  CHECK(td->fragmenter == nullptr);
3384  }
3385  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
3386  // assuming deleteChunksWithPrefix is atomic
3387  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
3388  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
3389 
3390  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
3391 
3392  std::unique_ptr<StringDictionaryClient> client;
3393  if (SysCatalog::instance().isAggregator()) {
3394  CHECK(!string_dict_hosts_.empty());
3395  DictRef dict_ref(currentDB_.dbId, -1);
3396  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
3397  }
3398  // clean up any dictionaries
3399  // delete all column descriptors for the table
3400  for (const auto& columnDescriptor : columnDescriptorMapById_) {
3401  auto cd = columnDescriptor.second;
3402  if (cd->tableId != td->tableId) {
3403  continue;
3404  }
3405  const int dict_id = cd->columnType.get_comp_param();
3406  // Dummy dictionaries created for a shard of a logical table have the id set to zero.
3407  if (cd->columnType.get_compression() == kENCODING_DICT && dict_id) {
3408  const DictRef dict_ref(currentDB_.dbId, dict_id);
3409  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
3410  CHECK(dictIt != dictDescriptorMapByRef_.end());
3411  const auto& dd = dictIt->second;
3412  CHECK_GE(dd->refcount, 1);
3413  // if this is the only table using this dict reset the dict
3414  if (dd->refcount == 1) {
3415  // close the dictionary
3416  dd->stringDict.reset();
3417  File_Namespace::renameForDelete(dd->dictFolderPath);
3418  if (client) {
3419  client->drop(dd->dictRef);
3420  }
3421  if (!dd->dictIsTemp) {
3422  boost::filesystem::create_directory(dd->dictFolderPath);
3423  }
3424  }
3425 
3426  DictDescriptor* new_dd = new DictDescriptor(dd->dictRef,
3427  dd->dictName,
3428  dd->dictNBits,
3429  dd->dictIsShared,
3430  dd->refcount,
3431  dd->dictFolderPath,
3432  dd->dictIsTemp);
3433  dictDescriptorMapByRef_.erase(dictIt);
3434  // now create new Dict -- need to figure out what to do here for temp tables
3435  if (client) {
3436  client->create(new_dd->dictRef, new_dd->dictIsTemp);
3437  }
3438  dictDescriptorMapByRef_[new_dd->dictRef].reset(new_dd);
3440  }
3441  }
3442 }
std::vector< int > ChunkKey
Definition: types.h:37
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:634
const DBMetadata currentDB_
Definition: Catalog.h:631
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:622
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:632
#define CHECK_GE(x, y)
Definition: Logger.h:216
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:621
static SysCatalog & instance()
Definition: SysCatalog.h:292
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:619
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1494
int32_t dictId
Definition: DictRef.h:10
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:203
Descriptor for a dictionary for a string columne.
mapd_unique_lock< mapd_shared_mutex > write_lock
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:218

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::dropColumn ( const TableDescriptor td,
const ColumnDescriptor cd 
)

Definition at line 1966 of file Catalog.cpp.

References CHECK, columnDescriptorMap_, columnDescriptorMapById_, columnDescriptorsForRoll, ColumnDescriptor::columnId, ColumnDescriptor::columnName, getMetadataForColumn(), getObjForLock(), getPhysicalTablesDescriptors(), TableDescriptor::nShards, SqliteConnector::query_with_text_params(), TableDescriptor::shard, sqliteConnector_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), and to_upper().

1966  {
1967  cat_write_lock write_lock(this);
1968  cat_sqlite_lock sqlite_lock(getObjForLock());
1969  // caller must handle sqlite/chunk transaction TOGETHER
1971  "DELETE FROM mapd_columns where tableid = ? and columnid = ?",
1972  std::vector<std::string>{std::to_string(td.tableId), std::to_string(cd.columnId)});
1973 
1975  "UPDATE mapd_tables SET ncolumns = ncolumns - 1 WHERE tableid = ?",
1976  std::vector<std::string>{std::to_string(td.tableId)});
1977 
1978  ColumnDescriptorMap::iterator columnDescIt =
1980  CHECK(columnDescIt != columnDescriptorMap_.end());
1981 
1982  columnDescriptorsForRoll.emplace_back(columnDescIt->second, nullptr);
1983 
1984  columnDescriptorMap_.erase(columnDescIt);
1986  --tableDescriptorMapById_[td.tableId]->nColumns;
1987  // for each shard
1988  if (td.nShards > 0 && td.shard < 0) {
1989  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
1990  const auto shard_cd = getMetadataForColumn(shard->tableId, cd.columnId);
1991  CHECK(shard_cd);
1992  dropColumn(*shard, *shard_cd);
1993  }
1994  }
1995 }
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:620
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:630
void dropColumn(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:1966
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:621
std::string to_string(char const *&&v)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:619
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4063
std::string to_upper(const std::string &str)
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:203
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:649
std::string columnName
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropForeignServer ( const std::string &  server_name)

Drops/deletes a foreign server DB object.

Parameters
server_name- Name of foreign server that will be deleted

Definition at line 2670 of file Catalog.cpp.

References CHECK_EQ, foreignServerMap_, foreignServerMapById_, SqliteConnector::getData(), SqliteConnector::getNumRows(), getObjForLock(), SqliteConnector::query(), SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), sqliteConnector_, and to_string().

2670  {
2671  cat_write_lock write_lock(this);
2672  cat_sqlite_lock sqlite_lock(getObjForLock());
2673 
2675  "SELECT id from omnisci_foreign_servers where name = ?",
2676  std::vector<std::string>{server_name});
2677  auto num_rows = sqliteConnector_.getNumRows();
2678  if (num_rows > 0) {
2679  CHECK_EQ(size_t(1), num_rows);
2680  auto server_id = sqliteConnector_.getData<int32_t>(0, 0);
2682  "SELECT table_id from omnisci_foreign_tables where server_id = ?",
2683  std::to_string(server_id));
2684  if (sqliteConnector_.getNumRows() > 0) {
2685  throw std::runtime_error{"Foreign server \"" + server_name +
2686  "\" is referenced "
2687  "by existing foreign tables and cannot be dropped."};
2688  }
2689  sqliteConnector_.query("BEGIN TRANSACTION");
2690  try {
2692  "DELETE FROM omnisci_foreign_servers WHERE name = ?",
2693  std::vector<std::string>{server_name});
2694  } catch (const std::exception& e) {
2695  sqliteConnector_.query("ROLLBACK TRANSACTION");
2696  throw;
2697  }
2698  sqliteConnector_.query("END TRANSACTION");
2699  foreignServerMap_.erase(server_name);
2700  foreignServerMapById_.erase(server_id);
2701  }
2702 }
#define CHECK_EQ(x, y)
Definition: Logger.h:211
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:630
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:627
ForeignServerMap foreignServerMap_
Definition: Catalog.h:626
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropTable ( const TableDescriptor td)

Definition at line 3474 of file Catalog.cpp.

References CHECK, doDropTable(), getMetadataForTable(), getObjForLock(), i, Catalog_Namespace::SysCatalog::instance(), TableDescriptor::isView, logicalToPhysicalTableMapById_, SqliteConnector::query(), SqliteConnector::query_with_text_param(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivilegesFromAll(), sqliteConnector_, TableDBObjectType, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and ViewDBObjectType.

3474  {
3477  cat_write_lock write_lock(this);
3478  cat_sqlite_lock sqlite_lock(getObjForLock());
3479  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
3480  sqliteConnector_.query("BEGIN TRANSACTION");
3481  try {
3482  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
3483  // remove all corresponding physical tables if this is a logical table
3484  const auto physicalTables = physicalTableIt->second;
3485  CHECK(!physicalTables.empty());
3486  for (size_t i = 0; i < physicalTables.size(); i++) {
3487  int32_t physical_tb_id = physicalTables[i];
3488  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
3489  CHECK(phys_td);
3490  doDropTable(phys_td);
3491  }
3492 
3493  // remove corresponding record from the logicalToPhysicalTableMap in sqlite database
3495  "DELETE FROM mapd_logical_to_physical WHERE logical_table_id = ?",
3496  std::to_string(td->tableId));
3498  }
3499  doDropTable(td);
3500  } catch (std::exception& e) {
3501  sqliteConnector_.query("ROLLBACK TRANSACTION");
3502  throw;
3503  }
3504  sqliteConnector_.query("END TRANSACTION");
3505 }
void doDropTable(const TableDescriptor *td)
Definition: Catalog.cpp:3507
std::string tableName
SqliteConnector sqliteConnector_
Definition: Catalog.h:630
void revokeDBObjectPrivilegesFromAll(DBObject object, Catalog *catalog)
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
static SysCatalog & instance()
Definition: SysCatalog.h:292
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:203
mapd_unique_lock< mapd_shared_mutex > write_lock
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:637

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropTableFromJsonUnlocked ( const std::string &  table_name) const
protected

Definition at line 2477 of file Catalog.cpp.

References basePath_, CHECK, currentDB_, test_fsi::d, Catalog_Namespace::DBMetadata::dbName, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and VLOG.

Referenced by doDropTable().

2477  {
2478  // relies on the catalog write lock
2479  using namespace rapidjson;
2480 
2481  VLOG(1) << "Dropping temporary table " << table_name << " to JSON for Calcite.";
2482 
2483  const auto db_name = currentDB_.dbName;
2484  const auto file_path = table_json_filepath(basePath_, db_name);
2485 
2486  CHECK(boost::filesystem::exists(file_path));
2487  Document d;
2488 
2489  std::ifstream reader(file_path.string());
2490  CHECK(reader.is_open());
2491  IStreamWrapper json_read_wrapper(reader);
2492  d.ParseStream(json_read_wrapper);
2493 
2494  CHECK(d.IsObject());
2495  auto table_name_ref = StringRef(table_name.c_str());
2496  CHECK(d.HasMember(table_name_ref));
2497  CHECK(d.RemoveMember(table_name_ref));
2498 
2499  // Overwrite the existing file
2500  std::ofstream writer(file_path.string(), std::ios::trunc | std::ios::out);
2501  CHECK(writer.is_open());
2502  OStreamWrapper json_wrapper(writer);
2503 
2504  Writer<OStreamWrapper> json_writer(json_wrapper);
2505  d.Accept(json_writer);
2506  writer.close();
2507 }
tuple d
Definition: test_fsi.py:9
const DBMetadata currentDB_
Definition: Catalog.h:631
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:154
#define CHECK(condition)
Definition: Logger.h:203
#define VLOG(n)
Definition: Logger.h:297

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string Catalog_Namespace::Catalog::dumpCreateTable ( const TableDescriptor td,
bool  multiline_formatting = true,
bool  dump_defaults = false 
) const

Definition at line 4533 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, Data_Namespace::CPU_LEVEL, DEFAULT_FRAGMENT_ROWS, DEFAULT_MAX_CHUNK_SIZE, DEFAULT_MAX_ROLLBACK_EPOCHS, DEFAULT_MAX_ROWS, DEFAULT_PAGE_SIZE, TableDescriptor::fragPageSize, g_leaf_count, gatherAdditionalInfo(), getAllColumnMetadataForTable(), getMetadataForColumn(), TableDescriptor::hasDeletedCol, ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, join(), kCHAR, kENCODING_DICT, kENCODING_GEOINT, kTEXT, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, quoteIfRequired(), TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and TableDescriptor::viewSQL.

4535  {
4536  cat_read_lock read_lock(this);
4537 
4538  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
4539  std::ostringstream os;
4540 
4541  if (foreign_table) {
4542  os << "CREATE FOREIGN TABLE " << td->tableName << " (";
4543  } else if (!td->isView) {
4544  os << "CREATE ";
4546  os << "TEMPORARY ";
4547  }
4548  os << "TABLE " + td->tableName + " (";
4549  } else {
4550  os << "CREATE VIEW " + td->tableName + " AS " << td->viewSQL;
4551  return os.str();
4552  }
4553  // scan column defines
4554  std::vector<std::string> additional_info;
4555  std::set<std::string> shared_dict_column_names;
4556 
4557  gatherAdditionalInfo(additional_info, shared_dict_column_names, td);
4558 
4559  // gather column defines
4560  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
4561  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
4562  bool first = true;
4563  for (const auto cd : cds) {
4564  if (!(cd->isSystemCol || cd->isVirtualCol)) {
4565  const auto& ti = cd->columnType;
4566  if (!first) {
4567  os << ",";
4568  if (!multiline_formatting) {
4569  os << " ";
4570  }
4571  } else {
4572  first = false;
4573  }
4574  if (multiline_formatting) {
4575  os << "\n ";
4576  }
4577  // column name
4578  os << quoteIfRequired(cd->columnName);
4579  // CHAR is perculiar... better dump it as TEXT(32) like \d does
4580  if (ti.get_type() == SQLTypes::kCHAR) {
4581  os << " "
4582  << "TEXT";
4583  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
4584  os << " "
4585  << "TEXT[]";
4586  } else {
4587  os << " " << ti.get_type_name();
4588  }
4589  os << (ti.get_notnull() ? " NOT NULL" : "");
4590  if (shared_dict_column_names.find(cd->columnName) ==
4591  shared_dict_column_names.end()) {
4592  // avoids "Exception: Column ... shouldn't specify an encoding, it borrows it
4593  // from the referenced column"
4594  if (ti.is_string() || (ti.is_array() && ti.get_subtype() == kTEXT)) {
4595  auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
4596  if (ti.get_compression() == kENCODING_DICT) {
4597  os << " ENCODING " << ti.get_compression_name() << "(" << (size * 8) << ")";
4598  } else {
4599  os << " ENCODING NONE";
4600  }
4601  } else if (ti.is_date_in_days() ||
4602  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
4603  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
4604  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
4605  } else if (ti.is_geometry()) {
4606  if (ti.get_compression() == kENCODING_GEOINT) {
4607  os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
4608  << ")";
4609  } else {
4610  os << " ENCODING NONE";
4611  }
4612  }
4613  }
4614  }
4615  }
4616  // gather SHARED DICTIONARYs
4617  if (additional_info.size()) {
4618  std::string comma;
4619  if (!multiline_formatting) {
4620  comma = ", ";
4621  } else {
4622  comma = ",\n ";
4623  }
4624  os << comma;
4625  os << boost::algorithm::join(additional_info, comma);
4626  }
4627  os << ")";
4628 
4629  std::vector<std::string> with_options;
4630  if (foreign_table) {
4631  if (multiline_formatting) {
4632  os << "\n";
4633  } else {
4634  os << " ";
4635  }
4636  os << "SERVER " << foreign_table->foreign_server->name;
4637 
4638  // gather WITH options ...
4639  for (const auto& [option, value] : foreign_table->options) {
4640  with_options.emplace_back(option + "='" + value + "'");
4641  }
4642  }
4643 
4644  if (dump_defaults || td->maxFragRows != DEFAULT_FRAGMENT_ROWS) {
4645  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
4646  }
4647  if (!foreign_table && (dump_defaults || td->maxChunkSize != DEFAULT_MAX_CHUNK_SIZE)) {
4648  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
4649  }
4650  if (!foreign_table && (dump_defaults || td->fragPageSize != DEFAULT_PAGE_SIZE)) {
4651  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
4652  }
4653  if (!foreign_table && (dump_defaults || td->maxRows != DEFAULT_MAX_ROWS)) {
4654  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
4655  }
4656  if ((dump_defaults || td->maxRollbackEpochs != DEFAULT_MAX_ROLLBACK_EPOCHS) &&
4657  td->maxRollbackEpochs != -1) {
4658  with_options.push_back("MAX_ROLLBACK_EPOCHS=" +
4660  }
4661  if (!foreign_table && (dump_defaults || !td->hasDeletedCol)) {
4662  with_options.push_back(td->hasDeletedCol ? "VACUUM='DELAYED'" : "VACUUM='IMMEDIATE'");
4663  }
4664  if (!foreign_table && !td->partitions.empty()) {
4665  with_options.push_back("PARTITIONS='" + td->partitions + "'");
4666  }
4667  if (!foreign_table && td->nShards > 0) {
4668  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
4669  CHECK(shard_cd);
4670  with_options.push_back(
4671  "SHARD_COUNT=" +
4672  std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
4673  }
4674  if (!foreign_table && td->sortedColumnId > 0) {
4675  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
4676  CHECK(sort_cd);
4677  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
4678  }
4679 
4680  if (!with_options.empty()) {
4681  if (!multiline_formatting) {
4682  os << " ";
4683  } else {
4684  os << "\n";
4685  }
4686  os << "WITH (" + boost::algorithm::join(with_options, ", ") + ")";
4687  }
4688  os << ";";
4689  return os.str();
4690 }
int32_t maxRollbackEpochs
std::string partitions
std::string tableName
std::string join(T const &container, std::string const &delim)
#define DEFAULT_MAX_CHUNK_SIZE
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:113
std::string to_string(char const *&&v)
std::string quoteIfRequired(const std::string &column_name) const
Definition: Catalog.cpp:4792
#define DEFAULT_MAX_ROWS
void gatherAdditionalInfo(std::vector< std::string > &additional_info, std::set< std::string > &shared_dict_column_names, const TableDescriptor *td) const
Definition: Catalog.cpp:4803
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
#define DEFAULT_MAX_ROLLBACK_EPOCHS
specifies the content in-memory of a row in the table metadata table
#define DEFAULT_PAGE_SIZE
Definition: sqltypes.h:51
#define DEFAULT_FRAGMENT_ROWS
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1771
Data_Namespace::MemoryLevel persistenceLevel
Definition: sqltypes.h:40
mapd_shared_lock< mapd_shared_mutex > read_lock
#define CHECK(condition)
Definition: Logger.h:203
std::string viewSQL
size_t g_leaf_count
Definition: ParserNode.cpp:76

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::dumpSchema ( const TableDescriptor td) const

Definition at line 4410 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, DEFAULT_MAX_ROLLBACK_EPOCHS, dictDescriptorMapByRef_, TableDescriptor::fragPageSize, g_leaf_count, getAllColumnMetadataForTable(), getMetadataForColumn(), TableDescriptor::hasDeletedCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, join(), kCHAR, kENCODING_DICT, kENCODING_GEOINT, kTEXT, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::tableId, and to_string().

Referenced by TableArchiver::dumpTable().

4410  {
4411  cat_read_lock read_lock(this);
4412 
4413  std::ostringstream os;
4414  os << "CREATE TABLE @T (";
4415  // gather column defines
4416  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
4417  std::string comma;
4418  std::vector<std::string> shared_dicts;
4419  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
4420  for (const auto cd : cds) {
4421  if (!(cd->isSystemCol || cd->isVirtualCol)) {
4422  const auto& ti = cd->columnType;
4423  os << comma << cd->columnName;
4424  // CHAR is perculiar... better dump it as TEXT(32) like \d does
4425  if (ti.get_type() == SQLTypes::kCHAR) {
4426  os << " "
4427  << "TEXT";
4428  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
4429  os << " "
4430  << "TEXT[]";
4431  } else {
4432  os << " " << ti.get_type_name();
4433  }
4434  os << (ti.get_notnull() ? " NOT NULL" : "");
4435  if (ti.is_string() || (ti.is_array() && ti.get_subtype() == kTEXT)) {
4436  auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
4437  if (ti.get_compression() == kENCODING_DICT) {
4438  // if foreign reference, get referenced tab.col
4439  const auto dict_id = ti.get_comp_param();
4440  const DictRef dict_ref(currentDB_.dbId, dict_id);
4441  const auto dict_it = dictDescriptorMapByRef_.find(dict_ref);
4442  CHECK(dict_it != dictDescriptorMapByRef_.end());
4443  const auto dict_name = dict_it->second->dictName;
4444  // when migrating a table, any foreign dict ref will be dropped
4445  // and the first cd of a dict will become root of the dict
4446  if (dict_root_cds.end() == dict_root_cds.find(dict_name)) {
4447  dict_root_cds[dict_name] = cd;
4448  os << " ENCODING " << ti.get_compression_name() << "(" << (size * 8) << ")";
4449  } else {
4450  const auto dict_root_cd = dict_root_cds[dict_name];
4451  shared_dicts.push_back("SHARED DICTIONARY (" + cd->columnName +
4452  ") REFERENCES @T(" + dict_root_cd->columnName + ")");
4453  // "... shouldn't specify an encoding, it borrows from the referenced
4454  // column"
4455  }
4456  } else {
4457  os << " ENCODING NONE";
4458  }
4459  } else if (ti.is_date_in_days() ||
4460  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
4461  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
4462  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
4463  } else if (ti.is_geometry()) {
4464  if (ti.get_compression() == kENCODING_GEOINT) {
4465  os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
4466  << ")";
4467  } else {
4468  os << " ENCODING NONE";
4469  }
4470  }
4471  comma = ", ";
4472  }
4473  }
4474  // gather SHARED DICTIONARYs
4475  if (shared_dicts.size()) {
4476  os << ", " << boost::algorithm::join(shared_dicts, ", ");
4477  }
4478  // gather WITH options ...
4479  std::vector<std::string> with_options;
4480  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
4481  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
4482  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
4483  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
4484  with_options.emplace_back(td->hasDeletedCol ? "VACUUM='DELAYED'"
4485  : "VACUUM='IMMEDIATE'");
4486  if (!td->partitions.empty()) {
4487  with_options.push_back("PARTITIONS='" + td->partitions + "'");
4488  }
4489  if (td->nShards > 0) {
4490  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
4491  CHECK(shard_cd);
4492  os << ", SHARD KEY(" << shard_cd->columnName << ")";
4493  with_options.push_back(
4494  "SHARD_COUNT=" +
4495  std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
4496  }
4497  if (td->sortedColumnId > 0) {
4498  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
4499  CHECK(sort_cd);
4500  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
4501  }
4503  td->maxRollbackEpochs != -1) {
4504  with_options.push_back("MAX_ROLLBACK_EPOCHS=" +
4506  }
4507  os << ") WITH (" + boost::algorithm::join(with_options, ", ") + ");";
4508  return os.str();
4509 }
int32_t maxRollbackEpochs
std::string partitions
const DBMetadata currentDB_
Definition: Catalog.h:631
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:622
std::string join(T const &container, std::string const &delim)
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:113
std::string to_string(char const *&&v)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
#define DEFAULT_MAX_ROLLBACK_EPOCHS
specifies the content in-memory of a row in the table metadata table
Definition: sqltypes.h:51
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1771
Definition: sqltypes.h:40
mapd_shared_lock< mapd_shared_mutex > read_lock
#define CHECK(condition)
Definition: Logger.h:203
size_t g_leaf_count
Definition: ParserNode.cpp:76

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::eraseDBData ( )

Definition at line 4220 of file Catalog.cpp.

References basePath_, calciteMgr_, currentDB_, Catalog_Namespace::DBMetadata::dbName, eraseTablePhysicalData(), getAllTableMetadata(), test_readcsv::table, and run_benchmark_import::tables.

4220  {
4221  cat_write_lock write_lock(this);
4222  // Physically erase all tables and dictionaries from disc and memory
4223  const auto tables = getAllTableMetadata();
4224  for (const auto table : tables) {
4226  }
4227  // Physically erase database metadata
4228  boost::filesystem::remove(basePath_ + "/mapd_catalogs/" + currentDB_.dbName);
4229  calciteMgr_->updateMetadata(currentDB_.dbName, "");
4230 }
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:4232
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:1797
const DBMetadata currentDB_
Definition: Catalog.h:631
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:635
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::eraseTablePhysicalData ( const TableDescriptor td)

Definition at line 4232 of file Catalog.cpp.

References calciteMgr_, CHECK, Data_Namespace::CPU_LEVEL, currentDB_, dataMgr_, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, TableDescriptor::fragmenter, Data_Namespace::GPU_LEVEL, INJECT_TIMER, TableDescriptor::isView, removeTableFromMap(), tableDescriptorMapById_, TableDescriptor::tableId, and TableDescriptor::tableName.

Referenced by doDropTable(), and eraseDBData().

4232  {
4233  const int tableId = td->tableId;
4234  // must destroy fragmenter before deleteChunks is called.
4235  if (td->fragmenter != nullptr) {
4236  auto tableDescIt = tableDescriptorMapById_.find(tableId);
4237  CHECK(tableDescIt != tableDescriptorMapById_.end());
4238  {
4239  INJECT_TIMER(deleting_fragmenter);
4240  tableDescIt->second->fragmenter = nullptr;
4241  }
4242  }
4243  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
4244  {
4245  INJECT_TIMER(deleteChunksWithPrefix);
4246  // assuming deleteChunksWithPrefix is atomic
4247  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
4248  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
4249  }
4250  if (!td->isView) {
4251  INJECT_TIMER(Remove_Table);
4252  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
4253  }
4254  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
4255  {
4256  INJECT_TIMER(removeTableFromMap_);
4257  removeTableFromMap(td->tableName, tableId);
4258  }
4259 }
std::vector< int > ChunkKey
Definition: types.h:37
std::string tableName
const DBMetadata currentDB_
Definition: Catalog.h:631
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:632
#define INJECT_TIMER(DESC)
Definition: measure.h:93
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:619
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:635
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1223
#define CHECK(condition)
Definition: Logger.h:203

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::executeDropTableSqliteQueries ( const TableDescriptor td)
protected

Definition at line 3515 of file Catalog.cpp.

References StorageType::FOREIGN_TABLE, SqliteConnector::getData(), SqliteConnector::getNumRows(), TableDescriptor::isView, kENCODING_DICT, SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), test_fsi::r, sqliteConnector_, TableDescriptor::storageType, TableDescriptor::tableId, and to_string().

Referenced by doDropTable().

3515  {
3516  const int tableId = td->tableId;
3517  sqliteConnector_.query_with_text_param("DELETE FROM mapd_tables WHERE tableid = ?",
3518  std::to_string(tableId));
3520  "select comp_param from mapd_columns where compression = ? and tableid = ?",
3521  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
3522  int numRows = sqliteConnector_.getNumRows();
3523  std::vector<int> dict_id_list;
3524  for (int r = 0; r < numRows; ++r) {
3525  dict_id_list.push_back(sqliteConnector_.getData<int>(r, 0));
3526  }
3527  for (auto dict_id : dict_id_list) {
3529  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
3530  std::vector<std::string>{std::to_string(dict_id)});
3531  }
3533  "DELETE FROM mapd_dictionaries WHERE dictid in (select comp_param from "
3534  "mapd_columns where compression = ? "
3535  "and tableid = ?) and refcount = 0",
3536  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
3537  sqliteConnector_.query_with_text_param("DELETE FROM mapd_columns WHERE tableid = ?",
3538  std::to_string(tableId));
3539  if (td->isView) {
3540  sqliteConnector_.query_with_text_param("DELETE FROM mapd_views WHERE tableid = ?",
3541  std::to_string(tableId));
3542  }
3545  "DELETE FROM omnisci_foreign_tables WHERE table_id = ?", std::to_string(tableId));
3546  }
3547 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:630
std::string storageType
tuple r
Definition: test_fsi.py:16
std::string to_string(char const *&&v)
static constexpr char const * FOREIGN_TABLE
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::expandGeoColumn ( const ColumnDescriptor cd,
std::list< ColumnDescriptor > &  columns 
)
static