OmniSciDB  21ac014ffc
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Catalog_Namespace::Catalog Class Referencefinal

class for a per-database catalog. also includes metadata for the current database and the current user. More...

#include <Catalog.h>

+ Collaboration diagram for Catalog_Namespace::Catalog:

Public Member Functions

 Catalog (const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
 Constructor - takes basePath to already extant data directory for writing. More...
 
 Catalog ()
 Constructor builds a hollow catalog used during constructor of other catalogs. More...
 
 ~Catalog ()
 Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite. More...
 
void createTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
 
void createShardedTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
int32_t createDashboard (DashboardDescriptor &vd)
 
void replaceDashboard (DashboardDescriptor &vd)
 
std::string createLink (LinkDescriptor &ld, size_t min_length)
 
void dropTable (const TableDescriptor *td)
 
void truncateTable (const TableDescriptor *td)
 
void renameTable (const TableDescriptor *td, const std::string &newTableName)
 
void renameTable (std::vector< std::pair< std::string, std::string >> &names)
 
void renameColumn (const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
 
void addColumn (const TableDescriptor &td, ColumnDescriptor &cd)
 
void dropColumn (const TableDescriptor &td, const ColumnDescriptor &cd)
 
void removeFragmenterForTable (const int table_id) const
 
const std::map< int, const
ColumnDescriptor * > 
getDictionaryToColumnMapping ()
 
const TableDescriptorgetMetadataForTable (const std::string &tableName, const bool populateFragmenter=true) const
 Returns a pointer to a const TableDescriptor struct matching the provided tableName. More...
 
const TableDescriptorgetMetadataForTableImpl (int tableId, const bool populateFragmenter) const
 
const TableDescriptorgetMetadataForTable (int tableId, bool populateFragmenter=true) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, const std::string &colName) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, int columnId) const
 
const ColumnDescriptorgetMetadataForColumnUnlocked (int tableId, int columnId) const
 
const int getColumnIdBySpi (const int tableId, const size_t spi) const
 
const ColumnDescriptorgetMetadataForColumnBySpi (const int tableId, const size_t spi) const
 
const DashboardDescriptorgetMetadataForDashboard (const std::string &userId, const std::string &dashName) const
 
const DashboardDescriptorgetMetadataForDashboard (const int32_t dashboard_id) const
 
void deleteMetadataForDashboards (const std::vector< int32_t > ids, const UserMetadata &user)
 
const LinkDescriptorgetMetadataForLink (const std::string &link) const
 
const LinkDescriptorgetMetadataForLink (int linkId) const
 
const
foreign_storage::ForeignTable
getForeignTableUnlocked (int tableId) const
 
const
foreign_storage::ForeignTable
getForeignTable (const std::string &tableName) const
 
const
foreign_storage::ForeignTable
getForeignTable (int table_id) const
 
std::list< const
ColumnDescriptor * > 
getAllColumnMetadataForTable (const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular table specified by table id. More...
 
std::list< const
ColumnDescriptor * > 
getAllColumnMetadataForTableUnlocked (const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 
std::list< const
TableDescriptor * > 
getAllTableMetadata () const
 
std::list< const
DashboardDescriptor * > 
getAllDashboardsMetadata () const
 
const DBMetadatagetCurrentDB () const
 
Data_Namespace::DataMgrgetDataMgr () const
 
std::shared_ptr< CalcitegetCalciteMgr () const
 
const std::string & getCatalogBasePath () const
 
const DictDescriptorgetMetadataForDict (int dict_ref, bool loadDict=true) const
 
const DictDescriptorgetMetadataForDictUnlocked (int dict_ref, bool loadDict) const
 
const std::vector< LeafHostInfo > & getStringDictionaryHosts () const
 
const ColumnDescriptorgetShardColumnMetadataForTable (const TableDescriptor *td) const
 
std::vector< const
TableDescriptor * > 
getPhysicalTablesDescriptors (const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
 
std::vector< std::string > getTableNamesForUser (const UserMetadata &user, const GetTablesType get_tables_type) const
 
std::vector< TableMetadatagetTablesMetadataForUser (const UserMetadata &user_metadata, const GetTablesType get_tables_type, const std::string &filter_table_name) const
 
int32_t getTableEpoch (const int32_t db_id, const int32_t table_id) const
 
void setTableEpoch (const int db_id, const int table_id, const int new_epoch)
 
void setMaxRollbackEpochs (const int32_t table_id, const int32_t max_rollback_epochs)
 
void setMaxRows (const int32_t table_id, const int64_t max_rows)
 
std::vector< TableEpochInfogetTableEpochs (const int32_t db_id, const int32_t table_id) const
 
void setTableEpochs (const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
 
void setTableEpochsLogExceptions (const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
 
int getDatabaseId () const
 
SqliteConnectorgetSqliteConnector ()
 
void roll (const bool forward)
 
DictRef addDictionary (ColumnDescriptor &cd)
 
void delDictionary (const ColumnDescriptor &cd)
 
void getDictionary (const ColumnDescriptor &cd, std::map< int, StringDictionary * > &stringDicts)
 
const bool checkMetadataForDeletedRecs (const TableDescriptor *td, int column_id) const
 
const ColumnDescriptorgetDeletedColumn (const TableDescriptor *td) const
 
const ColumnDescriptorgetDeletedColumnIfRowsDeleted (const TableDescriptor *td) const
 
void setDeletedColumn (const TableDescriptor *td, const ColumnDescriptor *cd)
 
void setDeletedColumnUnlocked (const TableDescriptor *td, const ColumnDescriptor *cd)
 
int getLogicalTableId (const int physicalTableId) const
 
void checkpoint (const int logicalTableId) const
 
void checkpointWithAutoRollback (const int logical_table_id) const
 
std::string name () const
 
void eraseDBData ()
 
void eraseTablePhysicalData (const TableDescriptor *td)
 
void setForReload (const int32_t tableId)
 
std::vector< std::string > getTableDataDirectories (const TableDescriptor *td) const
 
std::vector< std::string > getTableDictDirectories (const TableDescriptor *td) const
 
std::string getColumnDictDirectory (const ColumnDescriptor *cd) const
 
std::string dumpSchema (const TableDescriptor *td) const
 
std::string dumpCreateTable (const TableDescriptor *td, bool multiline_formatting=true, bool dump_defaults=false) const
 
void createForeignServer (std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
 
const
foreign_storage::ForeignServer
getForeignServer (const std::string &server_name) const
 
const std::unique_ptr< const
foreign_storage::ForeignServer
getForeignServerFromStorage (const std::string &server_name)
 
const std::unique_ptr< const
foreign_storage::ForeignTable
getForeignTableFromStorage (int table_id)
 
void changeForeignServerOwner (const std::string &server_name, const int new_owner_id)
 
void setForeignServerDataWrapper (const std::string &server_name, const std::string &data_wrapper)
 
void setForeignServerOptions (const std::string &server_name, const std::string &options)
 
void renameForeignServer (const std::string &server_name, const std::string &name)
 
void dropForeignServer (const std::string &server_name)
 
void getForeignServersForUser (const rapidjson::Value *filters, const UserMetadata &user, std::vector< const foreign_storage::ForeignServer * > &results)
 
void createDefaultServersIfNotExists ()
 
bool validateNonExistentTableOrView (const std::string &name, const bool if_not_exists)
 
std::vector< const
TableDescriptor * > 
getAllForeignTablesForRefresh () const
 
void updateForeignTableRefreshTimes (const int32_t table_id)
 
void setForeignTableOptions (const std::string &table_name, foreign_storage::OptionsMap &options_map, bool clear_existing_options=true)
 
void updateLeaf (const LeafHostInfo &string_dict_host)
 
void setUncappedTableEpoch (const std::string &table_name)
 
int32_t createCustomExpression (std::unique_ptr< CustomExpression > custom_expression)
 
const CustomExpressiongetCustomExpression (int32_t custom_expression_id) const
 
const std::unique_ptr< const
CustomExpression
getCustomExpressionFromStorage (int32_t custom_expression_id)
 
std::vector< const
CustomExpression * > 
getCustomExpressionsForUser (const UserMetadata &user) const
 
void updateCustomExpression (int32_t custom_expression_id, const std::string &expression_json)
 
void deleteCustomExpressions (const std::vector< int32_t > &custom_expression_ids, bool do_soft_delete)
 

Static Public Member Functions

static void expandGeoColumn (const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
 
static const std::string getForeignTableSchema (bool if_not_exists=false)
 
static const std::string getForeignServerSchema (bool if_not_exists=false)
 
static const std::string getCustomExpressionsSchema (bool if_not_exists=false)
 

Public Attributes

std::mutex sqliteMutex_
 
mapd_shared_mutex sharedMutex_
 
std::atomic< std::thread::id > thread_holding_sqlite_lock
 
std::atomic< std::thread::id > thread_holding_write_lock
 
bool initialized_ = false
 

Static Public Attributes

static thread_local bool thread_holds_read_lock = false
 

Protected Types

using ColumnDescriptorsForRoll = std::vector< std::pair< ColumnDescriptor *, ColumnDescriptor * >>
 

Protected Member Functions

void CheckAndExecuteMigrations ()
 
void CheckAndExecuteMigrationsPostBuildMaps ()
 
void updateDictionaryNames ()
 
void updateTableDescriptorSchema ()
 
void updateFixlenArrayColumns ()
 
void updateGeoColumns ()
 
void updateFrontendViewSchema ()
 
void updateLinkSchema ()
 
void updateFrontendViewAndLinkUsers ()
 
void updateLogicalToPhysicalTableLinkSchema ()
 
void updateLogicalToPhysicalTableMap (const int32_t logical_tb_id)
 
void updateDictionarySchema ()
 
void updatePageSize ()
 
void updateDeletedColumnIndicator ()
 
void updateDefaultColumnValues ()
 
void updateFrontendViewsToDashboards ()
 
void updateCustomExpressionsSchema ()
 
void updateFsiSchemas ()
 
void recordOwnershipOfObjectsInObjectPermissions ()
 
void checkDateInDaysColumnMigration ()
 
void createDashboardSystemRoles ()
 
void buildMaps ()
 
void addTableToMap (const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
 
void addReferenceToForeignDict (ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def, const bool persist_reference)
 
bool setColumnSharedDictionary (ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
void setColumnDictionary (ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
 
void addFrontendViewToMap (DashboardDescriptor &vd)
 
void addFrontendViewToMapNoLock (DashboardDescriptor &vd)
 
void addLinkToMap (LinkDescriptor &ld)
 
void removeTableFromMap (const std::string &tableName, const int tableId, const bool is_on_error=false)
 
void doDropTable (const TableDescriptor *td)
 
void executeDropTableSqliteQueries (const TableDescriptor *td)
 
void doTruncateTable (const TableDescriptor *td)
 
void renamePhysicalTable (const TableDescriptor *td, const std::string &newTableName)
 
void renamePhysicalTable (std::vector< std::pair< std::string, std::string >> &names, std::vector< int > &tableIds)
 
void instantiateFragmenter (TableDescriptor *td) const
 
void getAllColumnMetadataForTableImpl (const TableDescriptor *td, std::list< const ColumnDescriptor * > &colDescs, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 
std::string calculateSHA1 (const std::string &data)
 
std::string generatePhysicalTableName (const std::string &logicalTableName, const int32_t &shardNumber)
 
std::vector< DBObjectparseDashboardObjects (const std::string &view_meta, const int &user_id)
 
void createOrUpdateDashboardSystemRole (const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
 
const int getColumnIdBySpiUnlocked (const int table_id, const size_t spi) const
 
void serializeTableJsonUnlocked (const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
 
void dropTableFromJsonUnlocked (const std::string &table_name) const
 

Protected Attributes

std::string basePath_
 
TableDescriptorMap tableDescriptorMap_
 
TableDescriptorMapById tableDescriptorMapById_
 
ColumnDescriptorMap columnDescriptorMap_
 
ColumnDescriptorMapById columnDescriptorMapById_
 
DictDescriptorMapById dictDescriptorMapByRef_
 
DashboardDescriptorMap dashboardDescriptorMap_
 
LinkDescriptorMap linkDescriptorMap_
 
LinkDescriptorMapById linkDescriptorMapById_
 
ForeignServerMap foreignServerMap_
 
ForeignServerMapById foreignServerMapById_
 
CustomExpressionMapById custom_expr_map_by_id_
 
SqliteConnector sqliteConnector_
 
const DBMetadata currentDB_
 
std::shared_ptr
< Data_Namespace::DataMgr
dataMgr_
 
const std::vector< LeafHostInfostring_dict_hosts_
 
std::shared_ptr< CalcitecalciteMgr_
 
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
 
int nextTempTableId_
 
int nextTempDictId_
 
ColumnDescriptorsForRoll columnDescriptorsForRoll
 

Static Protected Attributes

static const std::string physicalTableNameTag_
 

Private Member Functions

void gatherAdditionalInfo (std::vector< std::string > &additional_info, std::set< std::string > &shared_dict_column_names, const TableDescriptor *td) const
 
std::string quoteIfRequired (const std::string &column_name) const
 
void adjustAlteredTableFiles (const std::string &temp_data_dir, const std::unordered_map< int, int > &all_column_ids_map) const
 
void renameTableDirectories (const std::string &temp_data_dir, const std::vector< std::string > &target_paths, const std::string &name_prefix) const
 
void buildForeignServerMap ()
 
void addForeignTableDetails ()
 
void setForeignServerProperty (const std::string &server_name, const std::string &property, const std::string &value)
 
void setForeignTableProperty (const foreign_storage::ForeignTable *table, const std::string &property, const std::string &value)
 
void alterPhysicalTableMetadata (const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
 
void alterTableMetadata (const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
 
void setTableFileMgrParams (const int table_id, const File_Namespace::FileMgrParams &file_mgr_params)
 
bool filterTableByTypeAndUser (const TableDescriptor *td, const UserMetadata &user_metadata, const GetTablesType get_tables_type) const
 
TableDescriptorgetMutableMetadataForTableUnlocked (int tableId)
 
void createForeignServerNoLocks (std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
 
foreign_storage::ForeignTablegetForeignTableUnlocked (const std::string &tableName) const
 
const CataloggetObjForLock ()
 
void removeChunksUnlocked (const int table_id) const
 
void buildCustomExpressionsMap ()
 
std::unique_ptr< CustomExpressiongetCustomExpressionFromConnector (size_t row)
 

Private Attributes

DeletedColumnPerTableMap deletedColumnPerTable_
 

Detailed Description

class for a per-database catalog. also includes metadata for the current database and the current user.

Catalog

Definition at line 102 of file Catalog.h.

Member Typedef Documentation

Definition at line 649 of file Catalog.h.

Constructor & Destructor Documentation

Catalog_Namespace::Catalog::Catalog ( const std::string &  basePath,
const DBMetadata curDB,
std::shared_ptr< Data_Namespace::DataMgr dataMgr,
const std::vector< LeafHostInfo > &  string_dict_hosts,
std::shared_ptr< Calcite calcite,
bool  is_new_db 
)

Constructor - takes basePath to already extant data directory for writing.

Parameters
basePathdirectory path for writing catalog
dbNamename of the database
fragmenterFragmenter object metadata - expects for this directory to already exist
Catalog_Namespace::Catalog::Catalog ( )

Constructor builds a hollow catalog used during constructor of other catalogs.

Definition at line 194 of file Catalog.cpp.

194 {}
Catalog_Namespace::Catalog::~Catalog ( )

Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite.

Definition at line 196 of file Catalog.cpp.

References basePath_, columnDescriptorMap_, currentDB_, Catalog_Namespace::DBMetadata::dbName, g_serialize_temp_tables, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and tableDescriptorMap_.

196  {
198  // must clean up heap-allocated TableDescriptor and ColumnDescriptor structs
199  for (TableDescriptorMap::iterator tableDescIt = tableDescriptorMap_.begin();
200  tableDescIt != tableDescriptorMap_.end();
201  ++tableDescIt) {
202  tableDescIt->second->fragmenter = nullptr;
203  delete tableDescIt->second;
204  }
205 
206  // TableDescriptorMapById points to the same descriptors. No need to delete
207 
208  for (ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.begin();
209  columnDescIt != columnDescriptorMap_.end();
210  ++columnDescIt) {
211  delete columnDescIt->second;
212  }
213 
214  // ColumnDescriptorMapById points to the same descriptors. No need to delete
215 
217  boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
218  }
219 }
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:621
const DBMetadata currentDB_
Definition: Catalog.h:632
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:154
bool g_serialize_temp_tables
Definition: Catalog.cpp:98
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:619

+ Here is the call graph for this function:

Member Function Documentation

void Catalog_Namespace::Catalog::addColumn ( const TableDescriptor td,
ColumnDescriptor cd 
)

Definition at line 1937 of file Catalog.cpp.

References addDictionary(), columnDescriptorMap_, columnDescriptorMapById_, columnDescriptorsForRoll, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::default_value, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), SqliteConnector::getData(), getPhysicalTablesDescriptors(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, kENCODING_DICT, TableDescriptor::nShards, SqliteConnector::query_with_text_params(), TableDescriptor::shard, sqliteConnector_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, TEXT, to_string(), to_upper(), and ColumnDescriptor::virtualExpr.

1937  {
1938  cat_write_lock write_lock(this);
1939  // caller must handle sqlite/chunk transaction TOGETHER
1940  cd.tableId = td.tableId;
1941  if (td.nShards > 0 && td.shard < 0) {
1942  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
1943  auto shard_cd = cd;
1944  addColumn(*shard, shard_cd);
1945  }
1946  }
1948  addDictionary(cd);
1949  }
1950 
1951  using BindType = SqliteConnector::BindType;
1952  std::vector<BindType> types(17, BindType::TEXT);
1953  if (!cd.default_value.has_value()) {
1954  types[16] = BindType::NULL_TYPE;
1955  }
1957  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
1958  "colscale, is_notnull, "
1959  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
1960  "is_deletedcol, default_value) "
1961  "VALUES (?, "
1962  "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
1963  "?, ?, ?, "
1964  "?, "
1965  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1966  std::vector<std::string>{std::to_string(td.tableId),
1967  std::to_string(td.tableId),
1968  cd.columnName,
1977  "",
1980  cd.virtualExpr,
1982  cd.default_value.value_or("NULL")},
1983  types);
1984 
1986  "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
1987  std::vector<std::string>{std::to_string(td.tableId)});
1988 
1990  "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
1991  std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
1992  cd.columnId = sqliteConnector_.getData<int>(0, 0);
1993 
1994  ++tableDescriptorMapById_[td.tableId]->nColumns;
1995  auto ncd = new ColumnDescriptor(cd);
1998  columnDescriptorsForRoll.emplace_back(nullptr, ncd);
1999 }
std::string virtualExpr
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:324
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
HOST DEVICE int get_size() const
Definition: sqltypes.h:333
T getData(const int row, const int col)
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:621
virtual void query_with_text_params(std::string const &query_only)
void addColumn(const TableDescriptor &td, ColumnDescriptor &cd)
Definition: Catalog.cpp:1937
SqliteConnector sqliteConnector_
Definition: Catalog.h:631
HOST DEVICE int get_scale() const
Definition: sqltypes.h:328
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:323
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:622
std::string to_string(char const *&&v)
#define TEXT
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:620
specifies the content in-memory of a row in the column metadata table
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4119
std::string to_upper(const std::string &str)
std::optional< std::string > default_value
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:331
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:325
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:332
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
DictRef addDictionary(ColumnDescriptor &cd)
Definition: Catalog.cpp:1842
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:330
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:650
std::string columnName
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39

+ Here is the call graph for this function:

DictRef Catalog_Namespace::Catalog::addDictionary ( ColumnDescriptor cd)

Definition at line 1842 of file Catalog.cpp.

References CHECK, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, DictDescriptor::dictFolderPath, setColumnDictionary(), string_dict_hosts_, tableDescriptorMapById_, and ColumnDescriptor::tableId.

Referenced by addColumn().

1842  {
1843  const auto& td = *tableDescriptorMapById_[cd.tableId];
1844  list<DictDescriptor> dds;
1845  setColumnDictionary(cd, dds, td, true);
1846  auto& dd = dds.back();
1847  CHECK(dd.dictRef.dictId);
1848 
1849  std::unique_ptr<StringDictionaryClient> client;
1850  if (!string_dict_hosts_.empty()) {
1851  client.reset(new StringDictionaryClient(
1852  string_dict_hosts_.front(), DictRef(currentDB_.dbId, -1), true));
1853  }
1854  if (client) {
1855  client->create(dd.dictRef, dd.dictIsTemp);
1856  }
1857 
1858  DictDescriptor* new_dd = new DictDescriptor(dd);
1859  dictDescriptorMapByRef_[dd.dictRef].reset(new_dd);
1860  if (!dd.dictIsTemp) {
1861  boost::filesystem::create_directory(new_dd->dictFolderPath);
1862  }
1863  return dd.dictRef;
1864 }
std::string dictFolderPath
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:635
const DBMetadata currentDB_
Definition: Catalog.h:632
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:623
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:620
struct dict_ref_t DictRef
Definition: DictRef.h:37
#define CHECK(condition)
Definition: Logger.h:206
Descriptor for a dictionary for a string columne.
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:3334

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addForeignTableDetails ( )
private

Definition at line 4342 of file Catalog.cpp.

References CHECK, and test_fsi::r.

Referenced by buildMaps().

4342  {
4344  "SELECT table_id, server_id, options, last_refresh_time, next_refresh_time from "
4345  "omnisci_foreign_tables");
4346  auto num_rows = sqliteConnector_.getNumRows();
4347  for (size_t r = 0; r < num_rows; r++) {
4348  const auto table_id = sqliteConnector_.getData<int32_t>(r, 0);
4349  const auto server_id = sqliteConnector_.getData<int32_t>(r, 1);
4350  const auto& options = sqliteConnector_.getData<std::string>(r, 2);
4351  const auto last_refresh_time = sqliteConnector_.getData<int>(r, 3);
4352  const auto next_refresh_time = sqliteConnector_.getData<int>(r, 4);
4353 
4354  CHECK(tableDescriptorMapById_.find(table_id) != tableDescriptorMapById_.end());
4355  auto foreign_table =
4356  dynamic_cast<foreign_storage::ForeignTable*>(tableDescriptorMapById_[table_id]);
4357  CHECK(foreign_table);
4358  foreign_table->foreign_server = foreignServerMapById_[server_id].get();
4359  CHECK(foreign_table->foreign_server);
4360  foreign_table->populateOptionsMap(options);
4361  foreign_table->last_refresh_time = last_refresh_time;
4362  foreign_table->next_refresh_time = next_refresh_time;
4363  }
4364 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:631
tuple r
Definition: test_fsi.py:16
virtual void query(const std::string &queryString)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:620
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:628
#define CHECK(condition)
Definition: Logger.h:206
virtual size_t getNumRows() const

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addFrontendViewToMap ( DashboardDescriptor vd)
protected

Definition at line 1332 of file Catalog.cpp.

References addFrontendViewToMapNoLock().

1332  {
1333  cat_write_lock write_lock(this);
1335 }
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1337
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addFrontendViewToMapNoLock ( DashboardDescriptor vd)
protected

Definition at line 1337 of file Catalog.cpp.

References dashboardDescriptorMap_, DashboardDescriptor::dashboardName, to_string(), and DashboardDescriptor::userId.

Referenced by addFrontendViewToMap().

1337  {
1338  cat_write_lock write_lock(this);
1340  std::make_shared<DashboardDescriptor>(vd);
1341 }
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:624
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addLinkToMap ( LinkDescriptor ld)
protected

Definition at line 1418 of file Catalog.cpp.

References currentDB_, Catalog_Namespace::DBMetadata::dbId, LinkDescriptor::link, linkDescriptorMap_, linkDescriptorMapById_, LinkDescriptor::linkId, and to_string().

1418  {
1419  cat_write_lock write_lock(this);
1420  LinkDescriptor* new_ld = new LinkDescriptor();
1421  *new_ld = ld;
1423  linkDescriptorMapById_[ld.linkId] = new_ld;
1424 }
const DBMetadata currentDB_
Definition: Catalog.h:632
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:626
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:625

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addReferenceToForeignDict ( ColumnDescriptor referencing_column,
Parser::SharedDictionaryDef  shared_dict_def,
const bool  persist_reference 
)
protected

Definition at line 3243 of file Catalog.cpp.

References CHECK, CHECK_GE, ColumnDescriptor::columnType, SQLTypeInfo::get_comp_param(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_foreign_col(), and to_string().

3245  {
3246  cat_write_lock write_lock(this);
3247  const auto foreign_ref_col = get_foreign_col(*this, shared_dict_def);
3248  CHECK(foreign_ref_col);
3249  referencing_column.columnType = foreign_ref_col->columnType;
3250  const int dict_id = referencing_column.columnType.get_comp_param();
3251  const DictRef dict_ref(currentDB_.dbId, dict_id);
3252  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
3253  CHECK(dictIt != dictDescriptorMapByRef_.end());
3254  const auto& dd = dictIt->second;
3255  CHECK_GE(dd->refcount, 1);
3256  ++dd->refcount;
3257  if (persist_reference) {
3258  cat_sqlite_lock sqlite_lock(getObjForLock());
3260  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
3261  {std::to_string(dict_id)});
3262  }
3263 }
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:631
const DBMetadata currentDB_
Definition: Catalog.h:632
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:623
#define CHECK_GE(x, y)
Definition: Logger.h:219
const ColumnDescriptor * get_foreign_col(const Catalog &cat, const Parser::SharedDictionaryDef &shared_dict_def)
Definition: Catalog.cpp:3231
std::string to_string(char const *&&v)
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:332
#define CHECK(condition)
Definition: Logger.h:206
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::addTableToMap ( const TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::list< DictDescriptor > &  dicts 
)
protected

Definition at line 1191 of file Catalog.cpp.

References CHECK, columnDescriptorMap_, columnDescriptorMapById_, ColumnDescriptor::columnId, TableDescriptor::columnIdBySpi_, ColumnDescriptor::columnName, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, DictDescriptor::dictFolderPath, dict_ref_t::dictId, TableDescriptor::hasDeletedCol, TableDescriptor::mutex_, setDeletedColumnUnlocked(), gpu_enabled::sort(), string_dict_hosts_, tableDescriptorMap_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, and to_upper().

Referenced by createTable().

1193  {
1194  cat_write_lock write_lock(this);
1195  TableDescriptor* new_td;
1196 
1197  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
1198  if (foreign_table) {
1199  auto new_foreign_table = new foreign_storage::ForeignTable();
1200  *new_foreign_table = *foreign_table;
1201  new_td = new_foreign_table;
1202  } else {
1203  new_td = new TableDescriptor();
1204  *new_td = *td;
1205  }
1206 
1207  new_td->mutex_ = std::make_shared<std::mutex>();
1208  tableDescriptorMap_[to_upper(td->tableName)] = new_td;
1209  tableDescriptorMapById_[td->tableId] = new_td;
1210  for (auto cd : columns) {
1211  ColumnDescriptor* new_cd = new ColumnDescriptor();
1212  *new_cd = cd;
1213  ColumnKey columnKey(new_cd->tableId, to_upper(new_cd->columnName));
1214  columnDescriptorMap_[columnKey] = new_cd;
1215  ColumnIdKey columnIdKey(new_cd->tableId, new_cd->columnId);
1216  columnDescriptorMapById_[columnIdKey] = new_cd;
1217 
1218  // Add deleted column to the map
1219  if (cd.isDeletedCol) {
1220  CHECK(new_td->hasDeletedCol);
1221  setDeletedColumnUnlocked(new_td, new_cd);
1222  }
1223  }
1224 
1225  std::sort(new_td->columnIdBySpi_.begin(),
1226  new_td->columnIdBySpi_.end(),
1227  [](const size_t a, const size_t b) -> bool { return a < b; });
1228 
1229  std::unique_ptr<StringDictionaryClient> client;
1230  DictRef dict_ref(currentDB_.dbId, -1);
1231  if (!string_dict_hosts_.empty()) {
1232  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1233  }
1234  for (auto dd : dicts) {
1235  if (!dd.dictRef.dictId) {
1236  // Dummy entry created for a shard of a logical table, nothing to do.
1237  continue;
1238  }
1239  dict_ref.dictId = dd.dictRef.dictId;
1240  if (client) {
1241  client->create(dict_ref, dd.dictIsTemp);
1242  }
1243  DictDescriptor* new_dd = new DictDescriptor(dd);
1244  dictDescriptorMapByRef_[dict_ref].reset(new_dd);
1245  if (!dd.dictIsTemp) {
1246  boost::filesystem::create_directory(new_dd->dictFolderPath);
1247  }
1248  }
1249 }
std::string dictFolderPath
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
std::string tableName
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:621
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:635
const DBMetadata currentDB_
Definition: Catalog.h:632
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:623
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
std::vector< int > columnIdBySpi_
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:622
std::shared_ptr< std::mutex > mutex_
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3222
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:620
specifies the content in-memory of a row in the column metadata table
std::string to_upper(const std::string &str)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:206
Descriptor for a dictionary for a string columne.
mapd_unique_lock< mapd_shared_mutex > write_lock
std::string columnName
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:619
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::adjustAlteredTableFiles ( const std::string &  temp_data_dir,
const std::unordered_map< int, int > &  all_column_ids_map 
) const
private
void Catalog_Namespace::Catalog::alterPhysicalTableMetadata ( const TableDescriptor td,
const TableDescriptorUpdateParams table_update_params 
)
private

Definition at line 2947 of file Catalog.cpp.

References CHECK, TableDescriptorUpdateParams::max_rollback_epochs, TableDescriptorUpdateParams::max_rows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::tableId, and to_string().

2949  {
2950  // Only called from parent alterTableParamMetadata, expect already to have catalog and
2951  // sqlite write locks
2952 
2953  // Sqlite transaction should have already been begun in parent alterTableCatalogMetadata
2954 
2956  CHECK(mutable_td);
2957  if (td->maxRollbackEpochs != table_update_params.max_rollback_epochs) {
2959  "UPDATE mapd_tables SET max_rollback_epochs = ? WHERE tableid = ?",
2960  std::vector<std::string>{std::to_string(table_update_params.max_rollback_epochs),
2961  std::to_string(td->tableId)});
2962  mutable_td->maxRollbackEpochs = table_update_params.max_rollback_epochs;
2963  }
2964 
2965  if (td->maxRows != table_update_params.max_rows) {
2967  "UPDATE mapd_tables SET max_rows = ? WHERE tableid = ?",
2968  std::vector<std::string>{std::to_string(table_update_params.max_rows),
2969  std::to_string(td->tableId)});
2970  mutable_td->maxRows = table_update_params.max_rows;
2971  }
2972 }
int32_t maxRollbackEpochs
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:631
std::string to_string(char const *&&v)
TableDescriptor * getMutableMetadataForTableUnlocked(int tableId)
Definition: Catalog.cpp:1528
#define CHECK(condition)
Definition: Logger.h:206

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::alterTableMetadata ( const TableDescriptor td,
const TableDescriptorUpdateParams table_update_params 
)
private

Definition at line 2974 of file Catalog.cpp.

References CHECK, logger::FATAL, i, LOG, TableDescriptor::tableId, and TableDescriptor::tableName.

2975  {
2976  cat_write_lock write_lock(this);
2977  cat_sqlite_lock sqlite_lock(getObjForLock());
2978  sqliteConnector_.query("BEGIN TRANSACTION");
2979  try {
2980  const auto physical_table_it = logicalToPhysicalTableMapById_.find(td->tableId);
2981  if (physical_table_it != logicalToPhysicalTableMapById_.end()) {
2982  const auto physical_tables = physical_table_it->second;
2983  CHECK(!physical_tables.empty());
2984  for (size_t i = 0; i < physical_tables.size(); i++) {
2985  int32_t physical_tb_id = physical_tables[i];
2986  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id, false);
2987  CHECK(phys_td);
2988  alterPhysicalTableMetadata(phys_td, table_update_params);
2989  }
2990  }
2991  alterPhysicalTableMetadata(td, table_update_params);
2992  } catch (std::exception& e) {
2993  sqliteConnector_.query("ROLLBACK TRANSACTION");
2994  LOG(FATAL) << "Table '" << td->tableName << "' catalog update failed";
2995  }
2996  sqliteConnector_.query("END TRANSACTION");
2997 }
std::string tableName
#define LOG(tag)
Definition: Logger.h:200
SqliteConnector sqliteConnector_
Definition: Catalog.h:631
virtual void query(const std::string &queryString)
void alterPhysicalTableMetadata(const TableDescriptor *td, const TableDescriptorUpdateParams &table_update_params)
Definition: Catalog.cpp:2947
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:206
mapd_unique_lock< mapd_shared_mutex > write_lock
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:638
void Catalog_Namespace::Catalog::buildCustomExpressionsMap ( )
private

Definition at line 1163 of file Catalog.cpp.

References custom_expr_map_by_id_, getCustomExpressionFromConnector(), SqliteConnector::getNumRows(), SqliteConnector::query(), and sqliteConnector_.

Referenced by buildMaps().

1163  {
1165  "SELECT id, name, expression_json, data_source_type, data_source_id, "
1166  "is_deleted "
1167  "FROM omnisci_custom_expressions");
1168  auto num_rows = sqliteConnector_.getNumRows();
1169  for (size_t row = 0; row < num_rows; row++) {
1170  auto custom_expr = getCustomExpressionFromConnector(row);
1171  custom_expr_map_by_id_[custom_expr->id] = std::move(custom_expr);
1172  }
1173 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:631
virtual void query(const std::string &queryString)
std::unique_ptr< CustomExpression > getCustomExpressionFromConnector(size_t row)
Definition: Catalog.cpp:1175
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:629
virtual size_t getNumRows() const

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::buildForeignServerMap ( )
private

Definition at line 4324 of file Catalog.cpp.

Referenced by buildMaps().

4324  {
4326  "SELECT id, name, data_wrapper_type, options, owner_user_id, creation_time FROM "
4327  "omnisci_foreign_servers");
4328  auto num_rows = sqliteConnector_.getNumRows();
4329  for (size_t row = 0; row < num_rows; row++) {
4330  auto foreign_server = std::make_shared<foreign_storage::ForeignServer>(
4331  sqliteConnector_.getData<int>(row, 0),
4332  sqliteConnector_.getData<std::string>(row, 1),
4333  sqliteConnector_.getData<std::string>(row, 2),
4334  sqliteConnector_.getData<std::string>(row, 3),
4335  sqliteConnector_.getData<std::int32_t>(row, 4),
4336  sqliteConnector_.getData<std::int32_t>(row, 5));
4337  foreignServerMap_[foreign_server->name] = foreign_server;
4338  foreignServerMapById_[foreign_server->id] = foreign_server;
4339  }
4340 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:631
virtual void query(const std::string &queryString)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:628
ForeignServerMap foreignServerMap_
Definition: Catalog.h:627
virtual size_t getNumRows() const

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::buildMaps ( )
protected

Definition at line 943 of file Catalog.cpp.

References addForeignTableDetails(), buildCustomExpressionsMap(), buildForeignServerMap(), CHECK, ColumnDescriptor::chunks, columnDescriptorMap_, columnDescriptorMapById_, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, createDefaultServersIfNotExists(), currentDB_, dashboardDescriptorMap_, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, ColumnDescriptor::default_value, dictDescriptorMapByRef_, logger::FATAL, StorageType::FOREIGN_TABLE, TableDescriptor::fragmenter, TableDescriptor::fragments, TableDescriptor::fragPageSize, TableDescriptor::fragType, g_base_path, g_enable_fsi, generate_dashboard_system_rolename(), SQLTypeInfo::get_physical_cols(), SqliteConnector::getData(), SqliteConnector::getNumRows(), getObjForLock(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::getUserFromId(), TableDescriptor::hasDeletedCol, SQLTypeInfo::is_geometry(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, SqliteConnector::isNull(), ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, TableDescriptor::keyMetainfo, LinkDescriptor::link, linkDescriptorMap_, linkDescriptorMapById_, LinkDescriptor::linkId, LOG, logicalToPhysicalTableMapById_, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nColumns, TableDescriptor::nShards, TableDescriptor::partitions, SqliteConnector::query(), test_fsi::r, SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_compression(), SQLTypeInfo::set_dimension(), SQLTypeInfo::set_notnull(), SQLTypeInfo::set_scale(), SQLTypeInfo::set_size(), SQLTypeInfo::set_subtype(), SQLTypeInfo::set_type(), setDeletedColumnUnlocked(), TableDescriptor::shard, TableDescriptor::shardedColumnId, gpu_enabled::sort(), TableDescriptor::sortedColumnId, sqliteConnector_, TableDescriptor::storageType, tableDescriptorMap_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), to_upper(), LinkDescriptor::updateTime, LinkDescriptor::userId, TableDescriptor::userId, LinkDescriptor::viewMetadata, TableDescriptor::viewSQL, LinkDescriptor::viewState, and ColumnDescriptor::virtualExpr.

943  {
945  cat_sqlite_lock sqlite_lock(getObjForLock());
946 
947  string dictQuery(
948  "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
949  sqliteConnector_.query(dictQuery);
950  size_t numRows = sqliteConnector_.getNumRows();
951  for (size_t r = 0; r < numRows; ++r) {
952  int dictId = sqliteConnector_.getData<int>(r, 0);
953  std::string dictName = sqliteConnector_.getData<string>(r, 1);
954  int dictNBits = sqliteConnector_.getData<int>(r, 2);
955  bool is_shared = sqliteConnector_.getData<bool>(r, 3);
956  int refcount = sqliteConnector_.getData<int>(r, 4);
957  std::string fname = g_base_path + "/mapd_data/DB_" + std::to_string(currentDB_.dbId) +
958  "_DICT_" + std::to_string(dictId);
959  DictRef dict_ref(currentDB_.dbId, dictId);
960  DictDescriptor* dd = new DictDescriptor(
961  dict_ref, dictName, dictNBits, is_shared, refcount, fname, false);
962  dictDescriptorMapByRef_[dict_ref].reset(dd);
963  }
964 
965  string tableQuery(
966  "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
967  "max_chunk_size, frag_page_size, "
968  "max_rows, partitions, shard_column_id, shard, num_shards, key_metainfo, userid, "
969  "sort_column_id, storage_type, max_rollback_epochs "
970  "from mapd_tables");
971  sqliteConnector_.query(tableQuery);
972  numRows = sqliteConnector_.getNumRows();
973  for (size_t r = 0; r < numRows; ++r) {
974  TableDescriptor* td;
975  const auto& storage_type = sqliteConnector_.getData<string>(r, 17);
976  if (!storage_type.empty() && storage_type != StorageType::FOREIGN_TABLE) {
977  const auto table_id = sqliteConnector_.getData<int>(r, 0);
978  const auto& table_name = sqliteConnector_.getData<string>(r, 1);
979  LOG(FATAL) << "Unable to read Catalog metadata: storage type is currently not a "
980  "supported table option (table "
981  << table_name << " [" << table_id << "] in database "
982  << currentDB_.dbName << ").";
983  }
984 
985  if (storage_type == StorageType::FOREIGN_TABLE) {
987  } else {
988  td = new TableDescriptor();
989  }
990 
991  td->storageType = storage_type;
992  td->tableId = sqliteConnector_.getData<int>(r, 0);
993  td->tableName = sqliteConnector_.getData<string>(r, 1);
994  td->nColumns = sqliteConnector_.getData<int>(r, 2);
995  td->isView = sqliteConnector_.getData<bool>(r, 3);
996  td->fragments = sqliteConnector_.getData<string>(r, 4);
997  td->fragType =
999  td->maxFragRows = sqliteConnector_.getData<int>(r, 6);
1000  td->maxChunkSize = sqliteConnector_.getData<int64_t>(r, 7);
1001  td->fragPageSize = sqliteConnector_.getData<int>(r, 8);
1002  td->maxRows = sqliteConnector_.getData<int64_t>(r, 9);
1003  td->partitions = sqliteConnector_.getData<string>(r, 10);
1004  td->shardedColumnId = sqliteConnector_.getData<int>(r, 11);
1005  td->shard = sqliteConnector_.getData<int>(r, 12);
1006  td->nShards = sqliteConnector_.getData<int>(r, 13);
1007  td->keyMetainfo = sqliteConnector_.getData<string>(r, 14);
1008  td->userId = sqliteConnector_.getData<int>(r, 15);
1009  td->sortedColumnId =
1010  sqliteConnector_.isNull(r, 16) ? 0 : sqliteConnector_.getData<int>(r, 16);
1011  if (!td->isView) {
1012  td->fragmenter = nullptr;
1013  }
1014  td->maxRollbackEpochs = sqliteConnector_.getData<int>(r, 18);
1015  td->hasDeletedCol = false;
1016 
1018  tableDescriptorMapById_[td->tableId] = td;
1019  }
1020 
1021  if (g_enable_fsi) {
1025  }
1026 
1027  string columnQuery(
1028  "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
1029  "is_notnull, compression, comp_param, "
1030  "size, chunks, is_systemcol, is_virtualcol, virtual_expr, is_deletedcol, "
1031  "default_value from "
1032  "mapd_columns ORDER BY tableid, "
1033  "columnid");
1034  sqliteConnector_.query(columnQuery);
1035  numRows = sqliteConnector_.getNumRows();
1036  int32_t skip_physical_cols = 0;
1037  for (size_t r = 0; r < numRows; ++r) {
1038  ColumnDescriptor* cd = new ColumnDescriptor();
1039  cd->tableId = sqliteConnector_.getData<int>(r, 0);
1040  cd->columnId = sqliteConnector_.getData<int>(r, 1);
1041  cd->columnName = sqliteConnector_.getData<string>(r, 2);
1045  cd->columnType.set_scale(sqliteConnector_.getData<int>(r, 6));
1049  cd->columnType.set_size(sqliteConnector_.getData<int>(r, 10));
1050  cd->chunks = sqliteConnector_.getData<string>(r, 11);
1051  cd->isSystemCol = sqliteConnector_.getData<bool>(r, 12);
1052  cd->isVirtualCol = sqliteConnector_.getData<bool>(r, 13);
1053  cd->virtualExpr = sqliteConnector_.getData<string>(r, 14);
1054  cd->isDeletedCol = sqliteConnector_.getData<bool>(r, 15);
1055  if (sqliteConnector_.isNull(r, 16)) {
1056  cd->default_value = std::nullopt;
1057  } else {
1058  cd->default_value = std::make_optional(sqliteConnector_.getData<string>(r, 16));
1059  }
1060  cd->isGeoPhyCol = skip_physical_cols > 0;
1061  ColumnKey columnKey(cd->tableId, to_upper(cd->columnName));
1062  columnDescriptorMap_[columnKey] = cd;
1063  ColumnIdKey columnIdKey(cd->tableId, cd->columnId);
1064  columnDescriptorMapById_[columnIdKey] = cd;
1065 
1066  if (skip_physical_cols <= 0) {
1067  skip_physical_cols = cd->columnType.get_physical_cols();
1068  }
1069 
1070  auto td_itr = tableDescriptorMapById_.find(cd->tableId);
1071  CHECK(td_itr != tableDescriptorMapById_.end());
1072 
1073  if (cd->isDeletedCol) {
1074  td_itr->second->hasDeletedCol = true;
1075  setDeletedColumnUnlocked(td_itr->second, cd);
1076  } else if (cd->columnType.is_geometry() || skip_physical_cols-- <= 0) {
1077  tableDescriptorMapById_[cd->tableId]->columnIdBySpi_.push_back(cd->columnId);
1078  }
1079  }
1080  // sort columnIdBySpi_ based on columnId
1081  for (auto& tit : tableDescriptorMapById_) {
1082  std::sort(tit.second->columnIdBySpi_.begin(),
1083  tit.second->columnIdBySpi_.end(),
1084  [](const size_t a, const size_t b) -> bool { return a < b; });
1085  }
1086 
1087  string viewQuery("SELECT tableid, sql FROM mapd_views");
1088  sqliteConnector_.query(viewQuery);
1089  numRows = sqliteConnector_.getNumRows();
1090  for (size_t r = 0; r < numRows; ++r) {
1091  int32_t tableId = sqliteConnector_.getData<int>(r, 0);
1092  TableDescriptor* td = tableDescriptorMapById_[tableId];
1093  td->viewSQL = sqliteConnector_.getData<string>(r, 1);
1094  td->fragmenter = nullptr;
1095  }
1096 
1097  string frontendViewQuery(
1098  "SELECT id, state, name, image_hash, strftime('%Y-%m-%dT%H:%M:%SZ', update_time), "
1099  "userid, "
1100  "metadata "
1101  "FROM mapd_dashboards");
1102  sqliteConnector_.query(frontendViewQuery);
1103  numRows = sqliteConnector_.getNumRows();
1104  for (size_t r = 0; r < numRows; ++r) {
1105  std::shared_ptr<DashboardDescriptor> vd = std::make_shared<DashboardDescriptor>();
1106  vd->dashboardId = sqliteConnector_.getData<int>(r, 0);
1107  vd->dashboardState = sqliteConnector_.getData<string>(r, 1);
1108  vd->dashboardName = sqliteConnector_.getData<string>(r, 2);
1109  vd->imageHash = sqliteConnector_.getData<string>(r, 3);
1110  vd->updateTime = sqliteConnector_.getData<string>(r, 4);
1111  vd->userId = sqliteConnector_.getData<int>(r, 5);
1112  vd->dashboardMetadata = sqliteConnector_.getData<string>(r, 6);
1113  vd->user = getUserFromId(vd->userId);
1114  vd->dashboardSystemRoleName = generate_dashboard_system_rolename(
1116  dashboardDescriptorMap_[std::to_string(vd->userId) + ":" + vd->dashboardName] = vd;
1117  }
1118 
1119  string linkQuery(
1120  "SELECT linkid, userid, link, view_state, strftime('%Y-%m-%dT%H:%M:%SZ', "
1121  "update_time), view_metadata "
1122  "FROM mapd_links");
1123  sqliteConnector_.query(linkQuery);
1124  numRows = sqliteConnector_.getNumRows();
1125  for (size_t r = 0; r < numRows; ++r) {
1126  LinkDescriptor* ld = new LinkDescriptor();
1127  ld->linkId = sqliteConnector_.getData<int>(r, 0);
1128  ld->userId = sqliteConnector_.getData<int>(r, 1);
1129  ld->link = sqliteConnector_.getData<string>(r, 2);
1130  ld->viewState = sqliteConnector_.getData<string>(r, 3);
1131  ld->updateTime = sqliteConnector_.getData<string>(r, 4);
1132  ld->viewMetadata = sqliteConnector_.getData<string>(r, 5);
1134  linkDescriptorMapById_[ld->linkId] = ld;
1135  }
1136 
1137  /* rebuild map linking logical tables to corresponding physical ones */
1138  string logicalToPhysicalTableMapQuery(
1139  "SELECT logical_table_id, physical_table_id "
1140  "FROM mapd_logical_to_physical");
1141  sqliteConnector_.query(logicalToPhysicalTableMapQuery);
1142  numRows = sqliteConnector_.getNumRows();
1143  for (size_t r = 0; r < numRows; ++r) {
1144  int32_t logical_tb_id = sqliteConnector_.getData<int>(r, 0);
1145  int32_t physical_tb_id = sqliteConnector_.getData<int>(r, 1);
1146  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
1147  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
1148  /* add new entity to the map logicalToPhysicalTableMapById_ */
1149  std::vector<int32_t> physicalTables;
1150  physicalTables.push_back(physical_tb_id);
1151  const auto it_ok =
1152  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
1153  CHECK(it_ok.second);
1154  } else {
1155  /* update map logicalToPhysicalTableMapById_ */
1156  physicalTableIt->second.push_back(physical_tb_id);
1157  }
1158  }
1159 
1161 }
int32_t maxRollbackEpochs
std::string virtualExpr
void set_compression(EncodingType c)
Definition: sqltypes.h:423
void set_size(int s)
Definition: sqltypes.h:421
std::string partitions
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
T getData(const int row, const int col)
SQLTypes
Definition: sqltypes.h:37
std::string tableName
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:621
#define LOG(tag)
Definition: Logger.h:200
SqliteConnector sqliteConnector_
Definition: Catalog.h:631
const DBMetadata currentDB_
Definition: Catalog.h:632
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:623
std::string storageType
tuple r
Definition: test_fsi.py:16
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:414
DEVICE void sort(ARGS &&...args)
Definition: gpu_enabled.h:105
virtual void query(const std::string &queryString)
std::string fragments
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:622
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:624
std::string to_string(char const *&&v)
std::string chunks
EncodingType
Definition: sqltypes.h:227
int get_physical_cols() const
Definition: sqltypes.h:344
std::string getUserFromId(const int32_t id)
Definition: Catalog.cpp:933
std::string g_base_path
Definition: SysCatalog.cpp:62
void set_scale(int s)
Definition: sqltypes.h:418
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:3222
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:620
specifies the content in-memory of a row in the column metadata table
std::string keyMetainfo
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
std::string to_upper(const std::string &str)
void set_comp_param(int p)
Definition: sqltypes.h:424
std::optional< std::string > default_value
bool isNull(const int row, const int col) const
void set_dimension(int d)
Definition: sqltypes.h:415
Fragmenter_Namespace::FragmenterType fragType
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
void set_notnull(bool n)
Definition: sqltypes.h:420
#define CHECK(condition)
Definition: Logger.h:206
bool is_geometry() const
Definition: sqltypes.h:510
Descriptor for a dictionary for a string columne.
std::string viewSQL
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:626
static constexpr char const * FOREIGN_TABLE
bool g_enable_fsi
Definition: Catalog.cpp:92
std::string columnName
virtual size_t getNumRows() const
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:638
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:619
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:625
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)
void createDefaultServersIfNotExists()
Definition: Catalog.cpp:4387
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39
HOST DEVICE void set_type(SQLTypes t)
Definition: sqltypes.h:413

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::calculateSHA1 ( const std::string &  data)
protected

Definition at line 4051 of file Catalog.cpp.

References i.

4051  {
4052  boost::uuids::detail::sha1 sha1;
4053  unsigned int digest[5];
4054  sha1.process_bytes(data.c_str(), data.length());
4055  sha1.get_digest(digest);
4056  std::stringstream ss;
4057  for (size_t i = 0; i < 5; i++) {
4058  ss << std::hex << digest[i];
4059  }
4060  return ss.str();
4061 }
void Catalog_Namespace::Catalog::changeForeignServerOwner ( const std::string &  server_name,
const int  new_owner_id 
)

Change the owner of a Foreign Server to a new owner.

Parameters
server_name- Name of the foreign server whose owner to change
new_owner_id- New owner's user id

Definition at line 2662 of file Catalog.cpp.

References CHECK, to_string(), and foreign_storage::ForeignServer::user_id.

2663  {
2664  cat_write_lock write_lock(this);
2665  foreign_storage::ForeignServer* foreign_server =
2666  foreignServerMap_.find(server_name)->second.get();
2667  CHECK(foreign_server);
2668  setForeignServerProperty(server_name, "owner_user_id", std::to_string(new_owner_id));
2669  // update in-memory server
2670  foreign_server->user_id = new_owner_id;
2671 }
std::string to_string(char const *&&v)
ForeignServerMap foreignServerMap_
Definition: Catalog.h:627
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:206
mapd_unique_lock< mapd_shared_mutex > write_lock
void setForeignServerProperty(const std::string &server_name, const std::string &property, const std::string &value)
Definition: Catalog.cpp:4366

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::CheckAndExecuteMigrations ( )
protected

Definition at line 906 of file Catalog.cpp.

References g_enable_fsi, recordOwnershipOfObjectsInObjectPermissions(), updateCustomExpressionsSchema(), updateDefaultColumnValues(), updateDeletedColumnIndicator(), updateDictionaryNames(), updateDictionarySchema(), updateFixlenArrayColumns(), updateFrontendViewAndLinkUsers(), updateFrontendViewSchema(), updateFrontendViewsToDashboards(), updateFsiSchemas(), updateGeoColumns(), updateLinkSchema(), updateLogicalToPhysicalTableLinkSchema(), updatePageSize(), and updateTableDescriptorSchema().

906  {
916  updatePageSize();
920  if (g_enable_fsi) {
922  }
925 }
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:427
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:696
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:121
void updateCustomExpressionsSchema()
Definition: Catalog.cpp:662
void updateLogicalToPhysicalTableLinkSchema()
Definition: Catalog.cpp:588
bool g_enable_fsi
Definition: Catalog.cpp:92

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::CheckAndExecuteMigrationsPostBuildMaps ( )
protected

Definition at line 927 of file Catalog.cpp.

References checkDateInDaysColumnMigration(), and createDashboardSystemRoles().

927  {
930 }
void checkDateInDaysColumnMigration()
Definition: Catalog.cpp:824

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::checkDateInDaysColumnMigration ( )
protected

Definition at line 824 of file Catalog.cpp.

References getCurrentDB(), getObjForLock(), migrations::MigrationMgr::migrateDateInDaysMetadata(), sqliteConnector_, and tableDescriptorMapById_.

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

824  {
825  cat_sqlite_lock sqlite_lock(getObjForLock());
828 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:631
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:222
static void migrateDateInDaysMetadata(const Catalog_Namespace::TableDescriptorMapById &table_descriptors_by_id, const int database_id, const Catalog_Namespace::Catalog *cat, SqliteConnector &sqlite)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:620
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const bool Catalog_Namespace::Catalog::checkMetadataForDeletedRecs ( const TableDescriptor td,
int  column_id 
) const

Definition at line 3180 of file Catalog.cpp.

References CHECK, and TableDescriptor::fragmenter.

3181  {
3182  // check if there are rows deleted by examining the deletedColumn metadata
3183  CHECK(td);
3184  auto fragmenter = td->fragmenter;
3185  if (fragmenter) {
3186  return fragmenter->hasDeletedRows(delete_column_id);
3187  } else {
3188  return false;
3189  }
3190 }
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
#define CHECK(condition)
Definition: Logger.h:206
void Catalog_Namespace::Catalog::checkpoint ( const int  logicalTableId) const

Definition at line 4258 of file Catalog.cpp.

Referenced by UpdelRoll::commitUpdate(), TableOptimizer::vacuumDeletedRows(), and TableOptimizer::vacuumFragmentsAboveMinSelectivity().

4258  {
4259  const auto td = getMetadataForTable(logicalTableId);
4260  const auto shards = getPhysicalTablesDescriptors(td);
4261  for (const auto shard : shards) {
4262  getDataMgr().checkpoint(getCurrentDB().dbId, shard->tableId);
4263  }
4264 }
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:223
void checkpoint(const int db_id, const int tb_id)
Definition: DataMgr.cpp:489
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:222
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4119
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::checkpointWithAutoRollback ( const int  logical_table_id) const

Definition at line 4266 of file Catalog.cpp.

Referenced by import_export::Loader::checkpoint(), Parser::LocalConnector::checkpoint(), StorageIOFacility::TransactionParameters::finalizeTransaction(), and TableOptimizer::vacuumFragmentsAboveMinSelectivity().

4266  {
4267  auto table_epochs = getTableEpochs(getDatabaseId(), logical_table_id);
4268  try {
4269  checkpoint(logical_table_id);
4270  } catch (...) {
4271  setTableEpochsLogExceptions(getDatabaseId(), table_epochs);
4272  throw;
4273  }
4274 }
int getDatabaseId() const
Definition: Catalog.h:277
void checkpoint(const int logicalTableId) const
Definition: Catalog.cpp:4258
void setTableEpochsLogExceptions(const int32_t db_id, const std::vector< TableEpochInfo > &table_epochs) const
Definition: Catalog.cpp:3162
std::vector< TableEpochInfo > getTableEpochs(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:3097

+ Here is the caller graph for this function:

int32_t Catalog_Namespace::Catalog::createCustomExpression ( std::unique_ptr< CustomExpression custom_expression)

Creates a new custom expression.

Parameters
custom_expression- unique pointer to struct containing custom expression details.
Returns
id of created custom expression

Definition at line 4925 of file Catalog.cpp.

References CHECK, CHECK_EQ, CHECK_GT, and to_string().

4926  {
4927  cat_write_lock write_lock(this);
4928  cat_sqlite_lock sqlite_lock(getObjForLock());
4929  sqliteConnector_.query("BEGIN TRANSACTION");
4930  int32_t custom_expression_id{-1};
4931  try {
4932  auto data_source_type_str =
4933  CustomExpression::dataSourceTypeToString(custom_expression->data_source_type);
4934  auto data_source_id_str = std::to_string(custom_expression->data_source_id);
4935  std::string custom_expr_select_query{
4936  "SELECT id FROM omnisci_custom_expressions WHERE name = ? and data_source_type = "
4937  "? and data_source_id = ? and is_deleted = ?"};
4938  std::vector<std::string> custom_expr_select_params{custom_expression->name,
4939  data_source_type_str,
4940  data_source_id_str,
4941  std::to_string(false)};
4942  sqliteConnector_.query_with_text_params(custom_expr_select_query,
4943  custom_expr_select_params);
4944  if (sqliteConnector_.getNumRows() > 0) {
4945  throw std::runtime_error{
4946  "A custom expression with the given "
4947  "name and data source already exists."};
4948  }
4950  "INSERT INTO omnisci_custom_expressions(name, expression_json, "
4951  "data_source_type, data_source_id, is_deleted) VALUES (?,?,?,?,?)",
4952  std::vector<std::string>{custom_expression->name,
4953  custom_expression->expression_json,
4954  data_source_type_str,
4955  data_source_id_str,
4956  std::to_string(false)});
4957  sqliteConnector_.query_with_text_params(custom_expr_select_query,
4958  custom_expr_select_params);
4959  CHECK_EQ(sqliteConnector_.getNumRows(), static_cast<size_t>(1));
4960  custom_expression->id = sqliteConnector_.getData<int32_t>(0, 0);
4961  custom_expression_id = custom_expression->id;
4962  CHECK(custom_expr_map_by_id_.find(custom_expression->id) ==
4963  custom_expr_map_by_id_.end());
4964  custom_expr_map_by_id_[custom_expression->id] = std::move(custom_expression);
4965  } catch (std::exception& e) {
4966  sqliteConnector_.query("ROLLBACK TRANSACTION");
4967  throw;
4968  }
4969  sqliteConnector_.query("END TRANSACTION");
4970  CHECK_GT(custom_expression_id, 0);
4971  return custom_expression_id;
4972 }
#define CHECK_EQ(x, y)
Definition: Logger.h:214
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:631
virtual void query(const std::string &queryString)
#define CHECK_GT(x, y)
Definition: Logger.h:218
std::string to_string(char const *&&v)
static std::string dataSourceTypeToString(DataSourceType type_enum)
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:206
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:629
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
virtual size_t getNumRows() const

+ Here is the call graph for this function:

int32_t Catalog_Namespace::Catalog::createDashboard ( DashboardDescriptor vd)

Definition at line 3914 of file Catalog.cpp.

References DashboardDescriptor::dashboardId, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, DashboardDescriptor::dashboardSystemRoleName, generate_dashboard_system_rolename(), DashboardDescriptor::imageHash, to_string(), Catalog_Namespace::sqlite_lock< T >::unlock(), Catalog_Namespace::write_lock< T >::unlock(), DashboardDescriptor::updateTime, and DashboardDescriptor::userId.

3914  {
3915  cat_write_lock write_lock(this);
3916  cat_sqlite_lock sqlite_lock(getObjForLock());
3917  sqliteConnector_.query("BEGIN TRANSACTION");
3918  try {
3919  // TODO(andrew): this should be an upsert
3921  "SELECT id FROM mapd_dashboards WHERE name = ? and userid = ?",
3922  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
3923  if (sqliteConnector_.getNumRows() > 0) {
3925  "UPDATE mapd_dashboards SET state = ?, image_hash = ?, metadata = ?, "
3926  "update_time = "
3927  "datetime('now') where name = ? "
3928  "and userid = ?",
3929  std::vector<std::string>{vd.dashboardState,
3930  vd.imageHash,
3931  vd.dashboardMetadata,
3932  vd.dashboardName,
3933  std::to_string(vd.userId)});
3934  } else {
3936  "INSERT INTO mapd_dashboards (name, state, image_hash, metadata, "
3937  "update_time, "
3938  "userid) "
3939  "VALUES "
3940  "(?,?,?,?, "
3941  "datetime('now'), ?)",
3942  std::vector<std::string>{vd.dashboardName,
3943  vd.dashboardState,
3944  vd.imageHash,
3945  vd.dashboardMetadata,
3946  std::to_string(vd.userId)});
3947  }
3948  } catch (std::exception& e) {
3949  sqliteConnector_.query("ROLLBACK TRANSACTION");
3950  throw;
3951  }
3952  sqliteConnector_.query("END TRANSACTION");
3953 
3954  // now get the auto generated dashboardId
3955  try {
3957  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_dashboards "
3958  "WHERE name = ? and userid = ?",
3959  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
3960  vd.dashboardId = sqliteConnector_.getData<int>(0, 0);
3961  vd.updateTime = sqliteConnector_.getData<std::string>(0, 1);
3962  } catch (std::exception& e) {
3963  throw;
3964  }
3968  sqlite_lock.unlock();
3969  write_lock.unlock();
3970  // NOTE(wamsi): Transactionally unsafe
3973  return vd.dashboardId;
3974 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:631
const DBMetadata currentDB_
Definition: Catalog.h:632
virtual void query(const std::string &queryString)
void addFrontendViewToMap(DashboardDescriptor &vd)
Definition: Catalog.cpp:1332
std::string to_string(char const *&&v)
std::string dashboardSystemRoleName
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1373
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createDashboardSystemRoles ( )
protected

Definition at line 830 of file Catalog.cpp.

References createOrUpdateDashboardSystemRole(), currentDB_, Catalog_Namespace::DBMetadata::dbId, logger::ERROR, generate_dashboard_system_rolename(), SqliteConnector::getData(), Catalog_Namespace::SysCatalog::getGranteesOfSharedDashboards(), SqliteConnector::getNumRows(), getObjForLock(), Catalog_Namespace::SysCatalog::getRoleGrantee(), Catalog_Namespace::SysCatalog::grantRoleBatch(), i, logger::INFO, Catalog_Namespace::SysCatalog::instance(), LOG, MAPD_VERSION, SqliteConnector::query(), SqliteConnector::query_with_text_params(), run_benchmark_import::result, sqliteConnector_, and to_string().

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

830  {
831  std::unordered_map<std::string, std::pair<int, std::string>> dashboards;
832  std::vector<std::string> dashboard_ids;
833  static const std::string migration_name{"dashboard_roles_migration"};
834  {
835  cat_sqlite_lock sqlite_lock(getObjForLock());
836  sqliteConnector_.query("BEGIN TRANSACTION");
837  try {
838  // migration_history should be present in all catalogs by now
839  // if not then would be created before this migration
841  "select * from mapd_version_history where migration_history = '" +
842  migration_name + "'");
843  if (sqliteConnector_.getNumRows() != 0) {
844  // no need for further execution
845  sqliteConnector_.query("END TRANSACTION");
846  return;
847  }
848  LOG(INFO) << "Performing dashboard internal roles Migration.";
849  sqliteConnector_.query("select id, userid, metadata from mapd_dashboards");
850  for (size_t i = 0; i < sqliteConnector_.getNumRows(); ++i) {
853  sqliteConnector_.getData<string>(i, 0)))) {
854  // Successfully created roles during previous migration/crash
855  // No need to include them
856  continue;
857  }
858  dashboards[sqliteConnector_.getData<string>(i, 0)] = std::make_pair(
859  sqliteConnector_.getData<int>(i, 1), sqliteConnector_.getData<string>(i, 2));
860  dashboard_ids.push_back(sqliteConnector_.getData<string>(i, 0));
861  }
862  } catch (const std::exception& e) {
863  sqliteConnector_.query("ROLLBACK TRANSACTION");
864  throw;
865  }
866  sqliteConnector_.query("END TRANSACTION");
867  }
868  // All current grantees with shared dashboards.
869  const auto active_grantees =
871 
872  try {
873  // NOTE(wamsi): Transactionally unsafe
874  for (auto dash : dashboards) {
875  createOrUpdateDashboardSystemRole(dash.second.second,
876  dash.second.first,
878  std::to_string(currentDB_.dbId), dash.first));
879  auto result = active_grantees.find(dash.first);
880  if (result != active_grantees.end()) {
883  dash.first)},
884  result->second);
885  }
886  }
887  cat_sqlite_lock sqlite_lock(getObjForLock());
888  // check if this has already been completed
890  "select * from mapd_version_history where migration_history = '" +
891  migration_name + "'");
892  if (sqliteConnector_.getNumRows() != 0) {
893  return;
894  }
896  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
897  std::vector<std::string>{std::to_string(MAPD_VERSION), migration_name});
898  } catch (const std::exception& e) {
899  LOG(ERROR) << "Failed to create dashboard system roles during migration: "
900  << e.what();
901  throw;
902  }
903  LOG(INFO) << "Successfully created dashboard system roles during migration.";
904 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
#define LOG(tag)
Definition: Logger.h:200
SqliteConnector sqliteConnector_
Definition: Catalog.h:631
const DBMetadata currentDB_
Definition: Catalog.h:632
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
void grantRoleBatch(const std::vector< std::string > &roles, const std::vector< std::string > &grantees)
static SysCatalog & instance()
Definition: SysCatalog.h:293
static const int32_t MAPD_VERSION
Definition: release.h:33
Role * getRoleGrantee(const std::string &name) const
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1373
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
std::unordered_map< std::string, std::vector< std::string > > getGranteesOfSharedDashboards(const std::vector< std::string > &dashboard_ids)
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
virtual size_t getNumRows() const
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createDefaultServersIfNotExists ( )

Creates default local file servers (if they don't already exist).

Definition at line 4387 of file Catalog.cpp.

References foreign_storage::DataWrapperType::CSV, foreign_storage::AbstractFileStorageDataWrapper::LOCAL_FILE_STORAGE_TYPE, OMNISCI_ROOT_USER_ID, foreign_storage::DataWrapperType::PARQUET, and foreign_storage::AbstractFileStorageDataWrapper::STORAGE_TYPE_KEY.

Referenced by buildMaps().

4387  {
4391 
4392  auto local_csv_server = std::make_unique<foreign_storage::ForeignServer>(
4393  "omnisci_local_csv",
4395  options,
4397  local_csv_server->validate();
4398  createForeignServerNoLocks(std::move(local_csv_server), true);
4399 
4400  auto local_parquet_server = std::make_unique<foreign_storage::ForeignServer>(
4401  "omnisci_local_parquet",
4403  options,
4405  local_parquet_server->validate();
4406  createForeignServerNoLocks(std::move(local_parquet_server), true);
4407 }
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2573
const int OMNISCI_ROOT_USER_ID
Definition: SysCatalog.h:61
static constexpr char const * CSV
std::map< std::string, std::string, std::less<>> OptionsMap
static constexpr char const * PARQUET

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createForeignServer ( std::unique_ptr< foreign_storage::ForeignServer foreign_server,
bool  if_not_exists 
)

Creates a new foreign server DB object.

Parameters
foreign_server- unique pointer to struct containing foreign server details
if_not_exists- flag indicating whether or not an attempt to create a new foreign server should occur if a server with the same name already exists. An exception is thrown if this flag is set to "false" and an attempt is made to create a pre-existing foreign server

Definition at line 2565 of file Catalog.cpp.

2567  {
2568  cat_write_lock write_lock(this);
2569  cat_sqlite_lock sqlite_lock(getObjForLock());
2570  createForeignServerNoLocks(std::move(foreign_server), if_not_exists);
2571 }
void createForeignServerNoLocks(std::unique_ptr< foreign_storage::ForeignServer > foreign_server, bool if_not_exists)
Definition: Catalog.cpp:2573
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
void Catalog_Namespace::Catalog::createForeignServerNoLocks ( std::unique_ptr< foreign_storage::ForeignServer foreign_server,
bool  if_not_exists 
)
private

Same as createForeignServer() but without acquiring locks. This should only be called from within a function/code block that already acquires appropriate locks.

Definition at line 2573 of file Catalog.cpp.

References CHECK, CHECK_EQ, and to_string().

2575  {
2577  "SELECT name from omnisci_foreign_servers where name = ?",
2578  std::vector<std::string>{foreign_server->name});
2579 
2580  if (sqliteConnector_.getNumRows() == 0) {
2581  foreign_server->creation_time = std::time(nullptr);
2583  "INSERT INTO omnisci_foreign_servers (name, data_wrapper_type, owner_user_id, "
2584  "creation_time, "
2585  "options) "
2586  "VALUES (?, ?, ?, ?, ?)",
2587  std::vector<std::string>{foreign_server->name,
2588  foreign_server->data_wrapper_type,
2589  std::to_string(foreign_server->user_id),
2590  std::to_string(foreign_server->creation_time),
2591  foreign_server->getOptionsAsJsonString()});
2593  "SELECT id from omnisci_foreign_servers where name = ?",
2594  std::vector<std::string>{foreign_server->name});
2595  CHECK_EQ(sqliteConnector_.getNumRows(), size_t(1));
2596  foreign_server->id = sqliteConnector_.getData<int32_t>(0, 0);
2597  std::shared_ptr<foreign_storage::ForeignServer> foreign_server_shared =
2598  std::move(foreign_server);
2599  CHECK(foreignServerMap_.find(foreign_server_shared->name) == foreignServerMap_.end())
2600  << "Attempting to insert a foreign server into foreign server map that already "
2601  "exists.";
2602  foreignServerMap_[foreign_server_shared->name] = foreign_server_shared;
2603  foreignServerMapById_[foreign_server_shared->id] = foreign_server_shared;
2604  } else if (!if_not_exists) {
2605  throw std::runtime_error{"A foreign server with name \"" + foreign_server->name +
2606  "\" already exists."};
2607  }
2608 }
#define CHECK_EQ(x, y)
Definition: Logger.h:214
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:631
std::string to_string(char const *&&v)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:628
ForeignServerMap foreignServerMap_
Definition: Catalog.h:627
#define CHECK(condition)
Definition: Logger.h:206
virtual size_t getNumRows() const

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::createLink ( LinkDescriptor ld,
size_t  min_length 
)

Definition at line 4063 of file Catalog.cpp.

References LinkDescriptor::link, LinkDescriptor::linkId, to_string(), LinkDescriptor::updateTime, LinkDescriptor::userId, LinkDescriptor::viewMetadata, and LinkDescriptor::viewState.

4063  {
4064  cat_write_lock write_lock(this);
4065  cat_sqlite_lock sqlite_lock(getObjForLock());
4066  sqliteConnector_.query("BEGIN TRANSACTION");
4067  try {
4069  .substr(0, 8);
4071  "SELECT linkid FROM mapd_links WHERE link = ? and userid = ?",
4072  std::vector<std::string>{ld.link, std::to_string(ld.userId)});
4073  if (sqliteConnector_.getNumRows() > 0) {
4075  "UPDATE mapd_links SET update_time = datetime('now') WHERE userid = ? AND "
4076  "link = ?",
4077  std::vector<std::string>{std::to_string(ld.userId), ld.link});
4078  } else {
4080  "INSERT INTO mapd_links (userid, link, view_state, view_metadata, "
4081  "update_time) VALUES (?,?,?,?, datetime('now'))",
4082  std::vector<std::string>{
4083  std::to_string(ld.userId), ld.link, ld.viewState, ld.viewMetadata});
4084  }
4085  // now get the auto generated dashid
4087  "SELECT linkid, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_links "
4088  "WHERE link = ?",
4089  ld.link);
4090  ld.linkId = sqliteConnector_.getData<int>(0, 0);
4091  ld.updateTime = sqliteConnector_.getData<std::string>(0, 1);
4092  } catch (std::exception& e) {
4093  sqliteConnector_.query("ROLLBACK TRANSACTION");
4094  throw;
4095  }
4096  sqliteConnector_.query("END TRANSACTION");
4097  addLinkToMap(ld);
4098  return ld.link;
4099 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:631
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
std::string calculateSHA1(const std::string &data)
Definition: Catalog.cpp:4051
void addLinkToMap(LinkDescriptor &ld)
Definition: Catalog.cpp:1418
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createOrUpdateDashboardSystemRole ( const std::string &  view_meta,
const int32_t &  user_id,
const std::string &  dash_role_name 
)
protected

Definition at line 1373 of file Catalog.cpp.

References Catalog_Namespace::SysCatalog::createRole(), Grantee::findDbObject(), Grantee::getDbObjects(), Catalog_Namespace::SysCatalog::getRoleGrantee(), Catalog_Namespace::SysCatalog::grantDBObjectPrivilegesBatch(), Catalog_Namespace::SysCatalog::instance(), parseDashboardObjects(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivileges(), TableDBObjectType, and ViewDBObjectType.

Referenced by createDashboardSystemRoles().

1375  {
1376  auto objects = parseDashboardObjects(view_meta, user_id);
1377  Role* rl = SysCatalog::instance().getRoleGrantee(dash_role_name);
1378  if (!rl) {
1379  // Dashboard role does not exist
1380  // create role and grant privileges
1381  // NOTE(wamsi): Transactionally unsafe
1382  SysCatalog::instance().createRole(dash_role_name, false);
1383  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1384  } else {
1385  // Dashboard system role already exists
1386  // Add/remove privileges on objects
1387  std::set<DBObjectKey> revoke_keys;
1388  auto ex_objects = rl->getDbObjects(true);
1389  for (auto key : *ex_objects | boost::adaptors::map_keys) {
1390  if (key.permissionType != TableDBObjectType &&
1391  key.permissionType != ViewDBObjectType) {
1392  continue;
1393  }
1394  bool found = false;
1395  for (auto obj : objects) {
1396  found = key == obj.getObjectKey() ? true : false;
1397  if (found) {
1398  break;
1399  }
1400  }
1401  if (!found) {
1402  revoke_keys.insert(key);
1403  }
1404  }
1405  for (auto& key : revoke_keys) {
1406  // revoke privs on object since the object is no
1407  // longer used by the dashboard as source
1408  // NOTE(wamsi): Transactionally unsafe
1410  dash_role_name, *rl->findDbObject(key, true), *this);
1411  }
1412  // Update privileges on remaining objects
1413  // NOTE(wamsi): Transactionally unsafe
1414  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1415  }
1416 }
void revokeDBObjectPrivileges(const std::string &grantee, const DBObject &object, const Catalog_Namespace::Catalog &catalog)
Definition: Grantee.h:76
DBObject * findDbObject(const DBObjectKey &objectKey, bool only_direct) const
Definition: Grantee.cpp:72
void createRole(const std::string &roleName, const bool &userPrivateRole=false)
static SysCatalog & instance()
Definition: SysCatalog.h:293
Role * getRoleGrantee(const std::string &name) const
void grantDBObjectPrivilegesBatch(const std::vector< std::string > &grantees, const std::vector< DBObject > &objects, const Catalog_Namespace::Catalog &catalog)
const DBObjectMap * getDbObjects(bool only_direct) const
Definition: Grantee.h:56
std::vector< DBObject > parseDashboardObjects(const std::string &view_meta, const int &user_id)
Definition: Catalog.cpp:1343

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createShardedTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs 
)

Definition at line 3375 of file Catalog.cpp.

References CHECK, i, TableDescriptor::nShards, TableDescriptor::shard, table_is_temporary(), TableDescriptor::tableId, and TableDescriptor::tableName.

3378  {
3379  cat_write_lock write_lock(this);
3380 
3381  /* create logical table */
3382  TableDescriptor* tdl = &td;
3383  createTable(*tdl, cols, shared_dict_defs, true); // create logical table
3384  int32_t logical_tb_id = tdl->tableId;
3385  std::string logical_table_name = tdl->tableName;
3386 
3387  /* create physical tables and link them to the logical table */
3388  std::vector<int32_t> physicalTables;
3389  for (int32_t i = 1; i <= td.nShards; i++) {
3390  TableDescriptor* tdp = &td;
3391  tdp->tableName = generatePhysicalTableName(logical_table_name, i);
3392  tdp->shard = i - 1;
3393  createTable(*tdp, cols, shared_dict_defs, false); // create physical table
3394  int32_t physical_tb_id = tdp->tableId;
3395 
3396  /* add physical table to the vector of physical tables */
3397  physicalTables.push_back(physical_tb_id);
3398  }
3399 
3400  if (!physicalTables.empty()) {
3401  /* add logical to physical tables correspondence to the map */
3402  const auto it_ok =
3403  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
3404  CHECK(it_ok.second);
3405  /* update sqlite mapd_logical_to_physical in sqlite database */
3406  if (!table_is_temporary(&td)) {
3407  updateLogicalToPhysicalTableMap(logical_tb_id);
3408  }
3409  }
3410 }
std::string tableName
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:2233
std::string generatePhysicalTableName(const std::string &logicalTableName, const int32_t &shardNumber)
Definition: Catalog.cpp:4317
bool table_is_temporary(const TableDescriptor *const td)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:206
mapd_unique_lock< mapd_shared_mutex > write_lock
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:638
void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id)
Definition: Catalog.cpp:602

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs,
bool  isLogicalTable 
)

Definition at line 2233 of file Catalog.cpp.

References addTableToMap(), calciteMgr_, CHECK, ColumnDescriptor::columnId, TableDescriptor::columnIdBySpi_, ColumnDescriptor::columnName, ColumnDescriptor::columnType, currentDB_, dataMgr_, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, ColumnDescriptor::default_value, dict_ref_t::dictId, Data_Namespace::DISK_LEVEL, expandGeoColumn(), StorageType::FOREIGN_TABLE, TableDescriptor::fragPageSize, TableDescriptor::fragType, g_enable_fsi, g_serialize_temp_tables, g_test_against_columnId_gap, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SQLTypeInfo::get_dimension(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_next_refresh_time(), SQLTypeInfo::get_notnull(), SQLTypeInfo::get_scale(), SQLTypeInfo::get_size(), SQLTypeInfo::get_subtype(), SQLTypeInfo::get_type(), getCurrentDB(), SqliteConnector::getData(), getObjForLock(), TableDescriptor::hasDeletedCol, SQLTypeInfo::is_array(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, kBIGINT, kBOOLEAN, kENCODING_DICT, TableDescriptor::keyMetainfo, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nColumns, foreign_storage::ForeignTable::next_refresh_time, nextTempDictId_, nextTempTableId_, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, SqliteConnector::query(), SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), serializeTableJsonUnlocked(), SQLTypeInfo::set_comp_param(), SQLTypeInfo::set_size(), setColumnDictionary(), setColumnSharedDictionary(), TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, sqliteConnector_, TableDescriptor::storageType, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, TEXT, to_string(), TableDescriptor::userId, TableDescriptor::viewSQL, and ColumnDescriptor::virtualExpr.

2237  {
2238  cat_write_lock write_lock(this);
2239  list<ColumnDescriptor> cds = cols;
2240  list<DictDescriptor> dds;
2241  std::set<std::string> toplevel_column_names;
2242  list<ColumnDescriptor> columns;
2243 
2244  if (!td.storageType.empty() &&
2247  throw std::runtime_error("Only temporary tables can be backed by foreign storage.");
2248  }
2249  dataMgr_->getForeignStorageInterface()->prepareTable(getCurrentDB().dbId, td, cds);
2250  }
2251 
2252  for (auto cd : cds) {
2253  if (cd.columnName == "rowid") {
2254  throw std::runtime_error(
2255  "Cannot create column with name rowid. rowid is a system defined column.");
2256  }
2257  columns.push_back(cd);
2258  toplevel_column_names.insert(cd.columnName);
2259  if (cd.columnType.is_geometry()) {
2260  expandGeoColumn(cd, columns);
2261  }
2262  }
2263  cds.clear();
2264 
2265  ColumnDescriptor cd;
2266  // add row_id column -- Must be last column in the table
2267  cd.columnName = "rowid";
2268  cd.isSystemCol = true;
2269  cd.columnType = SQLTypeInfo(kBIGINT, true);
2270 #ifdef MATERIALIZED_ROWID
2271  cd.isVirtualCol = false;
2272 #else
2273  cd.isVirtualCol = true;
2274  cd.virtualExpr = "MAPD_FRAG_ID * MAPD_ROWS_PER_FRAG + MAPD_FRAG_ROW_ID";
2275 #endif
2276  columns.push_back(cd);
2277  toplevel_column_names.insert(cd.columnName);
2278 
2279  if (td.hasDeletedCol) {
2280  ColumnDescriptor cd_del;
2281  cd_del.columnName = "$deleted$";
2282  cd_del.isSystemCol = true;
2283  cd_del.isVirtualCol = false;
2284  cd_del.columnType = SQLTypeInfo(kBOOLEAN, true);
2285  cd_del.isDeletedCol = true;
2286 
2287  columns.push_back(cd_del);
2288  }
2289 
2290  td.nColumns = columns.size();
2291  cat_sqlite_lock sqlite_lock(getObjForLock());
2292  sqliteConnector_.query("BEGIN TRANSACTION");
2294  try {
2296  R"(INSERT INTO mapd_tables (name, userid, ncolumns, isview, fragments, frag_type, max_frag_rows, max_chunk_size, frag_page_size, max_rows, partitions, shard_column_id, shard, num_shards, sort_column_id, storage_type, max_rollback_epochs, key_metainfo) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?))",
2297  std::vector<std::string>{td.tableName,
2298  std::to_string(td.userId),
2300  std::to_string(td.isView),
2301  "",
2306  std::to_string(td.maxRows),
2307  td.partitions,
2309  std::to_string(td.shard),
2310  std::to_string(td.nShards),
2312  td.storageType,
2314  td.keyMetainfo});
2315 
2316  // now get the auto generated tableid
2318  "SELECT tableid FROM mapd_tables WHERE name = ?", td.tableName);
2319  td.tableId = sqliteConnector_.getData<int>(0, 0);
2320  int colId = 1;
2321  for (auto cd : columns) {
2323  const bool is_foreign_col =
2324  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2325  if (!is_foreign_col) {
2326  setColumnDictionary(cd, dds, td, isLogicalTable);
2327  }
2328  }
2329 
2330  if (toplevel_column_names.count(cd.columnName)) {
2331  // make up colId gap for sanity test (begin with 1 bc much code depends on it!)
2332  if (colId > 1) {
2333  colId += g_test_against_columnId_gap;
2334  }
2335  if (!cd.isGeoPhyCol) {
2336  td.columnIdBySpi_.push_back(colId);
2337  }
2338  }
2339 
2340  using BindType = SqliteConnector::BindType;
2341  std::vector<BindType> types(17, BindType::TEXT);
2342  if (!cd.default_value.has_value()) {
2343  types[16] = BindType::NULL_TYPE;
2344  }
2346  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, "
2347  "coldim, colscale, is_notnull, "
2348  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
2349  "virtual_expr, is_deletedcol, default_value) "
2350  "VALUES (?, ?, ?, ?, ?, "
2351  "?, "
2352  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2353  std::vector<std::string>{std::to_string(td.tableId),
2354  std::to_string(colId),
2355  cd.columnName,
2364  "",
2367  cd.virtualExpr,
2369  cd.default_value.value_or("NULL")},
2370  types);
2371  cd.tableId = td.tableId;
2372  cd.columnId = colId++;
2373  cds.push_back(cd);
2374  }
2375  if (td.isView) {
2377  "INSERT INTO mapd_views (tableid, sql) VALUES (?,?)",
2378  std::vector<std::string>{std::to_string(td.tableId), td.viewSQL});
2379  }
2381  auto& foreign_table = dynamic_cast<foreign_storage::ForeignTable&>(td);
2382  foreign_table.next_refresh_time = get_next_refresh_time(foreign_table);
2384  "INSERT INTO omnisci_foreign_tables (table_id, server_id, options, "
2385  "last_refresh_time, next_refresh_time) VALUES (?, ?, ?, ?, ?)",
2386  std::vector<std::string>{std::to_string(foreign_table.tableId),
2387  std::to_string(foreign_table.foreign_server->id),
2388  foreign_table.getOptionsAsJsonString(),
2389  std::to_string(foreign_table.last_refresh_time),
2390  std::to_string(foreign_table.next_refresh_time)});
2391  }
2392  } catch (std::exception& e) {
2393  sqliteConnector_.query("ROLLBACK TRANSACTION");
2394  throw;
2395  }
2396  } else { // Temporary table
2397  td.tableId = nextTempTableId_++;
2398  int colId = 1;
2399  for (auto cd : columns) {
2401  const bool is_foreign_col =
2402  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2403 
2404  if (!is_foreign_col) {
2405  // Create a new temporary dictionary
2406  std::string fileName("");
2407  std::string folderPath("");
2409  nextTempDictId_++;
2410  DictDescriptor dd(dict_ref,
2411  fileName,
2413  false,
2414  1,
2415  folderPath,
2416  true); // Is dictName (2nd argument) used?
2417  dds.push_back(dd);
2418  if (!cd.columnType.is_array()) {
2420  }
2421  cd.columnType.set_comp_param(dict_ref.dictId);
2422  }
2423  }
2424  if (toplevel_column_names.count(cd.columnName)) {
2425  // make up colId gap for sanity test (begin with 1 bc much code depends on it!)
2426  if (colId > 1) {
2427  colId += g_test_against_columnId_gap;
2428  }
2429  if (!cd.isGeoPhyCol) {
2430  td.columnIdBySpi_.push_back(colId);
2431  }
2432  }
2433  cd.tableId = td.tableId;
2434  cd.columnId = colId++;
2435  cds.push_back(cd);
2436  }
2437 
2439  serializeTableJsonUnlocked(&td, cds);
2440  }
2441  }
2442 
2443  try {
2444  auto cache = dataMgr_->getPersistentStorageMgr()->getDiskCache();
2445  if (cache) {
2446  CHECK(!cache->hasCachedMetadataForKeyPrefix({getCurrentDB().dbId, td.tableId}))
2447  << "Disk cache at " + cache->getCacheDirectory()
2448  << " contains preexisting data for new table. Please "
2449  "delete or clear cache before continuing";
2450  }
2451 
2452  addTableToMap(&td, cds, dds);
2453  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
2454  if (!td.storageType.empty() && td.storageType != StorageType::FOREIGN_TABLE) {
2455  dataMgr_->getForeignStorageInterface()->registerTable(this, td, cds);
2456  }
2457  } catch (std::exception& e) {
2458  sqliteConnector_.query("ROLLBACK TRANSACTION");
2459  removeTableFromMap(td.tableName, td.tableId, true);
2460  throw;
2461  }
2462  sqliteConnector_.query("END TRANSACTION");
2463 
2465  sqlite_lock.unlock();
2467  true); // cause instantiateFragmenter() to be called
2468  }
2469 }
int32_t maxRollbackEpochs
void serializeTableJsonUnlocked(const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
Definition: Catalog.cpp:2471
int64_t get_next_refresh_time(const foreign_storage::ForeignTable &foreign_table)
Definition: Catalog.cpp:2221
std::string virtualExpr
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:324
void set_size(int s)
Definition: sqltypes.h:421
std::string partitions
HOST DEVICE int get_size() const
Definition: sqltypes.h:333
T getData(const int row, const int col)
std::string tableName
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:631
const DBMetadata currentDB_
Definition: Catalog.h:632
HOST DEVICE int get_scale() const
Definition: sqltypes.h:328
std::string storageType
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:633
virtual void query(const std::string &queryString)
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:323
std::vector< int > columnIdBySpi_
std::string to_string(char const *&&v)
void addTableToMap(const TableDescriptor *td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
Definition: Catalog.cpp:1191
#define TEXT
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:222
specifies the content in-memory of a row in the column metadata table
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:2090
std::string keyMetainfo
int g_test_against_columnId_gap
Definition: Catalog.cpp:91
bool g_serialize_temp_tables
Definition: Catalog.cpp:98
void set_comp_param(int p)
Definition: sqltypes.h:424
std::optional< std::string > default_value
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:636
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:331
Fragmenter_Namespace::FragmenterType fragType
Data_Namespace::MemoryLevel persistenceLevel
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:325
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1251
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:332
bool setColumnSharedDictionary(ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:3265
#define CHECK(condition)
Definition: Logger.h:206
Descriptor for a dictionary for a string columne.
std::string viewSQL
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:330
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:3334
static constexpr char const * FOREIGN_TABLE
bool g_enable_fsi
Definition: Catalog.cpp:92
std::string columnName
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
bool is_array() const
Definition: sqltypes.h:506

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::delDictionary ( const ColumnDescriptor cd)

Definition at line 1866 of file Catalog.cpp.

References CHECK, CHECK_GT, ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, g_base_path, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), SqliteConnector::getData(), getMetadataForTable(), getObjForLock(), SQLTypeInfo::is_string(), SQLTypeInfo::is_string_array(), kENCODING_DICT, SqliteConnector::query_with_text_param(), File_Namespace::renameForDelete(), sqliteConnector_, string_dict_hosts_, ColumnDescriptor::tableId, to_string(), and VLOG.

Referenced by roll().

1866  {
1867  cat_write_lock write_lock(this);
1868  cat_sqlite_lock sqlite_lock(getObjForLock());
1869  if (!(cd.columnType.is_string() || cd.columnType.is_string_array())) {
1870  return;
1871  }
1872  if (!(cd.columnType.get_compression() == kENCODING_DICT)) {
1873  return;
1874  }
1875  const auto dictId = cd.columnType.get_comp_param();
1876  CHECK_GT(dictId, 0);
1877  // decrement and zero check dict ref count
1878  const auto td = getMetadataForTable(cd.tableId);
1879  CHECK(td);
1881  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
1882  std::to_string(dictId));
1884  "SELECT refcount FROM mapd_dictionaries WHERE dictid = ?", std::to_string(dictId));
1885  const auto refcount = sqliteConnector_.getData<int>(0, 0);
1886  VLOG(3) << "Dictionary " << dictId << "from dropped table has reference count "
1887  << refcount;
1888  if (refcount > 0) {
1889  return;
1890  }
1891  const DictRef dictRef(currentDB_.dbId, dictId);
1892  sqliteConnector_.query_with_text_param("DELETE FROM mapd_dictionaries WHERE dictid = ?",
1893  std::to_string(dictId));
1894  File_Namespace::renameForDelete(g_base_path + "/mapd_data/DB_" +
1895  std::to_string(currentDB_.dbId) + "_DICT_" +
1896  std::to_string(dictId));
1897 
1898  std::unique_ptr<StringDictionaryClient> client;
1899  if (!string_dict_hosts_.empty()) {
1900  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dictRef, true));
1901  }
1902  if (client) {
1903  client->drop(dictRef);
1904  }
1905 
1906  dictDescriptorMapByRef_.erase(dictRef);
1907 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:631
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:635
const DBMetadata currentDB_
Definition: Catalog.h:632
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:623
#define CHECK_GT(x, y)
Definition: Logger.h:218
std::string to_string(char const *&&v)
std::string g_base_path
Definition: SysCatalog.cpp:62
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:331
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:332
#define CHECK(condition)
Definition: Logger.h:206
mapd_unique_lock< mapd_shared_mutex > write_lock
SQLTypeInfo columnType
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
bool is_string() const
Definition: sqltypes.h:498
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:218
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
bool is_string_array() const
Definition: sqltypes.h:499
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
#define VLOG(n)
Definition: Logger.h:300

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::deleteCustomExpressions ( const std::vector< int32_t > &  custom_expression_ids,
bool  do_soft_delete 
)

Deletes custom expressions with the given ids.

Parameters
custom_expression_ids- ids of custom expressions to delete
do_soft_delete- flag indicating whether or not to do a soft delete

Definition at line 5050 of file Catalog.cpp.

References CHECK, CHECK_EQ, join(), and to_string().

5051  {
5052  cat_write_lock write_lock(this);
5053  cat_sqlite_lock sqlite_lock(getObjForLock());
5054 
5055  std::vector<int32_t> invalid_ids;
5056  for (const auto id : custom_expression_ids) {
5057  if (custom_expr_map_by_id_.find(id) == custom_expr_map_by_id_.end()) {
5058  invalid_ids.emplace_back(id);
5059  }
5060  }
5061  if (!invalid_ids.empty()) {
5062  throw std::runtime_error{"Custom expressions with ids: " + join(invalid_ids, ",") +
5063  " do not exist."};
5064  }
5065  sqliteConnector_.query("BEGIN TRANSACTION");
5066  try {
5067  for (const auto id : custom_expression_ids) {
5069  "SELECT id FROM omnisci_custom_expressions WHERE id = ?",
5070  std::vector<std::string>{std::to_string(id)});
5071  CHECK_EQ(sqliteConnector_.getNumRows(), static_cast<size_t>(1));
5072  if (do_soft_delete) {
5074  "UPDATE omnisci_custom_expressions SET is_deleted = ? WHERE id = ?",
5075  std::vector<std::string>{std::to_string(true), std::to_string(id)});
5076  } else {
5078  "DELETE FROM omnisci_custom_expressions WHERE id = ?",
5079  std::vector<std::string>{std::to_string(id)});
5080  }
5081  }
5082 
5083  for (const auto id : custom_expression_ids) {
5084  if (do_soft_delete) {
5085  auto it = custom_expr_map_by_id_.find(id);
5086  CHECK(it != custom_expr_map_by_id_.end());
5087  it->second->is_deleted = true;
5088  } else {
5089  custom_expr_map_by_id_.erase(id);
5090  }
5091  }
5092  } catch (std::exception& e) {
5093  sqliteConnector_.query("ROLLBACK TRANSACTION");
5094  throw;
5095  }
5096  sqliteConnector_.query("END TRANSACTION");
5097 }
#define CHECK_EQ(x, y)
Definition: Logger.h:214
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:631
std::string join(T const &container, std::string const &delim)
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:206
CustomExpressionMapById custom_expr_map_by_id_
Definition: Catalog.h:629
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::deleteMetadataForDashboards ( const std::vector< int32_t >  ids,
const UserMetadata user 
)

Definition at line 1636 of file Catalog.cpp.

References DashboardDBObjectType, dashboardDescriptorMap_, AccessPrivileges::DELETE_DASHBOARD, getMetadataForDashboard(), getObjForLock(), Catalog_Namespace::SysCatalog::instance(), SqliteConnector::query(), SqliteConnector::query_with_text_params(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivilegesFromAllBatch(), sqliteConnector_, and to_string().

1637  {
1638  std::stringstream invalid_ids, restricted_ids;
1639 
1640  for (int32_t dashboard_id : dashboard_ids) {
1641  if (!getMetadataForDashboard(dashboard_id)) {
1642  invalid_ids << (!invalid_ids.str().empty() ? ", " : "") << dashboard_id;
1643  continue;
1644  }
1645  DBObject object(dashboard_id, DashboardDBObjectType);
1646  object.loadKey(*this);
1647  object.setPrivileges(AccessPrivileges::DELETE_DASHBOARD);
1648  std::vector<DBObject> privs = {object};
1649  if (!SysCatalog::instance().checkPrivileges(user, privs))
1650  restricted_ids << (!restricted_ids.str().empty() ? ", " : "") << dashboard_id;
1651  }
1652 
1653  if (invalid_ids.str().size() > 0 || restricted_ids.str().size() > 0) {
1654  std::stringstream error_message;
1655  error_message << "Delete dashboard(s) failed with error(s):";
1656  if (invalid_ids.str().size() > 0)
1657  error_message << "\nDashboard id: " << invalid_ids.str()
1658  << " - Dashboard id does not exist";
1659  if (restricted_ids.str().size() > 0)
1660  error_message
1661  << "\nDashboard id: " << restricted_ids.str()
1662  << " - User should be either owner of dashboard or super user to delete it";
1663  throw std::runtime_error(error_message.str());
1664  }
1665  std::vector<DBObject> dash_objs;
1666 
1667  for (int32_t dashboard_id : dashboard_ids) {
1668  dash_objs.push_back(DBObject(dashboard_id, DashboardDBObjectType));
1669  }
1670  // BE-5245: Transactionally unsafe (like other combined Catalog/Syscatalog operations)
1672  {
1673  cat_write_lock write_lock(this);
1674  cat_sqlite_lock sqlite_lock(getObjForLock());
1675 
1676  sqliteConnector_.query("BEGIN TRANSACTION");
1677  try {
1678  for (int32_t dashboard_id : dashboard_ids) {
1679  auto dash = getMetadataForDashboard(dashboard_id);
1680  // Dash should still exist if revokeDBObjectPrivileges passed but throw and
1681  // rollback if already deleted
1682  if (!dash) {
1683  throw std::runtime_error(
1684  std::string("Delete dashboard(s) failed with error(s):\nDashboard id: ") +
1685  std::to_string(dashboard_id) + " - Dashboard id does not exist ");
1686  }
1687  std::string user_id = std::to_string(dash->userId);
1688  std::string dash_name = dash->dashboardName;
1689  auto viewDescIt = dashboardDescriptorMap_.find(user_id + ":" + dash_name);
1690  dashboardDescriptorMap_.erase(viewDescIt);
1692  "DELETE FROM mapd_dashboards WHERE name = ? and userid = ?",
1693  std::vector<std::string>{dash_name, user_id});
1694  }
1695  } catch (std::exception& e) {
1696  sqliteConnector_.query("ROLLBACK TRANSACTION");
1697  throw;
1698  }
1699  sqliteConnector_.query("END TRANSACTION");
1700  }
1701 }
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:631
virtual void query(const std::string &queryString)
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:624
std::string to_string(char const *&&v)
static SysCatalog & instance()
Definition: SysCatalog.h:293
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
static const AccessPrivileges DELETE_DASHBOARD
Definition: DBObject.h:175
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
void revokeDBObjectPrivilegesFromAllBatch(std::vector< DBObject > &objects, Catalog *catalog)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::doDropTable ( const TableDescriptor td)
protected

Definition at line 3563 of file Catalog.cpp.

References g_serialize_temp_tables, table_is_temporary(), and TableDescriptor::tableName.

3563  {
3567  }
3569 }
void executeDropTableSqliteQueries(const TableDescriptor *td)
Definition: Catalog.cpp:3571
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:4288
std::string tableName
void dropTableFromJsonUnlocked(const std::string &table_name) const
Definition: Catalog.cpp:2533
bool g_serialize_temp_tables
Definition: Catalog.cpp:98
bool table_is_temporary(const TableDescriptor *const td)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::doTruncateTable ( const TableDescriptor td)
protected

Definition at line 3430 of file Catalog.cpp.

References CHECK, CHECK_GE, ColumnDescriptor::columnType, Data_Namespace::CPU_LEVEL, dict_ref_t::dictId, DictDescriptor::dictIsTemp, DictDescriptor::dictRef, TableDescriptor::fragmenter, SQLTypeInfo::get_comp_param(), SQLTypeInfo::get_compression(), Data_Namespace::GPU_LEVEL, kENCODING_DICT, File_Namespace::renameForDelete(), ColumnDescriptor::tableId, and TableDescriptor::tableId.

3430  {
3431  cat_write_lock write_lock(this);
3432 
3433  const int tableId = td->tableId;
3434  // must destroy fragmenter before deleteChunks is called.
3435  if (td->fragmenter != nullptr) {
3436  auto tableDescIt = tableDescriptorMapById_.find(tableId);
3437  CHECK(tableDescIt != tableDescriptorMapById_.end());
3438  tableDescIt->second->fragmenter = nullptr;
3439  CHECK(td->fragmenter == nullptr);
3440  }
3441  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
3442  // assuming deleteChunksWithPrefix is atomic
3443  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
3444  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
3445 
3446  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
3447 
3448  std::unique_ptr<StringDictionaryClient> client;
3449  if (SysCatalog::instance().isAggregator()) {
3450  CHECK(!string_dict_hosts_.empty());
3451  DictRef dict_ref(currentDB_.dbId, -1);
3452  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
3453  }
3454  // clean up any dictionaries
3455  // delete all column descriptors for the table
3456  for (const auto& columnDescriptor : columnDescriptorMapById_) {
3457  auto cd = columnDescriptor.second;
3458  if (cd->tableId != td->tableId) {
3459  continue;
3460  }
3461  const int dict_id = cd->columnType.get_comp_param();
3462  // Dummy dictionaries created for a shard of a logical table have the id set to zero.
3463  if (cd->columnType.get_compression() == kENCODING_DICT && dict_id) {
3464  const DictRef dict_ref(currentDB_.dbId, dict_id);
3465  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
3466  CHECK(dictIt != dictDescriptorMapByRef_.end());
3467  const auto& dd = dictIt->second;
3468  CHECK_GE(dd->refcount, 1);
3469  // if this is the only table using this dict reset the dict
3470  if (dd->refcount == 1) {
3471  // close the dictionary
3472  dd->stringDict.reset();
3473  File_Namespace::renameForDelete(dd->dictFolderPath);
3474  if (client) {
3475  client->drop(dd->dictRef);
3476  }
3477  if (!dd->dictIsTemp) {
3478  boost::filesystem::create_directory(dd->dictFolderPath);
3479  }
3480  }
3481 
3482  DictDescriptor* new_dd = new DictDescriptor(dd->dictRef,
3483  dd->dictName,
3484  dd->dictNBits,
3485  dd->dictIsShared,
3486  dd->refcount,
3487  dd->dictFolderPath,
3488  dd->dictIsTemp);
3489  dictDescriptorMapByRef_.erase(dictIt);
3490  // now create new Dict -- need to figure out what to do here for temp tables
3491  if (client) {
3492  client->create(new_dd->dictRef, new_dd->dictIsTemp);
3493  }
3494  dictDescriptorMapByRef_[new_dd->dictRef].reset(new_dd);
3496  }
3497  }
3498 }
std::vector< int > ChunkKey
Definition: types.h:37
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:635
const DBMetadata currentDB_
Definition: Catalog.h:632
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:623
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:633
#define CHECK_GE(x, y)
Definition: Logger.h:219
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:622
static SysCatalog & instance()
Definition: SysCatalog.h:293
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:620
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1522
int32_t dictId
Definition: DictRef.h:10
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:206
Descriptor for a dictionary for a string columne.
mapd_unique_lock< mapd_shared_mutex > write_lock
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:218

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropColumn ( const TableDescriptor td,
const ColumnDescriptor cd 
)

Definition at line 2001 of file Catalog.cpp.

References CHECK, columnDescriptorMap_, columnDescriptorMapById_, columnDescriptorsForRoll, ColumnDescriptor::columnId, ColumnDescriptor::columnName, getMetadataForColumn(), getObjForLock(), getPhysicalTablesDescriptors(), TableDescriptor::nShards, SqliteConnector::query_with_text_params(), TableDescriptor::shard, sqliteConnector_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), and to_upper().

2001  {
2002  cat_write_lock write_lock(this);
2003  cat_sqlite_lock sqlite_lock(getObjForLock());
2004  // caller must handle sqlite/chunk transaction TOGETHER
2006  "DELETE FROM mapd_columns where tableid = ? and columnid = ?",
2007  std::vector<std::string>{std::to_string(td.tableId), std::to_string(cd.columnId)});
2008 
2010  "UPDATE mapd_tables SET ncolumns = ncolumns - 1 WHERE tableid = ?",
2011  std::vector<std::string>{std::to_string(td.tableId)});
2012 
2013  ColumnDescriptorMap::iterator columnDescIt =
2015  CHECK(columnDescIt != columnDescriptorMap_.end());
2016 
2017  columnDescriptorsForRoll.emplace_back(columnDescIt->second, nullptr);
2018 
2019  columnDescriptorMap_.erase(columnDescIt);
2021  --tableDescriptorMapById_[td.tableId]->nColumns;
2022  // for each shard
2023  if (td.nShards > 0 && td.shard < 0) {
2024  for (const auto shard : getPhysicalTablesDescriptors(&td)) {
2025  const auto shard_cd = getMetadataForColumn(shard->tableId, cd.columnId);
2026  CHECK(shard_cd);
2027  dropColumn(*shard, *shard_cd);
2028  }
2029  }
2030 }
std::tuple< int, std::string > ColumnKey
Definition: Types.h:37
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:621
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:631
void dropColumn(const TableDescriptor &td, const ColumnDescriptor &cd)
Definition: Catalog.cpp:2001
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:622
std::string to_string(char const *&&v)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:620
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logical_table_desc, bool populate_fragmenter=true) const
Definition: Catalog.cpp:4119
std::string to_upper(const std::string &str)
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:206
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:650
std::string columnName
std::tuple< int, int > ColumnIdKey
Definition: Types.h:39

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropForeignServer ( const std::string &  server_name)

Drops/deletes a foreign server DB object.

Parameters
server_name- Name of foreign server that will be deleted

Definition at line 2726 of file Catalog.cpp.

References CHECK_EQ, and to_string().

2726  {
2727  cat_write_lock write_lock(this);
2728  cat_sqlite_lock sqlite_lock(getObjForLock());
2729 
2731  "SELECT id from omnisci_foreign_servers where name = ?",
2732  std::vector<std::string>{server_name});
2733  auto num_rows = sqliteConnector_.getNumRows();
2734  if (num_rows > 0) {
2735  CHECK_EQ(size_t(1), num_rows);
2736  auto server_id = sqliteConnector_.getData<int32_t>(0, 0);
2738  "SELECT table_id from omnisci_foreign_tables where server_id = ?",
2739  std::to_string(server_id));
2740  if (sqliteConnector_.getNumRows() > 0) {
2741  throw std::runtime_error{"Foreign server \"" + server_name +
2742  "\" is referenced "
2743  "by existing foreign tables and cannot be dropped."};
2744  }
2745  sqliteConnector_.query("BEGIN TRANSACTION");
2746  try {
2748  "DELETE FROM omnisci_foreign_servers WHERE name = ?",
2749  std::vector<std::string>{server_name});
2750  } catch (const std::exception& e) {
2751  sqliteConnector_.query("ROLLBACK TRANSACTION");
2752  throw;
2753  }
2754  sqliteConnector_.query("END TRANSACTION");
2755  foreignServerMap_.erase(server_name);
2756  foreignServerMapById_.erase(server_id);
2757  }
2758 }
#define CHECK_EQ(x, y)
Definition: Logger.h:214
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:631
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
ForeignServerMapById foreignServerMapById_
Definition: Catalog.h:628
ForeignServerMap foreignServerMap_
Definition: Catalog.h:627
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropTable ( const TableDescriptor td)

Definition at line 3530 of file Catalog.cpp.

References CHECK, i, TableDescriptor::isView, TableDBObjectType, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and ViewDBObjectType.

3530  {
3533  cat_write_lock write_lock(this);
3534  cat_sqlite_lock sqlite_lock(getObjForLock());
3535  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
3536  sqliteConnector_.query("BEGIN TRANSACTION");
3537  try {
3538  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
3539  // remove all corresponding physical tables if this is a logical table
3540  const auto physicalTables = physicalTableIt->second;
3541  CHECK(!physicalTables.empty());
3542  for (size_t i = 0; i < physicalTables.size(); i++) {
3543  int32_t physical_tb_id = physicalTables[i];
3544  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
3545  CHECK(phys_td);
3546  doDropTable(phys_td);
3547  }
3548 
3549  // remove corresponding record from the logicalToPhysicalTableMap in sqlite database
3551  "DELETE FROM mapd_logical_to_physical WHERE logical_table_id = ?",
3552  std::to_string(td->tableId));
3554  }
3555  doDropTable(td);
3556  } catch (std::exception& e) {
3557  sqliteConnector_.query("ROLLBACK TRANSACTION");
3558  throw;
3559  }
3560  sqliteConnector_.query("END TRANSACTION");
3561 }
void doDropTable(const TableDescriptor *td)
Definition: Catalog.cpp:3563
std::string tableName
SqliteConnector sqliteConnector_
Definition: Catalog.h:631
void revokeDBObjectPrivilegesFromAll(DBObject object, Catalog *catalog)
virtual void query(const std::string &queryString)
std::string to_string(char const *&&v)
static SysCatalog & instance()
Definition: SysCatalog.h:293
const Catalog * getObjForLock()
Definition: Catalog.cpp:221
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
#define CHECK(condition)
Definition: Logger.h:206
mapd_unique_lock< mapd_shared_mutex > write_lock
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:115
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:638

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::dropTableFromJsonUnlocked ( const std::string &  table_name) const
protected

Definition at line 2533 of file Catalog.cpp.

References CHECK, test_fsi::d, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and VLOG.

2533  {
2534  // relies on the catalog write lock
2535  using namespace rapidjson;
2536 
2537  VLOG(1) << "Dropping temporary table " << table_name << " to JSON for Calcite.";
2538 
2539  const auto db_name = currentDB_.dbName;
2540  const auto file_path = table_json_filepath(basePath_, db_name);
2541 
2542  CHECK(boost::filesystem::exists(file_path));
2543  Document d;
2544 
2545  std::ifstream reader(file_path.string());
2546  CHECK(reader.is_open());
2547  IStreamWrapper json_read_wrapper(reader);
2548  d.ParseStream(json_read_wrapper);
2549 
2550  CHECK(d.IsObject());
2551  auto table_name_ref = StringRef(table_name.c_str());
2552  CHECK(d.HasMember(table_name_ref));
2553  CHECK(d.RemoveMember(table_name_ref));
2554 
2555  // Overwrite the existing file
2556  std::ofstream writer(file_path.string(), std::ios::trunc | std::ios::out);
2557  CHECK(writer.is_open());
2558  OStreamWrapper json_wrapper(writer);
2559 
2560  Writer<OStreamWrapper> json_writer(json_wrapper);
2561  d.Accept(json_writer);
2562  writer.close();
2563 }
tuple d
Definition: test_fsi.py:9
const DBMetadata currentDB_
Definition: Catalog.h:632
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:154
#define CHECK(condition)
Definition: Logger.h:206
#define VLOG(n)
Definition: Logger.h:300

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::dumpCreateTable ( const TableDescriptor td,
bool  multiline_formatting = true,
bool  dump_defaults = false 
) const

Definition at line 4589 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, Data_Namespace::CPU_LEVEL, DEFAULT_FRAGMENT_ROWS, DEFAULT_MAX_CHUNK_SIZE, DEFAULT_MAX_ROLLBACK_EPOCHS, DEFAULT_MAX_ROWS, DEFAULT_PAGE_SIZE, TableDescriptor::fragPageSize, g_leaf_count, TableDescriptor::hasDeletedCol, ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, join(), kCHAR, kENCODING_DICT, kENCODING_GEOINT, kTEXT, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and TableDescriptor::viewSQL.

4591  {
4592  cat_read_lock read_lock(this);
4593 
4594  auto foreign_table = dynamic_cast<const foreign_storage::ForeignTable*>(td);
4595  std::ostringstream os;
4596 
4597  if (foreign_table) {
4598  os << "CREATE FOREIGN TABLE " << td->tableName << " (";
4599  } else if (!td->isView) {
4600  os << "CREATE ";
4602  os << "TEMPORARY ";
4603  }
4604  os << "TABLE " + td->tableName + " (";
4605  } else {
4606  os << "CREATE VIEW " + td->tableName + " AS " << td->viewSQL;
4607  return os.str();
4608  }
4609  // scan column defines
4610  std::vector<std::string> additional_info;
4611  std::set<std::string> shared_dict_column_names;
4612 
4613  gatherAdditionalInfo(additional_info, shared_dict_column_names, td);
4614 
4615  // gather column defines
4616  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
4617  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
4618  bool first = true;
4619  for (const auto cd : cds) {
4620  if (!(cd->isSystemCol || cd->isVirtualCol)) {
4621  const auto& ti = cd->columnType;
4622  if (!first) {
4623  os << ",";
4624  if (!multiline_formatting) {
4625  os << " ";
4626  }
4627  } else {
4628  first = false;
4629  }
4630  if (multiline_formatting) {
4631  os << "\n ";
4632  }
4633  // column name
4634  os << quoteIfRequired(cd->columnName);
4635  // CHAR is perculiar... better dump it as TEXT(32) like \d does
4636  if (ti.get_type() == SQLTypes::kCHAR) {
4637  os << " "
4638  << "TEXT";
4639  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
4640  os << " "
4641  << "TEXT[]";
4642  } else {
4643  os << " " << ti.get_type_name();
4644  }
4645  os << (ti.get_notnull() ? " NOT NULL" : "");
4646  if (shared_dict_column_names.find(cd->columnName) ==
4647  shared_dict_column_names.end()) {
4648  // avoids "Column ... shouldn't specify an encoding, it borrows it
4649  // from the referenced column"
4650  if (ti.is_string() || (ti.is_array() && ti.get_subtype() == kTEXT)) {
4651  auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
4652  if (ti.get_compression() == kENCODING_DICT) {
4653  os << " ENCODING " << ti.get_compression_name() << "(" << (size * 8) << ")";
4654  } else {
4655  os << " ENCODING NONE";
4656  }
4657  } else if (ti.is_date_in_days() ||
4658  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
4659  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
4660  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
4661  } else if (ti.is_geometry()) {
4662  if (ti.get_compression() == kENCODING_GEOINT) {
4663  os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
4664  << ")";
4665  } else {
4666  os << " ENCODING NONE";
4667  }
4668  }
4669  }
4670  }
4671  }
4672  // gather SHARED DICTIONARYs
4673  if (additional_info.size()) {
4674  std::string comma;
4675  if (!multiline_formatting) {
4676  comma = ", ";
4677  } else {
4678  comma = ",\n ";
4679  }
4680  os << comma;
4681  os << boost::algorithm::join(additional_info, comma);
4682  }
4683  os << ")";
4684 
4685  std::vector<std::string> with_options;
4686  if (foreign_table) {
4687  if (multiline_formatting) {
4688  os << "\n";
4689  } else {
4690  os << " ";
4691  }
4692  os << "SERVER " << foreign_table->foreign_server->name;
4693 
4694  // gather WITH options ...
4695  for (const auto& [option, value] : foreign_table->options) {
4696  with_options.emplace_back(option + "='" + value + "'");
4697  }
4698  }
4699 
4700  if (dump_defaults || td->maxFragRows != DEFAULT_FRAGMENT_ROWS) {
4701  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
4702  }
4703  if (!foreign_table && (dump_defaults || td->maxChunkSize != DEFAULT_MAX_CHUNK_SIZE)) {
4704  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
4705  }
4706  if (!foreign_table && (dump_defaults || td->fragPageSize != DEFAULT_PAGE_SIZE)) {
4707  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
4708  }
4709  if (!foreign_table && (dump_defaults || td->maxRows != DEFAULT_MAX_ROWS)) {
4710  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
4711  }
4712  if ((dump_defaults || td->maxRollbackEpochs != DEFAULT_MAX_ROLLBACK_EPOCHS) &&
4713  td->maxRollbackEpochs != -1) {
4714  with_options.push_back("MAX_ROLLBACK_EPOCHS=" +
4716  }
4717  if (!foreign_table && (dump_defaults || !td->hasDeletedCol)) {
4718  with_options.push_back(td->hasDeletedCol ? "VACUUM='DELAYED'" : "VACUUM='IMMEDIATE'");
4719  }
4720  if (!foreign_table && !td->partitions.empty()) {
4721  with_options.push_back("PARTITIONS='" + td->partitions + "'");
4722  }
4723  if (!foreign_table && td->nShards > 0) {
4724  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
4725  CHECK(shard_cd);
4726  with_options.push_back(
4727  "SHARD_COUNT=" +
4728  std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
4729  }
4730  if (!foreign_table && td->sortedColumnId > 0) {
4731  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
4732  CHECK(sort_cd);
4733  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
4734  }
4735 
4736  if (!with_options.empty()) {
4737  if (!multiline_formatting) {
4738  os << " ";
4739  } else {
4740  os << "\n";
4741  }
4742  os << "WITH (" + boost::algorithm::join(with_options, ", ") + ")";
4743  }
4744  os << ";";
4745  return os.str();
4746 }
int32_t maxRollbackEpochs
std::string partitions
std::string tableName
std::string join(T const &container, std::string const &delim)
#define DEFAULT_MAX_CHUNK_SIZE
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:113
std::string to_string(char const *&&v)
std::string quoteIfRequired(const std::string &column_name) const
Definition: Catalog.cpp:4848
#define DEFAULT_MAX_ROWS
void gatherAdditionalInfo(std::vector< std::string > &additional_info, std::set< std::string > &shared_dict_column_names, const TableDescriptor *td) const
Definition: Catalog.cpp:4859
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
#define DEFAULT_MAX_ROLLBACK_EPOCHS
specifies the content in-memory of a row in the table metadata table
#define DEFAULT_PAGE_SIZE
Definition: sqltypes.h:51
#define DEFAULT_FRAGMENT_ROWS
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1799
Data_Namespace::MemoryLevel persistenceLevel
Definition: sqltypes.h:40
mapd_shared_lock< mapd_shared_mutex > read_lock
#define CHECK(condition)
Definition: Logger.h:206
std::string viewSQL
size_t g_leaf_count
Definition: ParserNode.cpp:76

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::dumpSchema ( const TableDescriptor td) const

Definition at line 4466 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, DEFAULT_MAX_ROLLBACK_EPOCHS, TableDescriptor::fragPageSize, g_leaf_count, TableDescriptor::hasDeletedCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, join(), kCHAR, kENCODING_DICT, kENCODING_GEOINT, kTEXT, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRollbackEpochs, TableDescriptor::maxRows, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::tableId, and to_string().

Referenced by TableArchiver::dumpTable().

4466  {
4467  cat_read_lock read_lock(this);
4468 
4469  std::ostringstream os;
4470  os << "CREATE TABLE @T (";
4471  // gather column defines
4472  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
4473  std::string comma;
4474  std::vector<std::string> shared_dicts;
4475  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
4476  for (const auto cd : cds) {
4477  if (!(cd->isSystemCol || cd->isVirtualCol)) {
4478  const auto& ti = cd->columnType;
4479  os << comma << cd->columnName;
4480  // CHAR is perculiar... better dump it as TEXT(32) like \d does
4481  if (ti.get_type() == SQLTypes::kCHAR) {
4482  os << " "
4483  << "TEXT";
4484  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
4485  os << " "
4486  << "TEXT[]";
4487  } else {
4488  os << " " << ti.get_type_name();
4489  }
4490  os << (ti.get_notnull() ? " NOT NULL" : "");
4491  if (ti.is_string() || (ti.is_array() && ti.get_subtype() == kTEXT)) {
4492  auto size = ti.is_array() ? ti.get_logical_size() : ti.get_size();
4493  if (ti.get_compression() == kENCODING_DICT) {
4494  // if foreign reference, get referenced tab.col
4495  const auto dict_id = ti.get_comp_param();
4496  const DictRef dict_ref(currentDB_.dbId, dict_id);
4497  const auto dict_it = dictDescriptorMapByRef_.find(dict_ref);
4498  CHECK(dict_it != dictDescriptorMapByRef_.end());
4499  const auto dict_name = dict_it->second->dictName;
4500  // when migrating a table, any foreign dict ref will be dropped
4501  // and the first cd of a dict will become root of the dict
4502  if (dict_root_cds.end() == dict_root_cds.find(dict_name)) {
4503  dict_root_cds[dict_name] = cd;
4504  os << " ENCODING " << ti.get_compression_name() << "(" << (size * 8) << ")";
4505  } else {
4506  const auto dict_root_cd = dict_root_cds[dict_name];
4507  shared_dicts.push_back("SHARED DICTIONARY (" + cd->columnName +
4508  ") REFERENCES @T(" + dict_root_cd->columnName + ")");
4509  // "... shouldn't specify an encoding, it borrows from the referenced
4510  // column"
4511  }
4512  } else {
4513  os << " ENCODING NONE";
4514  }
4515  } else if (ti.is_date_in_days() ||
4516  (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size())) {
4517  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
4518  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
4519  } else if (ti.is_geometry()) {
4520  if (ti.get_compression() == kENCODING_GEOINT) {
4521  os << " ENCODING " << ti.get_compression_name() << "(" << ti.get_comp_param()
4522  << ")";
4523  } else {
4524  os << " ENCODING NONE";
4525  }
4526  }
4527  comma = ", ";
4528  }
4529  }
4530  // gather SHARED DICTIONARYs
4531  if (shared_dicts.size()) {
4532  os << ", " << boost::algorithm::join(shared_dicts, ", ");
4533  }
4534  // gather WITH options ...
4535  std::vector<std::string> with_options;
4536  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
4537  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
4538  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
4539  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
4540  with_options.emplace_back(td->hasDeletedCol ? "VACUUM='DELAYED'"
4541  : "VACUUM='IMMEDIATE'");
4542  if (!td->partitions.empty()) {
4543  with_options.push_back("PARTITIONS='" + td->partitions + "'");
4544  }
4545  if (td->nShards > 0) {
4546  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
4547  CHECK(shard_cd);
4548  os << ", SHARD KEY(" << shard_cd->columnName << ")";
4549  with_options.push_back(
4550  "SHARD_COUNT=" +
4551  std::to_string(td->nShards * std::max(g_leaf_count, static_cast<size_t>(1))));
4552  }
4553  if (td->sortedColumnId > 0) {
4554  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
4555  CHECK(sort_cd);
4556  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
4557  }
4559  td->maxRollbackEpochs != -1) {
4560  with_options.push_back("MAX_ROLLBACK_EPOCHS=" +
4562  }
4563  os << ") WITH (" + boost::algorithm::join(with_options, ", ") + ");";
4564  return os.str();
4565 }
int32_t maxRollbackEpochs
std::string partitions
const DBMetadata currentDB_
Definition: Catalog.h:632
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:623
std::string join(T const &container, std::string const &delim)
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:113
std::string to_string(char const *&&v)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
#define DEFAULT_MAX_ROLLBACK_EPOCHS
specifies the content in-memory of a row in the table metadata table
Definition: sqltypes.h:51
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1799
Definition: sqltypes.h:40
mapd_shared_lock< mapd_shared_mutex > read_lock
#define CHECK(condition)
Definition: Logger.h:206
size_t g_leaf_count
Definition: ParserNode.cpp:76

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::eraseDBData ( )

Definition at line 4276 of file Catalog.cpp.

References test_readcsv::table, and run_benchmark_import::tables.

4276  {
4277  cat_write_lock write_lock(this);
4278  // Physically erase all tables and dictionaries from disc and memory
4279  const auto tables = getAllTableMetadata();
4280  for (const auto table : tables) {
4282  }
4283  // Physically erase database metadata
4284  boost::filesystem::remove(basePath_ + "/mapd_catalogs/" + currentDB_.dbName);
4285  calciteMgr_->updateMetadata(currentDB_.dbName, "");
4286 }
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:4288
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:1825
const DBMetadata currentDB_
Definition: Catalog.h:632
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:636
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:114
mapd_unique_lock< mapd_shared_mutex > write_lock
void Catalog_Namespace::Catalog::eraseTablePhysicalData ( const TableDescriptor td)

Definition at line 4288 of file Catalog.cpp.

References CHECK, Data_Namespace::CPU_LEVEL, TableDescriptor::fragmenter, Data_Namespace::GPU_LEVEL, INJECT_TIMER, TableDescriptor::isView, TableDescriptor::tableId, and TableDescriptor::tableName.

4288  {
4289  const int tableId = td->tableId;
4290  // must destroy fragmenter before deleteChunks is called.
4291  if (td->fragmenter != nullptr) {
4292  auto tableDescIt = tableDescriptorMapById_.find(tableId);
4293  CHECK(tableDescIt != tableDescriptorMapById_.end());
4294  {
4295  INJECT_TIMER(deleting_fragmenter);
4296  tableDescIt->second->fragmenter = nullptr;
4297  }
4298  }
4299  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
4300  {
4301  INJECT_TIMER(deleteChunksWithPrefix);
4302  // assuming deleteChunksWithPrefix is atomic
4303  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
4304  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
4305  }
4306  if (!td->isView) {
4307  INJECT_TIMER(Remove_Table);
4308  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
4309  }
4310  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
4311  {
4312  INJECT_TIMER(removeTableFromMap_);
4313  removeTableFromMap(td->tableName, tableId);
4314  }
4315 }
std::vector< int > ChunkKey
Definition: types.h:37
std::string tableName
const DBMetadata currentDB_
Definition: Catalog.h:632
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:633
#define INJECT_TIMER(DESC)
Definition: measure.h:93
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:620
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:636
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1251
#define CHECK(condition)
Definition: Logger.h:206
void Catalog_Namespace::Catalog::executeDropTableSqliteQueries ( const TableDescriptor td)
protected

Definition at line 3571 of file Catalog.cpp.

References StorageType::FOREIGN_TABLE, TableDescriptor::isView, kENCODING_DICT, test_fsi::r, TableDescriptor::storageType, TableDescriptor::tableId, and to_string().

3571  {
3572  const int tableId = td->tableId;
3573  sqliteConnector_.query_with_text_param("DELETE FROM mapd_tables WHERE tableid = ?",
3574  std::to_string(tableId));
3576  "select comp_param from mapd_columns where compression = ? and tableid = ?",
3577  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
3578  int numRows = sqliteConnector_.getNumRows();
3579  std::vector<int> dict_id_list;
3580  for (int r = 0; r < numRows; ++r) {
3581  dict_id_list.push_back(sqliteConnector_.getData<int>(r, 0));
3582  }
3583  for (auto dict_id : dict_id_list) {
3585  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
3586  std::vector<std::string>{std::to_string(dict_id)});
3587  }
3589  "DELETE FROM mapd_dictionaries WHERE dictid in (select comp_param from "
3590  "mapd_columns where compression = ? "
3591  "and tableid = ?) and refcount = 0",
3592  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
3593  sqliteConnector_.query_with_text_param("DELETE FROM mapd_columns WHERE tableid = ?",
3594  std::to_string(tableId));
3595  if (td->isView) {
3596  sqliteConnector_.query_with_text_param("DELETE FROM mapd_views WHERE tableid = ?",
3597  std::to_string(tableId));
3598  }
3601  "DELETE FROM omnisci_foreign_tables WHERE table_id = ?", std::to_string(tableId));
3602  }
3603 }
T getData(const int row, const int col)
virtual void query_with_text_params(std::string const &query_only)
SqliteConnector sqliteConnector_
Definition: Catalog.h:631
std::string storageType
tuple r
Definition: test_fsi.py:16
std::string to_string(char const *&&v)
static constexpr char const * FOREIGN_TABLE
virtual void query_with_text_param(const std::string &queryString, const std::string &text_param)
virtual size_t getNumRows() const

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::expandGeoColumn ( const ColumnDescriptor cd,
std::list< ColumnDescriptor > &  columns 
)
static

Definition at line 2090 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, IS_GEO, kARRAY, kDOUBLE, kENCODING_GEOINT, kENCODING_NONE, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kTINYINT, SQLTypeInfo::set_size(), and SQLTypeInfo::set_subtype().

Referenced by createTable().

2091  {
2092  const auto& col_ti = cd.columnType;
2093  if (IS_GEO(col_ti.get_type())) {
2094  switch (col_ti.get_type()) {
2095  case kPOINT: {
2096  ColumnDescriptor physical_cd_coords(true);
2097  physical_cd_coords.columnName = cd.columnName + "_coords";
2098  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2099  // Raw data: compressed/uncompressed coords
2100  coords_ti.set_subtype(kTINYINT);
2101  size_t unit_size;
2102  if (col_ti.get_compression() == kENCODING_GEOINT &&
2103  col_ti.get_comp_param() == 32) {
2104  unit_size = 4 * sizeof(int8_t);
2105  } else {
2106  CHECK(col_ti.get_compression() == kENCODING_NONE);
2107  unit_size = 8 * sizeof(int8_t);
2108  }
2109  coords_ti.set_size(2 * unit_size);
2110  physical_cd_coords.columnType = coords_ti;
2111  columns.push_back(physical_cd_coords);
2112 
2113  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2114 
2115  break;
2116  }
2117  case kLINESTRING: {
2118  ColumnDescriptor physical_cd_coords(true);
2119  physical_cd_coords.columnName = cd.columnName + "_coords";
2120  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2121  // Raw data: compressed/uncompressed coords
2122  coords_ti.set_subtype(kTINYINT);
2123  physical_cd_coords.columnType = coords_ti;
2124  columns.push_back(physical_cd_coords);
2125 
2126  ColumnDescriptor physical_cd_bounds(true);
2127  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2128  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2129  bounds_ti.set_subtype(kDOUBLE);
2130  bounds_ti.set_size(4 * sizeof(double));
2131  physical_cd_bounds.columnType = bounds_ti;
2132  columns.push_back(physical_cd_bounds);
2133 
2134  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2135 
2136  break;
2137  }
2138  case kPOLYGON: {
2139  ColumnDescriptor physical_cd_coords(true);
2140  physical_cd_coords.columnName = cd.columnName + "_coords";
2141  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2142  // Raw data: compressed/uncompressed coords
2143  coords_ti.set_subtype(kTINYINT);
2144  physical_cd_coords.columnType = coords_ti;
2145  columns.push_back(physical_cd_coords);
2146 
2147  ColumnDescriptor physical_cd_ring_sizes(true);
2148  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
2149  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2150  ring_sizes_ti.set_subtype(kINT);
2151  physical_cd_ring_sizes.columnType = ring_sizes_ti;
2152  columns.push_back(physical_cd_ring_sizes);
2153 
2154  ColumnDescriptor physical_cd_bounds(true);
2155  physical_cd_bounds.columnName = cd.columnName + "_bounds";
2156  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2157  bounds_ti.set_subtype(kDOUBLE);
2158  bounds_ti.set_size(4 * sizeof(double));
2159  physical_cd_bounds.columnType = bounds_ti;
2160  columns.push_back(physical_cd_bounds);
2161 
2162  ColumnDescriptor physical_cd_render_group(true);
2163  physical_cd_render_group.columnName = cd.columnName + "_render_group";
2164  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, col_ti.get_notnull());
2165  physical_cd_render_group.columnType = render_group_ti;
2166  columns.push_back(physical_cd_render_group);
2167 
2168  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
2169 
2170  break;
2171  }
2172  case kMULTIPOLYGON: {
2173  ColumnDescriptor physical_cd_coords(true);
2174  physical_cd_coords.columnName = cd.columnName + "_coords";
2175  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, col_ti.get_notnull());
2176  // Raw data: compressed/uncompressed coords
2177  coords_ti.set_subtype(kTINYINT);
2178  physical_cd_coords.columnType = coords_ti;