OmniSciDB  04ee39c94c
Catalog_Namespace::Catalog Class Reference

class for a per-database catalog. also includes metadata for the current database and the current user. More...

#include <Catalog.h>

+ Collaboration diagram for Catalog_Namespace::Catalog:

Public Member Functions

 Catalog (const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
 Constructor - takes basePath to already extant data directory for writing. More...
 
virtual ~Catalog ()
 Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite. More...
 
void createTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
 
void createShardedTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
int32_t createDashboard (DashboardDescriptor &vd)
 
void replaceDashboard (DashboardDescriptor &vd)
 
std::string createLink (LinkDescriptor &ld, size_t min_length)
 
void dropTable (const TableDescriptor *td)
 
void truncateTable (const TableDescriptor *td)
 
void renameTable (const TableDescriptor *td, const std::string &newTableName)
 
void renameColumn (const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
 
void addColumn (const TableDescriptor &td, ColumnDescriptor &cd)
 
void removeChunks (const int table_id)
 
const TableDescriptorgetMetadataForTable (const std::string &tableName, const bool populateFragmenter=true) const
 Returns a pointer to a const TableDescriptor struct matching the provided tableName. More...
 
const TableDescriptorgetMetadataForTableImpl (int tableId, const bool populateFragmenter) const
 
const TableDescriptorgetMetadataForTable (int tableId) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, const std::string &colName) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, int columnId) const
 
const int getColumnIdBySpi (const int tableId, const size_t spi) const
 
const ColumnDescriptorgetMetadataForColumnBySpi (const int tableId, const size_t spi) const
 
const DashboardDescriptorgetMetadataForDashboard (const std::string &userId, const std::string &dashName) const
 
void deleteMetadataForDashboard (const std::string &userId, const std::string &dashName)
 
const DashboardDescriptorgetMetadataForDashboard (const int32_t dashboard_id) const
 
void deleteMetadataForDashboard (const int32_t dashboard_id)
 
const LinkDescriptorgetMetadataForLink (const std::string &link) const
 
const LinkDescriptorgetMetadataForLink (int linkId) const
 
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable (const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular table specified by table id. More...
 
std::list< const TableDescriptor * > getAllTableMetadata () const
 
std::list< const DashboardDescriptor * > getAllDashboardsMetadata () const
 
const DBMetadatagetCurrentDB () const
 
Data_Namespace::DataMgrgetDataMgr () const
 
std::shared_ptr< CalcitegetCalciteMgr () const
 
const std::string & getBasePath () const
 
const DictDescriptorgetMetadataForDict (int dict_ref, bool loadDict=true) const
 
const std::vector< LeafHostInfo > & getStringDictionaryHosts () const
 
const ColumnDescriptorgetShardColumnMetadataForTable (const TableDescriptor *td) const
 
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors (const TableDescriptor *logicalTableDesc) const
 
int32_t getTableEpoch (const int32_t db_id, const int32_t table_id) const
 
void setTableEpoch (const int db_id, const int table_id, const int new_epoch)
 
int getDatabaseId () const
 
SqliteConnectorgetSqliteConnector ()
 
void roll (const bool forward)
 
DictRef addDictionary (ColumnDescriptor &cd)
 
void delDictionary (const ColumnDescriptor &cd)
 
void getDictionary (const ColumnDescriptor &cd, std::map< int, StringDictionary *> &stringDicts)
 
const bool checkMetadataForDeletedRecs (int dbId, int tableId, int columnId) const
 
const ColumnDescriptorgetDeletedColumn (const TableDescriptor *td) const
 
const ColumnDescriptorgetDeletedColumnIfRowsDeleted (const TableDescriptor *td) const
 
void setDeletedColumn (const TableDescriptor *td, const ColumnDescriptor *cd)
 
void setDeletedColumnUnlocked (const TableDescriptor *td, const ColumnDescriptor *cd)
 
int getLogicalTableId (const int physicalTableId) const
 
void checkpoint (const int logicalTableId) const
 
std::string name () const
 
void eraseDBData ()
 
void eraseTablePhysicalData (const TableDescriptor *td)
 
void vacuumDeletedRows (const TableDescriptor *td) const
 
void vacuumDeletedRows (const int logicalTableId) const
 

Static Public Member Functions

static void expandGeoColumn (const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
 
static void set (const std::string &dbName, std::shared_ptr< Catalog > cat)
 
static std::shared_ptr< Catalogget (const std::string &dbName)
 
static std::shared_ptr< Catalogget (const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
 
static void remove (const std::string &dbName)
 

Public Attributes

std::mutex sqliteMutex_
 
mapd_shared_mutex sharedMutex_
 
std::atomic< std::thread::id > thread_holding_sqlite_lock
 
std::atomic< std::thread::id > thread_holding_write_lock
 

Static Public Attributes

static thread_local bool thread_holds_read_lock = false
 

Protected Types

typedef std::map< std::string, TableDescriptor * > TableDescriptorMap
 
typedef std::map< int, TableDescriptor * > TableDescriptorMapById
 
typedef std::map< int32_t, std::vector< int32_t > > LogicalToPhysicalTableMapById
 
typedef std::tuple< int, std::string > ColumnKey
 
typedef std::map< ColumnKey, ColumnDescriptor * > ColumnDescriptorMap
 
typedef std::tuple< int, int > ColumnIdKey
 
typedef std::map< ColumnIdKey, ColumnDescriptor * > ColumnDescriptorMapById
 
typedef std::map< DictRef, std::unique_ptr< DictDescriptor > > DictDescriptorMapById
 
typedef std::map< std::string, std::shared_ptr< DashboardDescriptor > > DashboardDescriptorMap
 
typedef std::map< std::string, LinkDescriptor * > LinkDescriptorMap
 
typedef std::map< int, LinkDescriptor * > LinkDescriptorMapById
 
typedef std::unordered_map< const TableDescriptor *, const ColumnDescriptor * > DeletedColumnPerTableMap
 
using ColumnDescriptorsForRoll = std::vector< std::pair< ColumnDescriptor *, ColumnDescriptor * > >
 

Protected Member Functions

void CheckAndExecuteMigrations ()
 
void CheckAndExecuteMigrationsPostBuildMaps ()
 
void updateDictionaryNames ()
 
void updateTableDescriptorSchema ()
 
void updateFixlenArrayColumns ()
 
void updateFrontendViewSchema ()
 
void updateLinkSchema ()
 
void updateFrontendViewAndLinkUsers ()
 
void updateLogicalToPhysicalTableLinkSchema ()
 
void updateLogicalToPhysicalTableMap (const int32_t logical_tb_id)
 
void updateDictionarySchema ()
 
void updatePageSize ()
 
void updateDeletedColumnIndicator ()
 
void updateFrontendViewsToDashboards ()
 
void recordOwnershipOfObjectsInObjectPermissions ()
 
void checkDateInDaysColumnMigration ()
 
void createDashboardSystemRoles ()
 
void buildMaps ()
 
void addTableToMap (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
 
void addReferenceToForeignDict (ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def)
 
bool setColumnSharedDictionary (ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
void setColumnDictionary (ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
 
void addFrontendViewToMap (DashboardDescriptor &vd)
 
void addFrontendViewToMapNoLock (DashboardDescriptor &vd)
 
void addLinkToMap (LinkDescriptor &ld)
 
void removeTableFromMap (const std::string &tableName, const int tableId, const bool is_on_error=false)
 
void doDropTable (const TableDescriptor *td)
 
void doTruncateTable (const TableDescriptor *td)
 
void renamePhysicalTable (const TableDescriptor *td, const std::string &newTableName)
 
void instantiateFragmenter (TableDescriptor *td) const
 
void getAllColumnMetadataForTable (const TableDescriptor *td, std::list< const ColumnDescriptor *> &colDescs, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 
std::string calculateSHA1 (const std::string &data)
 
std::string generatePhysicalTableName (const std::string &logicalTableName, const int32_t &shardNumber)
 
std::vector< DBObjectparseDashboardObjects (const std::string &view_meta, const int &user_id)
 
void createOrUpdateDashboardSystemRole (const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
 
const int getColumnIdBySpiUnlocked (const int table_id, const size_t spi) const
 

Protected Attributes

std::string basePath_
 
TableDescriptorMap tableDescriptorMap_
 
TableDescriptorMapById tableDescriptorMapById_
 
ColumnDescriptorMap columnDescriptorMap_
 
ColumnDescriptorMapById columnDescriptorMapById_
 
DictDescriptorMapById dictDescriptorMapByRef_
 
DashboardDescriptorMap dashboardDescriptorMap_
 
LinkDescriptorMap linkDescriptorMap_
 
LinkDescriptorMapById linkDescriptorMapById_
 
SqliteConnector sqliteConnector_
 
DBMetadata currentDB_
 
std::shared_ptr< Data_Namespace::DataMgrdataMgr_
 
const std::vector< LeafHostInfostring_dict_hosts_
 
std::shared_ptr< CalcitecalciteMgr_
 
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
 
int nextTempTableId_
 
int nextTempDictId_
 
ColumnDescriptorsForRoll columnDescriptorsForRoll
 

Static Protected Attributes

static const std::string physicalTableNameTag_
 

Private Attributes

DeletedColumnPerTableMap deletedColumnPerTable_
 

Static Private Attributes

static std::map< std::string, std::shared_ptr< Catalog > > mapd_cat_map_
 

Detailed Description

class for a per-database catalog. also includes metadata for the current database and the current user.

Catalog

Definition at line 81 of file Catalog.h.

Member Typedef Documentation

◆ ColumnDescriptorMap

Definition at line 230 of file Catalog.h.

◆ ColumnDescriptorMapById

Definition at line 232 of file Catalog.h.

◆ ColumnDescriptorsForRoll

Definition at line 327 of file Catalog.h.

◆ ColumnIdKey

typedef std::tuple<int, int> Catalog_Namespace::Catalog::ColumnIdKey
protected

Definition at line 231 of file Catalog.h.

◆ ColumnKey

typedef std::tuple<int, std::string> Catalog_Namespace::Catalog::ColumnKey
protected

Definition at line 229 of file Catalog.h.

◆ DashboardDescriptorMap

typedef std::map<std::string, std::shared_ptr<DashboardDescriptor> > Catalog_Namespace::Catalog::DashboardDescriptorMap
protected

Definition at line 235 of file Catalog.h.

◆ DeletedColumnPerTableMap

typedef std::unordered_map<const TableDescriptor*, const ColumnDescriptor*> Catalog_Namespace::Catalog::DeletedColumnPerTableMap
protected

Definition at line 239 of file Catalog.h.

◆ DictDescriptorMapById

typedef std::map<DictRef, std::unique_ptr<DictDescriptor> > Catalog_Namespace::Catalog::DictDescriptorMapById
protected

Definition at line 233 of file Catalog.h.

◆ LinkDescriptorMap

typedef std::map<std::string, LinkDescriptor*> Catalog_Namespace::Catalog::LinkDescriptorMap
protected

Definition at line 236 of file Catalog.h.

◆ LinkDescriptorMapById

Definition at line 237 of file Catalog.h.

◆ LogicalToPhysicalTableMapById

typedef std::map<int32_t, std::vector<int32_t> > Catalog_Namespace::Catalog::LogicalToPhysicalTableMapById
protected

Definition at line 228 of file Catalog.h.

◆ TableDescriptorMap

typedef std::map<std::string, TableDescriptor*> Catalog_Namespace::Catalog::TableDescriptorMap
protected

Definition at line 226 of file Catalog.h.

◆ TableDescriptorMapById

Definition at line 227 of file Catalog.h.

Constructor & Destructor Documentation

◆ Catalog()

Catalog_Namespace::Catalog::Catalog ( const std::string &  basePath,
const DBMetadata curDB,
std::shared_ptr< Data_Namespace::DataMgr dataMgr,
const std::vector< LeafHostInfo > &  string_dict_hosts,
std::shared_ptr< Calcite calcite,
bool  is_new_db 
)

Constructor - takes basePath to already extant data directory for writing.

Parameters
basePathdirectory path for writing catalog
dbNamename of the database
fragmenterFragmenter object metadata - expects for this directory to already exist

Definition at line 127 of file Catalog.cpp.

References buildMaps(), CheckAndExecuteMigrations(), and CheckAndExecuteMigrationsPostBuildMaps().

133  : basePath_(basePath)
134  , sqliteConnector_(curDB.dbName, basePath + "/mapd_catalogs/")
135  , currentDB_(curDB)
136  , dataMgr_(dataMgr)
137  , string_dict_hosts_(string_dict_hosts)
138  , calciteMgr_(calcite)
141  , sqliteMutex_()
142  , sharedMutex_()
145  if (!is_new_db) {
147  }
148  buildMaps();
149  if (!is_new_db) {
151  }
152 }
const int MAPD_TEMP_TABLE_START_ID
Definition: Catalog.cpp:78
mapd_shared_mutex sharedMutex_
Definition: Catalog.h:336
SqliteConnector sqliteConnector_
Definition: Catalog.h:309
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:313
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:311
const int MAPD_TEMP_DICT_START_ID
Definition: Catalog.cpp:80
std::atomic< std::thread::id > thread_holding_sqlite_lock
Definition: Catalog.h:337
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:314
std::atomic< std::thread::id > thread_holding_write_lock
Definition: Catalog.h:338
void CheckAndExecuteMigrationsPostBuildMaps()
Definition: Catalog.cpp:847
+ Here is the call graph for this function:

◆ ~Catalog()

Catalog_Namespace::Catalog::~Catalog ( )
virtual

Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite.

Definition at line 154 of file Catalog.cpp.

References columnDescriptorMap_, and tableDescriptorMap_.

154  {
155  cat_write_lock write_lock(this);
156  // must clean up heap-allocated TableDescriptor and ColumnDescriptor structs
157  for (TableDescriptorMap::iterator tableDescIt = tableDescriptorMap_.begin();
158  tableDescIt != tableDescriptorMap_.end();
159  ++tableDescIt) {
160  if (tableDescIt->second->fragmenter != nullptr) {
161  delete tableDescIt->second->fragmenter;
162  }
163  delete tableDescIt->second;
164  }
165 
166  // TableDescriptorMapById points to the same descriptors. No need to delete
167 
168  for (ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.begin();
169  columnDescIt != columnDescriptorMap_.end();
170  ++columnDescIt) {
171  delete columnDescIt->second;
172  }
173 
174  // ColumnDescriptorMapById points to the same descriptors. No need to delete
175 }
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:303
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:89
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:301

Member Function Documentation

◆ addColumn()

void Catalog_Namespace::Catalog::addColumn ( const TableDescriptor td,
ColumnDescriptor cd 
)

Definition at line 1700 of file Catalog.cpp.

References addDictionary(), columnDescriptorMap_, columnDescriptorMapById_, columnDescriptorsForRoll, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfoCore< TYPE_FACET_PACK >::get_comp_param(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_compression(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_dimension(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_notnull(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_scale(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_size(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_subtype(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), SqliteConnector::getData(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, kENCODING_DICT, SqliteConnector::query_with_text_params(), sqliteConnector_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), to_upper(), and ColumnDescriptor::virtualExpr.

1700  {
1701  // caller must handle sqlite/chunk transaction TOGETHER
1702  cd.tableId = td.tableId;
1704  addDictionary(cd);
1705  }
1706 
1708  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
1709  "colscale, is_notnull, "
1710  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
1711  "is_deletedcol) "
1712  "VALUES (?, "
1713  "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
1714  "?, ?, ?, "
1715  "?, "
1716  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1717  std::vector<std::string>{std::to_string(td.tableId),
1718  std::to_string(td.tableId),
1719  cd.columnName,
1728  "",
1731  cd.virtualExpr,
1733 
1735  "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
1736  std::vector<std::string>{std::to_string(td.tableId)});
1737 
1739  "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
1740  std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
1741  cd.columnId = sqliteConnector_.getData<int>(0, 0);
1742 
1743  ++tableDescriptorMapById_[td.tableId]->nColumns;
1744  auto ncd = new ColumnDescriptor(cd);
1747  columnDescriptorsForRoll.emplace_back(nullptr, ncd);
1748 }
std::string virtualExpr
HOST DEVICE int get_size() const
Definition: sqltypes.h:333
T getData(const int row, const int col)
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:325
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:303
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:330
SqliteConnector sqliteConnector_
Definition: Catalog.h:309
void query_with_text_params(std::string const &query_only)
HOST DEVICE int get_scale() const
Definition: sqltypes.h:328
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:323
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:331
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:304
std::string to_string(char const *&&v)
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:324
std::tuple< int, int > ColumnIdKey
Definition: Catalog.h:231
std::tuple< int, std::string > ColumnKey
Definition: Catalog.h:229
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:302
specifies the content in-memory of a row in the column metadata table
std::string to_upper(const std::string &str)
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:332
SQLTypeInfo columnType
DictRef addDictionary(ColumnDescriptor &cd)
Definition: Catalog.cpp:1613
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:328
std::string columnName
+ Here is the call graph for this function:

◆ addDictionary()

DictRef Catalog_Namespace::Catalog::addDictionary ( ColumnDescriptor cd)

Definition at line 1613 of file Catalog.cpp.

References CHECK, client, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, DictDescriptor::dictFolderPath, setColumnDictionary(), string_dict_hosts_, tableDescriptorMapById_, and ColumnDescriptor::tableId.

Referenced by addColumn().

1613  {
1614  const auto& td = *tableDescriptorMapById_[cd.tableId];
1615  list<DictDescriptor> dds;
1616  setColumnDictionary(cd, dds, td, true);
1617  auto& dd = dds.back();
1618  CHECK(dd.dictRef.dictId);
1619 
1620  std::unique_ptr<StringDictionaryClient> client;
1621  if (!string_dict_hosts_.empty()) {
1622  client.reset(new StringDictionaryClient(
1623  string_dict_hosts_.front(), DictRef(currentDB_.dbId, -1), true));
1624  }
1625  if (client) {
1626  client->create(dd.dictRef, dd.dictIsTemp);
1627  }
1628 
1629  DictDescriptor* new_dd = new DictDescriptor(dd);
1630  dictDescriptorMapByRef_[dd.dictRef].reset(new_dd);
1631  if (!dd.dictIsTemp) {
1632  boost::filesystem::create_directory(new_dd->dictFolderPath);
1633  }
1634  return dd.dictRef;
1635 }
std::string dictFolderPath
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:313
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:305
mapd::shared_ptr< MapDClient > client
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:302
struct dict_ref_t DictRef
Definition: DictRef.h:37
#define CHECK(condition)
Definition: Logger.h:187
Descriptor for a dictionary for a string columne.
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:2343
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ addFrontendViewToMap()

void Catalog_Namespace::Catalog::addFrontendViewToMap ( DashboardDescriptor vd)
protected

Definition at line 1181 of file Catalog.cpp.

References addFrontendViewToMapNoLock().

Referenced by createDashboard().

1181  {
1182  cat_write_lock write_lock(this);
1184 }
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1186
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:89
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ addFrontendViewToMapNoLock()

void Catalog_Namespace::Catalog::addFrontendViewToMapNoLock ( DashboardDescriptor vd)
protected

Definition at line 1186 of file Catalog.cpp.

References dashboardDescriptorMap_, DashboardDescriptor::dashboardName, to_string(), and DashboardDescriptor::userId.

Referenced by addFrontendViewToMap(), and replaceDashboard().

1186  {
1187  cat_write_lock write_lock(this);
1189  std::make_shared<DashboardDescriptor>(vd);
1190 }
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:306
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:89
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ addLinkToMap()

void Catalog_Namespace::Catalog::addLinkToMap ( LinkDescriptor ld)
protected

Definition at line 1263 of file Catalog.cpp.

References currentDB_, Catalog_Namespace::DBMetadata::dbId, LinkDescriptor::link, linkDescriptorMap_, linkDescriptorMapById_, LinkDescriptor::linkId, and to_string().

Referenced by createLink().

1263  {
1264  cat_write_lock write_lock(this);
1265  LinkDescriptor* new_ld = new LinkDescriptor();
1266  *new_ld = ld;
1268  linkDescriptorMapById_[ld.linkId] = new_ld;
1269 }
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:89
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:308
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:307
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ addReferenceToForeignDict()

void Catalog_Namespace::Catalog::addReferenceToForeignDict ( ColumnDescriptor referencing_column,
Parser::SharedDictionaryDef  shared_dict_def 
)
protected

Definition at line 2262 of file Catalog.cpp.

References CHECK, CHECK_GE, ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, SQLTypeInfoCore< TYPE_FACET_PACK >::get_comp_param(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_foreign_col(), SqliteConnector::query_with_text_params(), sqliteConnector_, and to_string().

Referenced by setColumnSharedDictionary().

2263  {
2264  cat_write_lock write_lock(this);
2265  const auto foreign_ref_col = get_foreign_col(*this, shared_dict_def);
2266  CHECK(foreign_ref_col);
2267  referencing_column.columnType = foreign_ref_col->columnType;
2268  const int dict_id = referencing_column.columnType.get_comp_param();
2269  const DictRef dict_ref(currentDB_.dbId, dict_id);
2270  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
2271  CHECK(dictIt != dictDescriptorMapByRef_.end());
2272  const auto& dd = dictIt->second;
2273  CHECK_GE(dd->refcount, 1);
2274  ++dd->refcount;
2275  cat_sqlite_lock sqlite_lock(this);
2277  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
2278  {std::to_string(dict_id)});
2279 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:309
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:305
void query_with_text_params(std::string const &query_only)
#define CHECK_GE(x, y)
Definition: Logger.h:200
const ColumnDescriptor * get_foreign_col(const Catalog &cat, const Parser::SharedDictionaryDef &shared_dict_def)
Definition: Catalog.cpp:2250
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:89
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:332
#define CHECK(condition)
Definition: Logger.h:187
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:90
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ addTableToMap()

void Catalog_Namespace::Catalog::addTableToMap ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::list< DictDescriptor > &  dicts 
)
protected

Definition at line 1049 of file Catalog.cpp.

References CHECK, client, columnDescriptorMap_, columnDescriptorMapById_, ColumnDescriptor::columnId, TableDescriptor::columnIdBySpi_, ColumnDescriptor::columnName, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, DictDescriptor::dictFolderPath, dict_ref_t::dictId, TableDescriptor::hasDeletedCol, TableDescriptor::mutex_, setDeletedColumnUnlocked(), string_dict_hosts_, tableDescriptorMap_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, and to_upper().

Referenced by createTable().

1051  {
1052  cat_write_lock write_lock(this);
1053  TableDescriptor* new_td = new TableDescriptor();
1054  *new_td = td;
1055  new_td->mutex_ = std::make_shared<std::mutex>();
1056  tableDescriptorMap_[to_upper(td.tableName)] = new_td;
1057  tableDescriptorMapById_[td.tableId] = new_td;
1058  for (auto cd : columns) {
1059  ColumnDescriptor* new_cd = new ColumnDescriptor();
1060  *new_cd = cd;
1061  ColumnKey columnKey(new_cd->tableId, to_upper(new_cd->columnName));
1062  columnDescriptorMap_[columnKey] = new_cd;
1063  ColumnIdKey columnIdKey(new_cd->tableId, new_cd->columnId);
1064  columnDescriptorMapById_[columnIdKey] = new_cd;
1065 
1066  // Add deleted column to the map
1067  if (cd.isDeletedCol) {
1068  CHECK(new_td->hasDeletedCol);
1069  setDeletedColumnUnlocked(new_td, new_cd);
1070  }
1071  }
1072 
1073  std::sort(new_td->columnIdBySpi_.begin(),
1074  new_td->columnIdBySpi_.end(),
1075  [](const size_t a, const size_t b) -> bool { return a < b; });
1076 
1077  std::unique_ptr<StringDictionaryClient> client;
1078  DictRef dict_ref(currentDB_.dbId, -1);
1079  if (!string_dict_hosts_.empty()) {
1080  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1081  }
1082  for (auto dd : dicts) {
1083  if (!dd.dictRef.dictId) {
1084  // Dummy entry created for a shard of a logical table, nothing to do.
1085  continue;
1086  }
1087  dict_ref.dictId = dd.dictRef.dictId;
1088  if (client) {
1089  client->create(dict_ref, dd.dictIsTemp);
1090  }
1091  DictDescriptor* new_dd = new DictDescriptor(dd);
1092  dictDescriptorMapByRef_[dict_ref].reset(new_dd);
1093  if (!dd.dictIsTemp) {
1094  boost::filesystem::create_directory(new_dd->dictFolderPath);
1095  }
1096  }
1097 }
std::string dictFolderPath
std::string tableName
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:303
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:313
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:305
std::vector< int > columnIdBySpi_
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:304
std::shared_ptr< std::mutex > mutex_
mapd::shared_ptr< MapDClient > client
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:2241
std::tuple< int, int > ColumnIdKey
Definition: Catalog.h:231
std::tuple< int, std::string > ColumnKey
Definition: Catalog.h:229
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:302
specifies the content in-memory of a row in the column metadata table
std::string to_upper(const std::string &str)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:89
#define CHECK(condition)
Definition: Logger.h:187
Descriptor for a dictionary for a string columne.
specifies the content in-memory of a row in the table metadata table
std::string columnName
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:301
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ buildMaps()

void Catalog_Namespace::Catalog::buildMaps ( )
protected

Definition at line 863 of file Catalog.cpp.

References basePath_, CHECK, ColumnDescriptor::chunks, columnDescriptorMap_, columnDescriptorMapById_, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, currentDB_, dashboardDescriptorMap_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, TableDescriptor::fragmenter, TableDescriptor::fragments, TableDescriptor::fragPageSize, TableDescriptor::fragType, generate_dashboard_system_rolename(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_physical_cols(), SqliteConnector::getData(), SqliteConnector::getNumRows(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::getUserFromId(), TableDescriptor::hasDeletedCol, SQLTypeInfoCore< TYPE_FACET_PACK >::is_geometry(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, SqliteConnector::isNull(), ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, TableDescriptor::keyMetainfo, LinkDescriptor::link, linkDescriptorMap_, linkDescriptorMapById_, LinkDescriptor::linkId, logicalToPhysicalTableMapById_, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRows, TableDescriptor::nColumns, TableDescriptor::nShards, TableDescriptor::partitions, SqliteConnector::query(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_comp_param(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_compression(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_dimension(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_notnull(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_scale(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_size(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_subtype(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_type(), setDeletedColumnUnlocked(), TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, sqliteConnector_, tableDescriptorMap_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), to_upper(), LinkDescriptor::updateTime, LinkDescriptor::userId, TableDescriptor::userId, LinkDescriptor::viewMetadata, TableDescriptor::viewSQL, LinkDescriptor::viewState, and ColumnDescriptor::virtualExpr.

Referenced by Catalog().

863  {
864  cat_write_lock write_lock(this);
865  cat_sqlite_lock sqlite_lock(this);
866 
867  string dictQuery(
868  "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
869  sqliteConnector_.query(dictQuery);
870  size_t numRows = sqliteConnector_.getNumRows();
871  for (size_t r = 0; r < numRows; ++r) {
872  int dictId = sqliteConnector_.getData<int>(r, 0);
873  std::string dictName = sqliteConnector_.getData<string>(r, 1);
874  int dictNBits = sqliteConnector_.getData<int>(r, 2);
875  bool is_shared = sqliteConnector_.getData<bool>(r, 3);
876  int refcount = sqliteConnector_.getData<int>(r, 4);
877  std::string fname = basePath_ + "/mapd_data/DB_" + std::to_string(currentDB_.dbId) +
878  "_DICT_" + std::to_string(dictId);
879  DictRef dict_ref(currentDB_.dbId, dictId);
880  DictDescriptor* dd = new DictDescriptor(
881  dict_ref, dictName, dictNBits, is_shared, refcount, fname, false);
882  dictDescriptorMapByRef_[dict_ref].reset(dd);
883  }
884 
885  string tableQuery(
886  "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
887  "max_chunk_size, frag_page_size, "
888  "max_rows, partitions, shard_column_id, shard, num_shards, key_metainfo, userid, "
889  "sort_column_id "
890  "from mapd_tables");
891  sqliteConnector_.query(tableQuery);
892  numRows = sqliteConnector_.getNumRows();
893  for (size_t r = 0; r < numRows; ++r) {
894  TableDescriptor* td = new TableDescriptor();
895  td->tableId = sqliteConnector_.getData<int>(r, 0);
896  td->tableName = sqliteConnector_.getData<string>(r, 1);
897  td->nColumns = sqliteConnector_.getData<int>(r, 2);
898  td->isView = sqliteConnector_.getData<bool>(r, 3);
899  td->fragments = sqliteConnector_.getData<string>(r, 4);
900  td->fragType =
902  td->maxFragRows = sqliteConnector_.getData<int>(r, 6);
903  td->maxChunkSize = sqliteConnector_.getData<int>(r, 7);
904  td->fragPageSize = sqliteConnector_.getData<int>(r, 8);
905  td->maxRows = sqliteConnector_.getData<int64_t>(r, 9);
906  td->partitions = sqliteConnector_.getData<string>(r, 10);
907  td->shardedColumnId = sqliteConnector_.getData<int>(r, 11);
908  td->shard = sqliteConnector_.getData<int>(r, 12);
909  td->nShards = sqliteConnector_.getData<int>(r, 13);
910  td->keyMetainfo = sqliteConnector_.getData<string>(r, 14);
911  td->userId = sqliteConnector_.getData<int>(r, 15);
912  td->sortedColumnId =
913  sqliteConnector_.isNull(r, 16) ? 0 : sqliteConnector_.getData<int>(r, 16);
914  if (!td->isView) {
915  td->fragmenter = nullptr;
916  }
917  td->hasDeletedCol = false;
920  }
921  string columnQuery(
922  "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
923  "is_notnull, compression, comp_param, "
924  "size, chunks, is_systemcol, is_virtualcol, virtual_expr, is_deletedcol from "
925  "mapd_columns ORDER BY tableid, "
926  "columnid");
927  sqliteConnector_.query(columnQuery);
928  numRows = sqliteConnector_.getNumRows();
929  int32_t skip_physical_cols = 0;
930  for (size_t r = 0; r < numRows; ++r) {
932  cd->tableId = sqliteConnector_.getData<int>(r, 0);
933  cd->columnId = sqliteConnector_.getData<int>(r, 1);
934  cd->columnName = sqliteConnector_.getData<string>(r, 2);
942  cd->columnType.set_size(sqliteConnector_.getData<int>(r, 10));
943  cd->chunks = sqliteConnector_.getData<string>(r, 11);
944  cd->isSystemCol = sqliteConnector_.getData<bool>(r, 12);
945  cd->isVirtualCol = sqliteConnector_.getData<bool>(r, 13);
946  cd->virtualExpr = sqliteConnector_.getData<string>(r, 14);
947  cd->isDeletedCol = sqliteConnector_.getData<bool>(r, 15);
948  cd->isGeoPhyCol = skip_physical_cols > 0;
949  ColumnKey columnKey(cd->tableId, to_upper(cd->columnName));
950  columnDescriptorMap_[columnKey] = cd;
951  ColumnIdKey columnIdKey(cd->tableId, cd->columnId);
952  columnDescriptorMapById_[columnIdKey] = cd;
953 
954  if (skip_physical_cols <= 0) {
955  skip_physical_cols = cd->columnType.get_physical_cols();
956  }
957 
958  auto td_itr = tableDescriptorMapById_.find(cd->tableId);
959  CHECK(td_itr != tableDescriptorMapById_.end());
960 
961  if (cd->isDeletedCol) {
962  td_itr->second->hasDeletedCol = true;
963  setDeletedColumnUnlocked(td_itr->second, cd);
964  } else if (cd->columnType.is_geometry() || skip_physical_cols-- <= 0) {
965  tableDescriptorMapById_[cd->tableId]->columnIdBySpi_.push_back(cd->columnId);
966  }
967  }
968  // sort columnIdBySpi_ based on columnId
969  for (auto& tit : tableDescriptorMapById_) {
970  std::sort(tit.second->columnIdBySpi_.begin(),
971  tit.second->columnIdBySpi_.end(),
972  [](const size_t a, const size_t b) -> bool { return a < b; });
973  }
974 
975  string viewQuery("SELECT tableid, sql FROM mapd_views");
976  sqliteConnector_.query(viewQuery);
977  numRows = sqliteConnector_.getNumRows();
978  for (size_t r = 0; r < numRows; ++r) {
979  int32_t tableId = sqliteConnector_.getData<int>(r, 0);
980  TableDescriptor* td = tableDescriptorMapById_[tableId];
981  td->viewSQL = sqliteConnector_.getData<string>(r, 1);
982  td->fragmenter = nullptr;
983  }
984 
985  string frontendViewQuery(
986  "SELECT id, state, name, image_hash, strftime('%Y-%m-%dT%H:%M:%SZ', update_time), "
987  "userid, "
988  "metadata "
989  "FROM mapd_dashboards");
990  sqliteConnector_.query(frontendViewQuery);
991  numRows = sqliteConnector_.getNumRows();
992  for (size_t r = 0; r < numRows; ++r) {
993  std::shared_ptr<DashboardDescriptor> vd = std::make_shared<DashboardDescriptor>();
994  vd->dashboardId = sqliteConnector_.getData<int>(r, 0);
995  vd->dashboardState = sqliteConnector_.getData<string>(r, 1);
996  vd->dashboardName = sqliteConnector_.getData<string>(r, 2);
997  vd->imageHash = sqliteConnector_.getData<string>(r, 3);
998  vd->updateTime = sqliteConnector_.getData<string>(r, 4);
999  vd->userId = sqliteConnector_.getData<int>(r, 5);
1000  vd->dashboardMetadata = sqliteConnector_.getData<string>(r, 6);
1001  vd->user = getUserFromId(vd->userId);
1002  vd->dashboardSystemRoleName = generate_dashboard_system_rolename(
1004  dashboardDescriptorMap_[std::to_string(vd->userId) + ":" + vd->dashboardName] = vd;
1005  }
1006 
1007  string linkQuery(
1008  "SELECT linkid, userid, link, view_state, strftime('%Y-%m-%dT%H:%M:%SZ', "
1009  "update_time), view_metadata "
1010  "FROM mapd_links");
1011  sqliteConnector_.query(linkQuery);
1012  numRows = sqliteConnector_.getNumRows();
1013  for (size_t r = 0; r < numRows; ++r) {
1014  LinkDescriptor* ld = new LinkDescriptor();
1015  ld->linkId = sqliteConnector_.getData<int>(r, 0);
1016  ld->userId = sqliteConnector_.getData<int>(r, 1);
1017  ld->link = sqliteConnector_.getData<string>(r, 2);
1018  ld->viewState = sqliteConnector_.getData<string>(r, 3);
1019  ld->updateTime = sqliteConnector_.getData<string>(r, 4);
1020  ld->viewMetadata = sqliteConnector_.getData<string>(r, 5);
1022  linkDescriptorMapById_[ld->linkId] = ld;
1023  }
1024 
1025  /* rebuild map linking logical tables to corresponding physical ones */
1026  string logicalToPhysicalTableMapQuery(
1027  "SELECT logical_table_id, physical_table_id "
1028  "FROM mapd_logical_to_physical");
1029  sqliteConnector_.query(logicalToPhysicalTableMapQuery);
1030  numRows = sqliteConnector_.getNumRows();
1031  for (size_t r = 0; r < numRows; ++r) {
1032  int32_t logical_tb_id = sqliteConnector_.getData<int>(r, 0);
1033  int32_t physical_tb_id = sqliteConnector_.getData<int>(r, 1);
1034  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
1035  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
1036  /* add new entity to the map logicalToPhysicalTableMapById_ */
1037  std::vector<int32_t> physicalTables;
1038  physicalTables.push_back(physical_tb_id);
1039  const auto it_ok =
1040  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
1041  CHECK(it_ok.second);
1042  } else {
1043  /* update map logicalToPhysicalTableMapById_ */
1044  physicalTableIt->second.push_back(physical_tb_id);
1045  }
1046  }
1047 }
std::string virtualExpr
std::string partitions
T getData(const int row, const int col)
SQLTypes
Definition: sqltypes.h:40
std::string tableName
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:303
EncodingType
Definition: encodetypes.h:22
SqliteConnector sqliteConnector_
Definition: Catalog.h:309
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:305
void set_size(int s)
Definition: sqltypes.h:421
void query(const std::string &queryString)
std::string fragments
HOST DEVICE void set_type(SQLTypes t)
Definition: sqltypes.h:413
void set_dimension(int d)
Definition: sqltypes.h:415
void set_scale(int s)
Definition: sqltypes.h:418
void set_compression(EncodingType c)
Definition: sqltypes.h:423
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:304
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:306
std::string to_string(char const *&&v)
void set_notnull(bool n)
Definition: sqltypes.h:420
std::string chunks
size_t getNumRows() const
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:414
std::string getUserFromId(const int32_t id)
Definition: Catalog.cpp:853
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:2241
std::tuple< int, int > ColumnIdKey
Definition: Catalog.h:231
std::tuple< int, std::string > ColumnKey
Definition: Catalog.h:229
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:302
specifies the content in-memory of a row in the column metadata table
std::string keyMetainfo
std::string to_upper(const std::string &str)
Fragmenter_Namespace::FragmenterType fragType
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:89
bool isNull(const int row, const int col) const
void set_comp_param(int p)
Definition: sqltypes.h:424
int get_physical_cols() const
Definition: sqltypes.h:344
#define CHECK(condition)
Definition: Logger.h:187
Descriptor for a dictionary for a string columne.
bool is_geometry() const
Definition: sqltypes.h:462
std::string viewSQL
SQLTypeInfo columnType
specifies the content in-memory of a row in the table metadata table
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:90
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:308
Fragmenter_Namespace::AbstractFragmenter * fragmenter
std::string columnName
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:316
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:301
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:307
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ calculateSHA1()

std::string Catalog_Namespace::Catalog::calculateSHA1 ( const std::string &  data)
protected

Definition at line 2828 of file Catalog.cpp.

Referenced by createLink().

2828  {
2829  boost::uuids::detail::sha1 sha1;
2830  unsigned int digest[5];
2831  sha1.process_bytes(data.c_str(), data.length());
2832  sha1.get_digest(digest);
2833  std::stringstream ss;
2834  for (size_t i = 0; i < 5; i++) {
2835  ss << std::hex << digest[i];
2836  }
2837  return ss.str();
2838 }
+ Here is the caller graph for this function:

◆ CheckAndExecuteMigrations()

void Catalog_Namespace::Catalog::CheckAndExecuteMigrations ( )
protected

Definition at line 832 of file Catalog.cpp.

References recordOwnershipOfObjectsInObjectPermissions(), updateDeletedColumnIndicator(), updateDictionaryNames(), updateDictionarySchema(), updateFixlenArrayColumns(), updateFrontendViewAndLinkUsers(), updateFrontendViewSchema(), updateFrontendViewsToDashboards(), updateLinkSchema(), updateLogicalToPhysicalTableLinkSchema(), updatePageSize(), and updateTableDescriptorSchema().

Referenced by Catalog().

832  {
841  updatePageSize();
845 }
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:325
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:526
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:96
void updateLogicalToPhysicalTableLinkSchema()
Definition: Catalog.cpp:465
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ CheckAndExecuteMigrationsPostBuildMaps()

void Catalog_Namespace::Catalog::CheckAndExecuteMigrationsPostBuildMaps ( )
protected

Definition at line 847 of file Catalog.cpp.

References checkDateInDaysColumnMigration(), and createDashboardSystemRoles().

Referenced by Catalog().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ checkDateInDaysColumnMigration()

void Catalog_Namespace::Catalog::checkDateInDaysColumnMigration ( )
protected

Definition at line 653 of file Catalog.cpp.

References logger::ERROR, getCurrentDB(), SqliteConnector::getData(), Executor::getExecutor(), SqliteConnector::getNumRows(), logger::INFO, kDATE, kENCODING_DATE_IN_DAYS, LOG, MAPD_VERSION, SqliteConnector::query(), SqliteConnector::query_with_text_params(), TableOptimizer::recomputeMetadata(), sqliteConnector_, tableDescriptorMapById_, and to_string().

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

653  {
654  cat_sqlite_lock sqlite_lock(this);
655  std::vector<int> tables_migrated = {};
656  std::unordered_map<int, std::vector<std::string>> tables_to_migrate;
657  sqliteConnector_.query("BEGIN TRANSACTION");
658  try {
660  "select name from sqlite_master WHERE type='table' AND "
661  "name='mapd_version_history'");
662  if (sqliteConnector_.getNumRows() == 0) {
664  "CREATE TABLE mapd_version_history(version integer, migration_history text "
665  "unique)");
667  "CREATE TABLE mapd_date_in_days_column_migration_tmp(table_id integer primary "
668  "key)");
669  } else {
671  "select * from mapd_version_history where migration_history = "
672  "'date_in_days_column'");
673  if (sqliteConnector_.getNumRows() != 0) {
674  // no need for further execution
675  sqliteConnector_.query("END TRANSACTION");
676  return;
677  }
678  LOG(INFO) << "Performing Date in days columns migration.";
680  "select name from sqlite_master where type='table' AND "
681  "name='mapd_date_in_days_column_migration_tmp'");
682  if (sqliteConnector_.getNumRows() != 0) {
684  "select table_id from mapd_date_in_days_column_migration_tmp");
685  if (sqliteConnector_.getNumRows() != 0) {
686  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
687  tables_migrated.push_back(sqliteConnector_.getData<int>(i, 0));
688  }
689  }
690  } else {
692  "CREATE TABLE mapd_date_in_days_column_migration_tmp(table_id integer "
693  "primary key)");
694  }
695  }
697  "SELECT tables.tableid, tables.name, columns.name FROM mapd_tables tables, "
698  "mapd_columns columns where tables.tableid = columns.tableid AND "
699  "columns.coltype = ?1 AND columns.compression = ?2",
700  std::vector<std::string>{
701  std::to_string(static_cast<int>(SQLTypes::kDATE)),
703  if (sqliteConnector_.getNumRows() != 0) {
704  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
705  tables_to_migrate[sqliteConnector_.getData<int>(i, 0)] = {
706  sqliteConnector_.getData<std::string>(i, 1),
707  sqliteConnector_.getData<std::string>(i, 2)};
708  }
709  }
710  } catch (const std::exception& e) {
711  LOG(ERROR) << "Failed to complete migraion on date in days column: " << e.what();
712  sqliteConnector_.query("ROLLBACK");
713  throw;
714  }
715  sqliteConnector_.query("END TRANSACTION");
716 
717  for (auto& id_names : tables_to_migrate) {
718  if (std::find(tables_migrated.begin(), tables_migrated.end(), id_names.first) ==
719  tables_migrated.end()) {
720  sqliteConnector_.query("BEGIN TRANSACTION");
721  try {
722  LOG(INFO) << "Table: " << id_names.second[0]
723  << " may suffer from issues with DATE column: " << id_names.second[1]
724  << ". Running an OPTIMIZE command to solve any issues with metadata.";
725 
726  auto executor = Executor::getExecutor(getCurrentDB().dbId);
727  TableDescriptorMapById::iterator tableDescIt =
728  tableDescriptorMapById_.find(id_names.first);
729  if (tableDescIt == tableDescriptorMapById_.end()) {
730  throw runtime_error("Table descriptor does not exist for table " +
731  id_names.second[0] + " does not exist.");
732  }
733  auto td = tableDescIt->second;
734  TableOptimizer optimizer(td, executor.get(), *this);
735  optimizer.recomputeMetadata();
736 
738  "INSERT INTO mapd_date_in_days_column_migration_tmp VALUES(?)",
739  std::vector<std::string>{std::to_string(id_names.first)});
740  } catch (const std::exception& e) {
741  LOG(ERROR) << "Failed to complete migraion on date in days column: " << e.what();
742  sqliteConnector_.query("ROLLBACK");
743  throw;
744  }
745  sqliteConnector_.query("COMMIT");
746  }
747  }
748 
749  sqliteConnector_.query("BEGIN TRANSACTION");
750  try {
751  sqliteConnector_.query("DROP TABLE mapd_date_in_days_column_migration_tmp");
753  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
754  std::vector<std::string>{std::to_string(MAPD_VERSION), "date_in_days_column"});
755  } catch (const std::exception& e) {
756  LOG(ERROR) << "Failed to complete migraion on date in days column: " << e.what();
757  sqliteConnector_.query("ROLLBACK");
758  throw;
759  }
760  sqliteConnector_.query("END TRANSACTION");
761  LOG(INFO) << "Migration successfull on Date in days columns";
762 }
static std::shared_ptr< Executor > getExecutor(const int db_id, const std::string &debug_dir="", const std::string &debug_file="", const MapDParameters mapd_parameters=MapDParameters(), ::QueryRenderer::QueryRenderManager *render_manager=nullptr)
Definition: Execute.cpp:122
T getData(const int row, const int col)
#define LOG(tag)
Definition: Logger.h:182
SqliteConnector sqliteConnector_
Definition: Catalog.h:309
void query_with_text_params(std::string const &query_only)
void query(const std::string &queryString)
Driver for running cleanup processes on a table. TableOptimizer provides functions for various cleanu...
std::string to_string(char const *&&v)
size_t getNumRows() const
static const int32_t MAPD_VERSION
Definition: release.h:33
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:302
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:176
Definition: sqltypes.h:55
void recomputeMetadata() const
Recomputes per-chunk metadata for each fragment in the table. Updates and deletes can cause chunk met...
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:90
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ checkMetadataForDeletedRecs()

const bool Catalog_Namespace::Catalog::checkMetadataForDeletedRecs ( int  dbId,
int  tableId,
int  columnId 
) const

Definition at line 2183 of file Catalog.cpp.

References dataMgr_.

Referenced by getDeletedColumnIfRowsDeleted().

2185  {
2186  // check if there are rows deleted by examining metadata for the deletedColumn metadata
2187  ChunkKey chunkKeyPrefix = {dbId, tableId, columnId};
2188  std::vector<std::pair<ChunkKey, ChunkMetadata>> chunkMetadataVec;
2189  dataMgr_->getChunkMetadataVecForKeyPrefix(chunkMetadataVec, chunkKeyPrefix);
2190  int64_t chunk_max{0};
2191 
2192  for (auto cm : chunkMetadataVec) {
2193  chunk_max = cm.second.chunkStats.max.tinyintval;
2194  // delete has occured
2195  if (chunk_max == 1) {
2196  return true;
2197  }
2198  }
2199  return false;
2200 }
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:311
std::vector< int > ChunkKey
Definition: types.h:35
+ Here is the caller graph for this function:

◆ checkpoint()

void Catalog_Namespace::Catalog::checkpoint ( const int  logicalTableId) const

Definition at line 2927 of file Catalog.cpp.

References Data_Namespace::DataMgr::checkpoint(), getCurrentDB(), getDataMgr(), getMetadataForTable(), and getPhysicalTablesDescriptors().

Referenced by Importer_NS::Loader::checkpoint(), and TableOptimizer::vacuumDeletedRows().

2927  {
2928  const auto td = getMetadataForTable(logicalTableId);
2929  const auto shards = getPhysicalTablesDescriptors(td);
2930  for (const auto shard : shards) {
2931  getDataMgr().checkpoint(getCurrentDB().dbId, shard->tableId);
2932  }
2933 }
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:177
void checkpoint(const int db_id, const int tb_id)
Definition: DataMgr.cpp:406
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logicalTableDesc) const
Definition: Catalog.cpp:2895
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:176
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ createDashboard()

int32_t Catalog_Namespace::Catalog::createDashboard ( DashboardDescriptor vd)

Definition at line 2694 of file Catalog.cpp.

References addFrontendViewToMap(), createOrUpdateDashboardSystemRole(), currentDB_, DashboardDescriptor::dashboardId, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, DashboardDescriptor::dashboardSystemRoleName, Catalog_Namespace::DBMetadata::dbId, generate_dashboard_system_rolename(), SqliteConnector::getData(), SqliteConnector::getNumRows(), DashboardDescriptor::imageHash, SqliteConnector::query(), SqliteConnector::query_with_text_params(), sqliteConnector_, to_string(), DashboardDescriptor::updateTime, and DashboardDescriptor::userId.

2694  {
2695  {
2696  cat_write_lock write_lock(this);
2697  cat_sqlite_lock sqlite_lock(this);
2698  sqliteConnector_.query("BEGIN TRANSACTION");
2699  try {
2700  // TODO(andrew): this should be an upsert
2702  "SELECT id FROM mapd_dashboards WHERE name = ? and userid = ?",
2703  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
2704  if (sqliteConnector_.getNumRows() > 0) {
2706  "UPDATE mapd_dashboards SET state = ?, image_hash = ?, metadata = ?, "
2707  "update_time = "
2708  "datetime('now') where name = ? "
2709  "and userid = ?",
2710  std::vector<std::string>{vd.dashboardState,
2711  vd.imageHash,
2712  vd.dashboardMetadata,
2713  vd.dashboardName,
2714  std::to_string(vd.userId)});
2715  } else {
2717  "INSERT INTO mapd_dashboards (name, state, image_hash, metadata, "
2718  "update_time, "
2719  "userid) "
2720  "VALUES "
2721  "(?,?,?,?, "
2722  "datetime('now'), ?)",
2723  std::vector<std::string>{vd.dashboardName,
2724  vd.dashboardState,
2725  vd.imageHash,
2726  vd.dashboardMetadata,
2727  std::to_string(vd.userId)});
2728  }
2729  } catch (std::exception& e) {
2730  sqliteConnector_.query("ROLLBACK TRANSACTION");
2731  throw;
2732  }
2733  sqliteConnector_.query("END TRANSACTION");
2734 
2735  // now get the auto generated dashboardId
2736  try {
2738  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_dashboards "
2739  "WHERE name = ? and userid = ?",
2740  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
2741  vd.dashboardId = sqliteConnector_.getData<int>(0, 0);
2742  vd.updateTime = sqliteConnector_.getData<std::string>(0, 1);
2743  } catch (std::exception& e) {
2744  throw;
2745  }
2749  }
2750  // NOTE(wamsi): Transactionally unsafe
2753  return vd.dashboardId;
2754 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:309
void query_with_text_params(std::string const &query_only)
void query(const std::string &queryString)
void addFrontendViewToMap(DashboardDescriptor &vd)
Definition: Catalog.cpp:1181
std::string to_string(char const *&&v)
size_t getNumRows() const
std::string dashboardSystemRoleName
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1222
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:89
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:90
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)
+ Here is the call graph for this function:

◆ createDashboardSystemRoles()

void Catalog_Namespace::Catalog::createDashboardSystemRoles ( )
protected

Definition at line 764 of file Catalog.cpp.

References createOrUpdateDashboardSystemRole(), currentDB_, Catalog_Namespace::DBMetadata::dbId, logger::ERROR, generate_dashboard_system_rolename(), SqliteConnector::getData(), Catalog_Namespace::SysCatalog::getGranteesOfSharedDashboards(), SqliteConnector::getNumRows(), Catalog_Namespace::SysCatalog::getRoleGrantee(), Catalog_Namespace::SysCatalog::grantRoleBatch(), logger::INFO, Catalog_Namespace::SysCatalog::instance(), LOG, MAPD_VERSION, SqliteConnector::query(), SqliteConnector::query_with_text_params(), run-benchmark-import::result, sqliteConnector_, and to_string().

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

764  {
765  std::unordered_map<std::string, std::pair<int, std::string>> dashboards;
766  std::vector<std::string> dashboard_ids;
767  static const std::string migration_name{"dashboard_roles_migration"};
768  {
769  cat_sqlite_lock sqlite_lock(this);
770  sqliteConnector_.query("BEGIN TRANSACTION");
771  try {
772  // migration_history should be present in all catalogs by now
773  // if not then would be created before this migration
775  "select * from mapd_version_history where migration_history = '" +
776  migration_name + "'");
777  if (sqliteConnector_.getNumRows() != 0) {
778  // no need for further execution
779  sqliteConnector_.query("END TRANSACTION");
780  return;
781  }
782  LOG(INFO) << "Performing dashboard internal roles Migration.";
783  sqliteConnector_.query("select id, userid, metadata from mapd_dashboards");
784  for (size_t i = 0; i < sqliteConnector_.getNumRows(); ++i) {
787  sqliteConnector_.getData<string>(i, 0)))) {
788  // Successfully created roles during previous migration/crash
789  // No need to include them
790  continue;
791  }
792  dashboards[sqliteConnector_.getData<string>(i, 0)] = std::make_pair(
793  sqliteConnector_.getData<int>(i, 1), sqliteConnector_.getData<string>(i, 2));
794  dashboard_ids.push_back(sqliteConnector_.getData<string>(i, 0));
795  }
796  } catch (const std::exception& e) {
797  sqliteConnector_.query("ROLLBACK TRANSACTION");
798  throw;
799  }
800  sqliteConnector_.query("END TRANSACTION");
801  }
802  // All current grantees with shared dashboards.
803  const auto active_grantees =
805 
806  try {
807  // NOTE(wamsi): Transactionally unsafe
808  for (auto dash : dashboards) {
809  createOrUpdateDashboardSystemRole(dash.second.second,
810  dash.second.first,
812  std::to_string(currentDB_.dbId), dash.first));
813  auto result = active_grantees.find(dash.first);
814  if (result != active_grantees.end()) {
817  dash.first)},
818  result->second);
819  }
820  }
822  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
823  std::vector<std::string>{std::to_string(MAPD_VERSION), migration_name});
824  } catch (const std::exception& e) {
825  LOG(ERROR) << "Failed to create dashboard system roles during migration: "
826  << e.what();
827  throw;
828  }
829  LOG(INFO) << "Successfully created dashboard system roles during migration.";
830 }
T getData(const int row, const int col)
#define LOG(tag)
Definition: Logger.h:182
SqliteConnector sqliteConnector_
Definition: Catalog.h:309
void query_with_text_params(std::string const &query_only)
void query(const std::string &queryString)
std::string to_string(char const *&&v)
void grantRoleBatch(const std::vector< std::string > &roles, const std::vector< std::string > &grantees)
size_t getNumRows() const
static SysCatalog & instance()
Definition: SysCatalog.h:240
static const int32_t MAPD_VERSION
Definition: release.h:33
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1222
std::unordered_map< std::string, std::vector< std::string > > getGranteesOfSharedDashboards(const std::vector< std::string > &dashboard_ids)
Role * getRoleGrantee(const std::string &name) const
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:90
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ createLink()

std::string Catalog_Namespace::Catalog::createLink ( LinkDescriptor ld,
size_t  min_length 
)

Definition at line 2840 of file Catalog.cpp.

References addLinkToMap(), calculateSHA1(), SqliteConnector::getData(), SqliteConnector::getNumRows(), LinkDescriptor::link, LinkDescriptor::linkId, SqliteConnector::query(), SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), sqliteConnector_, to_string(), LinkDescriptor::updateTime, LinkDescriptor::userId, LinkDescriptor::viewMetadata, and LinkDescriptor::viewState.

2840  {
2841  cat_write_lock write_lock(this);
2842  cat_sqlite_lock sqlite_lock(this);
2843  sqliteConnector_.query("BEGIN TRANSACTION");
2844  try {
2846  .substr(0, 8);
2848  "SELECT linkid FROM mapd_links WHERE link = ? and userid = ?",
2849  std::vector<std::string>{ld.link, std::to_string(ld.userId)});
2850  if (sqliteConnector_.getNumRows() > 0) {
2852  "UPDATE mapd_links SET update_time = datetime('now') WHERE userid = ? AND link "
2853  "= ?",
2854  std::vector<std::string>{std::to_string(ld.userId), ld.link});
2855  } else {
2857  "INSERT INTO mapd_links (userid, link, view_state, view_metadata, update_time) "
2858  "VALUES (?,?,?,?, "
2859  "datetime('now'))",
2860  std::vector<std::string>{
2861  std::to_string(ld.userId), ld.link, ld.viewState, ld.viewMetadata});
2862  }
2863  // now get the auto generated dashid
2865  "SELECT linkid, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_links "
2866  "WHERE link = ?",
2867  ld.link);
2868  ld.linkId = sqliteConnector_.getData<int>(0, 0);
2869  ld.updateTime = sqliteConnector_.getData<std::string>(0, 1);
2870  } catch (std::exception& e) {
2871  sqliteConnector_.query("ROLLBACK TRANSACTION");
2872  throw;
2873  }
2874  sqliteConnector_.query("END TRANSACTION");
2875  addLinkToMap(ld);
2876  return ld.link;
2877 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:309
void query_with_text_params(std::string const &query_only)
void query(const std::string &queryString)
std::string to_string(char const *&&v)
size_t getNumRows() const
std::string calculateSHA1(const std::string &data)
Definition: Catalog.cpp:2828
void addLinkToMap(LinkDescriptor &ld)
Definition: Catalog.cpp:1263
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:89
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:90
void query_with_text_param(const std::string &queryString, const std::string &text_param)
+ Here is the call graph for this function:

◆ createOrUpdateDashboardSystemRole()

void Catalog_Namespace::Catalog::createOrUpdateDashboardSystemRole ( const std::string &  view_meta,
const int32_t &  user_id,
const std::string &  dash_role_name 
)
protected

Definition at line 1222 of file Catalog.cpp.

References Catalog_Namespace::SysCatalog::createRole(), Grantee::findDbObject(), Grantee::getDbObjects(), Catalog_Namespace::SysCatalog::getRoleGrantee(), Catalog_Namespace::SysCatalog::grantDBObjectPrivilegesBatch(), Catalog_Namespace::SysCatalog::instance(), parseDashboardObjects(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivileges(), TableDBObjectType, and ViewDBObjectType.

Referenced by createDashboard(), createDashboardSystemRoles(), and replaceDashboard().

1224  {
1225  auto objects = parseDashboardObjects(view_meta, user_id);
1226  Role* rl = SysCatalog::instance().getRoleGrantee(dash_role_name);
1227  if (!rl) {
1228  // Dashboard role does not exist
1229  // create role and grant privileges
1230  // NOTE(wamsi): Transactionally unsafe
1231  SysCatalog::instance().createRole(dash_role_name, false);
1232  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1233  } else {
1234  // Dashboard system role already exists
1235  // Add/remove privileges on objects
1236  auto ex_objects = rl->getDbObjects(true);
1237  for (auto key : *ex_objects | boost::adaptors::map_keys) {
1238  if (key.permissionType != TableDBObjectType &&
1239  key.permissionType != ViewDBObjectType) {
1240  continue;
1241  }
1242  bool found = false;
1243  for (auto obj : objects) {
1244  found = key == obj.getObjectKey() ? true : false;
1245  if (found) {
1246  break;
1247  }
1248  }
1249  if (!found) {
1250  // revoke privs on object since the object is no
1251  // longer used by the dashboard as source
1252  // NOTE(wamsi): Transactionally unsafe
1254  dash_role_name, *rl->findDbObject(key, true), *this);
1255  }
1256  }
1257  // Update privileges on remaining objects
1258  // NOTE(wamsi): Transactionally unsafe
1259  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1260  }
1261 }
void revokeDBObjectPrivileges(const std::string &grantee, const DBObject &object, const Catalog_Namespace::Catalog &catalog)
DBObject * findDbObject(const DBObjectKey &objectKey, bool only_direct) const
Definition: Grantee.cpp:72
Definition: Grantee.h:76
void createRole(const std::string &roleName, const bool &userPrivateRole=false)
static SysCatalog & instance()
Definition: SysCatalog.h:240
void grantDBObjectPrivilegesBatch(const std::vector< std::string > &grantees, const std::vector< DBObject > &objects, const Catalog_Namespace::Catalog &catalog)
const DBObjectMap * getDbObjects(bool only_direct) const
Definition: Grantee.h:56
Role * getRoleGrantee(const std::string &name) const
std::vector< DBObject > parseDashboardObjects(const std::string &view_meta, const int &user_id)
Definition: Catalog.cpp:1192
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ createShardedTable()

void Catalog_Namespace::Catalog::createShardedTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs 
)

Definition at line 2384 of file Catalog.cpp.

References CHECK, createTable(), generatePhysicalTableName(), logicalToPhysicalTableMapById_, TableDescriptor::nShards, TableDescriptor::shard, TableDescriptor::tableId, TableDescriptor::tableName, and updateLogicalToPhysicalTableMap().

2387  {
2388  cat_write_lock write_lock(this);
2389 
2390  /* create logical table */
2391  TableDescriptor tdl(td);
2392  createTable(tdl, cols, shared_dict_defs, true); // create logical table
2393  int32_t logical_tb_id = tdl.tableId;
2394 
2395  /* create physical tables and link them to the logical table */
2396  std::vector<int32_t> physicalTables;
2397  for (int32_t i = 1; i <= td.nShards; i++) {
2398  TableDescriptor tdp(td);
2399  tdp.tableName = generatePhysicalTableName(tdp.tableName, i);
2400  tdp.shard = i - 1;
2401  createTable(tdp, cols, shared_dict_defs, false); // create physical table
2402  int32_t physical_tb_id = tdp.tableId;
2403 
2404  /* add physical table to the vector of physical tables */
2405  physicalTables.push_back(physical_tb_id);
2406  }
2407 
2408  if (!physicalTables.empty()) {
2409  /* add logical to physical tables correspondence to the map */
2410  const auto it_ok =
2411  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
2412  CHECK(it_ok.second);
2413  /* update sqlite mapd_logical_to_physical in sqlite database */
2414  updateLogicalToPhysicalTableMap(logical_tb_id);
2415  }
2416 }
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:1937
std::string generatePhysicalTableName(const std::string &logicalTableName, const int32_t &shardNumber)
Definition: Catalog.cpp:2977
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:89
#define CHECK(condition)
Definition: Logger.h:187
specifies the content in-memory of a row in the table metadata table
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:316
void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id)
Definition: Catalog.cpp:479
+ Here is the call graph for this function:

◆ createTable()

void Catalog_Namespace::Catalog::createTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs,
bool  isLogicalTable 
)

Definition at line 1937 of file Catalog.cpp.

References addTableToMap(), calciteMgr_, ColumnDescriptor::columnId, TableDescriptor::columnIdBySpi_, ColumnDescriptor::columnName, ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, dict_ref_t::dictId, Data_Namespace::DISK_LEVEL, expandGeoColumn(), TableDescriptor::fragPageSize, TableDescriptor::fragType, g_test_against_columnId_gap, SQLTypeInfoCore< TYPE_FACET_PACK >::get_comp_param(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_compression(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_dimension(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_notnull(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_scale(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_size(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_subtype(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), SqliteConnector::getData(), TableDescriptor::hasDeletedCol, SQLTypeInfoCore< TYPE_FACET_PACK >::is_array(), IS_GEO, ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, kBIGINT, kBOOLEAN, kENCODING_DICT, TableDescriptor::keyMetainfo, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRows, TableDescriptor::nColumns, nextTempDictId_, nextTempTableId_, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, SqliteConnector::query(), SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), removeTableFromMap(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_comp_param(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_size(), setColumnDictionary(), setColumnSharedDictionary(), TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, sqliteConnector_, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), TableDescriptor::userId, TableDescriptor::viewSQL, and ColumnDescriptor::virtualExpr.

Referenced by createShardedTable().

1941  {
1942  cat_write_lock write_lock(this);
1943  list<ColumnDescriptor> cds;
1944  list<DictDescriptor> dds;
1945  std::set<std::string> toplevel_column_names;
1946  list<ColumnDescriptor> columns;
1947  for (auto cd : cols) {
1948  if (cd.columnName == "rowid") {
1949  throw std::runtime_error(
1950  "Cannot create column with name rowid. rowid is a system defined column.");
1951  }
1952  columns.push_back(cd);
1953  toplevel_column_names.insert(cd.columnName);
1954  if (cd.columnType.is_geometry()) {
1955  expandGeoColumn(cd, columns);
1956  }
1957  }
1958 
1959  ColumnDescriptor cd;
1960  // add row_id column -- Must be last column in the table
1961  cd.columnName = "rowid";
1962  cd.isSystemCol = true;
1963  cd.columnType = SQLTypeInfo(kBIGINT, true);
1964 #ifdef MATERIALIZED_ROWID
1965  cd.isVirtualCol = false;
1966 #else
1967  cd.isVirtualCol = true;
1968  cd.virtualExpr = "MAPD_FRAG_ID * MAPD_ROWS_PER_FRAG + MAPD_FRAG_ROW_ID";
1969 #endif
1970  columns.push_back(cd);
1971  toplevel_column_names.insert(cd.columnName);
1972 
1973  if (td.hasDeletedCol) {
1974  ColumnDescriptor cd_del;
1975  cd_del.columnName = "$deleted$";
1976  cd_del.isSystemCol = true;
1977  cd_del.isVirtualCol = false;
1978  cd_del.columnType = SQLTypeInfo(kBOOLEAN, true);
1979  cd_del.isDeletedCol = true;
1980 
1981  columns.push_back(cd_del);
1982  }
1983 
1984  td.nColumns = columns.size();
1985  cat_sqlite_lock sqlite_lock(this);
1986  sqliteConnector_.query("BEGIN TRANSACTION");
1988  try {
1990  "INSERT INTO mapd_tables (name, userid, ncolumns, isview, fragments, "
1991  "frag_type, max_frag_rows, "
1992  "max_chunk_size, "
1993  "frag_page_size, max_rows, partitions, shard_column_id, shard, num_shards, "
1994  "sort_column_id, "
1995  "key_metainfo) VALUES (?, ?, ?, "
1996  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1997 
1998  std::vector<std::string>{td.tableName,
1999  std::to_string(td.userId),
2001  std::to_string(td.isView),
2002  "",
2007  std::to_string(td.maxRows),
2008  td.partitions,
2010  std::to_string(td.shard),
2011  std::to_string(td.nShards),
2013  td.keyMetainfo});
2014 
2015  // now get the auto generated tableid
2017  "SELECT tableid FROM mapd_tables WHERE name = ?", td.tableName);
2018  td.tableId = sqliteConnector_.getData<int>(0, 0);
2019  int colId = 1;
2020  for (auto cd : columns) {
2022  const bool is_foreign_col =
2023  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2024  if (!is_foreign_col) {
2025  setColumnDictionary(cd, dds, td, isLogicalTable);
2026  }
2027  }
2028 
2029  if (toplevel_column_names.count(cd.columnName)) {
2030  // make up colId gap for sanity test (begin with 1 bc much code depends on it!)
2031  if (colId > 1) {
2032  colId += g_test_against_columnId_gap;
2033  }
2034  if (!cd.isGeoPhyCol) {
2035  td.columnIdBySpi_.push_back(colId);
2036  }
2037  }
2038 
2040  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, "
2041  "coldim, colscale, is_notnull, "
2042  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
2043  "virtual_expr, is_deletedcol) "
2044  "VALUES (?, ?, ?, ?, ?, "
2045  "?, "
2046  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2047  std::vector<std::string>{std::to_string(td.tableId),
2048  std::to_string(colId),
2049  cd.columnName,
2058  "",
2061  cd.virtualExpr,
2063  cd.tableId = td.tableId;
2064  cd.columnId = colId++;
2065  cds.push_back(cd);
2066  }
2067  if (td.isView) {
2069  "INSERT INTO mapd_views (tableid, sql) VALUES (?,?)",
2070  std::vector<std::string>{std::to_string(td.tableId), td.viewSQL});
2071  }
2072  } catch (std::exception& e) {
2073  sqliteConnector_.query("ROLLBACK TRANSACTION");
2074  throw;
2075  }
2076 
2077  } else { // Temporary table
2078  td.tableId = nextTempTableId_++;
2079  int colId = 1;
2080  for (auto cd : columns) {
2081  auto col_ti = cd.columnType;
2082  if (IS_GEO(col_ti.get_type())) {
2083  throw runtime_error("Geometry types in temporary tables are not supported.");
2084  }
2085 
2087  // TODO(vraj) : create shared dictionary for temp table if needed
2088  std::string fileName("");
2089  std::string folderPath("");
2091  nextTempDictId_++;
2092  DictDescriptor dd(dict_ref,
2093  fileName,
2095  false,
2096  1,
2097  folderPath,
2098  true); // Is dictName (2nd argument) used?
2099  dds.push_back(dd);
2100  if (!cd.columnType.is_array()) {
2102  }
2103  cd.columnType.set_comp_param(dict_ref.dictId);
2104  }
2105  cd.tableId = td.tableId;
2106  cd.columnId = colId++;
2107  cds.push_back(cd);
2108  }
2109  }
2110  try {
2111  addTableToMap(td, cds, dds);
2112  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
2113  } catch (std::exception& e) {
2114  sqliteConnector_.query("ROLLBACK TRANSACTION");
2115  removeTableFromMap(td.tableName, td.tableId, true);
2116  throw;
2117  }
2118 
2119  sqliteConnector_.query("END TRANSACTION");
2120 }
std::string virtualExpr
std::string partitions
HOST DEVICE int get_size() const
Definition: sqltypes.h:333
T getData(const int row, const int col)
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:325
std::string tableName
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:330
SqliteConnector sqliteConnector_
Definition: Catalog.h:309
void set_size(int s)
Definition: sqltypes.h:421
void query_with_text_params(std::string const &query_only)
HOST DEVICE int get_scale() const
Definition: sqltypes.h:328
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:323
void query(const std::string &queryString)
void addTableToMap(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
Definition: Catalog.cpp:1049
std::vector< int > columnIdBySpi_
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:331
std::string to_string(char const *&&v)
bool is_array() const
Definition: sqltypes.h:458
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:324
specifies the content in-memory of a row in the column metadata table
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:1807
SQLTypeInfoCore< ArrayContextTypeSizer, ExecutorTypePackaging, DateTimeFacilities > SQLTypeInfo
Definition: sqltypes.h:823
std::string keyMetainfo
int g_test_against_columnId_gap
Definition: Catalog.cpp:72
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:314
Fragmenter_Namespace::FragmenterType fragType
Data_Namespace::MemoryLevel persistenceLevel
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1099
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:89
void set_comp_param(int p)
Definition: sqltypes.h:424
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:332
bool setColumnSharedDictionary(ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:2281
Descriptor for a dictionary for a string columne.
std::string viewSQL
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:90
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:2343
std::string columnName
void query_with_text_param(const std::string &queryString, const std::string &text_param)
#define IS_GEO(T)
Definition: sqltypes.h:164
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ delDictionary()

void Catalog_Namespace::Catalog::delDictionary ( const ColumnDescriptor cd)

Definition at line 1639 of file Catalog.cpp.

References basePath_, client, ColumnDescriptor::columnName, ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, SQLTypeInfoCore< TYPE_FACET_PACK >::get_comp_param(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_compression(), SQLTypeInfoCore< TYPE_FACET_PACK >::is_string(), SQLTypeInfoCore< TYPE_FACET_PACK >::is_string_array(), kENCODING_DICT, SqliteConnector::query_with_text_param(), File_Namespace::renameForDelete(), sqliteConnector_, string_dict_hosts_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableName, and to_string().

Referenced by roll().

1639  {
1640  if (!(cd.columnType.is_string() || cd.columnType.is_string_array())) {
1641  return;
1642  }
1643  if (!(cd.columnType.get_compression() == kENCODING_DICT)) {
1644  return;
1645  }
1646  if (!(cd.columnType.get_comp_param() > 0)) {
1647  return;
1648  }
1649 
1650  const auto& td = *tableDescriptorMapById_[cd.tableId];
1651  const auto dictId = cd.columnType.get_comp_param();
1652  const DictRef dictRef(currentDB_.dbId, dictId);
1653  const auto dictName =
1654  td.tableName + "_" + cd.columnName + "_dict" + std::to_string(dictId);
1655  sqliteConnector_.query_with_text_param("DELETE FROM mapd_dictionaries WHERE name = ?",
1656  dictName);
1657  File_Namespace::renameForDelete(basePath_ + "/mapd_data/DB_" +
1658  std::to_string(currentDB_.dbId) + "_DICT_" +
1659  std::to_string(dictId));
1660 
1661  std::unique_ptr<StringDictionaryClient> client;
1662  if (!string_dict_hosts_.empty()) {
1663  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dictRef, true));
1664  }
1665  if (client) {
1666  client->drop(dictRef);
1667  }
1668 
1669  dictDescriptorMapByRef_.erase(dictRef);
1670 }
bool is_string_array() const
Definition: sqltypes.h:451
SqliteConnector sqliteConnector_
Definition: Catalog.h:309
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:313
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:305
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:331
std::string to_string(char const *&&v)
mapd::shared_ptr< MapDClient > client
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:302
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:332
SQLTypeInfo columnType
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_<EPOCH>_<oldname>.
Definition: File.cpp:183
std::string columnName
void query_with_text_param(const std::string &queryString, const std::string &text_param)
bool is_string() const
Definition: sqltypes.h:450
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ deleteMetadataForDashboard() [1/2]

void Catalog_Namespace::Catalog::deleteMetadataForDashboard ( const std::string &  userId,
const std::string &  dashName 
)

Definition at line 1445 of file Catalog.cpp.

References dashboardDescriptorMap_, logger::ERROR, getMetadataForDashboard(), LOG, SqliteConnector::query(), SqliteConnector::query_with_text_params(), and sqliteConnector_.

Referenced by deleteMetadataForDashboard().

1446  {
1447  cat_write_lock write_lock(this);
1448 
1449  auto viewDescIt = dashboardDescriptorMap_.find(userId + ":" + dashName);
1450  if (viewDescIt == dashboardDescriptorMap_.end()) { // check to make sure view exists
1451  LOG(ERROR) << "No metadata for dashboard for user " << userId << " dashboard "
1452  << dashName << " does not exist in map";
1453  throw runtime_error("No metadata for dashboard for user " + userId + " dashboard " +
1454  dashName + " does not exist in map");
1455  }
1456  // found view in Map now remove it
1457  dashboardDescriptorMap_.erase(viewDescIt);
1458  // remove from DB
1459  cat_sqlite_lock sqlite_lock(this);
1460  sqliteConnector_.query("BEGIN TRANSACTION");
1461  try {
1463  "DELETE FROM mapd_dashboards WHERE name = ? and userid = ?",
1464  std::vector<std::string>{dashName, userId});
1465  } catch (std::exception& e) {
1466  sqliteConnector_.query("ROLLBACK TRANSACTION");
1467  throw;
1468  }
1469  sqliteConnector_.query("END TRANSACTION");
1470 }
#define LOG(tag)
Definition: Logger.h:182
SqliteConnector sqliteConnector_
Definition: Catalog.h:309
void query_with_text_params(std::string const &query_only)
void query(const std::string &queryString)
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:306
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:89
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:90
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ deleteMetadataForDashboard() [2/2]

void Catalog_Namespace::Catalog::deleteMetadataForDashboard ( const int32_t  dashboard_id)

Definition at line 1506 of file Catalog.cpp.

References DashboardDBObjectType, dashboardDescriptorMap_, deleteMetadataForDashboard(), getMetadataForLink(), Catalog_Namespace::SysCatalog::instance(), linkDescriptorMap_, name(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivilegesFromAll(), and to_string().

1506  {
1507  std::string userId;
1508  std::string name;
1509  bool found{false};
1510  {
1511  cat_read_lock read_lock(this);
1512  for (auto descp : dashboardDescriptorMap_) {
1513  auto dash = descp.second.get();
1514  if (dash->dashboardId == id) {
1515  userId = std::to_string(dash->userId);
1516  name = dash->dashboardName;
1517  found = true;
1518  break;
1519  }
1520  }
1521  }
1522  if (found) {
1523  // TODO: transactionally unsafe
1525  DBObject(id, DashboardDBObjectType), this);
1526  deleteMetadataForDashboard(userId, name);
1527  }
1528 }
void revokeDBObjectPrivilegesFromAll(DBObject object, Catalog *catalog)
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:88
void deleteMetadataForDashboard(const std::string &userId, const std::string &dashName)
Definition: Catalog.cpp:1445
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:306
std::string to_string(char const *&&v)
static SysCatalog & instance()
Definition: SysCatalog.h:240
std::string name() const
Definition: Catalog.h:219
+ Here is the call graph for this function:

◆ doDropTable()

void Catalog_Namespace::Catalog::doDropTable ( const TableDescriptor td)
protected

Definition at line 2558 of file Catalog.cpp.

References eraseTablePhysicalData(), SqliteConnector::getData(), SqliteConnector::getNumRows(), TableDescriptor::isView, kENCODING_DICT, SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), sqliteConnector_, TableDescriptor::tableId, and to_string().

Referenced by dropTable().

2558  {
2559  const int tableId = td->tableId;
2560  sqliteConnector_.query_with_text_param("DELETE FROM mapd_tables WHERE tableid = ?",
2561  std::to_string(tableId));
2563  "select comp_param from mapd_columns where compression = ? and tableid = ?",
2564  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
2565  int numRows = sqliteConnector_.getNumRows();
2566  std::vector<int> dict_id_list;
2567  for (int r = 0; r < numRows; ++r) {
2568  dict_id_list.push_back(sqliteConnector_.getData<int>(r, 0));
2569  }
2570  for (auto dict_id : dict_id_list) {
2572  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
2573  std::vector<std::string>{std::to_string(dict_id)});
2574  }
2576  "DELETE FROM mapd_dictionaries WHERE dictid in (select comp_param from "
2577  "mapd_columns where compression = ? "
2578  "and tableid = ?) and refcount = 0",
2579  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
2580  sqliteConnector_.query_with_text_param("DELETE FROM mapd_columns WHERE tableid = ?",
2581  std::to_string(tableId));
2582  if (td->isView) {
2583  sqliteConnector_.query_with_text_param("DELETE FROM mapd_views WHERE tableid = ?",
2584  std::to_string(tableId));
2585  }
2587 }
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:2947
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:309
void query_with_text_params(std::string const &query_only)
std::string to_string(char const *&&v)
size_t getNumRows() const
void query_with_text_param(const std::string &queryString, const std::string &text_param)
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ doTruncateTable()

void Catalog_Namespace::Catalog::doTruncateTable ( const TableDescriptor td)
protected

Definition at line 2436 of file Catalog.cpp.

References CHECK, CHECK_GE, client, columnDescriptorMapById_, Data_Namespace::CPU_LEVEL, currentDB_, dataMgr_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, dict_ref_t::dictId, DictDescriptor::dictIsTemp, DictDescriptor::dictRef, TableDescriptor::fragmenter, getMetadataForDict(), Data_Namespace::GPU_LEVEL, Catalog_Namespace::SysCatalog::instance(), kENCODING_DICT, File_Namespace::renameForDelete(), string_dict_hosts_, tableDescriptorMapById_, and TableDescriptor::tableId.

Referenced by truncateTable().

2436  {
2437  cat_write_lock write_lock(this);
2438 
2439  const int tableId = td->tableId;
2440  // must destroy fragmenter before deleteChunks is called.
2441  if (td->fragmenter != nullptr) {
2442  auto tableDescIt = tableDescriptorMapById_.find(tableId);
2443  delete td->fragmenter;
2444  tableDescIt->second->fragmenter = nullptr; // get around const-ness
2445  }
2446  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
2447  // assuming deleteChunksWithPrefix is atomic
2448  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
2449  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
2450 
2451  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
2452 
2453  std::unique_ptr<StringDictionaryClient> client;
2454  if (SysCatalog::instance().isAggregator()) {
2455  CHECK(!string_dict_hosts_.empty());
2456  DictRef dict_ref(currentDB_.dbId, -1);
2457  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
2458  }
2459  // clean up any dictionaries
2460  // delete all column descriptors for the table
2461  for (const auto& columnDescriptor : columnDescriptorMapById_) {
2462  auto cd = columnDescriptor.second;
2463  if (cd->tableId != td->tableId) {
2464  continue;
2465  }
2466  const int dict_id = cd->columnType.get_comp_param();
2467  // Dummy dictionaries created for a shard of a logical table have the id set to zero.
2468  if (cd->columnType.get_compression() == kENCODING_DICT && dict_id) {
2469  const DictRef dict_ref(currentDB_.dbId, dict_id);
2470  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
2471  CHECK(dictIt != dictDescriptorMapByRef_.end());
2472  const auto& dd = dictIt->second;
2473  CHECK_GE(dd->refcount, 1);
2474  // if this is the only table using this dict reset the dict
2475  if (dd->refcount == 1) {
2476  // close the dictionary
2477  dd->stringDict.reset();
2478  File_Namespace::renameForDelete(dd->dictFolderPath);
2479  if (client) {
2480  client->drop(dd->dictRef);
2481  }
2482  if (!dd->dictIsTemp) {
2483  boost::filesystem::create_directory(dd->dictFolderPath);
2484  }
2485  }
2486 
2487  DictDescriptor* new_dd = new DictDescriptor(dd->dictRef,
2488  dd->dictName,
2489  dd->dictNBits,
2490  dd->dictIsShared,
2491  dd->refcount,
2492  dd->dictFolderPath,
2493  dd->dictIsTemp);
2494  dictDescriptorMapByRef_.erase(dictIt);
2495  // now create new Dict -- need to figure out what to do here for temp tables
2496  if (client) {
2497  client->create(new_dd->dictRef, new_dd->dictIsTemp);
2498  }
2499  dictDescriptorMapByRef_[new_dd->dictRef].reset(new_dd);
2501  }
2502  }
2503 }
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:313
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:305
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:311
#define CHECK_GE(x, y)
Definition: Logger.h:200
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:304
static SysCatalog & instance()
Definition: SysCatalog.h:240
mapd::shared_ptr< MapDClient > client
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:302
int32_t dictId
Definition: DictRef.h:10
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1348
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:89
#define CHECK(condition)
Definition: Logger.h:187
std::vector< int > ChunkKey
Definition: types.h:35
Descriptor for a dictionary for a string columne.
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_<EPOCH>_<oldname>.
Definition: File.cpp:183
Fragmenter_Namespace::AbstractFragmenter * fragmenter
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ dropTable()

void Catalog_Namespace::Catalog::dropTable ( const TableDescriptor td)

Definition at line 2525 of file Catalog.cpp.

References CHECK, doDropTable(), getMetadataForTable(), Catalog_Namespace::SysCatalog::instance(), TableDescriptor::isView, logicalToPhysicalTableMapById_, SqliteConnector::query(), SqliteConnector::query_with_text_param(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivilegesFromAll(), sqliteConnector_, TableDBObjectType, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and ViewDBObjectType.

2525  {
2528  cat_write_lock write_lock(this);
2529  cat_sqlite_lock sqlite_lock(this);
2530  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
2531  sqliteConnector_.query("BEGIN TRANSACTION");
2532  try {
2533  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2534  // remove all corresponding physical tables if this is a logical table
2535  const auto physicalTables = physicalTableIt->second;
2536  CHECK(!physicalTables.empty());
2537  for (size_t i = 0; i < physicalTables.size(); i++) {
2538  int32_t physical_tb_id = physicalTables[i];
2539  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2540  CHECK(phys_td);
2541  doDropTable(phys_td);
2542  }
2543 
2544  // remove corresponding record from the logicalToPhysicalTableMap in sqlite database
2546  "DELETE FROM mapd_logical_to_physical WHERE logical_table_id = ?",
2547  std::to_string(td->tableId));
2549  }
2550  doDropTable(td);
2551  } catch (std::exception& e) {
2552  sqliteConnector_.query("ROLLBACK TRANSACTION");
2553  throw;
2554  }
2555  sqliteConnector_.query("END TRANSACTION");
2556 }
void doDropTable(const TableDescriptor *td)
Definition: Catalog.cpp:2558
std::string tableName
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
SqliteConnector sqliteConnector_
Definition: Catalog.h:309
void revokeDBObjectPrivilegesFromAll(DBObject object, Catalog *catalog)
void query(const std::string &queryString)
std::string to_string(char const *&&v)
static SysCatalog & instance()
Definition: SysCatalog.h:240
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:89
#define CHECK(condition)
Definition: Logger.h:187
specifies the content in-memory of a row in the table metadata table
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:90
void query_with_text_param(const std::string &queryString, const std::string &text_param)
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:316
+ Here is the call graph for this function:

◆ eraseDBData()

void Catalog_Namespace::Catalog::eraseDBData ( )

Definition at line 2935 of file Catalog.cpp.

References basePath_, calciteMgr_, currentDB_, Catalog_Namespace::DBMetadata::dbName, eraseTablePhysicalData(), getAllTableMetadata(), and run-benchmark-import::tables.

2935  {
2936  cat_write_lock write_lock(this);
2937  // Physically erase all tables and dictionaries from disc and memory
2938  const auto tables = getAllTableMetadata();
2939  for (const auto table : tables) {
2940  eraseTablePhysicalData(table);
2941  }
2942  // Physically erase database metadata
2943  boost::filesystem::remove(basePath_ + "/mapd_catalogs/" + currentDB_.dbName);
2944  calciteMgr_->updateMetadata(currentDB_.dbName, "");
2945 }
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:1596
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:2947
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:314
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:89
+ Here is the call graph for this function:

◆ eraseTablePhysicalData()

void Catalog_Namespace::Catalog::eraseTablePhysicalData ( const TableDescriptor td)

Definition at line 2947 of file Catalog.cpp.

References calciteMgr_, Data_Namespace::CPU_LEVEL, currentDB_, dataMgr_, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, TableDescriptor::fragmenter, Data_Namespace::GPU_LEVEL, INJECT_TIMER, TableDescriptor::isView, removeTableFromMap(), tableDescriptorMapById_, TableDescriptor::tableId, and TableDescriptor::tableName.

Referenced by doDropTable(), and eraseDBData().

2947  {
2948  cat_write_lock write_lock(this);
2949  const int tableId = td->tableId;
2950  // must destroy fragmenter before deleteChunks is called.
2951  if (td->fragmenter != nullptr) {
2952  auto tableDescIt = tableDescriptorMapById_.find(tableId);
2953  {
2954  INJECT_TIMER(deleting_fragmenter);
2955  delete td->fragmenter;
2956  }
2957  tableDescIt->second->fragmenter = nullptr; // get around const-ness
2958  }
2959  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
2960  {
2961  INJECT_TIMER(deleteChunksWithPrefix);
2962  // assuming deleteChunksWithPrefix is atomic
2963  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
2964  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
2965  }
2966  if (!td->isView) {
2967  INJECT_TIMER(Remove_Table);
2968  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
2969  }
2970  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
2971  {
2972  INJECT_TIMER(removeTableFromMap_);
2973  removeTableFromMap(td->tableName, tableId);
2974  }
2975 }
std::string tableName
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:311
#define INJECT_TIMER(DESC)
Definition: measure.h:91
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:302
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:314
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1099
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:89
std::vector< int > ChunkKey
Definition: types.h:35
Fragmenter_Namespace::AbstractFragmenter * fragmenter
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ expandGeoColumn()

void Catalog_Namespace::Catalog::expandGeoColumn ( const ColumnDescriptor cd,
std::list< ColumnDescriptor > &  columns 
)
static

Definition at line 1807 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, IS_GEO, kARRAY, kDOUBLE, kENCODING_GEOINT, kENCODING_NONE, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kTINYINT, SQLTypeInfoCore< TYPE_FACET_PACK >::set_size(), and SQLTypeInfoCore< TYPE_FACET_PACK >::set_subtype().

Referenced by createTable().

1808  {
1809  const auto& col_ti = cd.columnType;
1810  if (IS_GEO(col_ti.get_type())) {
1811  switch (col_ti.get_type()) {
1812  case kPOINT: {
1813  ColumnDescriptor physical_cd_coords(true);
1814  physical_cd_coords.columnName = cd.columnName + "_coords";
1815  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, true);
1816  // Raw data: compressed/uncompressed coords
1817  coords_ti.set_subtype(kTINYINT);
1818  size_t unit_size;
1819  if (col_ti.get_compression() == kENCODING_GEOINT &&
1820  col_ti.get_comp_param() == 32) {
1821  unit_size = 4 * sizeof(int8_t);
1822  } else {
1823  CHECK(col_ti.get_compression() == kENCODING_NONE);
1824  unit_size = 8 * sizeof(int8_t);
1825  }
1826  coords_ti.set_size(2 * unit_size);
1827  physical_cd_coords.columnType = coords_ti;
1828  columns.push_back(physical_cd_coords);
1829 
1830  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1831 
1832  break;
1833  }
1834  case kLINESTRING: {
1835  ColumnDescriptor physical_cd_coords(true);
1836  physical_cd_coords.columnName = cd.columnName + "_coords";
1837  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, true);
1838  // Raw data: compressed/uncompressed coords
1839  coords_ti.set_subtype(kTINYINT);
1840  physical_cd_coords.columnType = coords_ti;
1841  columns.push_back(physical_cd_coords);
1842 
1843  ColumnDescriptor physical_cd_bounds(true);
1844  physical_cd_bounds.columnName = cd.columnName + "_bounds";
1845  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, true);
1846  bounds_ti.set_subtype(kDOUBLE);
1847  bounds_ti.set_size(4 * sizeof(double));
1848  physical_cd_bounds.columnType = bounds_ti;
1849  columns.push_back(physical_cd_bounds);
1850 
1851  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1852 
1853  break;
1854  }
1855  case kPOLYGON: {
1856  ColumnDescriptor physical_cd_coords(true);
1857  physical_cd_coords.columnName = cd.columnName + "_coords";
1858  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, true);
1859  // Raw data: compressed/uncompressed coords
1860  coords_ti.set_subtype(kTINYINT);
1861  physical_cd_coords.columnType = coords_ti;
1862  columns.push_back(physical_cd_coords);
1863 
1864  ColumnDescriptor physical_cd_ring_sizes(true);
1865  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
1866  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, true);
1867  ring_sizes_ti.set_subtype(kINT);
1868  physical_cd_ring_sizes.columnType = ring_sizes_ti;
1869  columns.push_back(physical_cd_ring_sizes);
1870 
1871  ColumnDescriptor physical_cd_bounds(true);
1872  physical_cd_bounds.columnName = cd.columnName + "_bounds";
1873  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, true);
1874  bounds_ti.set_subtype(kDOUBLE);
1875  bounds_ti.set_size(4 * sizeof(double));
1876  physical_cd_bounds.columnType = bounds_ti;
1877  columns.push_back(physical_cd_bounds);
1878 
1879  ColumnDescriptor physical_cd_render_group(true);
1880  physical_cd_render_group.columnName = cd.columnName + "_render_group";
1881  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, true);
1882  physical_cd_render_group.columnType = render_group_ti;
1883  columns.push_back(physical_cd_render_group);
1884 
1885  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1886 
1887  break;
1888  }
1889  case kMULTIPOLYGON: {
1890  ColumnDescriptor physical_cd_coords(true);
1891  physical_cd_coords.columnName = cd.columnName + "_coords";
1892  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, true);
1893  // Raw data: compressed/uncompressed coords
1894  coords_ti.set_subtype(kTINYINT);
1895  physical_cd_coords.columnType = coords_ti;
1896  columns.push_back(physical_cd_coords);
1897 
1898  ColumnDescriptor physical_cd_ring_sizes(true);
1899  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
1900  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, true);
1901  ring_sizes_ti.set_subtype(kINT);
1902  physical_cd_ring_sizes.columnType = ring_sizes_ti;
1903  columns.push_back(physical_cd_ring_sizes);
1904 
1905  ColumnDescriptor physical_cd_poly_rings(true);
1906  physical_cd_poly_rings.columnName = cd.columnName + "_poly_rings";
1907  SQLTypeInfo poly_rings_ti = SQLTypeInfo(kARRAY, true);
1908  poly_rings_ti.set_subtype(kINT);
1909  physical_cd_poly_rings.columnType = poly_rings_ti;
1910  columns.push_back(physical_cd_poly_rings);
1911 
1912  ColumnDescriptor physical_cd_bounds(true);
1913  physical_cd_bounds.columnName = cd.columnName + "_bounds";
1914  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, true);
1915  bounds_ti.set_subtype(kDOUBLE);
1916  bounds_ti.set_size(4 * sizeof(double));
1917  physical_cd_bounds.columnType = bounds_ti;
1918  columns.push_back(physical_cd_bounds);
1919 
1920  ColumnDescriptor physical_cd_render_group(true);
1921  physical_cd_render_group.columnName = cd.columnName + "_render_group";
1922  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, true);
1923  physical_cd_render_group.columnType = render_group_ti;
1924  columns.push_back(physical_cd_render_group);
1925 
1926  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1927 
1928  break;
1929  }
1930  default:
1931  throw runtime_error("Unrecognized geometry type.");
1932  break;
1933  }
1934  }
1935 }
void set_size(int s)
Definition: sqltypes.h:421
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:414
specifies the content in-memory of a row in the column metadata table
SQLTypeInfoCore< ArrayContextTypeSizer, ExecutorTypePackaging, DateTimeFacilities > SQLTypeInfo
Definition: sqltypes.h:823
#define CHECK(condition)
Definition: Logger.h:187
Definition: sqltypes.h:47
SQLTypeInfo columnType
std::string columnName
#define IS_GEO(T)
Definition: sqltypes.h:164
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ generatePhysicalTableName()

std::string Catalog_Namespace::Catalog::generatePhysicalTableName ( const std::string &  logicalTableName,
const int32_t &  shardNumber 
)
protected

Definition at line 2977 of file Catalog.cpp.

References physicalTableNameTag_, and to_string().

Referenced by createShardedTable(), and renameTable().

2978  {
2979  std::string physicalTableName =
2980  logicalTableName + physicalTableNameTag_ + std::to_string(shardNumber);
2981  return (physicalTableName);
2982 }
std::string to_string(char const *&&v)
static const std::string physicalTableNameTag_
Definition: Catalog.h:318
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ get() [1/2]

std::shared_ptr< Catalog > Catalog_Namespace::Catalog::get ( const std::string &  dbName)
static

Definition at line 2988 of file Catalog.cpp.

References Catalog_Namespace::DBMetadata::dbName, mapd_cat_map_, and set().

Referenced by Catalog_Namespace::SysCatalog::createDatabase(), Catalog_Namespace::SysCatalog::dropDatabase(), anonymous_namespace{DBObjectPrivilegesTest.cpp}::get_qr_for_user(), Catalog_Namespace::SysCatalog::login(), main(), Catalog_Namespace::SysCatalog::switchDatabase(), TEST(), and TEST_F().

2988  {
2989  auto cat_it = mapd_cat_map_.find(dbName);
2990  if (cat_it != mapd_cat_map_.end()) {
2991  return cat_it->second;
2992  }
2993  return nullptr;
2994 }
static std::map< std::string, std::shared_ptr< Catalog > > mapd_cat_map_
Definition: Catalog.h:331
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ get() [2/2]

static std::shared_ptr<Catalog> Catalog_Namespace::Catalog::get ( const std::string &  basePath,
const DBMetadata curDB,
std::shared_ptr< Data_Namespace::DataMgr dataMgr,
const std::vector< LeafHostInfo > &  string_dict_hosts,
std::shared_ptr< Calcite calcite,
bool  is_new_db 
)
static

◆ getAllColumnMetadataForTable() [1/2]

list< const ColumnDescriptor * > Catalog_Namespace::Catalog::getAllColumnMetadataForTable ( const int  tableId,
const bool  fetchSystemColumns,
const bool  fetchVirtualColumns,
const bool  fetchPhysicalColumns 
) const

Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular table specified by table id.

Parameters
tableIdtable id we want the column metadata for
Returns
list of pointers to const ColumnDescriptor structs - one for each and every column in the table

Definition at line 1579 of file Catalog.cpp.

References getMetadataForTableImpl().

Referenced by Analyzer::RangeTableEntry::add_all_column_descs(), Parser::InsertStmt::analyze(), Parser::InsertValuesStmt::analyze(), Fragmenter_Namespace::copyDataOfShard(), Parser::InsertValuesStmt::determineLeafIndex(), Analyzer::RangeTableEntry::expand_star_in_targetlist(), getMetadataForLink(), getShardColumnMetadataForTable(), instantiateFragmenter(), populate_table_random(), Parser::InsertIntoTableAsSelectStmt::populateData(), TableOptimizer::recomputeMetadata(), scan_table_return_hash(), scan_table_return_hash_non_iter(), and validate_shared_dictionary().

1583  {
1584  cat_read_lock read_lock(this);
1585  list<const ColumnDescriptor*> columnDescriptors;
1586  const TableDescriptor* td =
1587  getMetadataForTableImpl(tableId, false); // dont instantiate fragmenter
1589  columnDescriptors,
1590  fetchSystemColumns,
1591  fetchVirtualColumns,
1592  fetchPhysicalColumns);
1593  return columnDescriptors;
1594 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:88
const TableDescriptor * getMetadataForTableImpl(int tableId, const bool populateFragmenter) const
Definition: Catalog.cpp:1328
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1579
specifies the content in-memory of a row in the table metadata table
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getAllColumnMetadataForTable() [2/2]

void Catalog_Namespace::Catalog::getAllColumnMetadataForTable ( const TableDescriptor td,
std::list< const ColumnDescriptor *> &  colDescs,
const bool  fetchSystemColumns,
const bool  fetchVirtualColumns,
const bool  fetchPhysicalColumns 
) const
protected

◆ getAllDashboardsMetadata()

list< const DashboardDescriptor * > Catalog_Namespace::Catalog::getAllDashboardsMetadata ( ) const

Definition at line 1605 of file Catalog.cpp.

References dashboardDescriptorMap_.

1605  {
1606  list<const DashboardDescriptor*> view_list;
1607  for (auto p : dashboardDescriptorMap_) {
1608  view_list.push_back(p.second.get());
1609  }
1610  return view_list;
1611 }
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:306

◆ getAllTableMetadata()

list< const TableDescriptor * > Catalog_Namespace::Catalog::getAllTableMetadata ( ) const

Definition at line 1596 of file Catalog.cpp.

References tableDescriptorMapById_.

Referenced by eraseDBData().

1596  {
1597  cat_read_lock read_lock(this);
1598  list<const TableDescriptor*> table_list;
1599  for (auto p : tableDescriptorMapById_) {
1600  table_list.push_back(p.second);
1601  }
1602  return table_list;
1603 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:88
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:302
+ Here is the caller graph for this function:

◆ getBasePath()

const std::string& Catalog_Namespace::Catalog::getBasePath ( ) const
inline

Definition at line 179 of file Catalog.h.

179 { return basePath_; }

◆ getCalciteMgr()

std::shared_ptr<Calcite> Catalog_Namespace::Catalog::getCalciteMgr ( ) const
inline

Definition at line 178 of file Catalog.h.

Referenced by Lock_Namespace::parse_to_ra().

178 { return calciteMgr_; }
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:314
+ Here is the caller graph for this function:

◆ getColumnIdBySpi()

const int Catalog_Namespace::Catalog::getColumnIdBySpi ( const int  tableId,
const size_t  spi 
) const

Definition at line 1430 of file Catalog.cpp.

References getColumnIdBySpiUnlocked().

Referenced by anonymous_namespace{RelAlgExecutor.cpp}::collect_used_input_desc(), and anonymous_namespace{RelAlgExecutor.cpp}::get_physical_inputs().

1430  {
1431  cat_read_lock read_lock(this);
1432  return getColumnIdBySpiUnlocked(table_id, spi);
1433 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:88
const int getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const
Definition: Catalog.cpp:1413
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getColumnIdBySpiUnlocked()

const int Catalog_Namespace::Catalog::getColumnIdBySpiUnlocked ( const int  table_id,
const size_t  spi 
) const
protected

Definition at line 1413 of file Catalog.cpp.

References CHECK, SPIMAP_MAGIC1, SPIMAP_MAGIC2, and tableDescriptorMapById_.

Referenced by getColumnIdBySpi(), and getMetadataForColumnBySpi().

1413  {
1414  const auto tabDescIt = tableDescriptorMapById_.find(table_id);
1415  CHECK(tableDescriptorMapById_.end() != tabDescIt);
1416  const auto& columnIdBySpi = tabDescIt->second->columnIdBySpi_;
1417 
1418  auto spx = spi;
1419  int phi = 0;
1420  if (spx >= SPIMAP_MAGIC1) // see Catalog.h
1421  {
1422  phi = (spx - SPIMAP_MAGIC1) % SPIMAP_MAGIC2;
1423  spx = (spx - SPIMAP_MAGIC1) / SPIMAP_MAGIC2;
1424  }
1425 
1426  CHECK(0 < spx && spx <= columnIdBySpi.size());
1427  return columnIdBySpi[spx - 1] + phi;
1428 }
#define SPIMAP_MAGIC1
Definition: Catalog.h:68
#define SPIMAP_MAGIC2
Definition: Catalog.h:69
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:302
#define CHECK(condition)
Definition: Logger.h:187
+ Here is the caller graph for this function:

◆ getCurrentDB()

◆ getDatabaseId()

int Catalog_Namespace::Catalog::getDatabaseId ( ) const
inline

Definition at line 192 of file Catalog.h.

192 { return currentDB_.dbId; }

◆ getDataMgr()

◆ getDeletedColumn()

const ColumnDescriptor * Catalog_Namespace::Catalog::getDeletedColumn ( const TableDescriptor td) const

Definition at line 2177 of file Catalog.cpp.

References deletedColumnPerTable_.

Referenced by TableOptimizer::recomputeMetadata(), and vacuumDeletedRows().

2177  {
2178  cat_read_lock read_lock(this);
2179  const auto it = deletedColumnPerTable_.find(td);
2180  return it != deletedColumnPerTable_.end() ? it->second : nullptr;
2181 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:88
DeletedColumnPerTableMap deletedColumnPerTable_
Definition: Catalog.h:332
+ Here is the caller graph for this function:

◆ getDeletedColumnIfRowsDeleted()

const ColumnDescriptor * Catalog_Namespace::Catalog::getDeletedColumnIfRowsDeleted ( const TableDescriptor td) const

Definition at line 2202 of file Catalog.cpp.

References CHECK, checkMetadataForDeletedRecs(), ColumnDescriptor::columnId, currentDB_, Catalog_Namespace::DBMetadata::dbId, deletedColumnPerTable_, getMetadataForTable(), logicalToPhysicalTableMapById_, and TableDescriptor::tableId.

Referenced by Executor::addDeletedColumn().

2203  {
2204  cat_read_lock read_lock(this);
2205 
2206  const auto it = deletedColumnPerTable_.find(td);
2207  // if not a table that supports delete return nullptr, nothing more to do
2208  if (it == deletedColumnPerTable_.end()) {
2209  return nullptr;
2210  }
2211  const ColumnDescriptor* cd = it->second;
2212 
2213  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
2214 
2215  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2216  // check all shards
2217  const auto physicalTables = physicalTableIt->second;
2218  CHECK(!physicalTables.empty());
2219  for (size_t i = 0; i < physicalTables.size(); i++) {
2220  int32_t physical_tb_id = physicalTables[i];
2221  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2222  CHECK(phys_td);
2224  return cd;
2225  }
2226  }
2227  } else {
2229  return cd;
2230  }
2231  }
2232  // no deletes so far recorded in metadata
2233  return nullptr;
2234 }
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:88
const bool checkMetadataForDeletedRecs(int dbId, int tableId, int columnId) const
Definition: Catalog.cpp:2183
DeletedColumnPerTableMap deletedColumnPerTable_
Definition: Catalog.h:332
specifies the content in-memory of a row in the column metadata table
#define CHECK(condition)
Definition: Logger.h:187
specifies the content in-memory of a row in the table metadata table
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:316
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getDictionary()

void Catalog_Namespace::Catalog::getDictionary ( const ColumnDescriptor cd,
std::map< int, StringDictionary *> &  stringDicts 
)

Definition at line 1672 of file Catalog.cpp.

References CHECK, columnDescriptorMap_, ColumnDescriptor::columnName, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, getMetadataForDict(), kENCODING_DICT, ColumnDescriptor::tableId, and to_upper().

1673  {
1674  // learn 'committed' ColumnDescriptor of this column
1675  auto cit = columnDescriptorMap_.find(ColumnKey(cd.tableId, to_upper(cd.columnName)));
1676  CHECK(cit != columnDescriptorMap_.end());
1677  auto& ccd = *cit->second;
1678 
1679  if (!(ccd.columnType.is_string() || ccd.columnType.is_string_array())) {
1680  return;
1681  }
1682  if (!(ccd.columnType.get_compression() == kENCODING_DICT)) {
1683  return;
1684  }
1685  if (!(ccd.columnType.get_comp_param() > 0)) {
1686  return;
1687  }
1688 
1689  auto dictId = ccd.columnType.get_comp_param();
1690  getMetadataForDict(dictId);
1691 
1692  const DictRef dictRef(currentDB_.dbId, dictId);
1693  auto dit = dictDescriptorMapByRef_.find(dictRef);
1694  CHECK(dit != dictDescriptorMapByRef_.end());
1695  CHECK(dit->second);
1696  CHECK(dit->second.get()->stringDict);
1697  stringDicts[ccd.columnId] = dit->second.get()->stringDict.get();
1698 }
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:303
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:305
std::tuple< int, std::string > ColumnKey
Definition: Catalog.h:229
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1348
std::string to_upper(const std::string &str)
#define CHECK(condition)
Definition: Logger.h:187
std::string columnName
+ Here is the call graph for this function:

◆ getLogicalTableId()

int Catalog_Namespace::Catalog::getLogicalTableId ( const int  physicalTableId) const

Definition at line 2914 of file Catalog.cpp.

References logicalToPhysicalTableMapById_.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumns(), and vacuumDeletedRows().

2914  {
2915  for (const auto& l : logicalToPhysicalTableMapById_) {
2916  if (l.second.end() != std::find_if(l.second.begin(),
2917  l.second.end(),
2918  [&](decltype(*l.second.begin()) tid) -> bool {
2919  return physicalTableId == tid;
2920  })) {
2921  return l.first;
2922  }
2923  }
2924  return physicalTableId;
2925 }
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:316
+ Here is the caller graph for this function:

◆ getMetadataForColumn() [1/2]

◆ getMetadataForColumn() [2/2]

const ColumnDescriptor * Catalog_Namespace::Catalog::getMetadataForColumn ( int  tableId,
int  columnId 
) const

Definition at line 1401 of file Catalog.cpp.

References columnDescriptorMapById_.

1401  {
1402  cat_read_lock read_lock(this);
1403 
1404  ColumnIdKey columnIdKey(tableId, columnId);
1405  auto colDescIt = columnDescriptorMapById_.find(columnIdKey);
1406  if (colDescIt == columnDescriptorMapById_
1407  .end()) { // need to check to make sure column exists for table
1408  return nullptr;
1409  }
1410  return colDescIt->second;
1411 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:88
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:304
std::tuple< int, int > ColumnIdKey
Definition: Catalog.h:231

◆ getMetadataForColumnBySpi()

const ColumnDescriptor * Catalog_Namespace::Catalog::getMetadataForColumnBySpi ( const int  tableId,
const size_t  spi 
) const

Definition at line 1435 of file Catalog.cpp.

References columnDescriptorMapById_, and getColumnIdBySpiUnlocked().

Referenced by RelAlgTranslator::translateGeoColumn().

1436  {
1437  cat_read_lock read_lock(this);
1438 
1439  const auto columnId = getColumnIdBySpiUnlocked(tableId, spi);
1440  ColumnIdKey columnIdKey(tableId, columnId);
1441  const auto colDescIt = columnDescriptorMapById_.find(columnIdKey);
1442  return columnDescriptorMapById_.end() == colDescIt ? nullptr : colDescIt->second;
1443 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:88
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:304
std::tuple< int, int > ColumnIdKey
Definition: Catalog.h:231
const int getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const
Definition: Catalog.cpp:1413
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getMetadataForDashboard() [1/2]

const DashboardDescriptor* Catalog_Namespace::Catalog::getMetadataForDashboard ( const std::string &  userId,
const std::string &  dashName 
) const

Referenced by deleteMetadataForDashboard(), getMetadataForDashboard(), and DBObject::loadKey().

+ Here is the caller graph for this function:

◆ getMetadataForDashboard() [2/2]

const DashboardDescriptor * Catalog_Namespace::Catalog::getMetadataForDashboard ( const int32_t  dashboard_id) const

Definition at line 1484 of file Catalog.cpp.

References dashboardDescriptorMap_, getMetadataForDashboard(), name(), and to_string().

1484  {
1485  cat_read_lock read_lock(this);
1486  std::string userId;
1487  std::string name;
1488  bool found{false};
1489  {
1490  for (auto descp : dashboardDescriptorMap_) {
1491  auto dash = descp.second.get();
1492  if (dash->dashboardId == id) {
1493  userId = std::to_string(dash->userId);
1494  name = dash->dashboardName;
1495  found = true;
1496  break;
1497  }
1498  }
1499  }
1500  if (found) {
1501  return getMetadataForDashboard(userId, name);
1502  }
1503  return nullptr;
1504 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:88
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:306
std::string to_string(char const *&&v)
std::string name() const
Definition: Catalog.h:219
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const
+ Here is the call graph for this function:

◆ getMetadataForDict()

const DictDescriptor * Catalog_Namespace::Catalog::getMetadataForDict ( int  dict_ref,
bool  loadDict = true 
) const

Definition at line 1348 of file Catalog.cpp.

References currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, measure< TimeT >::execution(), g_cache_string_hash, logger::INFO, LOG, and string_dict_hosts_.

Referenced by Parser::InsertValuesStmt::determineLeafIndex(), DictionaryValueConverter< TARGET_TYPE >::DictionaryValueConverter(), doTruncateTable(), MapDHandler::fixup_row_descriptor(), getDictionary(), Executor::getStringDictionaryProxy(), anonymous_namespace{Execute.cpp}::insert_one_dict_str(), MapDHandler::populateThriftColumnType(), StringValueConverter::StringValueConverter(), and Fragmenter_Namespace::InsertOrderFragmenter::updateColumn().

1349  {
1350  const DictRef dictRef(currentDB_.dbId, dictId);
1351  cat_read_lock read_lock(this);
1352  auto dictDescIt = dictDescriptorMapByRef_.find(dictRef);
1353  if (dictDescIt ==
1354  dictDescriptorMapByRef_.end()) { // check to make sure dictionary exists
1355  return nullptr;
1356  }
1357  auto& dd = dictDescIt->second;
1358 
1359  if (loadDict) {
1360  cat_sqlite_lock sqlite_lock(this);
1361  if (!dd->stringDict) {
1362  auto time_ms = measure<>::execution([&]() {
1363  if (string_dict_hosts_.empty()) {
1364  if (dd->dictIsTemp) {
1365  dd->stringDict = std::make_shared<StringDictionary>(
1366  dd->dictFolderPath, true, true, g_cache_string_hash);
1367  } else {
1368  dd->stringDict = std::make_shared<StringDictionary>(
1369  dd->dictFolderPath, false, true, g_cache_string_hash);
1370  }
1371  } else {
1372  dd->stringDict =
1373  std::make_shared<StringDictionary>(string_dict_hosts_.front(), dd->dictRef);
1374  }
1375  });
1376  LOG(INFO) << "Time to load Dictionary " << dd->dictRef.dbId << "_"
1377  << dd->dictRef.dictId << " was " << time_ms << "ms";
1378  }
1379  }
1380 
1381  return dd.get();
1382 }
#define LOG(tag)
Definition: Logger.h:182
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:313
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:305
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:88
bool g_cache_string_hash
Definition: Execute.cpp:86
static TimeT::rep execution(F func, Args &&... args)
Definition: sample.cpp:29
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:90
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getMetadataForLink() [1/2]

const LinkDescriptor* Catalog_Namespace::Catalog::getMetadataForLink ( const std::string &  link) const

Referenced by deleteMetadataForDashboard().

+ Here is the caller graph for this function:

◆ getMetadataForLink() [2/2]

const LinkDescriptor * Catalog_Namespace::Catalog::getMetadataForLink ( int  linkId) const

Definition at line 1539 of file Catalog.cpp.

References columnDescriptorMapById_, getAllColumnMetadataForTable(), linkDescriptorMapById_, and TableDescriptor::tableId.

1539  {
1540  cat_read_lock read_lock(this);
1541  auto linkDescIt = linkDescriptorMapById_.find(linkId);
1542  if (linkDescIt == linkDescriptorMapById_.end()) { // check to make sure view exists
1543  return nullptr;
1544  }
1545  return linkDescIt->second;
1546 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:88
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:308
+ Here is the call graph for this function:

◆ getMetadataForTable() [1/2]

const TableDescriptor* Catalog_Namespace::Catalog::getMetadataForTable ( const std::string &  tableName,
const bool  populateFragmenter = true 
) const

Returns a pointer to a const TableDescriptor struct matching the provided tableName.

Parameters
tableNametable specified column belongs to
Returns
pointer to const TableDescriptor object queried for or nullptr if it does not exist.

Referenced by Executor::addDeletedColumn(), Parser::InsertStmt::analyze(), Parser::QuerySpec::analyze_from_clause(), UpdelRoll::cancelUpdate(), anonymous_namespace{MapDHandler.cpp}::check_table_not_sharded(), BaselineJoinHashTable::checkHashJoinReplicationConstraint(), checkPermissionForTables(), checkpoint(), Fragmenter_Namespace::computeRowIndicesOfShards(), Fragmenter_Namespace::copyDataOfShard(), anonymous_namespace{ExecuteTest.cpp}::create_and_populate_tables(), Parser::InsertValuesStmt::determineLeafIndex(), dropTable(), Executor::executeSimpleInsert(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_foreign_col(), anonymous_namespace{Execute.cpp}::get_table_name(), get_table_name_by_id(), getDeletedColumnIfRowsDeleted(), getPhysicalTablesDescriptors(), Lock_Namespace::getTableChunkKey(), getTableEpoch(), anonymous_namespace{ExecuteTest.cpp}::import_array_test(), Fragmenter_Namespace::InsertDataLoader::insertData(), Parser::InsertIntoTableAsSelectStmt::LocalConnector::insertDataToLeaf(), instantiateFragmenter(), DBObject::loadKey(), parseDashboardObjects(), populate_table_random(), removeChunks(), renameTable(), scan_table_return_hash(), scan_table_return_hash_non_iter(), Fragmenter_Namespace::set_chunk_metadata(), setTableEpoch(), GroupByAndAggregate::shard_count_for_top_groups(), Fragmenter_Namespace::SortedOrderFragmenter::sortData(), MapDHandler::sql_execute_impl(), TEST(), truncateTable(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumnMetadata(), vacuumDeletedRows(), and validate_shared_dictionary().

+ Here is the caller graph for this function:

◆ getMetadataForTable() [2/2]

const TableDescriptor * Catalog_Namespace::Catalog::getMetadataForTable ( int  tableId) const

Definition at line 1344 of file Catalog.cpp.

References getMetadataForTableImpl().

1344  {
1345  return getMetadataForTableImpl(tableId, true);
1346 }
const TableDescriptor * getMetadataForTableImpl(int tableId, const bool populateFragmenter) const
Definition: Catalog.cpp:1328
+ Here is the call graph for this function:

◆ getMetadataForTableImpl()

const TableDescriptor * Catalog_Namespace::Catalog::getMetadataForTableImpl ( int  tableId,
const bool  populateFragmenter 
) const

Definition at line 1328 of file Catalog.cpp.

References TableDescriptor::fragmenter, instantiateFragmenter(), TableDescriptor::isView, TableDescriptor::mutex_, and tableDescriptorMapById_.

Referenced by getAllColumnMetadataForTable(), and getMetadataForTable().

1330  {
1331  cat_read_lock read_lock(this);
1332  auto tableDescIt = tableDescriptorMapById_.find(tableId);
1333  if (tableDescIt == tableDescriptorMapById_.end()) { // check to make sure table exists
1334  return nullptr;
1335  }
1336  TableDescriptor* td = tableDescIt->second;
1337  std::unique_lock<std::mutex> td_lock(*td->mutex_.get());
1338  if (populateFragmenter && td->fragmenter == nullptr && !td->isView) {
1340  }
1341  return td; // returns pointer to table descriptor
1342 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:88
std::shared_ptr< std::mutex > mutex_
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:302
specifies the content in-memory of a row in the table metadata table
Fragmenter_Namespace::AbstractFragmenter * fragmenter
void instantiateFragmenter(TableDescriptor *td) const
Definition: Catalog.cpp:1271
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getPhysicalTablesDescriptors()

std::vector< const TableDescriptor * > Catalog_Namespace::Catalog::getPhysicalTablesDescriptors ( const TableDescriptor logicalTableDesc) const

Definition at line 2895 of file Catalog.cpp.

References CHECK, getMetadataForTable(), logicalToPhysicalTableMapById_, and TableDescriptor::tableId.

Referenced by checkpoint(), Fragmenter_Namespace::copyDataOfShard(), TableOptimizer::recomputeMetadata(), and vacuumDeletedRows().

2896  {
2897  cat_read_lock read_lock(this);
2898  const auto physicalTableIt =
2899  logicalToPhysicalTableMapById_.find(logicalTableDesc->tableId);
2900  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
2901  return {logicalTableDesc};
2902  }
2903 
2904  const auto physicalTablesIds = physicalTableIt->second;
2905  CHECK(!physicalTablesIds.empty());
2906  std::vector<const TableDescriptor*> physicalTables;
2907  for (size_t i = 0; i < physicalTablesIds.size(); i++) {
2908  physicalTables.push_back(getMetadataForTable(physicalTablesIds[i]));
2909  }
2910 
2911  return physicalTables;
2912 }
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:88
#define CHECK(condition)
Definition: Logger.h:187
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:316
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getShardColumnMetadataForTable()

const ColumnDescriptor * Catalog_Namespace::Catalog::getShardColumnMetadataForTable ( const TableDescriptor td) const

Definition at line 2879 of file Catalog.cpp.

References getAllColumnMetadataForTable(), TableDescriptor::shardedColumnId, and TableDescriptor::tableId.

Referenced by Fragmenter_Namespace::computeRowIndicesOfShards(), and Parser::InsertValuesStmt::determineLeafIndex().

2880  {
2881  const auto column_descriptors =
2882  getAllColumnMetadataForTable(td->tableId, false, true, true);
2883 
2884  const ColumnDescriptor* shard_cd{nullptr};
2885  int i = 1;
2886  for (auto cd_itr = column_descriptors.begin(); cd_itr != column_descriptors.end();
2887  ++cd_itr, ++i) {
2888  if (i == td->shardedColumnId) {
2889  shard_cd = *cd_itr;
2890  }
2891  }
2892  return shard_cd;
2893 }
specifies the content in-memory of a row in the column metadata table
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1579
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getSqliteConnector()

SqliteConnector& Catalog_Namespace::Catalog::getSqliteConnector ( )
inline

Definition at line 194 of file Catalog.h.

194 { return sqliteConnector_; }
SqliteConnector sqliteConnector_
Definition: Catalog.h:309

◆ getStringDictionaryHosts()

const std::vector< LeafHostInfo > & Catalog_Namespace::Catalog::getStringDictionaryHosts ( ) const

Definition at line 1384 of file Catalog.cpp.

References columnDescriptorMap_, getMetadataForColumn(), string_dict_hosts_, and to_upper().

1384  {
1385  return string_dict_hosts_;
1386 }
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:313
+ Here is the call graph for this function:

◆ getTableEpoch()

int32_t Catalog_Namespace::Catalog::getTableEpoch ( const int32_t  db_id,
const int32_t  table_id 
) const

Definition at line 2123 of file Catalog.cpp.

References CHECK, dataMgr_, logger::ERROR, getMetadataForTable(), LOG, and logicalToPhysicalTableMapById_.

Referenced by Importer_NS::Loader::getTableEpoch().

2123  {
2124  cat_read_lock read_lock(this);
2125  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(table_id);
2126  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2127  // check all shards have same checkpoint
2128  const auto physicalTables = physicalTableIt->second;
2129  CHECK(!physicalTables.empty());
2130  size_t curr_epoch = 0;
2131  for (size_t i = 0; i < physicalTables.size(); i++) {
2132  int32_t physical_tb_id = physicalTables[i];
2133  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2134  CHECK(phys_td);
2135  if (i == 0) {
2136  curr_epoch = dataMgr_->getTableEpoch(db_id, physical_tb_id);
2137  } else {
2138  if (curr_epoch != dataMgr_->getTableEpoch(db_id, physical_tb_id)) {
2139  // oh dear the leaves do not agree on the epoch for this table
2140  LOG(ERROR) << "Epochs on shards do not all agree on table id " << table_id
2141  << " db id " << db_id << " epoch " << curr_epoch << " leaf_epoch "
2142  << dataMgr_->getTableEpoch(db_id, physical_tb_id);
2143  return -1;
2144  }
2145  }
2146  }
2147  return curr_epoch;
2148  } else {
2149  return dataMgr_->getTableEpoch(db_id, table_id);
2150  }
2151 }
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
#define LOG(tag)
Definition: Logger.h:182
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:311
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:88
#define CHECK(condition)
Definition: Logger.h:187
specifies the content in-memory of a row in the table metadata table
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:316
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ instantiateFragmenter()

void Catalog_Namespace::Catalog::instantiateFragmenter ( TableDescriptor td) const
protected

Definition at line 1271 of file Catalog.cpp.

References currentDB_, dataMgr_, Catalog_Namespace::DBMetadata::dbId, measure< TimeT >::execution(), TableDescriptor::fragmenter, TableDescriptor::fragPageSize, TableDescriptor::fragType, getAllColumnMetadataForTable(), getMetadataForTable(), logger::INFO, Fragmenter_Namespace::INSERT_ORDER, TableDescriptor::isView, LOG, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRows, TableDescriptor::mutex_, TableDescriptor::persistenceLevel, TableDescriptor::shard, TableDescriptor::sortedColumnId, tableDescriptorMap_, TableDescriptor::tableId, TableDescriptor::tableName, and to_upper().

Referenced by getMetadataForTableImpl().

1271  {
1272  auto time_ms = measure<>::execution([&]() {
1273  // instanciate table fragmenter upon first use
1274  // assume only insert order fragmenter is supported
1276  vector<Chunk> chunkVec;
1277  list<const ColumnDescriptor*> columnDescs;
1278  getAllColumnMetadataForTable(td, columnDescs, true, false, true);
1279  Chunk::translateColumnDescriptorsToChunkVec(columnDescs, chunkVec);
1280  ChunkKey chunkKeyPrefix = {currentDB_.dbId, td->tableId};
1281  if (td->sortedColumnId > 0) {
1282  td->fragmenter = new SortedOrderFragmenter(chunkKeyPrefix,
1283  chunkVec,
1284  dataMgr_.get(),
1285  const_cast<Catalog*>(this),
1286  td->tableId,
1287  td->shard,
1288  td->maxFragRows,
1289  td->maxChunkSize,
1290  td->fragPageSize,
1291  td->maxRows,
1292  td->persistenceLevel);
1293  } else {
1294  td->fragmenter = new InsertOrderFragmenter(chunkKeyPrefix,
1295  chunkVec,
1296  dataMgr_.get(),
1297  const_cast<Catalog*>(this),
1298  td->tableId,
1299  td->shard,
1300  td->maxFragRows,
1301  td->maxChunkSize,
1302  td->fragPageSize,
1303  td->maxRows,
1304  td->persistenceLevel);
1305  }
1306  });
1307  LOG(INFO) << "Instantiating Fragmenter for table " << td->tableName << " took "
1308  << time_ms << "ms";
1309 }
std::string tableName
#define LOG(tag)
Definition: Logger.h:182
Catalog(const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
Constructor - takes basePath to already extant data directory for writing.
Definition: Catalog.cpp:127
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:311
The InsertOrderFragmenter is a child class of AbstractFragmenter, and fragments data in insert order...
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1579
Fragmenter_Namespace::FragmenterType fragType
Data_Namespace::MemoryLevel persistenceLevel
std::vector< int > ChunkKey
Definition: types.h:35
static TimeT::rep execution(F func, Args &&... args)
Definition: sample.cpp:29
Fragmenter_Namespace::AbstractFragmenter * fragmenter
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ name()

std::string Catalog_Namespace::Catalog::name ( ) const
inline

Definition at line 219 of file Catalog.h.

Referenced by deleteMetadataForDashboard(), getMetadataForDashboard(), and recordOwnershipOfObjectsInObjectPermissions().

219 { return getCurrentDB().dbName; }
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:176
+ Here is the caller graph for this function:

◆ parseDashboardObjects()

std::vector< DBObject > Catalog_Namespace::Catalog::parseDashboardObjects ( const std::string &  view_meta,
const int &  user_id 
)
protected

Definition at line 1192 of file Catalog.cpp.

References currentDB_, DBObjectKey::dbId, Catalog_Namespace::DBMetadata::dbId, getMetadataForTable(), logger::INFO, LOG, DBObjectKey::objectId, parse_underlying_dashboard_objects(), DBObjectKey::permissionType, AccessPrivileges::SELECT_FROM_TABLE, AccessPrivileges::SELECT_FROM_VIEW, TableDBObjectType, run-benchmark-import::type, and ViewDBObjectType.

Referenced by createOrUpdateDashboardSystemRole().

1193  {
1194  std::vector<DBObject> objects;
1195  DBObjectKey key;
1196  key.dbId = currentDB_.dbId;
1197  auto _key_place = [&key](auto type, auto id) {
1198  key.permissionType = type;
1199  key.objectId = id;
1200  return key;
1201  };
1202  for (auto object_name : parse_underlying_dashboard_objects(view_meta)) {
1203  auto td = getMetadataForTable(object_name);
1204  if (!td) {
1205  // Parsed object source is not present in current database
1206  // LOG the info and ignore
1207  LOG(INFO) << "Ignoring dashboard source Table/View: " << object_name
1208  << " no longer exists in current DB.";
1209  continue;
1210  }
1211  // Dashboard source can be Table or View
1212  const auto object_type = td->isView ? ViewDBObjectType : TableDBObjectType;
1213  const auto priv = td->isView ? AccessPrivileges::SELECT_FROM_VIEW
1215  objects.emplace_back(_key_place(object_type, td->tableId), priv, user_id);
1216  objects.back().setObjectType(td->isView ? ViewDBObjectType : TableDBObjectType);
1217  objects.back().setName(td->tableName);
1218  }
1219  return objects;
1220 }
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
#define LOG(tag)
Definition: Logger.h:182
int32_t objectId
Definition: DBObject.h:56
static const AccessPrivileges SELECT_FROM_TABLE
Definition: DBObject.h:153
std::vector< std::string > parse_underlying_dashboard_objects(const std::string &meta)
static const AccessPrivileges SELECT_FROM_VIEW
Definition: DBObject.h:173
int32_t dbId
Definition: DBObject.h:55
int32_t permissionType
Definition: DBObject.h:54
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ recordOwnershipOfObjectsInObjectPermissions()

void Catalog_Namespace::Catalog::recordOwnershipOfObjectsInObjectPermissions ( )
protected

Definition at line 526 of file Catalog.cpp.

References AccessPrivileges::ALL_DASHBOARD, AccessPrivileges::ALL_DATABASE, AccessPrivileges::ALL_TABLE, AccessPrivileges::ALL_VIEW, CHECK, currentDB_, DashboardDBObjectType, DatabaseDBObjectType, DBObjectKey::dbId, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, Catalog_Namespace::DBMetadata::dbOwner, logger::ERROR, SqliteConnector::getData(), SqliteConnector::getNumRows(), Catalog_Namespace::SysCatalog::instance(), LOG, name(), DBObjectKey::objectId, DBObjectKey::permissionType, Catalog_Namespace::SysCatalog::populateRoleDbObjects(), SqliteConnector::query(), SqliteConnector::query_with_text_params(), DBObject::setObjectKey(), DBObject::setOwner(), DBObject::setPrivileges(), sqliteConnector_, TableDBObjectType, to_string(), run-benchmark-import::type, and ViewDBObjectType.

Referenced by CheckAndExecuteMigrations().

526  {
527  cat_sqlite_lock sqlite_lock(this);
528  sqliteConnector_.query("BEGIN TRANSACTION");
529  std::vector<DBObject> objects;
530  try {
532  "SELECT name FROM sqlite_master WHERE type='table' AND "
533  "name='mapd_record_ownership_marker'");
534  // check if mapd catalog - marker exists
535  if (sqliteConnector_.getNumRows() != 0 && currentDB_.dbId == 1) {
536  // already done
537  sqliteConnector_.query("END TRANSACTION");
538  return;
539  }
540  // check if different catalog - marker exists
541  else if (sqliteConnector_.getNumRows() != 0 && currentDB_.dbId != 1) {
542  sqliteConnector_.query("SELECT dummy FROM mapd_record_ownership_marker");
543  // Check if migration is being performed on existing non mapd catalogs
544  // Older non mapd dbs will have table but no record in them
545  if (sqliteConnector_.getNumRows() != 0) {
546  // already done
547  sqliteConnector_.query("END TRANSACTION");
548  return;
549  }
550  }
551  // marker not exists - create one
552  else {
553  sqliteConnector_.query("CREATE TABLE mapd_record_ownership_marker (dummy integer)");
554  }
555 
556  DBMetadata db;
557  CHECK(SysCatalog::instance().getMetadataForDB(currentDB_.dbName, db));
558  // place dbId as a refernce for migration being performed
560  "INSERT INTO mapd_record_ownership_marker (dummy) VALUES (?1)",
561  std::vector<std::string>{std::to_string(db.dbOwner)});
562 
563  static const std::map<const DBObjectType, const AccessPrivileges>
564  object_level_all_privs_lookup{
569 
570  // grant owner all permissions on DB
571  DBObjectKey key;
572  key.dbId = currentDB_.dbId;
573  auto _key_place = [&key](auto type) {
574  key.permissionType = type;
575  return key;
576  };
577  for (auto& it : object_level_all_privs_lookup) {
578  objects.emplace_back(_key_place(it.first), it.second, db.dbOwner);
579  objects.back().setName(currentDB_.dbName);
580  }
581 
582  {
583  // other users tables and views
584  string tableQuery(
585  "SELECT tableid, name, userid, isview FROM mapd_tables WHERE userid > 0");
586  sqliteConnector_.query(tableQuery);
587  size_t numRows = sqliteConnector_.getNumRows();
588  for (size_t r = 0; r < numRows; ++r) {
589  int32_t tableid = sqliteConnector_.getData<int>(r, 0);
590  std::string tableName = sqliteConnector_.getData<string>(r, 1);
591  int32_t ownerid = sqliteConnector_.getData<int>(r, 2);
592  bool isview = sqliteConnector_.getData<bool>(r, 3);
593 
596  DBObjectKey key;
597  key.dbId = currentDB_.dbId;
598  key.objectId = tableid;
599  key.permissionType = type;
600 
601  DBObject obj(tableName, type);
602  obj.setObjectKey(key);
603  obj.setOwner(ownerid);
604  obj.setPrivileges(isview ? AccessPrivileges::ALL_VIEW
606 
607  objects.push_back(obj);
608  }
609  }
610 
611  {
612  // other users dashboards
613  string tableQuery("SELECT id, name, userid FROM mapd_dashboards WHERE userid > 0");
614  sqliteConnector_.query(tableQuery);
615  size_t numRows = sqliteConnector_.getNumRows();
616  for (size_t r = 0; r < numRows; ++r) {
617  int32_t dashId = sqliteConnector_.getData<int>(r, 0);
618  std::string dashName = sqliteConnector_.getData<string>(r, 1);
619  int32_t ownerid = sqliteConnector_.getData<int>(r, 2);
620 
622  DBObjectKey key;
623  key.dbId = currentDB_.dbId;
624  key.objectId = dashId;
625  key.permissionType = type;
626 
627  DBObject obj(dashName, type);
628  obj.setObjectKey(key);
629  obj.setOwner(ownerid);
630  obj.setPrivileges(AccessPrivileges::ALL_DASHBOARD);
631 
632  objects.push_back(obj);
633  }
634  }
635  } catch (const std::exception& e) {
636  sqliteConnector_.query("ROLLBACK TRANSACTION");
637  throw;
638  }
639  sqliteConnector_.query("END TRANSACTION");
640 
641  // now apply the objects to the syscat to track the permisisons
642  // moved outside transaction to avoid lock in sqlite
643  try {
645  } catch (const std::exception& e) {
646  LOG(ERROR) << " Issue during migration of DB " << name() << " issue was " << e.what();
647  throw std::runtime_error(" Issue during migration of DB " + name() + " issue was " +
648  e.what());
649  // will need to remove the mapd_record_ownership_marker table and retry
650  }
651 }
T getData(const int row, const int col)
static const AccessPrivileges ALL_DATABASE
Definition: DBObject.h:144
DBObjectType
Definition: DBObject.h:42
#define LOG(tag)
Definition: Logger.h:182
SqliteConnector sqliteConnector_
Definition: Catalog.h:309
void query_with_text_params(std::string const &query_only)
void query(const std::string &queryString)
int32_t objectId
Definition: DBObject.h:56
std::string to_string(char const *&&v)
static const AccessPrivileges ALL_VIEW
Definition: DBOb