OmniSciDB  1dac507f6e
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Catalog_Namespace::Catalog Class Referencefinal

class for a per-database catalog. also includes metadata for the current database and the current user. More...

#include <Catalog.h>

+ Collaboration diagram for Catalog_Namespace::Catalog:

Public Member Functions

 Catalog (const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
 Constructor - takes basePath to already extant data directory for writing. More...
 
 ~Catalog ()
 Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite. More...
 
void createTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
 
void createShardedTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
int32_t createDashboard (DashboardDescriptor &vd)
 
void replaceDashboard (DashboardDescriptor &vd)
 
std::string createLink (LinkDescriptor &ld, size_t min_length)
 
void dropTable (const TableDescriptor *td)
 
void truncateTable (const TableDescriptor *td)
 
void renameTable (const TableDescriptor *td, const std::string &newTableName)
 
void renameColumn (const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
 
void addColumn (const TableDescriptor &td, ColumnDescriptor &cd)
 
void removeChunks (const int table_id)
 
const TableDescriptorgetMetadataForTable (const std::string &tableName, const bool populateFragmenter=true) const
 Returns a pointer to a const TableDescriptor struct matching the provided tableName. More...
 
const TableDescriptorgetMetadataForTableImpl (int tableId, const bool populateFragmenter) const
 
const TableDescriptorgetMetadataForTable (int tableId) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, const std::string &colName) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, int columnId) const
 
const int getColumnIdBySpi (const int tableId, const size_t spi) const
 
const ColumnDescriptorgetMetadataForColumnBySpi (const int tableId, const size_t spi) const
 
const DashboardDescriptorgetMetadataForDashboard (const std::string &userId, const std::string &dashName) const
 
void deleteMetadataForDashboard (const std::string &userId, const std::string &dashName)
 
const DashboardDescriptorgetMetadataForDashboard (const int32_t dashboard_id) const
 
void deleteMetadataForDashboard (const int32_t dashboard_id)
 
const LinkDescriptorgetMetadataForLink (const std::string &link) const
 
const LinkDescriptorgetMetadataForLink (int linkId) const
 
std::list< const
ColumnDescriptor * > 
getAllColumnMetadataForTable (const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular table specified by table id. More...
 
std::list< const
TableDescriptor * > 
getAllTableMetadata () const
 
std::list< const
DashboardDescriptor * > 
getAllDashboardsMetadata () const
 
const DBMetadatagetCurrentDB () const
 
Data_Namespace::DataMgrgetDataMgr () const
 
std::shared_ptr< CalcitegetCalciteMgr () const
 
const std::string & getBasePath () const
 
const DictDescriptorgetMetadataForDict (int dict_ref, bool loadDict=true) const
 
const std::vector< LeafHostInfo > & getStringDictionaryHosts () const
 
const ColumnDescriptorgetShardColumnMetadataForTable (const TableDescriptor *td) const
 
std::vector< const
TableDescriptor * > 
getPhysicalTablesDescriptors (const TableDescriptor *logicalTableDesc) const
 
int32_t getTableEpoch (const int32_t db_id, const int32_t table_id) const
 
void setTableEpoch (const int db_id, const int table_id, const int new_epoch)
 
int getDatabaseId () const
 
SqliteConnectorgetSqliteConnector ()
 
void roll (const bool forward)
 
DictRef addDictionary (ColumnDescriptor &cd)
 
void delDictionary (const ColumnDescriptor &cd)
 
void getDictionary (const ColumnDescriptor &cd, std::map< int, StringDictionary * > &stringDicts)
 
const bool checkMetadataForDeletedRecs (int dbId, int tableId, int columnId) const
 
const ColumnDescriptorgetDeletedColumn (const TableDescriptor *td) const
 
const ColumnDescriptorgetDeletedColumnIfRowsDeleted (const TableDescriptor *td) const
 
void setDeletedColumn (const TableDescriptor *td, const ColumnDescriptor *cd)
 
void setDeletedColumnUnlocked (const TableDescriptor *td, const ColumnDescriptor *cd)
 
int getLogicalTableId (const int physicalTableId) const
 
void checkpoint (const int logicalTableId) const
 
std::string name () const
 
void eraseDBData ()
 
void eraseTablePhysicalData (const TableDescriptor *td)
 
void vacuumDeletedRows (const TableDescriptor *td) const
 
void vacuumDeletedRows (const int logicalTableId) const
 
void dumpTable (const TableDescriptor *td, const std::string &path, const std::string &compression) const
 
void restoreTable (const SessionInfo &session, const TableDescriptor *td, const std::string &file_path, const std::string &compression)
 
void restoreTable (const SessionInfo &session, const std::string &table_name, const std::string &file_path, const std::string &compression)
 
std::vector< std::string > getTableDataDirectories (const TableDescriptor *td) const
 
std::vector< std::string > getTableDictDirectories (const TableDescriptor *td) const
 
std::string getColumnDictDirectory (const ColumnDescriptor *cd) const
 
std::string dumpSchema (const TableDescriptor *td) const
 

Static Public Member Functions

static void expandGeoColumn (const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
 
static void set (const std::string &dbName, std::shared_ptr< Catalog > cat)
 
static std::shared_ptr< Catalogget (const std::string &dbName)
 
static std::shared_ptr< Catalogget (const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
 
static void remove (const std::string &dbName)
 

Public Attributes

std::mutex sqliteMutex_
 
mapd_shared_mutex sharedMutex_
 
std::atomic< std::thread::id > thread_holding_sqlite_lock
 
std::atomic< std::thread::id > thread_holding_write_lock
 

Static Public Attributes

static thread_local bool thread_holds_read_lock = false
 

Protected Types

typedef std::map< std::string,
TableDescriptor * > 
TableDescriptorMap
 
typedef std::map< int,
TableDescriptor * > 
TableDescriptorMapById
 
typedef std::map< int32_t,
std::vector< int32_t > > 
LogicalToPhysicalTableMapById
 
typedef std::tuple< int,
std::string > 
ColumnKey
 
typedef std::map< ColumnKey,
ColumnDescriptor * > 
ColumnDescriptorMap
 
typedef std::tuple< int, int > ColumnIdKey
 
typedef std::map< ColumnIdKey,
ColumnDescriptor * > 
ColumnDescriptorMapById
 
typedef std::map< DictRef,
std::unique_ptr
< DictDescriptor > > 
DictDescriptorMapById
 
typedef std::map< std::string,
std::shared_ptr
< DashboardDescriptor > > 
DashboardDescriptorMap
 
typedef std::map< std::string,
LinkDescriptor * > 
LinkDescriptorMap
 
typedef std::map< int,
LinkDescriptor * > 
LinkDescriptorMapById
 
typedef std::unordered_map
< const TableDescriptor
*, const ColumnDescriptor * > 
DeletedColumnPerTableMap
 
using ColumnDescriptorsForRoll = std::vector< std::pair< ColumnDescriptor *, ColumnDescriptor * >>
 

Protected Member Functions

void CheckAndExecuteMigrations ()
 
void CheckAndExecuteMigrationsPostBuildMaps ()
 
void updateDictionaryNames ()
 
void updateTableDescriptorSchema ()
 
void updateFixlenArrayColumns ()
 
void updateFrontendViewSchema ()
 
void updateLinkSchema ()
 
void updateFrontendViewAndLinkUsers ()
 
void updateLogicalToPhysicalTableLinkSchema ()
 
void updateLogicalToPhysicalTableMap (const int32_t logical_tb_id)
 
void updateDictionarySchema ()
 
void updatePageSize ()
 
void updateDeletedColumnIndicator ()
 
void updateFrontendViewsToDashboards ()
 
void recordOwnershipOfObjectsInObjectPermissions ()
 
void checkDateInDaysColumnMigration ()
 
void createDashboardSystemRoles ()
 
void buildMaps ()
 
void addTableToMap (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
 
void addReferenceToForeignDict (ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def)
 
bool setColumnSharedDictionary (ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
void setColumnDictionary (ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
 
void addFrontendViewToMap (DashboardDescriptor &vd)
 
void addFrontendViewToMapNoLock (DashboardDescriptor &vd)
 
void addLinkToMap (LinkDescriptor &ld)
 
void removeTableFromMap (const std::string &tableName, const int tableId, const bool is_on_error=false)
 
void doDropTable (const TableDescriptor *td)
 
void doTruncateTable (const TableDescriptor *td)
 
void renamePhysicalTable (const TableDescriptor *td, const std::string &newTableName)
 
void instantiateFragmenter (TableDescriptor *td) const
 
void getAllColumnMetadataForTable (const TableDescriptor *td, std::list< const ColumnDescriptor * > &colDescs, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 
std::string calculateSHA1 (const std::string &data)
 
std::string generatePhysicalTableName (const std::string &logicalTableName, const int32_t &shardNumber)
 
std::vector< DBObjectparseDashboardObjects (const std::string &view_meta, const int &user_id)
 
void createOrUpdateDashboardSystemRole (const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
 
const int getColumnIdBySpiUnlocked (const int table_id, const size_t spi) const
 

Protected Attributes

std::string basePath_
 
TableDescriptorMap tableDescriptorMap_
 
TableDescriptorMapById tableDescriptorMapById_
 
ColumnDescriptorMap columnDescriptorMap_
 
ColumnDescriptorMapById columnDescriptorMapById_
 
DictDescriptorMapById dictDescriptorMapByRef_
 
DashboardDescriptorMap dashboardDescriptorMap_
 
LinkDescriptorMap linkDescriptorMap_
 
LinkDescriptorMapById linkDescriptorMapById_
 
SqliteConnector sqliteConnector_
 
DBMetadata currentDB_
 
std::shared_ptr
< Data_Namespace::DataMgr
dataMgr_
 
const std::vector< LeafHostInfostring_dict_hosts_
 
std::shared_ptr< CalcitecalciteMgr_
 
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
 
int nextTempTableId_
 
int nextTempDictId_
 
ColumnDescriptorsForRoll columnDescriptorsForRoll
 

Static Protected Attributes

static const std::string physicalTableNameTag_
 

Private Member Functions

void adjustAlteredTableFiles (const std::string &temp_data_dir, const std::unordered_map< int, int > &all_column_ids_map) const
 
void renameTableDirectories (const std::string &temp_data_dir, const std::vector< std::string > &target_paths, const std::string &name_prefix) const
 

Private Attributes

DeletedColumnPerTableMap deletedColumnPerTable_
 

Static Private Attributes

static std::map< std::string,
std::shared_ptr< Catalog > > 
mapd_cat_map_
 

Detailed Description

class for a per-database catalog. also includes metadata for the current database and the current user.

Catalog

Definition at line 81 of file Catalog.h.

Member Typedef Documentation

Definition at line 247 of file Catalog.h.

Definition at line 249 of file Catalog.h.

Definition at line 344 of file Catalog.h.

typedef std::tuple<int, int> Catalog_Namespace::Catalog::ColumnIdKey
protected

Definition at line 248 of file Catalog.h.

typedef std::tuple<int, std::string> Catalog_Namespace::Catalog::ColumnKey
protected

Definition at line 246 of file Catalog.h.

typedef std::map<std::string, std::shared_ptr<DashboardDescriptor> > Catalog_Namespace::Catalog::DashboardDescriptorMap
protected

Definition at line 252 of file Catalog.h.

typedef std::unordered_map<const TableDescriptor*, const ColumnDescriptor*> Catalog_Namespace::Catalog::DeletedColumnPerTableMap
protected

Definition at line 256 of file Catalog.h.

typedef std::map<DictRef, std::unique_ptr<DictDescriptor> > Catalog_Namespace::Catalog::DictDescriptorMapById
protected

Definition at line 250 of file Catalog.h.

typedef std::map<std::string, LinkDescriptor*> Catalog_Namespace::Catalog::LinkDescriptorMap
protected

Definition at line 253 of file Catalog.h.

Definition at line 254 of file Catalog.h.

typedef std::map<int32_t, std::vector<int32_t> > Catalog_Namespace::Catalog::LogicalToPhysicalTableMapById
protected

Definition at line 245 of file Catalog.h.

typedef std::map<std::string, TableDescriptor*> Catalog_Namespace::Catalog::TableDescriptorMap
protected

Definition at line 243 of file Catalog.h.

Definition at line 244 of file Catalog.h.

Constructor & Destructor Documentation

Catalog_Namespace::Catalog::Catalog ( const std::string &  basePath,
const DBMetadata curDB,
std::shared_ptr< Data_Namespace::DataMgr dataMgr,
const std::vector< LeafHostInfo > &  string_dict_hosts,
std::shared_ptr< Calcite calcite,
bool  is_new_db 
)

Constructor - takes basePath to already extant data directory for writing.

Parameters
basePathdirectory path for writing catalog
dbNamename of the database
fragmenterFragmenter object metadata - expects for this directory to already exist

Definition at line 129 of file Catalog.cpp.

References buildMaps(), CheckAndExecuteMigrations(), and CheckAndExecuteMigrationsPostBuildMaps().

135  : basePath_(basePath)
136  , sqliteConnector_(curDB.dbName, basePath + "/mapd_catalogs/")
137  , currentDB_(curDB)
138  , dataMgr_(dataMgr)
139  , string_dict_hosts_(string_dict_hosts)
140  , calciteMgr_(calcite)
143  , sqliteMutex_()
144  , sharedMutex_()
147  if (!is_new_db) {
149  }
150  buildMaps();
151  if (!is_new_db) {
153  }
154 }
const int MAPD_TEMP_TABLE_START_ID
Definition: Catalog.cpp:80
mapd_shared_mutex sharedMutex_
Definition: Catalog.h:359
SqliteConnector sqliteConnector_
Definition: Catalog.h:326
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:330
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:328
const int MAPD_TEMP_DICT_START_ID
Definition: Catalog.cpp:82
std::atomic< std::thread::id > thread_holding_sqlite_lock
Definition: Catalog.h:360
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:331
std::atomic< std::thread::id > thread_holding_write_lock
Definition: Catalog.h:361
void CheckAndExecuteMigrationsPostBuildMaps()
Definition: Catalog.cpp:849

+ Here is the call graph for this function:

Catalog_Namespace::Catalog::~Catalog ( )

Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite.

Definition at line 156 of file Catalog.cpp.

References columnDescriptorMap_, and tableDescriptorMap_.

156  {
157  cat_write_lock write_lock(this);
158  // must clean up heap-allocated TableDescriptor and ColumnDescriptor structs
159  for (TableDescriptorMap::iterator tableDescIt = tableDescriptorMap_.begin();
160  tableDescIt != tableDescriptorMap_.end();
161  ++tableDescIt) {
162  if (tableDescIt->second->fragmenter != nullptr) {
163  delete tableDescIt->second->fragmenter;
164  }
165  delete tableDescIt->second;
166  }
167 
168  // TableDescriptorMapById points to the same descriptors. No need to delete
169 
170  for (ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.begin();
171  columnDescIt != columnDescriptorMap_.end();
172  ++columnDescIt) {
173  delete columnDescIt->second;
174  }
175 
176  // ColumnDescriptorMapById points to the same descriptors. No need to delete
177 }
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:320
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:91
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:318

Member Function Documentation

void Catalog_Namespace::Catalog::addColumn ( const TableDescriptor td,
ColumnDescriptor cd 
)

Definition at line 1702 of file Catalog.cpp.

References addDictionary(), columnDescriptorMap_, columnDescriptorsForRoll, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfoCore< TYPE_FACET_PACK >::get_comp_param(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_compression(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_dimension(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_notnull(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_scale(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_size(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_subtype(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), SqliteConnector::getData(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, kENCODING_DICT, SqliteConnector::query_with_text_params(), sqliteConnector_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), to_upper(), and ColumnDescriptor::virtualExpr.

1702  {
1703  // caller must handle sqlite/chunk transaction TOGETHER
1704  cd.tableId = td.tableId;
1706  addDictionary(cd);
1707  }
1708 
1710  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
1711  "colscale, is_notnull, "
1712  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
1713  "is_deletedcol) "
1714  "VALUES (?, "
1715  "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
1716  "?, ?, ?, "
1717  "?, "
1718  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1719  std::vector<std::string>{std::to_string(td.tableId),
1720  std::to_string(td.tableId),
1721  cd.columnName,
1730  "",
1733  cd.virtualExpr,
1735 
1737  "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
1738  std::vector<std::string>{std::to_string(td.tableId)});
1739 
1741  "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
1742  std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
1743  cd.columnId = sqliteConnector_.getData<int>(0, 0);
1744 
1745  ++tableDescriptorMapById_[td.tableId]->nColumns;
1746  auto ncd = new ColumnDescriptor(cd);
1749  columnDescriptorsForRoll.emplace_back(nullptr, ncd);
1750 }
std::string virtualExpr
T getData(const int row, const int col)
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:334
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:335
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:320
SqliteConnector sqliteConnector_
Definition: Catalog.h:326
void query_with_text_params(std::string const &query_only)
HOST DEVICE int get_scale() const
Definition: sqltypes.h:331
HOST DEVICE int get_size() const
Definition: sqltypes.h:336
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:321
std::string to_string(char const *&&v)
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:326
std::tuple< int, int > ColumnIdKey
Definition: Catalog.h:248
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:333
std::tuple< int, std::string > ColumnKey
Definition: Catalog.h:246
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:319
specifies the content in-memory of a row in the column metadata table
std::string to_upper(const std::string &str)
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:328
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:327
SQLTypeInfo columnType
DictRef addDictionary(ColumnDescriptor &cd)
Definition: Catalog.cpp:1615
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:345
std::string columnName

+ Here is the call graph for this function:

DictRef Catalog_Namespace::Catalog::addDictionary ( ColumnDescriptor cd)

Definition at line 1615 of file Catalog.cpp.

References CHECK(), currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, DictDescriptor::dictFolderPath, setColumnDictionary(), string_dict_hosts_, tableDescriptorMapById_, and ColumnDescriptor::tableId.

Referenced by addColumn().

1615  {
1616  const auto& td = *tableDescriptorMapById_[cd.tableId];
1617  list<DictDescriptor> dds;
1618  setColumnDictionary(cd, dds, td, true);
1619  auto& dd = dds.back();
1620  CHECK(dd.dictRef.dictId);
1621 
1622  std::unique_ptr<StringDictionaryClient> client;
1623  if (!string_dict_hosts_.empty()) {
1624  client.reset(new StringDictionaryClient(
1625  string_dict_hosts_.front(), DictRef(currentDB_.dbId, -1), true));
1626  }
1627  if (client) {
1628  client->create(dd.dictRef, dd.dictIsTemp);
1629  }
1630 
1631  DictDescriptor* new_dd = new DictDescriptor(dd);
1632  dictDescriptorMapByRef_[dd.dictRef].reset(new_dd);
1633  if (!dd.dictIsTemp) {
1634  boost::filesystem::create_directory(new_dd->dictFolderPath);
1635  }
1636  return dd.dictRef;
1637 }
std::string dictFolderPath
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:330
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:322
CHECK(cgen_state)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:319
struct dict_ref_t DictRef
Definition: DictRef.h:37
Descriptor for a dictionary for a string columne.
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:2345

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addFrontendViewToMap ( DashboardDescriptor vd)
protected

Definition at line 1183 of file Catalog.cpp.

References addFrontendViewToMapNoLock().

Referenced by createDashboard().

1183  {
1184  cat_write_lock write_lock(this);
1186 }
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1188
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:91

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addFrontendViewToMapNoLock ( DashboardDescriptor vd)
protected

Definition at line 1188 of file Catalog.cpp.

References dashboardDescriptorMap_, DashboardDescriptor::dashboardName, to_string(), and DashboardDescriptor::userId.

Referenced by addFrontendViewToMap(), and replaceDashboard().

1188  {
1189  cat_write_lock write_lock(this);
1191  std::make_shared<DashboardDescriptor>(vd);
1192 }
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:323
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:91

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addLinkToMap ( LinkDescriptor ld)
protected

Definition at line 1265 of file Catalog.cpp.

References currentDB_, Catalog_Namespace::DBMetadata::dbId, LinkDescriptor::link, linkDescriptorMap_, linkDescriptorMapById_, LinkDescriptor::linkId, and to_string().

Referenced by createLink().

1265  {
1266  cat_write_lock write_lock(this);
1267  LinkDescriptor* new_ld = new LinkDescriptor();
1268  *new_ld = ld;
1270  linkDescriptorMapById_[ld.linkId] = new_ld;
1271 }
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:91
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:325
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:324

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addReferenceToForeignDict ( ColumnDescriptor referencing_column,
Parser::SharedDictionaryDef  shared_dict_def 
)
protected

Definition at line 2264 of file Catalog.cpp.

References CHECK(), CHECK_GE, ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, SQLTypeInfoCore< TYPE_FACET_PACK >::get_comp_param(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_foreign_col(), SqliteConnector::query_with_text_params(), sqliteConnector_, and to_string().

Referenced by setColumnSharedDictionary().

2265  {
2266  cat_write_lock write_lock(this);
2267  const auto foreign_ref_col = get_foreign_col(*this, shared_dict_def);
2268  CHECK(foreign_ref_col);
2269  referencing_column.columnType = foreign_ref_col->columnType;
2270  const int dict_id = referencing_column.columnType.get_comp_param();
2271  const DictRef dict_ref(currentDB_.dbId, dict_id);
2272  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
2273  CHECK(dictIt != dictDescriptorMapByRef_.end());
2274  const auto& dd = dictIt->second;
2275  CHECK_GE(dd->refcount, 1);
2276  ++dd->refcount;
2277  cat_sqlite_lock sqlite_lock(this);
2279  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
2280  {std::to_string(dict_id)});
2281 }
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:335
SqliteConnector sqliteConnector_
Definition: Catalog.h:326
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:322
void query_with_text_params(std::string const &query_only)
#define CHECK_GE(x, y)
Definition: Logger.h:203
const ColumnDescriptor * get_foreign_col(const Catalog &cat, const Parser::SharedDictionaryDef &shared_dict_def)
Definition: Catalog.cpp:2252
std::string to_string(char const *&&v)
CHECK(cgen_state)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:91
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:92

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::addTableToMap ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::list< DictDescriptor > &  dicts 
)
protected

Definition at line 1051 of file Catalog.cpp.

References CHECK(), columnDescriptorMap_, columnDescriptorMapById_, ColumnDescriptor::columnId, TableDescriptor::columnIdBySpi_, ColumnDescriptor::columnName, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, DictDescriptor::dictFolderPath, dict_ref_t::dictId, TableDescriptor::hasDeletedCol, TableDescriptor::mutex_, setDeletedColumnUnlocked(), string_dict_hosts_, tableDescriptorMap_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, and to_upper().

Referenced by createTable().

1053  {
1054  cat_write_lock write_lock(this);
1055  TableDescriptor* new_td = new TableDescriptor();
1056  *new_td = td;
1057  new_td->mutex_ = std::make_shared<std::mutex>();
1058  tableDescriptorMap_[to_upper(td.tableName)] = new_td;
1059  tableDescriptorMapById_[td.tableId] = new_td;
1060  for (auto cd : columns) {
1061  ColumnDescriptor* new_cd = new ColumnDescriptor();
1062  *new_cd = cd;
1063  ColumnKey columnKey(new_cd->tableId, to_upper(new_cd->columnName));
1064  columnDescriptorMap_[columnKey] = new_cd;
1065  ColumnIdKey columnIdKey(new_cd->tableId, new_cd->columnId);
1066  columnDescriptorMapById_[columnIdKey] = new_cd;
1067 
1068  // Add deleted column to the map
1069  if (cd.isDeletedCol) {
1070  CHECK(new_td->hasDeletedCol);
1071  setDeletedColumnUnlocked(new_td, new_cd);
1072  }
1073  }
1074 
1075  std::sort(new_td->columnIdBySpi_.begin(),
1076  new_td->columnIdBySpi_.end(),
1077  [](const size_t a, const size_t b) -> bool { return a < b; });
1078 
1079  std::unique_ptr<StringDictionaryClient> client;
1080  DictRef dict_ref(currentDB_.dbId, -1);
1081  if (!string_dict_hosts_.empty()) {
1082  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1083  }
1084  for (auto dd : dicts) {
1085  if (!dd.dictRef.dictId) {
1086  // Dummy entry created for a shard of a logical table, nothing to do.
1087  continue;
1088  }
1089  dict_ref.dictId = dd.dictRef.dictId;
1090  if (client) {
1091  client->create(dict_ref, dd.dictIsTemp);
1092  }
1093  DictDescriptor* new_dd = new DictDescriptor(dd);
1094  dictDescriptorMapByRef_[dict_ref].reset(new_dd);
1095  if (!dd.dictIsTemp) {
1096  boost::filesystem::create_directory(new_dd->dictFolderPath);
1097  }
1098  }
1099 }
std::string dictFolderPath
std::string tableName
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:320
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:330
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:322
std::vector< int > columnIdBySpi_
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:321
std::shared_ptr< std::mutex > mutex_
CHECK(cgen_state)
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:2243
std::tuple< int, int > ColumnIdKey
Definition: Catalog.h:248
std::tuple< int, std::string > ColumnKey
Definition: Catalog.h:246
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:319
specifies the content in-memory of a row in the column metadata table
std::string to_upper(const std::string &str)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:91
Descriptor for a dictionary for a string columne.
specifies the content in-memory of a row in the table metadata table
std::string columnName
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:318

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::adjustAlteredTableFiles ( const std::string &  temp_data_dir,
const std::unordered_map< int, int > &  all_column_ids_map 
) const
private

Definition at line 360 of file DumpRestore.cpp.

References CHECK(), ThreadController_NS::SimpleThreadController< FutureReturnType >::checkThreadsStatus(), cpu_threads(), anonymous_namespace{StringDictionary.cpp}::file_size(), ThreadController_NS::SimpleThreadController< FutureReturnType >::finish(), MAPD_FILE_EXT, Catalog_Namespace::simple_file_closer, split(), ThreadController_NS::SimpleThreadController< FutureReturnType >::startThread(), and to_string().

Referenced by restoreTable().

362  {
363  boost::filesystem::path base_path(temp_data_dir);
364  boost::filesystem::recursive_directory_iterator end_it;
366  for (boost::filesystem::recursive_directory_iterator fit(base_path); fit != end_it;
367  ++fit) {
368  if (boost::filesystem::is_regular_file(fit->status())) {
369  const std::string file_path = fit->path().string();
370  const std::string file_name = fit->path().filename().string();
371  std::vector<std::string> tokens;
372  boost::split(tokens, file_name, boost::is_any_of("."));
373  // ref. FileMgr::init for hint of data file name layout
374  if (tokens.size() > 2 && MAPD_FILE_EXT == "." + tokens[2]) {
375  thread_controller.startThread([file_name, file_path, tokens, &column_ids_map] {
376  const auto page_size = boost::lexical_cast<int64_t>(tokens[1]);
377  const auto file_size = boost::filesystem::file_size(file_path);
378  std::unique_ptr<FILE, decltype(simple_file_closer)> fp(
379  std::fopen(file_path.c_str(), "r+"), simple_file_closer);
380  if (!fp) {
381  throw std::runtime_error("Failed to open " + file_path +
382  " for update: " + std::strerror(errno));
383  }
384  // ref. FileInfo::openExistingFile for hint of chunk header layout
385  for (size_t page = 0; page < file_size / page_size; ++page) {
386  int ints[8];
387  if (0 != std::fseek(fp.get(), page * page_size, SEEK_SET)) {
388  throw std::runtime_error("Failed to seek to page# " + std::to_string(page) +
389  file_path + " for read: " + std::strerror(errno));
390  }
391  if (1 != fread(ints, sizeof ints, 1, fp.get())) {
392  throw std::runtime_error("Failed to read " + file_path + ": " +
393  std::strerror(errno));
394  }
395  if (ints[0] > 0) { // header size
396  auto cit = column_ids_map.find(ints[3]);
397  CHECK(cit != column_ids_map.end());
398  if (ints[3] != cit->second) {
399  ints[3] = cit->second;
400  if (0 != std::fseek(fp.get(), page * page_size, SEEK_SET)) {
401  throw std::runtime_error("Failed to seek to page# " +
402  std::to_string(page) + file_path +
403  " for write: " + std::strerror(errno));
404  }
405  if (1 != fwrite(ints, sizeof ints, 1, fp.get())) {
406  throw std::runtime_error("Failed to write " + file_path + ": " +
407  std::strerror(errno));
408  }
409  }
410  }
411  }
412  });
413  thread_controller.checkThreadsStatus();
414  }
415  }
416  }
417  thread_controller.finish();
418 }
#define MAPD_FILE_EXT
Definition: File.h:26
std::string to_string(char const *&&v)
CHECK(cgen_state)
std::vector< std::string > split(const std::string &str, const std::string &delim)
split apart a string into a vector of substrings
int cpu_threads()
Definition: thread_count.h:25

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::buildMaps ( )
protected

Definition at line 865 of file Catalog.cpp.

References basePath_, CHECK(), ColumnDescriptor::chunks, columnDescriptorMap_, columnDescriptorMapById_, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, currentDB_, dashboardDescriptorMap_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, TableDescriptor::fragmenter, TableDescriptor::fragments, TableDescriptor::fragPageSize, TableDescriptor::fragType, generate_dashboard_system_rolename(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_physical_cols(), SqliteConnector::getData(), SqliteConnector::getNumRows(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::getUserFromId(), TableDescriptor::hasDeletedCol, SQLTypeInfoCore< TYPE_FACET_PACK >::is_geometry(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, SqliteConnector::isNull(), ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, TableDescriptor::keyMetainfo, LinkDescriptor::link, linkDescriptorMap_, linkDescriptorMapById_, LinkDescriptor::linkId, logicalToPhysicalTableMapById_, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRows, TableDescriptor::nColumns, TableDescriptor::nShards, TableDescriptor::partitions, SqliteConnector::query(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_comp_param(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_compression(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_dimension(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_notnull(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_scale(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_size(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_subtype(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_type(), setDeletedColumnUnlocked(), TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, sqliteConnector_, tableDescriptorMap_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), to_upper(), LinkDescriptor::updateTime, LinkDescriptor::userId, TableDescriptor::userId, LinkDescriptor::viewMetadata, TableDescriptor::viewSQL, LinkDescriptor::viewState, and ColumnDescriptor::virtualExpr.

Referenced by Catalog().

865  {
866  cat_write_lock write_lock(this);
867  cat_sqlite_lock sqlite_lock(this);
868 
869  string dictQuery(
870  "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
871  sqliteConnector_.query(dictQuery);
872  size_t numRows = sqliteConnector_.getNumRows();
873  for (size_t r = 0; r < numRows; ++r) {
874  int dictId = sqliteConnector_.getData<int>(r, 0);
875  std::string dictName = sqliteConnector_.getData<string>(r, 1);
876  int dictNBits = sqliteConnector_.getData<int>(r, 2);
877  bool is_shared = sqliteConnector_.getData<bool>(r, 3);
878  int refcount = sqliteConnector_.getData<int>(r, 4);
879  std::string fname = basePath_ + "/mapd_data/DB_" + std::to_string(currentDB_.dbId) +
880  "_DICT_" + std::to_string(dictId);
881  DictRef dict_ref(currentDB_.dbId, dictId);
882  DictDescriptor* dd = new DictDescriptor(
883  dict_ref, dictName, dictNBits, is_shared, refcount, fname, false);
884  dictDescriptorMapByRef_[dict_ref].reset(dd);
885  }
886 
887  string tableQuery(
888  "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
889  "max_chunk_size, frag_page_size, "
890  "max_rows, partitions, shard_column_id, shard, num_shards, key_metainfo, userid, "
891  "sort_column_id "
892  "from mapd_tables");
893  sqliteConnector_.query(tableQuery);
894  numRows = sqliteConnector_.getNumRows();
895  for (size_t r = 0; r < numRows; ++r) {
896  TableDescriptor* td = new TableDescriptor();
897  td->tableId = sqliteConnector_.getData<int>(r, 0);
898  td->tableName = sqliteConnector_.getData<string>(r, 1);
899  td->nColumns = sqliteConnector_.getData<int>(r, 2);
900  td->isView = sqliteConnector_.getData<bool>(r, 3);
901  td->fragments = sqliteConnector_.getData<string>(r, 4);
902  td->fragType =
904  td->maxFragRows = sqliteConnector_.getData<int>(r, 6);
905  td->maxChunkSize = sqliteConnector_.getData<int>(r, 7);
906  td->fragPageSize = sqliteConnector_.getData<int>(r, 8);
907  td->maxRows = sqliteConnector_.getData<int64_t>(r, 9);
908  td->partitions = sqliteConnector_.getData<string>(r, 10);
909  td->shardedColumnId = sqliteConnector_.getData<int>(r, 11);
910  td->shard = sqliteConnector_.getData<int>(r, 12);
911  td->nShards = sqliteConnector_.getData<int>(r, 13);
912  td->keyMetainfo = sqliteConnector_.getData<string>(r, 14);
913  td->userId = sqliteConnector_.getData<int>(r, 15);
914  td->sortedColumnId =
915  sqliteConnector_.isNull(r, 16) ? 0 : sqliteConnector_.getData<int>(r, 16);
916  if (!td->isView) {
917  td->fragmenter = nullptr;
918  }
919  td->hasDeletedCol = false;
922  }
923  string columnQuery(
924  "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
925  "is_notnull, compression, comp_param, "
926  "size, chunks, is_systemcol, is_virtualcol, virtual_expr, is_deletedcol from "
927  "mapd_columns ORDER BY tableid, "
928  "columnid");
929  sqliteConnector_.query(columnQuery);
930  numRows = sqliteConnector_.getNumRows();
931  int32_t skip_physical_cols = 0;
932  for (size_t r = 0; r < numRows; ++r) {
934  cd->tableId = sqliteConnector_.getData<int>(r, 0);
935  cd->columnId = sqliteConnector_.getData<int>(r, 1);
936  cd->columnName = sqliteConnector_.getData<string>(r, 2);
944  cd->columnType.set_size(sqliteConnector_.getData<int>(r, 10));
945  cd->chunks = sqliteConnector_.getData<string>(r, 11);
946  cd->isSystemCol = sqliteConnector_.getData<bool>(r, 12);
947  cd->isVirtualCol = sqliteConnector_.getData<bool>(r, 13);
948  cd->virtualExpr = sqliteConnector_.getData<string>(r, 14);
949  cd->isDeletedCol = sqliteConnector_.getData<bool>(r, 15);
950  cd->isGeoPhyCol = skip_physical_cols > 0;
951  ColumnKey columnKey(cd->tableId, to_upper(cd->columnName));
952  columnDescriptorMap_[columnKey] = cd;
953  ColumnIdKey columnIdKey(cd->tableId, cd->columnId);
954  columnDescriptorMapById_[columnIdKey] = cd;
955 
956  if (skip_physical_cols <= 0) {
957  skip_physical_cols = cd->columnType.get_physical_cols();
958  }
959 
960  auto td_itr = tableDescriptorMapById_.find(cd->tableId);
961  CHECK(td_itr != tableDescriptorMapById_.end());
962 
963  if (cd->isDeletedCol) {
964  td_itr->second->hasDeletedCol = true;
965  setDeletedColumnUnlocked(td_itr->second, cd);
966  } else if (cd->columnType.is_geometry() || skip_physical_cols-- <= 0) {
967  tableDescriptorMapById_[cd->tableId]->columnIdBySpi_.push_back(cd->columnId);
968  }
969  }
970  // sort columnIdBySpi_ based on columnId
971  for (auto& tit : tableDescriptorMapById_) {
972  std::sort(tit.second->columnIdBySpi_.begin(),
973  tit.second->columnIdBySpi_.end(),
974  [](const size_t a, const size_t b) -> bool { return a < b; });
975  }
976 
977  string viewQuery("SELECT tableid, sql FROM mapd_views");
978  sqliteConnector_.query(viewQuery);
979  numRows = sqliteConnector_.getNumRows();
980  for (size_t r = 0; r < numRows; ++r) {
981  int32_t tableId = sqliteConnector_.getData<int>(r, 0);
982  TableDescriptor* td = tableDescriptorMapById_[tableId];
983  td->viewSQL = sqliteConnector_.getData<string>(r, 1);
984  td->fragmenter = nullptr;
985  }
986 
987  string frontendViewQuery(
988  "SELECT id, state, name, image_hash, strftime('%Y-%m-%dT%H:%M:%SZ', update_time), "
989  "userid, "
990  "metadata "
991  "FROM mapd_dashboards");
992  sqliteConnector_.query(frontendViewQuery);
993  numRows = sqliteConnector_.getNumRows();
994  for (size_t r = 0; r < numRows; ++r) {
995  std::shared_ptr<DashboardDescriptor> vd = std::make_shared<DashboardDescriptor>();
996  vd->dashboardId = sqliteConnector_.getData<int>(r, 0);
997  vd->dashboardState = sqliteConnector_.getData<string>(r, 1);
998  vd->dashboardName = sqliteConnector_.getData<string>(r, 2);
999  vd->imageHash = sqliteConnector_.getData<string>(r, 3);
1000  vd->updateTime = sqliteConnector_.getData<string>(r, 4);
1001  vd->userId = sqliteConnector_.getData<int>(r, 5);
1002  vd->dashboardMetadata = sqliteConnector_.getData<string>(r, 6);
1003  vd->user = getUserFromId(vd->userId);
1004  vd->dashboardSystemRoleName = generate_dashboard_system_rolename(
1006  dashboardDescriptorMap_[std::to_string(vd->userId) + ":" + vd->dashboardName] = vd;
1007  }
1008 
1009  string linkQuery(
1010  "SELECT linkid, userid, link, view_state, strftime('%Y-%m-%dT%H:%M:%SZ', "
1011  "update_time), view_metadata "
1012  "FROM mapd_links");
1013  sqliteConnector_.query(linkQuery);
1014  numRows = sqliteConnector_.getNumRows();
1015  for (size_t r = 0; r < numRows; ++r) {
1016  LinkDescriptor* ld = new LinkDescriptor();
1017  ld->linkId = sqliteConnector_.getData<int>(r, 0);
1018  ld->userId = sqliteConnector_.getData<int>(r, 1);
1019  ld->link = sqliteConnector_.getData<string>(r, 2);
1020  ld->viewState = sqliteConnector_.getData<string>(r, 3);
1021  ld->updateTime = sqliteConnector_.getData<string>(r, 4);
1022  ld->viewMetadata = sqliteConnector_.getData<string>(r, 5);
1024  linkDescriptorMapById_[ld->linkId] = ld;
1025  }
1026 
1027  /* rebuild map linking logical tables to corresponding physical ones */
1028  string logicalToPhysicalTableMapQuery(
1029  "SELECT logical_table_id, physical_table_id "
1030  "FROM mapd_logical_to_physical");
1031  sqliteConnector_.query(logicalToPhysicalTableMapQuery);
1032  numRows = sqliteConnector_.getNumRows();
1033  for (size_t r = 0; r < numRows; ++r) {
1034  int32_t logical_tb_id = sqliteConnector_.getData<int>(r, 0);
1035  int32_t physical_tb_id = sqliteConnector_.getData<int>(r, 1);
1036  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
1037  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
1038  /* add new entity to the map logicalToPhysicalTableMapById_ */
1039  std::vector<int32_t> physicalTables;
1040  physicalTables.push_back(physical_tb_id);
1041  const auto it_ok =
1042  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
1043  CHECK(it_ok.second);
1044  } else {
1045  /* update map logicalToPhysicalTableMapById_ */
1046  physicalTableIt->second.push_back(physical_tb_id);
1047  }
1048  }
1049 }
std::string virtualExpr
std::string partitions
T getData(const int row, const int col)
SQLTypes
Definition: sqltypes.h:41
std::string tableName
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:320
EncodingType
Definition: encodetypes.h:22
SqliteConnector sqliteConnector_
Definition: Catalog.h:326
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:322
void set_size(int s)
Definition: sqltypes.h:424
void query(const std::string &queryString)
std::string fragments
HOST DEVICE void set_type(SQLTypes t)
Definition: sqltypes.h:416
void set_dimension(int d)
Definition: sqltypes.h:418
void set_scale(int s)
Definition: sqltypes.h:421
void set_compression(EncodingType c)
Definition: sqltypes.h:426
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:321
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:323
std::string to_string(char const *&&v)
void set_notnull(bool n)
Definition: sqltypes.h:423
std::string chunks
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:417
CHECK(cgen_state)
std::string getUserFromId(const int32_t id)
Definition: Catalog.cpp:855
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:2243
std::tuple< int, int > ColumnIdKey
Definition: Catalog.h:248
std::tuple< int, std::string > ColumnKey
Definition: Catalog.h:246
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:319
specifies the content in-memory of a row in the column metadata table
size_t getNumRows() const
std::string keyMetainfo
bool is_geometry() const
Definition: sqltypes.h:489
std::string to_upper(const std::string &str)
bool isNull(const int row, const int col) const
Fragmenter_Namespace::FragmenterType fragType
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:91
void set_comp_param(int p)
Definition: sqltypes.h:427
Descriptor for a dictionary for a string columne.
int get_physical_cols() const
Definition: sqltypes.h:347
std::string viewSQL
SQLTypeInfo columnType
specifies the content in-memory of a row in the table metadata table
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:92
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:325
Fragmenter_Namespace::AbstractFragmenter * fragmenter
std::string columnName
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:333
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:318
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:324
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string Catalog_Namespace::Catalog::calculateSHA1 ( const std::string &  data)
protected

Definition at line 2830 of file Catalog.cpp.

Referenced by createLink().

2830  {
2831  boost::uuids::detail::sha1 sha1;
2832  unsigned int digest[5];
2833  sha1.process_bytes(data.c_str(), data.length());
2834  sha1.get_digest(digest);
2835  std::stringstream ss;
2836  for (size_t i = 0; i < 5; i++) {
2837  ss << std::hex << digest[i];
2838  }
2839  return ss.str();
2840 }

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::CheckAndExecuteMigrations ( )
protected

Definition at line 834 of file Catalog.cpp.

References recordOwnershipOfObjectsInObjectPermissions(), updateDeletedColumnIndicator(), updateDictionaryNames(), updateDictionarySchema(), updateFixlenArrayColumns(), updateFrontendViewAndLinkUsers(), updateFrontendViewSchema(), updateFrontendViewsToDashboards(), updateLinkSchema(), updateLogicalToPhysicalTableLinkSchema(), updatePageSize(), and updateTableDescriptorSchema().

Referenced by Catalog().

834  {
843  updatePageSize();
847 }
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:327
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:528
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:98
void updateLogicalToPhysicalTableLinkSchema()
Definition: Catalog.cpp:467

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::CheckAndExecuteMigrationsPostBuildMaps ( )
protected

Definition at line 849 of file Catalog.cpp.

References checkDateInDaysColumnMigration(), and createDashboardSystemRoles().

Referenced by Catalog().

849  {
852 }
void checkDateInDaysColumnMigration()
Definition: Catalog.cpp:655

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::checkDateInDaysColumnMigration ( )
protected

Definition at line 655 of file Catalog.cpp.

References logger::ERROR, getCurrentDB(), SqliteConnector::getData(), Executor::getExecutor(), SqliteConnector::getNumRows(), logger::INFO, kDATE, kENCODING_DATE_IN_DAYS, LOG, MAPD_VERSION, SqliteConnector::query(), SqliteConnector::query_with_text_params(), TableOptimizer::recomputeMetadata(), sqliteConnector_, tableDescriptorMapById_, and to_string().

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

655  {
656  cat_sqlite_lock sqlite_lock(this);
657  std::vector<int> tables_migrated = {};
658  std::unordered_map<int, std::vector<std::string>> tables_to_migrate;
659  sqliteConnector_.query("BEGIN TRANSACTION");
660  try {
662  "select name from sqlite_master WHERE type='table' AND "
663  "name='mapd_version_history'");
664  if (sqliteConnector_.getNumRows() == 0) {
666  "CREATE TABLE mapd_version_history(version integer, migration_history text "
667  "unique)");
669  "CREATE TABLE mapd_date_in_days_column_migration_tmp(table_id integer primary "
670  "key)");
671  } else {
673  "select * from mapd_version_history where migration_history = "
674  "'date_in_days_column'");
675  if (sqliteConnector_.getNumRows() != 0) {
676  // no need for further execution
677  sqliteConnector_.query("END TRANSACTION");
678  return;
679  }
680  LOG(INFO) << "Performing Date in days columns migration.";
682  "select name from sqlite_master where type='table' AND "
683  "name='mapd_date_in_days_column_migration_tmp'");
684  if (sqliteConnector_.getNumRows() != 0) {
686  "select table_id from mapd_date_in_days_column_migration_tmp");
687  if (sqliteConnector_.getNumRows() != 0) {
688  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
689  tables_migrated.push_back(sqliteConnector_.getData<int>(i, 0));
690  }
691  }
692  } else {
694  "CREATE TABLE mapd_date_in_days_column_migration_tmp(table_id integer "
695  "primary key)");
696  }
697  }
699  "SELECT tables.tableid, tables.name, columns.name FROM mapd_tables tables, "
700  "mapd_columns columns where tables.tableid = columns.tableid AND "
701  "columns.coltype = ?1 AND columns.compression = ?2",
702  std::vector<std::string>{
703  std::to_string(static_cast<int>(SQLTypes::kDATE)),
705  if (sqliteConnector_.getNumRows() != 0) {
706  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
707  tables_to_migrate[sqliteConnector_.getData<int>(i, 0)] = {
708  sqliteConnector_.getData<std::string>(i, 1),
709  sqliteConnector_.getData<std::string>(i, 2)};
710  }
711  }
712  } catch (const std::exception& e) {
713  LOG(ERROR) << "Failed to complete migraion on date in days column: " << e.what();
714  sqliteConnector_.query("ROLLBACK");
715  throw;
716  }
717  sqliteConnector_.query("END TRANSACTION");
718 
719  for (auto& id_names : tables_to_migrate) {
720  if (std::find(tables_migrated.begin(), tables_migrated.end(), id_names.first) ==
721  tables_migrated.end()) {
722  sqliteConnector_.query("BEGIN TRANSACTION");
723  try {
724  LOG(INFO) << "Table: " << id_names.second[0]
725  << " may suffer from issues with DATE column: " << id_names.second[1]
726  << ". Running an OPTIMIZE command to solve any issues with metadata.";
727 
728  auto executor = Executor::getExecutor(getCurrentDB().dbId);
729  TableDescriptorMapById::iterator tableDescIt =
730  tableDescriptorMapById_.find(id_names.first);
731  if (tableDescIt == tableDescriptorMapById_.end()) {
732  throw runtime_error("Table descriptor does not exist for table " +
733  id_names.second[0] + " does not exist.");
734  }
735  auto td = tableDescIt->second;
736  TableOptimizer optimizer(td, executor.get(), *this);
737  optimizer.recomputeMetadata();
738 
740  "INSERT INTO mapd_date_in_days_column_migration_tmp VALUES(?)",
741  std::vector<std::string>{std::to_string(id_names.first)});
742  } catch (const std::exception& e) {
743  LOG(ERROR) << "Failed to complete migraion on date in days column: " << e.what();
744  sqliteConnector_.query("ROLLBACK");
745  throw;
746  }
747  sqliteConnector_.query("COMMIT");
748  }
749  }
750 
751  sqliteConnector_.query("BEGIN TRANSACTION");
752  try {
753  sqliteConnector_.query("DROP TABLE mapd_date_in_days_column_migration_tmp");
755  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
756  std::vector<std::string>{std::to_string(MAPD_VERSION), "date_in_days_column"});
757  } catch (const std::exception& e) {
758  LOG(ERROR) << "Failed to complete migraion on date in days column: " << e.what();
759  sqliteConnector_.query("ROLLBACK");
760  throw;
761  }
762  sqliteConnector_.query("END TRANSACTION");
763  LOG(INFO) << "Migration successfull on Date in days columns";
764 }
T getData(const int row, const int col)
static std::shared_ptr< Executor > getExecutor(const int db_id, const std::string &debug_dir="", const std::string &debug_file="", const MapDParameters mapd_parameters=MapDParameters(),::QueryRenderer::QueryRenderManager *render_manager=nullptr)
Definition: Execute.cpp:127
#define LOG(tag)
Definition: Logger.h:185
SqliteConnector sqliteConnector_
Definition: Catalog.h:326
void query_with_text_params(std::string const &query_only)
void query(const std::string &queryString)
Driver for running cleanup processes on a table. TableOptimizer provides functions for various cleanu...
std::string to_string(char const *&&v)
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:176
static const int32_t MAPD_VERSION
Definition: release.h:33
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:319
size_t getNumRows() const
Definition: sqltypes.h:56
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:92
void recomputeMetadata() const
Recomputes per-chunk metadata for each fragment in the table. Updates and deletes can cause chunk met...

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const bool Catalog_Namespace::Catalog::checkMetadataForDeletedRecs ( int  dbId,
int  tableId,
int  columnId 
) const

Definition at line 2185 of file Catalog.cpp.

References dataMgr_.

Referenced by getDeletedColumnIfRowsDeleted().

2187  {
2188  // check if there are rows deleted by examining metadata for the deletedColumn metadata
2189  ChunkKey chunkKeyPrefix = {dbId, tableId, columnId};
2190  std::vector<std::pair<ChunkKey, ChunkMetadata>> chunkMetadataVec;
2191  dataMgr_->getChunkMetadataVecForKeyPrefix(chunkMetadataVec, chunkKeyPrefix);
2192  int64_t chunk_max{0};
2193 
2194  for (auto cm : chunkMetadataVec) {
2195  chunk_max = cm.second.chunkStats.max.tinyintval;
2196  // delete has occured
2197  if (chunk_max == 1) {
2198  return true;
2199  }
2200  }
2201  return false;
2202 }
std::vector< int > ChunkKey
Definition: types.h:35
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:328

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::checkpoint ( const int  logicalTableId) const

Definition at line 2929 of file Catalog.cpp.

References Data_Namespace::DataMgr::checkpoint(), getCurrentDB(), getDataMgr(), getMetadataForTable(), and getPhysicalTablesDescriptors().

Referenced by Importer_NS::Loader::checkpoint(), and TableOptimizer::vacuumDeletedRows().

2929  {
2930  const auto td = getMetadataForTable(logicalTableId);
2931  const auto shards = getPhysicalTablesDescriptors(td);
2932  for (const auto shard : shards) {
2933  getDataMgr().checkpoint(getCurrentDB().dbId, shard->tableId);
2934  }
2935 }
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:177
void checkpoint(const int db_id, const int tb_id)
Definition: DataMgr.cpp:406
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logicalTableDesc) const
Definition: Catalog.cpp:2897
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:176
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

int32_t Catalog_Namespace::Catalog::createDashboard ( DashboardDescriptor vd)

Definition at line 2696 of file Catalog.cpp.

References addFrontendViewToMap(), createOrUpdateDashboardSystemRole(), currentDB_, DashboardDescriptor::dashboardId, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, DashboardDescriptor::dashboardSystemRoleName, Catalog_Namespace::DBMetadata::dbId, generate_dashboard_system_rolename(), SqliteConnector::getData(), SqliteConnector::getNumRows(), DashboardDescriptor::imageHash, SqliteConnector::query(), SqliteConnector::query_with_text_params(), sqliteConnector_, to_string(), DashboardDescriptor::updateTime, and DashboardDescriptor::userId.

2696  {
2697  {
2698  cat_write_lock write_lock(this);
2699  cat_sqlite_lock sqlite_lock(this);
2700  sqliteConnector_.query("BEGIN TRANSACTION");
2701  try {
2702  // TODO(andrew): this should be an upsert
2704  "SELECT id FROM mapd_dashboards WHERE name = ? and userid = ?",
2705  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
2706  if (sqliteConnector_.getNumRows() > 0) {
2708  "UPDATE mapd_dashboards SET state = ?, image_hash = ?, metadata = ?, "
2709  "update_time = "
2710  "datetime('now') where name = ? "
2711  "and userid = ?",
2712  std::vector<std::string>{vd.dashboardState,
2713  vd.imageHash,
2714  vd.dashboardMetadata,
2715  vd.dashboardName,
2716  std::to_string(vd.userId)});
2717  } else {
2719  "INSERT INTO mapd_dashboards (name, state, image_hash, metadata, "
2720  "update_time, "
2721  "userid) "
2722  "VALUES "
2723  "(?,?,?,?, "
2724  "datetime('now'), ?)",
2725  std::vector<std::string>{vd.dashboardName,
2726  vd.dashboardState,
2727  vd.imageHash,
2728  vd.dashboardMetadata,
2729  std::to_string(vd.userId)});
2730  }
2731  } catch (std::exception& e) {
2732  sqliteConnector_.query("ROLLBACK TRANSACTION");
2733  throw;
2734  }
2735  sqliteConnector_.query("END TRANSACTION");
2736 
2737  // now get the auto generated dashboardId
2738  try {
2740  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_dashboards "
2741  "WHERE name = ? and userid = ?",
2742  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
2743  vd.dashboardId = sqliteConnector_.getData<int>(0, 0);
2744  vd.updateTime = sqliteConnector_.getData<std::string>(0, 1);
2745  } catch (std::exception& e) {
2746  throw;
2747  }
2751  }
2752  // NOTE(wamsi): Transactionally unsafe
2755  return vd.dashboardId;
2756 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:326
void query_with_text_params(std::string const &query_only)
void query(const std::string &queryString)
void addFrontendViewToMap(DashboardDescriptor &vd)
Definition: Catalog.cpp:1183
std::string to_string(char const *&&v)
std::string dashboardSystemRoleName
size_t getNumRows() const
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1224
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:91
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:92
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createDashboardSystemRoles ( )
protected

Definition at line 766 of file Catalog.cpp.

References createOrUpdateDashboardSystemRole(), currentDB_, Catalog_Namespace::DBMetadata::dbId, logger::ERROR, generate_dashboard_system_rolename(), SqliteConnector::getData(), Catalog_Namespace::SysCatalog::getGranteesOfSharedDashboards(), SqliteConnector::getNumRows(), Catalog_Namespace::SysCatalog::getRoleGrantee(), Catalog_Namespace::SysCatalog::grantRoleBatch(), logger::INFO, Catalog_Namespace::SysCatalog::instance(), LOG, MAPD_VERSION, SqliteConnector::query(), SqliteConnector::query_with_text_params(), run_benchmark_import::result, sqliteConnector_, and to_string().

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

766  {
767  std::unordered_map<std::string, std::pair<int, std::string>> dashboards;
768  std::vector<std::string> dashboard_ids;
769  static const std::string migration_name{"dashboard_roles_migration"};
770  {
771  cat_sqlite_lock sqlite_lock(this);
772  sqliteConnector_.query("BEGIN TRANSACTION");
773  try {
774  // migration_history should be present in all catalogs by now
775  // if not then would be created before this migration
777  "select * from mapd_version_history where migration_history = '" +
778  migration_name + "'");
779  if (sqliteConnector_.getNumRows() != 0) {
780  // no need for further execution
781  sqliteConnector_.query("END TRANSACTION");
782  return;
783  }
784  LOG(INFO) << "Performing dashboard internal roles Migration.";
785  sqliteConnector_.query("select id, userid, metadata from mapd_dashboards");
786  for (size_t i = 0; i < sqliteConnector_.getNumRows(); ++i) {
789  sqliteConnector_.getData<string>(i, 0)))) {
790  // Successfully created roles during previous migration/crash
791  // No need to include them
792  continue;
793  }
794  dashboards[sqliteConnector_.getData<string>(i, 0)] = std::make_pair(
795  sqliteConnector_.getData<int>(i, 1), sqliteConnector_.getData<string>(i, 2));
796  dashboard_ids.push_back(sqliteConnector_.getData<string>(i, 0));
797  }
798  } catch (const std::exception& e) {
799  sqliteConnector_.query("ROLLBACK TRANSACTION");
800  throw;
801  }
802  sqliteConnector_.query("END TRANSACTION");
803  }
804  // All current grantees with shared dashboards.
805  const auto active_grantees =
807 
808  try {
809  // NOTE(wamsi): Transactionally unsafe
810  for (auto dash : dashboards) {
811  createOrUpdateDashboardSystemRole(dash.second.second,
812  dash.second.first,
814  std::to_string(currentDB_.dbId), dash.first));
815  auto result = active_grantees.find(dash.first);
816  if (result != active_grantees.end()) {
819  dash.first)},
820  result->second);
821  }
822  }
824  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
825  std::vector<std::string>{std::to_string(MAPD_VERSION), migration_name});
826  } catch (const std::exception& e) {
827  LOG(ERROR) << "Failed to create dashboard system roles during migration: "
828  << e.what();
829  throw;
830  }
831  LOG(INFO) << "Successfully created dashboard system roles during migration.";
832 }
T getData(const int row, const int col)
#define LOG(tag)
Definition: Logger.h:185
SqliteConnector sqliteConnector_
Definition: Catalog.h:326
void query_with_text_params(std::string const &query_only)
void query(const std::string &queryString)
std::string to_string(char const *&&v)
void grantRoleBatch(const std::vector< std::string > &roles, const std::vector< std::string > &grantees)
static SysCatalog & instance()
Definition: SysCatalog.h:242
static const int32_t MAPD_VERSION
Definition: release.h:33
Role * getRoleGrantee(const std::string &name) const
size_t getNumRows() const
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1224
std::unordered_map< std::string, std::vector< std::string > > getGranteesOfSharedDashboards(const std::vector< std::string > &dashboard_ids)
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:92
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string Catalog_Namespace::Catalog::createLink ( LinkDescriptor ld,
size_t  min_length 
)

Definition at line 2842 of file Catalog.cpp.

References addLinkToMap(), calculateSHA1(), SqliteConnector::getData(), SqliteConnector::getNumRows(), LinkDescriptor::link, LinkDescriptor::linkId, SqliteConnector::query(), SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), sqliteConnector_, to_string(), LinkDescriptor::updateTime, LinkDescriptor::userId, LinkDescriptor::viewMetadata, and LinkDescriptor::viewState.

2842  {
2843  cat_write_lock write_lock(this);
2844  cat_sqlite_lock sqlite_lock(this);
2845  sqliteConnector_.query("BEGIN TRANSACTION");
2846  try {
2848  .substr(0, 8);
2850  "SELECT linkid FROM mapd_links WHERE link = ? and userid = ?",
2851  std::vector<std::string>{ld.link, std::to_string(ld.userId)});
2852  if (sqliteConnector_.getNumRows() > 0) {
2854  "UPDATE mapd_links SET update_time = datetime('now') WHERE userid = ? AND link "
2855  "= ?",
2856  std::vector<std::string>{std::to_string(ld.userId), ld.link});
2857  } else {
2859  "INSERT INTO mapd_links (userid, link, view_state, view_metadata, update_time) "
2860  "VALUES (?,?,?,?, "
2861  "datetime('now'))",
2862  std::vector<std::string>{
2863  std::to_string(ld.userId), ld.link, ld.viewState, ld.viewMetadata});
2864  }
2865  // now get the auto generated dashid
2867  "SELECT linkid, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_links "
2868  "WHERE link = ?",
2869  ld.link);
2870  ld.linkId = sqliteConnector_.getData<int>(0, 0);
2871  ld.updateTime = sqliteConnector_.getData<std::string>(0, 1);
2872  } catch (std::exception& e) {
2873  sqliteConnector_.query("ROLLBACK TRANSACTION");
2874  throw;
2875  }
2876  sqliteConnector_.query("END TRANSACTION");
2877  addLinkToMap(ld);
2878  return ld.link;
2879 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:326
void query_with_text_params(std::string const &query_only)
void query(const std::string &queryString)
std::string to_string(char const *&&v)
std::string calculateSHA1(const std::string &data)
Definition: Catalog.cpp:2830
void addLinkToMap(LinkDescriptor &ld)
Definition: Catalog.cpp:1265
size_t getNumRows() const
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:91
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:92
void query_with_text_param(const std::string &queryString, const std::string &text_param)

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createOrUpdateDashboardSystemRole ( const std::string &  view_meta,
const int32_t &  user_id,
const std::string &  dash_role_name 
)
protected

Definition at line 1224 of file Catalog.cpp.

References Catalog_Namespace::SysCatalog::createRole(), Grantee::findDbObject(), Grantee::getDbObjects(), Catalog_Namespace::SysCatalog::getRoleGrantee(), Catalog_Namespace::SysCatalog::grantDBObjectPrivilegesBatch(), Catalog_Namespace::SysCatalog::instance(), parseDashboardObjects(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivileges(), TableDBObjectType, and ViewDBObjectType.

Referenced by createDashboard(), createDashboardSystemRoles(), and replaceDashboard().

1226  {
1227  auto objects = parseDashboardObjects(view_meta, user_id);
1228  Role* rl = SysCatalog::instance().getRoleGrantee(dash_role_name);
1229  if (!rl) {
1230  // Dashboard role does not exist
1231  // create role and grant privileges
1232  // NOTE(wamsi): Transactionally unsafe
1233  SysCatalog::instance().createRole(dash_role_name, false);
1234  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1235  } else {
1236  // Dashboard system role already exists
1237  // Add/remove privileges on objects
1238  auto ex_objects = rl->getDbObjects(true);
1239  for (auto key : *ex_objects | boost::adaptors::map_keys) {
1240  if (key.permissionType != TableDBObjectType &&
1241  key.permissionType != ViewDBObjectType) {
1242  continue;
1243  }
1244  bool found = false;
1245  for (auto obj : objects) {
1246  found = key == obj.getObjectKey() ? true : false;
1247  if (found) {
1248  break;
1249  }
1250  }
1251  if (!found) {
1252  // revoke privs on object since the object is no
1253  // longer used by the dashboard as source
1254  // NOTE(wamsi): Transactionally unsafe
1256  dash_role_name, *rl->findDbObject(key, true), *this);
1257  }
1258  }
1259  // Update privileges on remaining objects
1260  // NOTE(wamsi): Transactionally unsafe
1261  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1262  }
1263 }
void revokeDBObjectPrivileges(const std::string &grantee, const DBObject &object, const Catalog_Namespace::Catalog &catalog)
Definition: Grantee.h:76
DBObject * findDbObject(const DBObjectKey &objectKey, bool only_direct) const
Definition: Grantee.cpp:72
void createRole(const std::string &roleName, const bool &userPrivateRole=false)
static SysCatalog & instance()
Definition: SysCatalog.h:242
Role * getRoleGrantee(const std::string &name) const
void grantDBObjectPrivilegesBatch(const std::vector< std::string > &grantees, const std::vector< DBObject > &objects, const Catalog_Namespace::Catalog &catalog)
const DBObjectMap * getDbObjects(bool only_direct) const
Definition: Grantee.h:56
std::vector< DBObject > parseDashboardObjects(const std::string &view_meta, const int &user_id)
Definition: Catalog.cpp:1194

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::createShardedTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs 
)

Definition at line 2386 of file Catalog.cpp.

References CHECK(), createTable(), generatePhysicalTableName(), logicalToPhysicalTableMapById_, TableDescriptor::nShards, TableDescriptor::shard, TableDescriptor::tableId, TableDescriptor::tableName, and updateLogicalToPhysicalTableMap().

2389  {
2390  cat_write_lock write_lock(this);
2391 
2392  /* create logical table */
2393  TableDescriptor tdl(td);
2394  createTable(tdl, cols, shared_dict_defs, true); // create logical table
2395  int32_t logical_tb_id = tdl.tableId;
2396 
2397  /* create physical tables and link them to the logical table */
2398  std::vector<int32_t> physicalTables;
2399  for (int32_t i = 1; i <= td.nShards; i++) {
2400  TableDescriptor tdp(td);
2401  tdp.tableName = generatePhysicalTableName(tdp.tableName, i);
2402  tdp.shard = i - 1;
2403  createTable(tdp, cols, shared_dict_defs, false); // create physical table
2404  int32_t physical_tb_id = tdp.tableId;
2405 
2406  /* add physical table to the vector of physical tables */
2407  physicalTables.push_back(physical_tb_id);
2408  }
2409 
2410  if (!physicalTables.empty()) {
2411  /* add logical to physical tables correspondence to the map */
2412  const auto it_ok =
2413  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
2414  CHECK(it_ok.second);
2415  /* update sqlite mapd_logical_to_physical in sqlite database */
2416  updateLogicalToPhysicalTableMap(logical_tb_id);
2417  }
2418 }
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:1939
std::string generatePhysicalTableName(const std::string &logicalTableName, const int32_t &shardNumber)
Definition: Catalog.cpp:2979
CHECK(cgen_state)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:91
specifies the content in-memory of a row in the table metadata table
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:333
void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id)
Definition: Catalog.cpp:481

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::createTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs,
bool  isLogicalTable 
)

Definition at line 1939 of file Catalog.cpp.

References addTableToMap(), calciteMgr_, ColumnDescriptor::columnId, TableDescriptor::columnIdBySpi_, ColumnDescriptor::columnName, ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, dict_ref_t::dictId, Data_Namespace::DISK_LEVEL, expandGeoColumn(), TableDescriptor::fragPageSize, TableDescriptor::fragType, g_test_against_columnId_gap, SQLTypeInfoCore< TYPE_FACET_PACK >::get_comp_param(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_compression(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_dimension(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_notnull(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_scale(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_size(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_subtype(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), SqliteConnector::getData(), TableDescriptor::hasDeletedCol, SQLTypeInfoCore< TYPE_FACET_PACK >::is_array(), IS_GEO, ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, kBIGINT, kBOOLEAN, kENCODING_DICT, TableDescriptor::keyMetainfo, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRows, TableDescriptor::nColumns, nextTempDictId_, nextTempTableId_, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, SqliteConnector::query(), SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), removeTableFromMap(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_comp_param(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_size(), setColumnDictionary(), setColumnSharedDictionary(), TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, sqliteConnector_, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), TableDescriptor::userId, TableDescriptor::viewSQL, and ColumnDescriptor::virtualExpr.

Referenced by createShardedTable().

1943  {
1944  cat_write_lock write_lock(this);
1945  list<ColumnDescriptor> cds;
1946  list<DictDescriptor> dds;
1947  std::set<std::string> toplevel_column_names;
1948  list<ColumnDescriptor> columns;
1949  for (auto cd : cols) {
1950  if (cd.columnName == "rowid") {
1951  throw std::runtime_error(
1952  "Cannot create column with name rowid. rowid is a system defined column.");
1953  }
1954  columns.push_back(cd);
1955  toplevel_column_names.insert(cd.columnName);
1956  if (cd.columnType.is_geometry()) {
1957  expandGeoColumn(cd, columns);
1958  }
1959  }
1960 
1961  ColumnDescriptor cd;
1962  // add row_id column -- Must be last column in the table
1963  cd.columnName = "rowid";
1964  cd.isSystemCol = true;
1965  cd.columnType = SQLTypeInfo(kBIGINT, true);
1966 #ifdef MATERIALIZED_ROWID
1967  cd.isVirtualCol = false;
1968 #else
1969  cd.isVirtualCol = true;
1970  cd.virtualExpr = "MAPD_FRAG_ID * MAPD_ROWS_PER_FRAG + MAPD_FRAG_ROW_ID";
1971 #endif
1972  columns.push_back(cd);
1973  toplevel_column_names.insert(cd.columnName);
1974 
1975  if (td.hasDeletedCol) {
1976  ColumnDescriptor cd_del;
1977  cd_del.columnName = "$deleted$";
1978  cd_del.isSystemCol = true;
1979  cd_del.isVirtualCol = false;
1980  cd_del.columnType = SQLTypeInfo(kBOOLEAN, true);
1981  cd_del.isDeletedCol = true;
1982 
1983  columns.push_back(cd_del);
1984  }
1985 
1986  td.nColumns = columns.size();
1987  cat_sqlite_lock sqlite_lock(this);
1988  sqliteConnector_.query("BEGIN TRANSACTION");
1990  try {
1992  "INSERT INTO mapd_tables (name, userid, ncolumns, isview, fragments, "
1993  "frag_type, max_frag_rows, "
1994  "max_chunk_size, "
1995  "frag_page_size, max_rows, partitions, shard_column_id, shard, num_shards, "
1996  "sort_column_id, "
1997  "key_metainfo) VALUES (?, ?, ?, "
1998  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1999 
2000  std::vector<std::string>{td.tableName,
2001  std::to_string(td.userId),
2003  std::to_string(td.isView),
2004  "",
2009  std::to_string(td.maxRows),
2010  td.partitions,
2012  std::to_string(td.shard),
2013  std::to_string(td.nShards),
2015  td.keyMetainfo});
2016 
2017  // now get the auto generated tableid
2019  "SELECT tableid FROM mapd_tables WHERE name = ?", td.tableName);
2020  td.tableId = sqliteConnector_.getData<int>(0, 0);
2021  int colId = 1;
2022  for (auto cd : columns) {
2024  const bool is_foreign_col =
2025  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2026  if (!is_foreign_col) {
2027  setColumnDictionary(cd, dds, td, isLogicalTable);
2028  }
2029  }
2030 
2031  if (toplevel_column_names.count(cd.columnName)) {
2032  // make up colId gap for sanity test (begin with 1 bc much code depends on it!)
2033  if (colId > 1) {
2034  colId += g_test_against_columnId_gap;
2035  }
2036  if (!cd.isGeoPhyCol) {
2037  td.columnIdBySpi_.push_back(colId);
2038  }
2039  }
2040 
2042  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, "
2043  "coldim, colscale, is_notnull, "
2044  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
2045  "virtual_expr, is_deletedcol) "
2046  "VALUES (?, ?, ?, ?, ?, "
2047  "?, "
2048  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2049  std::vector<std::string>{std::to_string(td.tableId),
2050  std::to_string(colId),
2051  cd.columnName,
2060  "",
2063  cd.virtualExpr,
2065  cd.tableId = td.tableId;
2066  cd.columnId = colId++;
2067  cds.push_back(cd);
2068  }
2069  if (td.isView) {
2071  "INSERT INTO mapd_views (tableid, sql) VALUES (?,?)",
2072  std::vector<std::string>{std::to_string(td.tableId), td.viewSQL});
2073  }
2074  } catch (std::exception& e) {
2075  sqliteConnector_.query("ROLLBACK TRANSACTION");
2076  throw;
2077  }
2078 
2079  } else { // Temporary table
2080  td.tableId = nextTempTableId_++;
2081  int colId = 1;
2082  for (auto cd : columns) {
2083  auto col_ti = cd.columnType;
2084  if (IS_GEO(col_ti.get_type())) {
2085  throw runtime_error("Geometry types in temporary tables are not supported.");
2086  }
2087 
2089  // TODO(vraj) : create shared dictionary for temp table if needed
2090  std::string fileName("");
2091  std::string folderPath("");
2093  nextTempDictId_++;
2094  DictDescriptor dd(dict_ref,
2095  fileName,
2097  false,
2098  1,
2099  folderPath,
2100  true); // Is dictName (2nd argument) used?
2101  dds.push_back(dd);
2102  if (!cd.columnType.is_array()) {
2104  }
2105  cd.columnType.set_comp_param(dict_ref.dictId);
2106  }
2107  cd.tableId = td.tableId;
2108  cd.columnId = colId++;
2109  cds.push_back(cd);
2110  }
2111  }
2112  try {
2113  addTableToMap(td, cds, dds);
2114  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
2115  } catch (std::exception& e) {
2116  sqliteConnector_.query("ROLLBACK TRANSACTION");
2117  removeTableFromMap(td.tableName, td.tableId, true);
2118  throw;
2119  }
2120 
2121  sqliteConnector_.query("END TRANSACTION");
2122 }
std::string virtualExpr
std::string partitions
T getData(const int row, const int col)
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:334
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:335
std::string tableName
SqliteConnector sqliteConnector_
Definition: Catalog.h:326
void set_size(int s)
Definition: sqltypes.h:424
void query_with_text_params(std::string const &query_only)
HOST DEVICE int get_scale() const
Definition: sqltypes.h:331
HOST DEVICE int get_size() const
Definition: sqltypes.h:336
void query(const std::string &queryString)
void addTableToMap(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
Definition: Catalog.cpp:1051
std::vector< int > columnIdBySpi_
std::string to_string(char const *&&v)
bool is_array() const
Definition: sqltypes.h:485
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:326
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:333
specifies the content in-memory of a row in the column metadata table
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:1809
SQLTypeInfoCore< ArrayContextTypeSizer, ExecutorTypePackaging, DateTimeFacilities > SQLTypeInfo
Definition: sqltypes.h:852
std::string keyMetainfo
int g_test_against_columnId_gap
Definition: Catalog.cpp:74
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:331
Fragmenter_Namespace::FragmenterType fragType
Data_Namespace::MemoryLevel persistenceLevel
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:328
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1101
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:91
void set_comp_param(int p)
Definition: sqltypes.h:427
bool setColumnSharedDictionary(ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:2283
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:327
Descriptor for a dictionary for a string columne.
std::string viewSQL
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:92
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:2345
std::string columnName
void query_with_text_param(const std::string &queryString, const std::string &text_param)
#define IS_GEO(T)
Definition: sqltypes.h:167

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::delDictionary ( const ColumnDescriptor cd)

Definition at line 1641 of file Catalog.cpp.

References basePath_, ColumnDescriptor::columnName, ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, SQLTypeInfoCore< TYPE_FACET_PACK >::get_comp_param(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_compression(), SQLTypeInfoCore< TYPE_FACET_PACK >::is_string(), SQLTypeInfoCore< TYPE_FACET_PACK >::is_string_array(), kENCODING_DICT, SqliteConnector::query_with_text_param(), File_Namespace::renameForDelete(), sqliteConnector_, string_dict_hosts_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableName, and to_string().

Referenced by roll().

1641  {
1642  if (!(cd.columnType.is_string() || cd.columnType.is_string_array())) {
1643  return;
1644  }
1645  if (!(cd.columnType.get_compression() == kENCODING_DICT)) {
1646  return;
1647  }
1648  if (!(cd.columnType.get_comp_param() > 0)) {
1649  return;
1650  }
1651 
1652  const auto& td = *tableDescriptorMapById_[cd.tableId];
1653  const auto dictId = cd.columnType.get_comp_param();
1654  const DictRef dictRef(currentDB_.dbId, dictId);
1655  const auto dictName =
1656  td.tableName + "_" + cd.columnName + "_dict" + std::to_string(dictId);
1657  sqliteConnector_.query_with_text_param("DELETE FROM mapd_dictionaries WHERE name = ?",
1658  dictName);
1659  File_Namespace::renameForDelete(basePath_ + "/mapd_data/DB_" +
1660  std::to_string(currentDB_.dbId) + "_DICT_" +
1661  std::to_string(dictId));
1662 
1663  std::unique_ptr<StringDictionaryClient> client;
1664  if (!string_dict_hosts_.empty()) {
1665  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dictRef, true));
1666  }
1667  if (client) {
1668  client->drop(dictRef);
1669  }
1670 
1671  dictDescriptorMapByRef_.erase(dictRef);
1672 }
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:334
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:335
SqliteConnector sqliteConnector_
Definition: Catalog.h:326
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:330
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:322
std::string to_string(char const *&&v)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:319
bool is_string() const
Definition: sqltypes.h:477
SQLTypeInfo columnType
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:183
std::string columnName
void query_with_text_param(const std::string &queryString, const std::string &text_param)
bool is_string_array() const
Definition: sqltypes.h:478

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::deleteMetadataForDashboard ( const std::string &  userId,
const std::string &  dashName 
)

Definition at line 1447 of file Catalog.cpp.

References dashboardDescriptorMap_, logger::ERROR, LOG, SqliteConnector::query(), SqliteConnector::query_with_text_params(), and sqliteConnector_.

Referenced by deleteMetadataForDashboard().

1448  {
1449  cat_write_lock write_lock(this);
1450 
1451  auto viewDescIt = dashboardDescriptorMap_.find(userId + ":" + dashName);
1452  if (viewDescIt == dashboardDescriptorMap_.end()) { // check to make sure view exists
1453  LOG(ERROR) << "No metadata for dashboard for user " << userId << " dashboard "
1454  << dashName << " does not exist in map";
1455  throw runtime_error("No metadata for dashboard for user " + userId + " dashboard " +
1456  dashName + " does not exist in map");
1457  }
1458  // found view in Map now remove it
1459  dashboardDescriptorMap_.erase(viewDescIt);
1460  // remove from DB
1461  cat_sqlite_lock sqlite_lock(this);
1462  sqliteConnector_.query("BEGIN TRANSACTION");
1463  try {
1465  "DELETE FROM mapd_dashboards WHERE name = ? and userid = ?",
1466  std::vector<std::string>{dashName, userId});
1467  } catch (std::exception& e) {
1468  sqliteConnector_.query("ROLLBACK TRANSACTION");
1469  throw;
1470  }
1471  sqliteConnector_.query("END TRANSACTION");
1472 }
#define LOG(tag)
Definition: Logger.h:185
SqliteConnector sqliteConnector_
Definition: Catalog.h:326
void query_with_text_params(std::string const &query_only)
void query(const std::string &queryString)
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:323
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:91
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:92

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::deleteMetadataForDashboard ( const int32_t  dashboard_id)

Definition at line 1508 of file Catalog.cpp.

References DashboardDBObjectType, dashboardDescriptorMap_, deleteMetadataForDashboard(), Catalog_Namespace::SysCatalog::instance(), name(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivilegesFromAll(), and to_string().

1508  {
1509  std::string userId;
1510  std::string name;
1511  bool found{false};
1512  {
1513  cat_read_lock read_lock(this);
1514  for (auto descp : dashboardDescriptorMap_) {
1515  auto dash = descp.second.get();
1516  if (dash->dashboardId == id) {
1517  userId = std::to_string(dash->userId);
1518  name = dash->dashboardName;
1519  found = true;
1520  break;
1521  }
1522  }
1523  }
1524  if (found) {
1525  // TODO: transactionally unsafe
1527  DBObject(id, DashboardDBObjectType), this);
1528  deleteMetadataForDashboard(userId, name);
1529  }
1530 }
void revokeDBObjectPrivilegesFromAll(DBObject object, Catalog *catalog)
void deleteMetadataForDashboard(const std::string &userId, const std::string &dashName)
Definition: Catalog.cpp:1447
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:323
std::string to_string(char const *&&v)
std::string name() const
Definition: Catalog.h:219
static SysCatalog & instance()
Definition: SysCatalog.h:242
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:90

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::doDropTable ( const TableDescriptor td)
protected

Definition at line 2560 of file Catalog.cpp.

References eraseTablePhysicalData(), SqliteConnector::getData(), SqliteConnector::getNumRows(), TableDescriptor::isView, kENCODING_DICT, SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), sqliteConnector_, TableDescriptor::tableId, and to_string().

Referenced by dropTable().

2560  {
2561  const int tableId = td->tableId;
2562  sqliteConnector_.query_with_text_param("DELETE FROM mapd_tables WHERE tableid = ?",
2563  std::to_string(tableId));
2565  "select comp_param from mapd_columns where compression = ? and tableid = ?",
2566  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
2567  int numRows = sqliteConnector_.getNumRows();
2568  std::vector<int> dict_id_list;
2569  for (int r = 0; r < numRows; ++r) {
2570  dict_id_list.push_back(sqliteConnector_.getData<int>(r, 0));
2571  }
2572  for (auto dict_id : dict_id_list) {
2574  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
2575  std::vector<std::string>{std::to_string(dict_id)});
2576  }
2578  "DELETE FROM mapd_dictionaries WHERE dictid in (select comp_param from "
2579  "mapd_columns where compression = ? "
2580  "and tableid = ?) and refcount = 0",
2581  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
2582  sqliteConnector_.query_with_text_param("DELETE FROM mapd_columns WHERE tableid = ?",
2583  std::to_string(tableId));
2584  if (td->isView) {
2585  sqliteConnector_.query_with_text_param("DELETE FROM mapd_views WHERE tableid = ?",
2586  std::to_string(tableId));
2587  }
2589 }
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:2949
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:326
void query_with_text_params(std::string const &query_only)
std::string to_string(char const *&&v)
size_t getNumRows() const
void query_with_text_param(const std::string &queryString, const std::string &text_param)

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::doTruncateTable ( const TableDescriptor td)
protected

Definition at line 2438 of file Catalog.cpp.

References CHECK(), CHECK_GE, ColumnDescriptor::columnType, Data_Namespace::CPU_LEVEL, currentDB_, dataMgr_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, dict_ref_t::dictId, DictDescriptor::dictIsTemp, DictDescriptor::dictRef, TableDescriptor::fragmenter, SQLTypeInfoCore< TYPE_FACET_PACK >::get_comp_param(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_compression(), getMetadataForDict(), Data_Namespace::GPU_LEVEL, Catalog_Namespace::SysCatalog::instance(), kENCODING_DICT, File_Namespace::renameForDelete(), string_dict_hosts_, tableDescriptorMapById_, ColumnDescriptor::tableId, and TableDescriptor::tableId.

Referenced by truncateTable().

2438  {
2439  cat_write_lock write_lock(this);
2440 
2441  const int tableId = td->tableId;
2442  // must destroy fragmenter before deleteChunks is called.
2443  if (td->fragmenter != nullptr) {
2444  auto tableDescIt = tableDescriptorMapById_.find(tableId);
2445  delete td->fragmenter;
2446  tableDescIt->second->fragmenter = nullptr; // get around const-ness
2447  }
2448  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
2449  // assuming deleteChunksWithPrefix is atomic
2450  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
2451  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
2452 
2453  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
2454 
2455  std::unique_ptr<StringDictionaryClient> client;
2456  if (SysCatalog::instance().isAggregator()) {
2457  CHECK(!string_dict_hosts_.empty());
2458  DictRef dict_ref(currentDB_.dbId, -1);
2459  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
2460  }
2461  // clean up any dictionaries
2462  // delete all column descriptors for the table
2463  for (const auto& columnDescriptor : columnDescriptorMapById_) {
2464  auto cd = columnDescriptor.second;
2465  if (cd->tableId != td->tableId) {
2466  continue;
2467  }
2468  const int dict_id = cd->columnType.get_comp_param();
2469  // Dummy dictionaries created for a shard of a logical table have the id set to zero.
2470  if (cd->columnType.get_compression() == kENCODING_DICT && dict_id) {
2471  const DictRef dict_ref(currentDB_.dbId, dict_id);
2472  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
2473  CHECK(dictIt != dictDescriptorMapByRef_.end());
2474  const auto& dd = dictIt->second;
2475  CHECK_GE(dd->refcount, 1);
2476  // if this is the only table using this dict reset the dict
2477  if (dd->refcount == 1) {
2478  // close the dictionary
2479  dd->stringDict.reset();
2480  File_Namespace::renameForDelete(dd->dictFolderPath);
2481  if (client) {
2482  client->drop(dd->dictRef);
2483  }
2484  if (!dd->dictIsTemp) {
2485  boost::filesystem::create_directory(dd->dictFolderPath);
2486  }
2487  }
2488 
2489  DictDescriptor* new_dd = new DictDescriptor(dd->dictRef,
2490  dd->dictName,
2491  dd->dictNBits,
2492  dd->dictIsShared,
2493  dd->refcount,
2494  dd->dictFolderPath,
2495  dd->dictIsTemp);
2496  dictDescriptorMapByRef_.erase(dictIt);
2497  // now create new Dict -- need to figure out what to do here for temp tables
2498  if (client) {
2499  client->create(new_dd->dictRef, new_dd->dictIsTemp);
2500  }
2501  dictDescriptorMapByRef_[new_dd->dictRef].reset(new_dd);
2503  }
2504  }
2505 }
std::vector< int > ChunkKey
Definition: types.h:35
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:330
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:322
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:328
#define CHECK_GE(x, y)
Definition: Logger.h:203
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:321
static SysCatalog & instance()
Definition: SysCatalog.h:242
CHECK(cgen_state)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:319
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1350
int32_t dictId
Definition: DictRef.h:10
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:91
Descriptor for a dictionary for a string columne.
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_&lt;EPOCH&gt;_&lt;oldname&gt;.
Definition: File.cpp:183
Fragmenter_Namespace::AbstractFragmenter * fragmenter

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::dropTable ( const TableDescriptor td)

Definition at line 2527 of file Catalog.cpp.

References CHECK(), doDropTable(), getMetadataForTable(), Catalog_Namespace::SysCatalog::instance(), TableDescriptor::isView, logicalToPhysicalTableMapById_, SqliteConnector::query(), SqliteConnector::query_with_text_param(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivilegesFromAll(), sqliteConnector_, TableDBObjectType, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and ViewDBObjectType.

2527  {
2530  cat_write_lock write_lock(this);
2531  cat_sqlite_lock sqlite_lock(this);
2532  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
2533  sqliteConnector_.query("BEGIN TRANSACTION");
2534  try {
2535  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2536  // remove all corresponding physical tables if this is a logical table
2537  const auto physicalTables = physicalTableIt->second;
2538  CHECK(!physicalTables.empty());
2539  for (size_t i = 0; i < physicalTables.size(); i++) {
2540  int32_t physical_tb_id = physicalTables[i];
2541  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2542  CHECK(phys_td);
2543  doDropTable(phys_td);
2544  }
2545 
2546  // remove corresponding record from the logicalToPhysicalTableMap in sqlite database
2548  "DELETE FROM mapd_logical_to_physical WHERE logical_table_id = ?",
2549  std::to_string(td->tableId));
2551  }
2552  doDropTable(td);
2553  } catch (std::exception& e) {
2554  sqliteConnector_.query("ROLLBACK TRANSACTION");
2555  throw;
2556  }
2557  sqliteConnector_.query("END TRANSACTION");
2558 }
void doDropTable(const TableDescriptor *td)
Definition: Catalog.cpp:2560
std::string tableName
SqliteConnector sqliteConnector_
Definition: Catalog.h:326
void revokeDBObjectPrivilegesFromAll(DBObject object, Catalog *catalog)
void query(const std::string &queryString)
std::string to_string(char const *&&v)
static SysCatalog & instance()
Definition: SysCatalog.h:242
CHECK(cgen_state)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:91
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
specifies the content in-memory of a row in the table metadata table
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:92
void query_with_text_param(const std::string &queryString, const std::string &text_param)
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:333

+ Here is the call graph for this function:

std::string Catalog_Namespace::Catalog::dumpSchema ( const TableDescriptor td) const

Definition at line 274 of file DumpRestore.cpp.

References CHECK(), currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, TableDescriptor::fragPageSize, getAllColumnMetadataForTable(), getMetadataForColumn(), TableDescriptor::hasDeletedCol, join(), kCHAR, kENCODING_DICT, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRows, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::tableId, and to_string().

Referenced by dumpTable().

274  {
275  std::ostringstream os;
276  os << "CREATE TABLE @T (";
277  // gather column defines
278  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
279  std::string comma;
280  std::vector<std::string> shared_dicts;
281  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
282  for (const auto cd : cds) {
283  if (!(cd->isSystemCol || cd->isVirtualCol)) {
284  const auto& ti = cd->columnType;
285  os << comma << cd->columnName;
286  // CHAR is perculiar... better dump it as TEXT(32) like \d does
287  if (ti.get_type() == SQLTypes::kCHAR) {
288  os << " "
289  << "TEXT";
290  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
291  os << " "
292  << "TEXT[]";
293  } else {
294  os << " " << ti.get_type_name();
295  }
296  os << (ti.get_notnull() ? " NOT NULL" : "");
297  if (ti.is_string()) {
298  if (ti.get_compression() == kENCODING_DICT) {
299  // if foreign reference, get referenced tab.col
300  const auto dict_id = ti.get_comp_param();
301  const DictRef dict_ref(currentDB_.dbId, dict_id);
302  const auto dict_it = dictDescriptorMapByRef_.find(dict_ref);
303  CHECK(dict_it != dictDescriptorMapByRef_.end());
304  const auto dict_name = dict_it->second->dictName;
305  // when migrating a table, any foreign dict ref will be dropped
306  // and the first cd of a dict will become root of the dict
307  if (dict_root_cds.end() == dict_root_cds.find(dict_name)) {
308  dict_root_cds[dict_name] = cd;
309  os << " ENCODING " << ti.get_compression_name() << "(" << (ti.get_size() * 8)
310  << ")";
311  } else {
312  const auto dict_root_cd = dict_root_cds[dict_name];
313  shared_dicts.push_back("SHARED DICTIONARY (" + cd->columnName +
314  ") REFERENCES @T(" + dict_root_cd->columnName + ")");
315  // "... shouldn't specify an encoding, it borrows from the referenced column"
316  }
317  } else {
318  os << " ENCODING NONE";
319  }
320  } else if (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size()) {
321  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
322  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
323  }
324  comma = ", ";
325  }
326  }
327  // gather SHARED DICTIONARYs
328  if (shared_dicts.size()) {
329  os << ", " << boost::algorithm::join(shared_dicts, ", ");
330  }
331  // gather WITH options ...
332  std::vector<std::string> with_options;
333  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
334  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
335  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
336  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
337  with_options.emplace_back(td->hasDeletedCol ? "VACUUM='DELAYED'"
338  : "VACUUM='IMMEDIATE'");
339  if (!td->partitions.empty()) {
340  with_options.push_back("PARTITIONS='" + td->partitions + "'");
341  }
342  if (td->nShards > 0) {
343  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
344  CHECK(shard_cd);
345  os << ", SHARD KEY(" << shard_cd->columnName << ")";
346  with_options.push_back("SHARD_COUNT=" + std::to_string(td->nShards));
347  }
348  if (td->sortedColumnId > 0) {
349  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
350  CHECK(sort_cd);
351  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
352  }
353  os << ") WITH (" + boost::algorithm::join(with_options, ", ") + ");";
354  return os.str();
355 }
std::string partitions
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:322
std::string join(T const &container, std::string const &delim)
std::string to_string(char const *&&v)
CHECK(cgen_state)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1581
Definition: sqltypes.h:44

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::dumpTable ( const TableDescriptor td,
const std::string &  path,
const std::string &  compression 
) const

Definition at line 201 of file DumpRestore.cpp.

References Catalog_Namespace::abs_path(), currentDB_, Catalog_Namespace::DBMetadata::dbId, Data_Namespace::DISK_LEVEL, dumpSchema(), g_cluster, getAllColumnMetadataForTable(), getColumnDictDirectory(), getDataMgr(), Data_Namespace::DataMgr::getGlobalFileMgr(), Lock_Namespace::TableLockMgrImpl< TableLockMgr >::getReadLockForTable(), getTableDataDirectories(), getTableDictDirectories(), getTableEpoch(), TableDescriptor::isView, join(), TableDescriptor::persistenceLevel, Catalog_Namespace::run(), Catalog_Namespace::simple_file_closer, Catalog_Namespace::table_epoch_filename, Catalog_Namespace::table_oldinfo_filename, Catalog_Namespace::table_schema_filename, TableDescriptor::tableId, TableDescriptor::tableName, and to_string().

203  {
204  if (g_cluster) {
205  throw std::runtime_error("DUMP/RESTORE is not supported yet on distributed setup.");
206  }
207  if (boost::filesystem::exists(archive_path)) {
208  throw std::runtime_error("Archive " + archive_path + " already exists.");
209  }
211  throw std::runtime_error("Dumping view or temporary table is not supported.");
212  }
213  // collect paths of files to archive
214  const auto global_file_mgr = getDataMgr().getGlobalFileMgr();
215  std::vector<std::string> file_paths;
216  auto file_writer = [&file_paths, global_file_mgr](const std::string& file_name,
217  const std::string& file_type,
218  const std::string& file_data) {
219  const auto file_path = abs_path(global_file_mgr) + "/" + file_name;
220  std::unique_ptr<FILE, decltype(simple_file_closer)> fp(
221  std::fopen(file_path.c_str(), "w"), simple_file_closer);
222  if (!fp) {
223  throw std::runtime_error("Failed to create " + file_type + " file '" + file_path +
224  "': " + std::strerror(errno));
225  }
226  if (std::fwrite(file_data.data(), 1, file_data.size(), fp.get()) < file_data.size()) {
227  throw std::runtime_error("Failed to write " + file_type + " file '" + file_path +
228  "': " + std::strerror(errno));
229  }
230  file_paths.push_back(file_name);
231  };
232  // grab table read lock for concurrent SELECT (but would block capped COPY or INSERT)
233  auto table_read_lock =
235 
236  // grab catalog read lock only. no need table read or checkpoint lock
237  // because want to allow concurrent inserts while this dump proceeds.
238  const auto table_name = td->tableName;
239  {
240  cat_read_lock read_lock(this);
241  // - gen schema file
242  const auto schema_str = dumpSchema(td);
243  file_writer(table_schema_filename, "table schema", schema_str);
244  // - gen column-old-info file
245  const auto cds = getAllColumnMetadataForTable(td->tableId, true, true, true);
246  std::vector<std::string> column_oldinfo;
247  std::transform(cds.begin(),
248  cds.end(),
249  std::back_inserter(column_oldinfo),
250  [&](const auto cd) -> std::string {
251  return cd->columnName + ":" + std::to_string(cd->columnId) + ":" +
253  });
254  const auto column_oldinfo_str = boost::algorithm::join(column_oldinfo, " ");
255  file_writer(table_oldinfo_filename, "table old info", column_oldinfo_str);
256  // - gen table epoch
257  const auto epoch = getTableEpoch(currentDB_.dbId, td->tableId);
258  file_writer(table_epoch_filename, "table epoch", std::to_string(epoch));
259  // - collect table data file paths ...
260  const auto data_file_dirs = getTableDataDirectories(td);
261  file_paths.insert(file_paths.end(), data_file_dirs.begin(), data_file_dirs.end());
262  // - collect table dict file paths ...
263  const auto dict_file_dirs = getTableDictDirectories(td);
264  file_paths.insert(file_paths.end(), dict_file_dirs.begin(), dict_file_dirs.end());
265  // tar takes time. release cat lock to yield the cat to concurrent CREATE statements.
266  }
267  // run tar to archive the files ... this may take a while !!
268  run("tar " + compression + " -cvf \"" + archive_path + "\" " +
269  boost::algorithm::join(file_paths, " "),
270  abs_path(global_file_mgr));
271 }
static ReadLock getReadLockForTable(const Catalog_Namespace::Catalog &cat, const std::string &table_name)
Definition: TableLockMgr.h:95
std::string run(const std::string &cmd, const std::string &chdir="")
Definition: DumpRestore.cpp:75
std::string tableName
bool g_cluster
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:177
std::string join(T const &container, std::string const &delim)
int32_t getTableEpoch(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:2125
std::vector< std::string > getTableDataDirectories(const TableDescriptor *td) const
static constexpr char const * table_schema_filename
Definition: DumpRestore.cpp:65
std::string to_string(char const *&&v)
std::string abs_path(const File_Namespace::GlobalFileMgr *global_file_mgr)
Definition: DumpRestore.cpp:71
std::string getColumnDictDirectory(const ColumnDescriptor *cd) const
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:90
static constexpr char const * table_oldinfo_filename
Definition: DumpRestore.cpp:66
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1581
static constexpr char const * table_epoch_filename
Definition: DumpRestore.cpp:67
std::vector< std::string > getTableDictDirectories(const TableDescriptor *td) const
Data_Namespace::MemoryLevel persistenceLevel
std::string dumpSchema(const TableDescriptor *td) const
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
Definition: DataMgr.cpp:437

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::eraseDBData ( )

Definition at line 2937 of file Catalog.cpp.

References basePath_, calciteMgr_, currentDB_, Catalog_Namespace::DBMetadata::dbName, eraseTablePhysicalData(), getAllTableMetadata(), and run_benchmark_import::tables.

2937  {
2938  cat_write_lock write_lock(this);
2939  // Physically erase all tables and dictionaries from disc and memory
2940  const auto tables = getAllTableMetadata();
2941  for (const auto table : tables) {
2942  eraseTablePhysicalData(table);
2943  }
2944  // Physically erase database metadata
2945  boost::filesystem::remove(basePath_ + "/mapd_catalogs/" + currentDB_.dbName);
2946  calciteMgr_->updateMetadata(currentDB_.dbName, "");
2947 }
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:2949
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:1598
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:331
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:91

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::eraseTablePhysicalData ( const TableDescriptor td)

Definition at line 2949 of file Catalog.cpp.

References calciteMgr_, Data_Namespace::CPU_LEVEL, currentDB_, dataMgr_, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, TableDescriptor::fragmenter, Data_Namespace::GPU_LEVEL, INJECT_TIMER, TableDescriptor::isView, removeTableFromMap(), tableDescriptorMapById_, TableDescriptor::tableId, and TableDescriptor::tableName.

Referenced by doDropTable(), and eraseDBData().

2949  {
2950  cat_write_lock write_lock(this);
2951  const int tableId = td->tableId;
2952  // must destroy fragmenter before deleteChunks is called.
2953  if (td->fragmenter != nullptr) {
2954  auto tableDescIt = tableDescriptorMapById_.find(tableId);
2955  {
2956  INJECT_TIMER(deleting_fragmenter);
2957  delete td->fragmenter;
2958  }
2959  tableDescIt->second->fragmenter = nullptr; // get around const-ness
2960  }
2961  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
2962  {
2963  INJECT_TIMER(deleteChunksWithPrefix);
2964  // assuming deleteChunksWithPrefix is atomic
2965  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
2966  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
2967  }
2968  if (!td->isView) {
2969  INJECT_TIMER(Remove_Table);
2970  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
2971  }
2972  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
2973  {
2974  INJECT_TIMER(removeTableFromMap_);
2975  removeTableFromMap(td->tableName, tableId);
2976  }
2977 }
std::vector< int > ChunkKey
Definition: types.h:35
std::string tableName
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:328
#define INJECT_TIMER(DESC)
Definition: measure.h:91
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:319
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:331
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1101
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:91
Fragmenter_Namespace::AbstractFragmenter * fragmenter

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::expandGeoColumn ( const ColumnDescriptor cd,
std::list< ColumnDescriptor > &  columns 
)
static

Definition at line 1809 of file Catalog.cpp.

References CHECK(), ColumnDescriptor::columnName, ColumnDescriptor::columnType, IS_GEO, kARRAY, kDOUBLE, kENCODING_GEOINT, kENCODING_NONE, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kTINYINT, SQLTypeInfoCore< TYPE_FACET_PACK >::set_size(), and SQLTypeInfoCore< TYPE_FACET_PACK >::set_subtype().

Referenced by createTable().

1810  {
1811  const auto& col_ti = cd.columnType;
1812  if (IS_GEO(col_ti.get_type())) {
1813  switch (col_ti.get_type()) {
1814  case kPOINT: {
1815  ColumnDescriptor physical_cd_coords(true);
1816  physical_cd_coords.columnName = cd.columnName + "_coords";
1817  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, true);
1818  // Raw data: compressed/uncompressed coords
1819  coords_ti.set_subtype(kTINYINT);
1820  size_t unit_size;
1821  if (col_ti.get_compression() == kENCODING_GEOINT &&
1822  col_ti.get_comp_param() == 32) {
1823  unit_size = 4 * sizeof(int8_t);
1824  } else {
1825  CHECK(col_ti.get_compression() == kENCODING_NONE);
1826  unit_size = 8 * sizeof(int8_t);
1827  }
1828  coords_ti.set_size(2 * unit_size);
1829  physical_cd_coords.columnType = coords_ti;
1830  columns.push_back(physical_cd_coords);
1831 
1832  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1833 
1834  break;
1835  }
1836  case kLINESTRING: {
1837  ColumnDescriptor physical_cd_coords(true);
1838  physical_cd_coords.columnName = cd.columnName + "_coords";
1839  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, true);
1840  // Raw data: compressed/uncompressed coords
1841  coords_ti.set_subtype(kTINYINT);
1842  physical_cd_coords.columnType = coords_ti;
1843  columns.push_back(physical_cd_coords);
1844 
1845  ColumnDescriptor physical_cd_bounds(true);
1846  physical_cd_bounds.columnName = cd.columnName + "_bounds";
1847  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, true);
1848  bounds_ti.set_subtype(kDOUBLE);
1849  bounds_ti.set_size(4 * sizeof(double));
1850  physical_cd_bounds.columnType = bounds_ti;
1851  columns.push_back(physical_cd_bounds);
1852 
1853  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1854 
1855  break;
1856  }
1857  case kPOLYGON: {
1858  ColumnDescriptor physical_cd_coords(true);
1859  physical_cd_coords.columnName = cd.columnName + "_coords";
1860  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, true);
1861  // Raw data: compressed/uncompressed coords
1862  coords_ti.set_subtype(kTINYINT);
1863  physical_cd_coords.columnType = coords_ti;
1864  columns.push_back(physical_cd_coords);
1865 
1866  ColumnDescriptor physical_cd_ring_sizes(true);
1867  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
1868  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, true);
1869  ring_sizes_ti.set_subtype(kINT);
1870  physical_cd_ring_sizes.columnType = ring_sizes_ti;
1871  columns.push_back(physical_cd_ring_sizes);
1872 
1873  ColumnDescriptor physical_cd_bounds(true);
1874  physical_cd_bounds.columnName = cd.columnName + "_bounds";
1875  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, true);
1876  bounds_ti.set_subtype(kDOUBLE);
1877  bounds_ti.set_size(4 * sizeof(double));
1878  physical_cd_bounds.columnType = bounds_ti;
1879  columns.push_back(physical_cd_bounds);
1880 
1881  ColumnDescriptor physical_cd_render_group(true);
1882  physical_cd_render_group.columnName = cd.columnName + "_render_group";
1883  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, true);
1884  physical_cd_render_group.columnType = render_group_ti;
1885  columns.push_back(physical_cd_render_group);
1886 
1887  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1888 
1889  break;
1890  }
1891  case kMULTIPOLYGON: {
1892  ColumnDescriptor physical_cd_coords(true);
1893  physical_cd_coords.columnName = cd.columnName + "_coords";
1894  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, true);
1895  // Raw data: compressed/uncompressed coords
1896  coords_ti.set_subtype(kTINYINT);
1897  physical_cd_coords.columnType = coords_ti;
1898  columns.push_back(physical_cd_coords);
1899 
1900  ColumnDescriptor physical_cd_ring_sizes(true);
1901  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
1902  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, true);
1903  ring_sizes_ti.set_subtype(kINT);
1904  physical_cd_ring_sizes.columnType = ring_sizes_ti;
1905  columns.push_back(physical_cd_ring_sizes);
1906 
1907  ColumnDescriptor physical_cd_poly_rings(true);
1908  physical_cd_poly_rings.columnName = cd.columnName + "_poly_rings";
1909  SQLTypeInfo poly_rings_ti = SQLTypeInfo(kARRAY, true);
1910  poly_rings_ti.set_subtype(kINT);
1911  physical_cd_poly_rings.columnType = poly_rings_ti;
1912  columns.push_back(physical_cd_poly_rings);
1913 
1914  ColumnDescriptor physical_cd_bounds(true);
1915  physical_cd_bounds.columnName = cd.columnName + "_bounds";
1916  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, true);
1917  bounds_ti.set_subtype(kDOUBLE);
1918  bounds_ti.set_size(4 * sizeof(double));
1919  physical_cd_bounds.columnType = bounds_ti;
1920  columns.push_back(physical_cd_bounds);
1921 
1922  ColumnDescriptor physical_cd_render_group(true);
1923  physical_cd_render_group.columnName = cd.columnName + "_render_group";
1924  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, true);
1925  physical_cd_render_group.columnType = render_group_ti;
1926  columns.push_back(physical_cd_render_group);
1927 
1928  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1929 
1930  break;
1931  }
1932  default:
1933  throw runtime_error("Unrecognized geometry type.");
1934  break;
1935  }
1936  }
1937 }
void set_size(int s)
Definition: sqltypes.h:424
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:417
CHECK(cgen_state)
specifies the content in-memory of a row in the column metadata table
SQLTypeInfoCore< ArrayContextTypeSizer, ExecutorTypePackaging, DateTimeFacilities > SQLTypeInfo
Definition: sqltypes.h:852
Definition: sqltypes.h:48
SQLTypeInfo columnType
std::string columnName
#define IS_GEO(T)
Definition: sqltypes.h:167

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string Catalog_Namespace::Catalog::generatePhysicalTableName ( const std::string &  logicalTableName,
const int32_t &  shardNumber 
)
protected

Definition at line 2979 of file Catalog.cpp.

References physicalTableNameTag_, and to_string().

Referenced by createShardedTable(), and renameTable().

2980  {
2981  std::string physicalTableName =
2982  logicalTableName + physicalTableNameTag_ + std::to_string(shardNumber);
2983  return (physicalTableName);
2984 }
std::string to_string(char const *&&v)
static const std::string physicalTableNameTag_
Definition: Catalog.h:335

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::shared_ptr< Catalog > Catalog_Namespace::Catalog::get ( const std::string &  dbName)
static

Definition at line 2990 of file Catalog.cpp.

References mapd_cat_map_.

Referenced by Catalog_Namespace::SysCatalog::createDatabase(), Catalog_Namespace::SysCatalog::dropDatabase(), RelAlgExecutor::executeRelAlgQueryNoRetry(), Catalog_Namespace::SysCatalog::login(), main(), and Catalog_Namespace::SysCatalog::switchDatabase().

2990  {
2991  auto cat_it = mapd_cat_map_.find(dbName);
2992  if (cat_it != mapd_cat_map_.end()) {
2993  return cat_it->second;
2994  }
2995  return nullptr;
2996 }
static std::map< std::string, std::shared_ptr< Catalog > > mapd_cat_map_
Definition: Catalog.h:348

+ Here is the caller graph for this function:

static std::shared_ptr<Catalog> Catalog_Namespace::Catalog::get ( const std::string &  basePath,
const DBMetadata curDB,
std::shared_ptr< Data_Namespace::DataMgr dataMgr,
const std::vector< LeafHostInfo > &  string_dict_hosts,
std::shared_ptr< Calcite calcite,
bool  is_new_db 
)
static
list< const ColumnDescriptor * > Catalog_Namespace::Catalog::getAllColumnMetadataForTable ( const int  tableId,
const bool  fetchSystemColumns,
const bool  fetchVirtualColumns,
const bool  fetchPhysicalColumns 
) const

Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular table specified by table id.

Parameters
tableIdtable id we want the column metadata for
Returns
list of pointers to const ColumnDescriptor structs - one for each and every column in the table

Definition at line 1581 of file Catalog.cpp.

References getMetadataForTableImpl().

Referenced by Analyzer::RangeTableEntry::add_all_column_descs(), Parser::InsertStmt::analyze(), Parser::InsertValuesStmt::analyze(), Fragmenter_Namespace::copyDataOfShard(), Parser::InsertValuesStmt::determineLeafIndex(), dumpSchema(), dumpTable(), Analyzer::RangeTableEntry::expand_star_in_targetlist(), getShardColumnMetadataForTable(), getTableDictDirectories(), instantiateFragmenter(), TableOptimizer::recomputeMetadata(), restoreTable(), and validate_shared_dictionary().

1585  {
1586  cat_read_lock read_lock(this);
1587  list<const ColumnDescriptor*> columnDescriptors;
1588  const TableDescriptor* td =
1589  getMetadataForTableImpl(tableId, false); // dont instantiate fragmenter
1591  columnDescriptors,
1592  fetchSystemColumns,
1593  fetchVirtualColumns,
1594  fetchPhysicalColumns);
1595  return columnDescriptors;
1596 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:90
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1581
const TableDescriptor * getMetadataForTableImpl(int tableId, const bool populateFragmenter) const
Definition: Catalog.cpp:1330
specifies the content in-memory of a row in the table metadata table

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::getAllColumnMetadataForTable ( const TableDescriptor td,
std::list< const ColumnDescriptor * > &  colDescs,
const bool  fetchSystemColumns,
const bool  fetchVirtualColumns,
const bool  fetchPhysicalColumns 
) const
protected
list< const DashboardDescriptor * > Catalog_Namespace::Catalog::getAllDashboardsMetadata ( ) const

Definition at line 1607 of file Catalog.cpp.

References dashboardDescriptorMap_.

1607  {
1608  list<const DashboardDescriptor*> view_list;
1609  for (auto p : dashboardDescriptorMap_) {
1610  view_list.push_back(p.second.get());
1611  }
1612  return view_list;
1613 }
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:323
list< const TableDescriptor * > Catalog_Namespace::Catalog::getAllTableMetadata ( ) const

Definition at line 1598 of file Catalog.cpp.

References tableDescriptorMapById_.

Referenced by eraseDBData().

1598  {
1599  cat_read_lock read_lock(this);
1600  list<const TableDescriptor*> table_list;
1601  for (auto p : tableDescriptorMapById_) {
1602  table_list.push_back(p.second);
1603  }
1604  return table_list;
1605 }
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:319
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:90

+ Here is the caller graph for this function:

const std::string& Catalog_Namespace::Catalog::getBasePath ( ) const
inline

Definition at line 179 of file Catalog.h.

References basePath_.

179 { return basePath_; }
std::shared_ptr<Calcite> Catalog_Namespace::Catalog::getCalciteMgr ( ) const
inline

Definition at line 178 of file Catalog.h.

References calciteMgr_.

Referenced by QueryRunner::QueryRunner::parsePlanCalcite().

178 { return calciteMgr_; }
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:331

+ Here is the caller graph for this function:

std::string Catalog_Namespace::Catalog::getColumnDictDirectory ( const ColumnDescriptor cd) const

Definition at line 171 of file DumpRestore.cpp.

References CHECK(), ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, SQLTypeInfoCore< TYPE_FACET_PACK >::get_comp_param(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_compression(), SQLTypeInfoCore< TYPE_FACET_PACK >::is_string(), SQLTypeInfoCore< TYPE_FACET_PACK >::is_string_array(), and kENCODING_DICT.

Referenced by dumpTable(), getTableDictDirectories(), and restoreTable().

171  {
172  if ((cd->columnType.is_string() || cd->columnType.is_string_array()) &&
174  cd->columnType.get_comp_param() > 0) {
175  const auto dictId = cd->columnType.get_comp_param();
176  const DictRef dictRef(currentDB_.dbId, dictId);
177  const auto dit = dictDescriptorMapByRef_.find(dictRef);
178  CHECK(dit != dictDescriptorMapByRef_.end());
179  CHECK(dit->second);
180  boost::filesystem::path file_path(dit->second->dictFolderPath);
181  return file_path.filename().string();
182  }
183  return std::string();
184 }
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:334
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:335
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:322
CHECK(cgen_state)
bool is_string() const
Definition: sqltypes.h:477
SQLTypeInfo columnType
bool is_string_array() const
Definition: sqltypes.h:478

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const int Catalog_Namespace::Catalog::getColumnIdBySpi ( const int  tableId,
const size_t  spi 
) const

Definition at line 1432 of file Catalog.cpp.

References getColumnIdBySpiUnlocked().

Referenced by anonymous_namespace{RelAlgExecutor.cpp}::collect_used_input_desc(), and anonymous_namespace{RelAlgExecutor.cpp}::get_physical_inputs().

1432  {
1433  cat_read_lock read_lock(this);
1434  return getColumnIdBySpiUnlocked(table_id, spi);
1435 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:90
const int getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const
Definition: Catalog.cpp:1415

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const int Catalog_Namespace::Catalog::getColumnIdBySpiUnlocked ( const int  table_id,
const size_t  spi 
) const
protected

Definition at line 1415 of file Catalog.cpp.

References CHECK(), SPIMAP_MAGIC1, SPIMAP_MAGIC2, and tableDescriptorMapById_.

Referenced by getColumnIdBySpi(), and getMetadataForColumnBySpi().

1415  {
1416  const auto tabDescIt = tableDescriptorMapById_.find(table_id);
1417  CHECK(tableDescriptorMapById_.end() != tabDescIt);
1418  const auto& columnIdBySpi = tabDescIt->second->columnIdBySpi_;
1419 
1420  auto spx = spi;
1421  int phi = 0;
1422  if (spx >= SPIMAP_MAGIC1) // see Catalog.h
1423  {
1424  phi = (spx - SPIMAP_MAGIC1) % SPIMAP_MAGIC2;
1425  spx = (spx - SPIMAP_MAGIC1) / SPIMAP_MAGIC2;
1426  }
1427 
1428  CHECK(0 < spx && spx <= columnIdBySpi.size());
1429  return columnIdBySpi[spx - 1] + phi;
1430 }
#define SPIMAP_MAGIC1
Definition: Catalog.h:68
#define SPIMAP_MAGIC2
Definition: Catalog.h:69
CHECK(cgen_state)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:319

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

int Catalog_Namespace::Catalog::getDatabaseId ( ) const
inline

Definition at line 192 of file Catalog.h.

References currentDB_, and Catalog_Namespace::DBMetadata::dbId.

Referenced by RelAlgTranslator::getInIntegerSetExpr().

192 { return currentDB_.dbId; }

+ Here is the caller graph for this function:

const ColumnDescriptor * Catalog_Namespace::Catalog::getDeletedColumn ( const TableDescriptor td) const

Definition at line 2179 of file Catalog.cpp.

References deletedColumnPerTable_.

Referenced by TableOptimizer::recomputeMetadata(), and vacuumDeletedRows().

2179  {
2180  cat_read_lock read_lock(this);
2181  const auto it = deletedColumnPerTable_.find(td);
2182  return it != deletedColumnPerTable_.end() ? it->second : nullptr;
2183 }
DeletedColumnPerTableMap deletedColumnPerTable_
Definition: Catalog.h:349
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:90

+ Here is the caller graph for this function:

const ColumnDescriptor * Catalog_Namespace::Catalog::getDeletedColumnIfRowsDeleted ( const TableDescriptor td) const

Definition at line 2204 of file Catalog.cpp.

References CHECK(), checkMetadataForDeletedRecs(), ColumnDescriptor::columnId, currentDB_, Catalog_Namespace::DBMetadata::dbId, deletedColumnPerTable_, getMetadataForTable(), logicalToPhysicalTableMapById_, and TableDescriptor::tableId.

2205  {
2206  cat_read_lock read_lock(this);
2207 
2208  const auto it = deletedColumnPerTable_.find(td);
2209  // if not a table that supports delete return nullptr, nothing more to do
2210  if (it == deletedColumnPerTable_.end()) {
2211  return nullptr;
2212  }
2213  const ColumnDescriptor* cd = it->second;
2214 
2215  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
2216 
2217  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2218  // check all shards
2219  const auto physicalTables = physicalTableIt->second;
2220  CHECK(!physicalTables.empty());
2221  for (size_t i = 0; i < physicalTables.size(); i++) {
2222  int32_t physical_tb_id = physicalTables[i];
2223  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2224  CHECK(phys_td);
2226  return cd;
2227  }
2228  }
2229  } else {
2231  return cd;
2232  }
2233  }
2234  // no deletes so far recorded in metadata
2235  return nullptr;
2236 }
DeletedColumnPerTableMap deletedColumnPerTable_
Definition: Catalog.h:349
CHECK(cgen_state)
const bool checkMetadataForDeletedRecs(int dbId, int tableId, int columnId) const
Definition: Catalog.cpp:2185
specifies the content in-memory of a row in the column metadata table
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:90
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
specifies the content in-memory of a row in the table metadata table
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:333

+ Here is the call graph for this function:

void Catalog_Namespace::Catalog::getDictionary ( const ColumnDescriptor cd,
std::map< int, StringDictionary * > &  stringDicts 
)

Definition at line 1674 of file Catalog.cpp.

References CHECK(), columnDescriptorMap_, ColumnDescriptor::columnName, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, getMetadataForDict(), kENCODING_DICT, ColumnDescriptor::tableId, and to_upper().

1675  {
1676  // learn 'committed' ColumnDescriptor of this column
1677  auto cit = columnDescriptorMap_.find(ColumnKey(cd.tableId, to_upper(cd.columnName)));
1678  CHECK(cit != columnDescriptorMap_.end());
1679  auto& ccd = *cit->second;
1680 
1681  if (!(ccd.columnType.is_string() || ccd.columnType.is_string_array())) {
1682  return;
1683  }
1684  if (!(ccd.columnType.get_compression() == kENCODING_DICT)) {
1685  return;
1686  }
1687  if (!(ccd.columnType.get_comp_param() > 0)) {
1688  return;
1689  }
1690 
1691  auto dictId = ccd.columnType.get_comp_param();
1692  getMetadataForDict(dictId);
1693 
1694  const DictRef dictRef(currentDB_.dbId, dictId);
1695  auto dit = dictDescriptorMapByRef_.find(dictRef);
1696  CHECK(dit != dictDescriptorMapByRef_.end());
1697  CHECK(dit->second);
1698  CHECK(dit->second.get()->stringDict);
1699  stringDicts[ccd.columnId] = dit->second.get()->stringDict.get();
1700 }
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:320
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:322
CHECK(cgen_state)
std::tuple< int, std::string > ColumnKey
Definition: Catalog.h:246
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1350
std::string to_upper(const std::string &str)
std::string columnName

+ Here is the call graph for this function:

int Catalog_Namespace::Catalog::getLogicalTableId ( const int  physicalTableId) const

Definition at line 2916 of file Catalog.cpp.

References logicalToPhysicalTableMapById_.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumns(), and vacuumDeletedRows().

2916  {
2917  for (const auto& l : logicalToPhysicalTableMapById_) {
2918  if (l.second.end() != std::find_if(l.second.begin(),
2919  l.second.end(),
2920  [&](decltype(*l.second.begin()) tid) -> bool {
2921  return physicalTableId == tid;
2922  })) {
2923  return l.first;
2924  }
2925  }
2926  return physicalTableId;
2927 }
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:333

+ Here is the caller graph for this function:

const ColumnDescriptor * Catalog_Namespace::Catalog::getMetadataForColumn ( int  tableId,
int  columnId 
) const

Definition at line 1403 of file Catalog.cpp.

References columnDescriptorMapById_.

1403  {
1404  cat_read_lock read_lock(this);
1405 
1406  ColumnIdKey columnIdKey(tableId, columnId);
1407  auto colDescIt = columnDescriptorMapById_.find(columnIdKey);
1408  if (colDescIt == columnDescriptorMapById_
1409  .end()) { // need to check to make sure column exists for table
1410  return nullptr;
1411  }
1412  return colDescIt->second;
1413 }
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:321
std::tuple< int, int > ColumnIdKey
Definition: Catalog.h:248
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:90
const ColumnDescriptor * Catalog_Namespace::Catalog::getMetadataForColumnBySpi ( const int  tableId,
const size_t  spi 
) const

Definition at line 1437 of file Catalog.cpp.

References columnDescriptorMapById_, and getColumnIdBySpiUnlocked().

Referenced by RelAlgTranslator::translateGeoColumn(), and RelAlgTranslator::translateInput().

1438  {
1439  cat_read_lock read_lock(this);
1440 
1441  const auto columnId = getColumnIdBySpiUnlocked(tableId, spi);
1442  ColumnIdKey columnIdKey(tableId, columnId);
1443  const auto colDescIt = columnDescriptorMapById_.find(columnIdKey);
1444  return columnDescriptorMapById_.end() == colDescIt ? nullptr : colDescIt->second;
1445 }
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:321
std::tuple< int, int > ColumnIdKey
Definition: Catalog.h:248
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:90
const int getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const
Definition: Catalog.cpp:1415

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const DashboardDescriptor* Catalog_Namespace::Catalog::getMetadataForDashboard ( const std::string &  userId,
const std::string &  dashName 
) const

Referenced by getMetadataForDashboard(), and DBObject::loadKey().

+ Here is the caller graph for this function:

const DashboardDescriptor * Catalog_Namespace::Catalog::getMetadataForDashboard ( const int32_t  dashboard_id) const

Definition at line 1486 of file Catalog.cpp.

References dashboardDescriptorMap_, getMetadataForDashboard(), name(), and to_string().

1486  {
1487  cat_read_lock read_lock(this);
1488  std::string userId;
1489  std::string name;
1490  bool found{false};
1491  {
1492  for (auto descp : dashboardDescriptorMap_) {
1493  auto dash = descp.second.get();
1494  if (dash->dashboardId == id) {
1495  userId = std::to_string(dash->userId);
1496  name = dash->dashboardName;
1497  found = true;
1498  break;
1499  }
1500  }
1501  }
1502  if (found) {
1503  return getMetadataForDashboard(userId, name);
1504  }
1505  return nullptr;
1506 }
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:323
std::string to_string(char const *&&v)
std::string name() const
Definition: Catalog.h:219
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:90

+ Here is the call graph for this function:

const DictDescriptor * Catalog_Namespace::Catalog::getMetadataForDict ( int  dict_ref,
bool  loadDict = true 
) const

Definition at line 1350 of file Catalog.cpp.

References currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, measure< TimeT >::execution(), g_cache_string_hash, logger::INFO, LOG, and string_dict_hosts_.

Referenced by Parser::InsertValuesStmt::determineLeafIndex(), DictionaryValueConverter< TARGET_TYPE >::DictionaryValueConverter(), doTruncateTable(), MapDHandler::fixup_row_descriptor(), getDictionary(), Importer_NS::Loader::init(), anonymous_namespace{Execute.cpp}::insert_one_dict_str(), MapDHandler::populateThriftColumnType(), StringValueConverter::StringValueConverter(), and Fragmenter_Namespace::InsertOrderFragmenter::updateColumn().

1351  {
1352  const DictRef dictRef(currentDB_.dbId, dictId);
1353  cat_read_lock read_lock(this);
1354  auto dictDescIt = dictDescriptorMapByRef_.find(dictRef);
1355  if (dictDescIt ==
1356  dictDescriptorMapByRef_.end()) { // check to make sure dictionary exists
1357  return nullptr;
1358  }
1359  auto& dd = dictDescIt->second;
1360 
1361  if (loadDict) {
1362  cat_sqlite_lock sqlite_lock(this);
1363  if (!dd->stringDict) {
1364  auto time_ms = measure<>::execution([&]() {
1365  if (string_dict_hosts_.empty()) {
1366  if (dd->dictIsTemp) {
1367  dd->stringDict = std::make_shared<StringDictionary>(
1368  dd->dictFolderPath, true, true, g_cache_string_hash);
1369  } else {
1370  dd->stringDict = std::make_shared<StringDictionary>(
1371  dd->dictFolderPath, false, true, g_cache_string_hash);
1372  }
1373  } else {
1374  dd->stringDict =
1375  std::make_shared<StringDictionary>(string_dict_hosts_.front(), dd->dictRef);
1376  }
1377  });
1378  LOG(INFO) << "Time to load Dictionary " << dd->dictRef.dbId << "_"
1379  << dd->dictRef.dictId << " was " << time_ms << "ms";
1380  }
1381  }
1382 
1383  return dd.get();
1384 }
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:185
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:330
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:322
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:90
bool g_cache_string_hash
Definition: Execute.cpp:88
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:92

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const LinkDescriptor* Catalog_Namespace::Catalog::getMetadataForLink ( const std::string &  link) const
const LinkDescriptor * Catalog_Namespace::Catalog::getMetadataForLink ( int  linkId) const

Definition at line 1541 of file Catalog.cpp.

References linkDescriptorMapById_.

1541  {
1542  cat_read_lock read_lock(this);
1543  auto linkDescIt = linkDescriptorMapById_.find(linkId);
1544  if (linkDescIt == linkDescriptorMapById_.end()) { // check to make sure view exists
1545  return nullptr;
1546  }
1547  return linkDescIt->second;
1548 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:90
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:325
const TableDescriptor* Catalog_Namespace::Catalog::getMetadataForTable ( const std::string &  tableName,
const bool  populateFragmenter = true 
) const

Returns a pointer to a const TableDescriptor struct matching the provided tableName.

Parameters
tableNametable specified column belongs to
Returns
pointer to const TableDescriptor object queried for or nullptr if it does not exist.

Referenced by Parser::InsertStmt::analyze(), Parser::QuerySpec::analyze_from_clause(), UpdelRoll::cancelUpdate(), anonymous_namespace{MapDHandler.cpp}::check_table_not_sharded(), BaselineJoinHashTable::checkHashJoinReplicationConstraint(), checkPermissionForTables(), checkpoint(), Fragmenter_Namespace::computeRowIndicesOfShards(), Fragmenter_Namespace::copyDataOfShard(), Parser::InsertValuesStmt::determineLeafIndex(), dropTable(), Executor::executeSimpleInsert(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_foreign_col(), anonymous_namespace{Execute.cpp}::get_table_name(), getDeletedColumnIfRowsDeleted(), getPhysicalTablesDescriptors(), Lock_Namespace::getTableChunkKey(), getTableEpoch(), Fragmenter_Namespace::InsertDataLoader::insertData(), Parser::InsertIntoTableAsSelectStmt::LocalConnector::insertDataToLeaf(), DBObject::loadKey(), parseDashboardObjects(), removeChunks(), renameTable(), restoreTable(), Fragmenter_Namespace::set_chunk_metadata(), setTableEpoch(), GroupByAndAggregate::shard_count_for_top_groups(), Fragmenter_Namespace::SortedOrderFragmenter::sortData(), truncateTable(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumnMetadata(), vacuumDeletedRows(), and validate_shared_dictionary().

+ Here is the caller graph for this function:

const TableDescriptor * Catalog_Namespace::Catalog::getMetadataForTable ( int  tableId) const

Definition at line 1346 of file Catalog.cpp.

References getMetadataForTableImpl().

1346  {
1347  return getMetadataForTableImpl(tableId, true);
1348 }
const TableDescriptor * getMetadataForTableImpl(int tableId, const bool populateFragmenter) const
Definition: Catalog.cpp:1330

+ Here is the call graph for this function:

const TableDescriptor * Catalog_Namespace::Catalog::getMetadataForTableImpl ( int  tableId,
const bool  populateFragmenter 
) const

Definition at line 1330 of file Catalog.cpp.

References TableDescriptor::fragmenter, instantiateFragmenter(), TableDescriptor::isView, TableDescriptor::mutex_, and tableDescriptorMapById_.

Referenced by getAllColumnMetadataForTable(), and getMetadataForTable().

1332  {
1333  cat_read_lock read_lock(this);
1334  auto tableDescIt = tableDescriptorMapById_.find(tableId);
1335  if (tableDescIt == tableDescriptorMapById_.end()) { // check to make sure table exists
1336  return nullptr;
1337  }
1338  TableDescriptor* td = tableDescIt->second;
1339  std::unique_lock<std::mutex> td_lock(*td->mutex_.get());
1340  if (populateFragmenter && td->fragmenter == nullptr && !td->isView) {
1342  }
1343  return td; // returns pointer to table descriptor
1344 }
void instantiateFragmenter(TableDescriptor *td) const
Definition: Catalog.cpp:1273
std::shared_ptr< std::mutex > mutex_
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:319
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:90
specifies the content in-memory of a row in the table metadata table
Fragmenter_Namespace::AbstractFragmenter * fragmenter

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::vector< const TableDescriptor * > Catalog_Namespace::Catalog::getPhysicalTablesDescriptors ( const TableDescriptor logicalTableDesc) const

Definition at line 2897 of file Catalog.cpp.

References CHECK(), getMetadataForTable(), logicalToPhysicalTableMapById_, and TableDescriptor::tableId.

Referenced by checkpoint(), Fragmenter_Namespace::copyDataOfShard(), Importer_NS::Loader::distributeToShards(), getTableDataDirectories(), Importer_NS::Loader::loadImpl(), TableOptimizer::recomputeMetadata(), and vacuumDeletedRows().

2898  {
2899  cat_read_lock read_lock(this);
2900  const auto physicalTableIt =
2901  logicalToPhysicalTableMapById_.find(logicalTableDesc->tableId);
2902  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
2903  return {logicalTableDesc};
2904  }
2905 
2906  const auto physicalTablesIds = physicalTableIt->second;
2907  CHECK(!physicalTablesIds.empty());
2908  std::vector<const TableDescriptor*> physicalTables;
2909  for (size_t i = 0; i < physicalTablesIds.size(); i++) {
2910  physicalTables.push_back(getMetadataForTable(physicalTablesIds[i]));
2911  }
2912 
2913  return physicalTables;
2914 }
CHECK(cgen_state)
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:90
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:333

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

const ColumnDescriptor * Catalog_Namespace::Catalog::getShardColumnMetadataForTable ( const TableDescriptor td) const

Definition at line 2881 of file Catalog.cpp.

References getAllColumnMetadataForTable(), TableDescriptor::shardedColumnId, and TableDescriptor::tableId.

Referenced by Fragmenter_Namespace::computeRowIndicesOfShards(), and Parser::InsertValuesStmt::determineLeafIndex().

2882  {
2883  const auto column_descriptors =
2884  getAllColumnMetadataForTable(td->tableId, false, true, true);
2885 
2886  const ColumnDescriptor* shard_cd{nullptr};
2887  int i = 1;
2888  for (auto cd_itr = column_descriptors.begin(); cd_itr != column_descriptors.end();
2889  ++cd_itr, ++i) {
2890  if (i == td->shardedColumnId) {
2891  shard_cd = *cd_itr;
2892  }
2893  }
2894  return shard_cd;
2895 }
specifies the content in-memory of a row in the column metadata table
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1581

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

SqliteConnector& Catalog_Namespace::Catalog::getSqliteConnector ( )
inline

Definition at line 194 of file Catalog.h.

References sqliteConnector_.

194 { return sqliteConnector_; }
SqliteConnector sqliteConnector_
Definition: Catalog.h:326
const std::vector< LeafHostInfo > & Catalog_Namespace::Catalog::getStringDictionaryHosts ( ) const

Definition at line 1386 of file Catalog.cpp.

References string_dict_hosts_.

Referenced by RelAlgTranslator::getInIntegerSetExpr().

1386  {
1387  return string_dict_hosts_;
1388 }
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:330

+ Here is the caller graph for this function:

std::vector< std::string > Catalog_Namespace::Catalog::getTableDataDirectories ( const TableDescriptor td) const

Definition at line 158 of file DumpRestore.cpp.

References currentDB_, Catalog_Namespace::DBMetadata::dbId, getDataMgr(), File_Namespace::GlobalFileMgr::getFileMgr(), Data_Namespace::DataMgr::getGlobalFileMgr(), and getPhysicalTablesDescriptors().

Referenced by dumpTable(), and restoreTable().

159  {
160  const auto global_file_mgr = getDataMgr().getGlobalFileMgr();
161  std::vector<std::string> file_paths;
162  for (auto shard : getPhysicalTablesDescriptors(td)) {
163  const auto file_mgr = global_file_mgr->getFileMgr(currentDB_.dbId, shard->tableId);
164  boost::filesystem::path file_path(file_mgr->getFileMgrBasePath());
165  file_paths.push_back(file_path.filename().string());
166  }
167  return file_paths;
168 }
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:177
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logicalTableDesc) const
Definition: Catalog.cpp:2897
FileMgr * getFileMgr(const int db_id, const int tb_id)
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
Definition: DataMgr.cpp:437

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::vector< std::string > Catalog_Namespace::Catalog::getTableDictDirectories ( const TableDescriptor td) const

Definition at line 187 of file DumpRestore.cpp.

References getAllColumnMetadataForTable(), getColumnDictDirectory(), and TableDescriptor::tableId.

Referenced by dumpTable(), and restoreTable().

188  {
189  std::vector<std::string> file_paths;
190  for (auto cd : getAllColumnMetadataForTable(td->tableId, false, false, true)) {
191  auto file_base = getColumnDictDirectory(cd);
192  if (!file_base.empty() &&
193  file_paths.end() == std::find(file_paths.begin(), file_paths.end(), file_base)) {
194  file_paths.push_back(file_base);
195  }
196  }
197  return file_paths;
198 }
std::string getColumnDictDirectory(const ColumnDescriptor *cd) const
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1581

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

int32_t Catalog_Namespace::Catalog::getTableEpoch ( const int32_t  db_id,
const int32_t  table_id 
) const

Definition at line 2125 of file Catalog.cpp.

References CHECK(), dataMgr_, logger::ERROR, getMetadataForTable(), LOG, and logicalToPhysicalTableMapById_.

Referenced by dumpTable(), and Importer_NS::Loader::getTableEpoch().

2125  {
2126  cat_read_lock read_lock(this);
2127  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(table_id);
2128  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2129  // check all shards have same checkpoint
2130  const auto physicalTables = physicalTableIt->second;
2131  CHECK(!physicalTables.empty());
2132  size_t curr_epoch = 0;
2133  for (size_t i = 0; i < physicalTables.size(); i++) {
2134  int32_t physical_tb_id = physicalTables[i];
2135  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2136  CHECK(phys_td);
2137  if (i == 0) {
2138  curr_epoch = dataMgr_->getTableEpoch(db_id, physical_tb_id);
2139  } else {
2140  if (curr_epoch != dataMgr_->getTableEpoch(db_id, physical_tb_id)) {
2141  // oh dear the leaves do not agree on the epoch for this table
2142  LOG(ERROR) << "Epochs on shards do not all agree on table id " << table_id
2143  << " db id " << db_id << " epoch " << curr_epoch << " leaf_epoch "
2144  << dataMgr_->getTableEpoch(db_id, physical_tb_id);
2145  return -1;
2146  }
2147  }
2148  }
2149  return curr_epoch;
2150  } else {
2151  return dataMgr_->getTableEpoch(db_id, table_id);
2152  }
2153 }
#define LOG(tag)
Definition: Logger.h:185
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:328
CHECK(cgen_state)
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:90
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
specifies the content in-memory of a row in the table metadata table
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:333

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void Catalog_Namespace::Catalog::instantiateFragmenter ( TableDescriptor td) const
protected

Definition at line 1273 of file Catalog.cpp.

References CHECK_EQ, currentDB_, dataMgr_, Catalog_Namespace::DBMetadata::dbId, measure< TimeT >::execution(), TableDescriptor::fragmenter, TableDescriptor::fragPageSize, TableDescriptor::fragType, getAllColumnMetadataForTable(), logger::INFO, Fragmenter_Namespace::INSERT_ORDER, LOG, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRows, TableDescriptor::persistenceLevel, TableDescriptor::shard, TableDescriptor::sortedColumnId, TableDescriptor::tableId, and TableDescriptor::tableName.

Referenced by getMetadataForTableImpl().

1273  {
1274  auto time_ms = measure<>::execution([&]() {
1275  // instanciate table fragmenter upon first use
1276  // assume only insert order fragmenter is supported
1278  vector<Chunk> chunkVec;
1279  list<const ColumnDescriptor*> columnDescs;
1280  getAllColumnMetadataForTable(td, columnDescs, true, false, true);
1281  Chunk::translateColumnDescriptorsToChunkVec(columnDescs, chunkVec);
1282  ChunkKey chunkKeyPrefix = {currentDB_.dbId, td->tableId};
1283  if (td->sortedColumnId > 0) {
1284  td->fragmenter = new SortedOrderFragmenter(chunkKeyPrefix,
1285  chunkVec,
1286  dataMgr_.get(),
1287  const_cast<Catalog*>(this),
1288  td->tableId,
1289  td->shard,
1290  td->maxFragRows,
1291  td->maxChunkSize,
1292  td->fragPageSize,
1293  td->maxRows,
1294  td->persistenceLevel);
1295  } else {
1296  td->fragmenter = new InsertOrderFragmenter(chunkKeyPrefix,
1297  chunkVec,
1298  dataMgr_.get(),
1299  const_cast<Catalog*>(this),
1300  td->tableId,
1301  td->shard,
1302  td->maxFragRows,
1303  td->maxChunkSize,
1304  td->fragPageSize,
1305  td->maxRows,
1306  td->persistenceLevel);
1307  }
1308  });
1309  LOG(INFO) << "Instantiating Fragmenter for table " << td->tableName << " took "
1310  << time_ms << "ms";
1311 }
#define CHECK_EQ(x, y)
Definition: Logger.h:198
std::vector< int > ChunkKey
Definition: types.h:35
std::string tableName
static TimeT::rep execution(F func, Args &&...args)
Definition: sample.cpp:29
#define LOG(tag)
Definition: Logger.h:185
Catalog(const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
Constructor - takes basePath to already extant data directory for writing.
Definition: Catalog.cpp:129
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:328
The InsertOrderFragmenter is a child class of AbstractFragmenter, and fragments data in insert order...
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1581
Fragmenter_Namespace::FragmenterType fragType
Data_Namespace::MemoryLevel persistenceLevel
Fragmenter_Namespace::AbstractFragmenter * fragmenter

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::string Catalog_Namespace::Catalog::name ( ) const
inline

Definition at line 219 of file Catalog.h.

References Catalog_Namespace::DBMetadata::dbName, and getCurrentDB().

Referenced by deleteMetadataForDashboard(), getMetadataForDashboard(), and recordOwnershipOfObjectsInObjectPermissions().

219 { return getCurrentDB().dbName; }
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:176

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

std::vector< DBObject > Catalog_Namespace::Catalog::parseDashboardObjects ( const std::string &  view_meta,
const int &  user_id 
)
protected

Definition at line 1194 of file Catalog.cpp.

References currentDB_, DBObjectKey::dbId, Catalog_Namespace::DBMetadata::dbId, getMetadataForTable(), logger::INFO, LOG, DBObjectKey::objectId, parse_underlying_dashboard_objects(), DBObjectKey::permissionType, AccessPrivileges::SELECT_FROM_TABLE, AccessPrivileges::SELECT_FROM_VIEW, TableDBObjectType, run_benchmark_import::type, and ViewDBObjectType.

Referenced by createOrUpdateDashboardSystemRole().

1195  {
1196  std::vector<DBObject> objects;
1197  DBObjectKey key;
1198  key.dbId = currentDB_.dbId;
1199  auto _key_place = [&key](auto type, auto id) {
1200  key.permissionType = type;
1201  key.objectId = id;
1202  return key;
1203  };
1204  for (auto object_name : parse_underlying_dashboard_objects(view_meta)) {
1205  auto td = getMetadataForTable(object_name);
1206  if (!td) {
1207  // Parsed object source is not present in current database
1208  // LOG the info and ignore
1209  LOG(INFO) << "Ignoring dashboard source Table/View: " << object_name
1210  << " no longer exists in current DB.";
1211  continue;
1212  }
1213  // Dashboard source can be Table or View
1214  const auto object_type = td->isView ? ViewDBObjectType : TableDBObjectType;
1215  const auto priv = td->isView ? AccessPrivileges::SELECT_FROM_VIEW
1217  objects.emplace_back(_key_place(object_type, td->tableId), priv, user_id);
1218  objects.back().setObjectType(td->isView ? ViewDBObjectType : TableDBObjectType);
1219  objects.back().setName(td->tableName);
1220  }
1221  return objects;
1222 }
#define LOG(tag)
Definition: Logger.h:185
int32_t objectId
Definition: DBObject.h:56
static const AccessPrivileges SELECT_FROM_TABLE
Definition: DBObject.h:153