OmniSciDB  29e35f4d58
Catalog_Namespace::Catalog Class Referencefinal

class for a per-database catalog. also includes metadata for the current database and the current user. More...

#include <Catalog.h>

+ Collaboration diagram for Catalog_Namespace::Catalog:

Public Member Functions

 Catalog (const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
 Constructor - takes basePath to already extant data directory for writing. More...
 
 ~Catalog ()
 Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite. More...
 
void createTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
 
void createShardedTable (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
int32_t createDashboard (DashboardDescriptor &vd)
 
void replaceDashboard (DashboardDescriptor &vd)
 
std::string createLink (LinkDescriptor &ld, size_t min_length)
 
void dropTable (const TableDescriptor *td)
 
void truncateTable (const TableDescriptor *td)
 
void renameTable (const TableDescriptor *td, const std::string &newTableName)
 
void renameColumn (const TableDescriptor *td, const ColumnDescriptor *cd, const std::string &newColumnName)
 
void addColumn (const TableDescriptor &td, ColumnDescriptor &cd)
 
void removeChunks (const int table_id)
 
const TableDescriptorgetMetadataForTable (const std::string &tableName, const bool populateFragmenter=true) const
 Returns a pointer to a const TableDescriptor struct matching the provided tableName. More...
 
const TableDescriptorgetMetadataForTableImpl (int tableId, const bool populateFragmenter) const
 
const TableDescriptorgetMetadataForTable (int tableId) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, const std::string &colName) const
 
const ColumnDescriptorgetMetadataForColumn (int tableId, int columnId) const
 
const int getColumnIdBySpi (const int tableId, const size_t spi) const
 
const ColumnDescriptorgetMetadataForColumnBySpi (const int tableId, const size_t spi) const
 
const DashboardDescriptorgetMetadataForDashboard (const std::string &userId, const std::string &dashName) const
 
void deleteMetadataForDashboard (const std::string &userId, const std::string &dashName)
 
const DashboardDescriptorgetMetadataForDashboard (const int32_t dashboard_id) const
 
void deleteMetadataForDashboard (const int32_t dashboard_id)
 
const LinkDescriptorgetMetadataForLink (const std::string &link) const
 
const LinkDescriptorgetMetadataForLink (int linkId) const
 
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable (const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular table specified by table id. More...
 
std::list< const TableDescriptor * > getAllTableMetadata () const
 
std::list< const DashboardDescriptor * > getAllDashboardsMetadata () const
 
const DBMetadatagetCurrentDB () const
 
Data_Namespace::DataMgrgetDataMgr () const
 
std::shared_ptr< CalcitegetCalciteMgr () const
 
const std::string & getBasePath () const
 
const DictDescriptorgetMetadataForDict (int dict_ref, bool loadDict=true) const
 
const std::vector< LeafHostInfo > & getStringDictionaryHosts () const
 
const ColumnDescriptorgetShardColumnMetadataForTable (const TableDescriptor *td) const
 
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors (const TableDescriptor *logicalTableDesc) const
 
int32_t getTableEpoch (const int32_t db_id, const int32_t table_id) const
 
void setTableEpoch (const int db_id, const int table_id, const int new_epoch)
 
int getDatabaseId () const
 
SqliteConnectorgetSqliteConnector ()
 
void roll (const bool forward)
 
DictRef addDictionary (ColumnDescriptor &cd)
 
void delDictionary (const ColumnDescriptor &cd)
 
void getDictionary (const ColumnDescriptor &cd, std::map< int, StringDictionary *> &stringDicts)
 
const bool checkMetadataForDeletedRecs (int dbId, int tableId, int columnId) const
 
const ColumnDescriptorgetDeletedColumn (const TableDescriptor *td) const
 
const ColumnDescriptorgetDeletedColumnIfRowsDeleted (const TableDescriptor *td) const
 
void setDeletedColumn (const TableDescriptor *td, const ColumnDescriptor *cd)
 
void setDeletedColumnUnlocked (const TableDescriptor *td, const ColumnDescriptor *cd)
 
int getLogicalTableId (const int physicalTableId) const
 
void checkpoint (const int logicalTableId) const
 
std::string name () const
 
void eraseDBData ()
 
void eraseTablePhysicalData (const TableDescriptor *td)
 
void vacuumDeletedRows (const TableDescriptor *td) const
 
void vacuumDeletedRows (const int logicalTableId) const
 
void dumpTable (const TableDescriptor *td, const std::string &path, const std::string &compression) const
 
void restoreTable (const SessionInfo &session, const TableDescriptor *td, const std::string &file_path, const std::string &compression)
 
void restoreTable (const SessionInfo &session, const std::string &table_name, const std::string &file_path, const std::string &compression)
 
std::vector< std::string > getTableDataDirectories (const TableDescriptor *td) const
 
std::vector< std::string > getTableDictDirectories (const TableDescriptor *td) const
 
std::string getColumnDictDirectory (const ColumnDescriptor *cd) const
 
std::string dumpSchema (const TableDescriptor *td) const
 

Static Public Member Functions

static void expandGeoColumn (const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
 
static void set (const std::string &dbName, std::shared_ptr< Catalog > cat)
 
static std::shared_ptr< Catalogget (const std::string &dbName)
 
static std::shared_ptr< Catalogget (const std::string &basePath, const DBMetadata &curDB, std::shared_ptr< Data_Namespace::DataMgr > dataMgr, const std::vector< LeafHostInfo > &string_dict_hosts, std::shared_ptr< Calcite > calcite, bool is_new_db)
 
static void remove (const std::string &dbName)
 

Public Attributes

std::mutex sqliteMutex_
 
mapd_shared_mutex sharedMutex_
 
std::atomic< std::thread::id > thread_holding_sqlite_lock
 
std::atomic< std::thread::id > thread_holding_write_lock
 

Static Public Attributes

static thread_local bool thread_holds_read_lock = false
 

Protected Types

typedef std::map< std::string, TableDescriptor * > TableDescriptorMap
 
typedef std::map< int, TableDescriptor * > TableDescriptorMapById
 
typedef std::map< int32_t, std::vector< int32_t > > LogicalToPhysicalTableMapById
 
typedef std::tuple< int, std::string > ColumnKey
 
typedef std::map< ColumnKey, ColumnDescriptor * > ColumnDescriptorMap
 
typedef std::tuple< int, int > ColumnIdKey
 
typedef std::map< ColumnIdKey, ColumnDescriptor * > ColumnDescriptorMapById
 
typedef std::map< DictRef, std::unique_ptr< DictDescriptor > > DictDescriptorMapById
 
typedef std::map< std::string, std::shared_ptr< DashboardDescriptor > > DashboardDescriptorMap
 
typedef std::map< std::string, LinkDescriptor * > LinkDescriptorMap
 
typedef std::map< int, LinkDescriptor * > LinkDescriptorMapById
 
typedef std::unordered_map< const TableDescriptor *, const ColumnDescriptor * > DeletedColumnPerTableMap
 
using ColumnDescriptorsForRoll = std::vector< std::pair< ColumnDescriptor *, ColumnDescriptor * > >
 

Protected Member Functions

void CheckAndExecuteMigrations ()
 
void CheckAndExecuteMigrationsPostBuildMaps ()
 
void updateDictionaryNames ()
 
void updateTableDescriptorSchema ()
 
void updateFixlenArrayColumns ()
 
void updateFrontendViewSchema ()
 
void updateLinkSchema ()
 
void updateFrontendViewAndLinkUsers ()
 
void updateLogicalToPhysicalTableLinkSchema ()
 
void updateLogicalToPhysicalTableMap (const int32_t logical_tb_id)
 
void updateDictionarySchema ()
 
void updatePageSize ()
 
void updateDeletedColumnIndicator ()
 
void updateFrontendViewsToDashboards ()
 
void recordOwnershipOfObjectsInObjectPermissions ()
 
void checkDateInDaysColumnMigration ()
 
void createDashboardSystemRoles ()
 
void buildMaps ()
 
void addTableToMap (TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
 
void addReferenceToForeignDict (ColumnDescriptor &referencing_column, Parser::SharedDictionaryDef shared_dict_def, const bool persist_reference)
 
bool setColumnSharedDictionary (ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
 
void setColumnDictionary (ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
 
void addFrontendViewToMap (DashboardDescriptor &vd)
 
void addFrontendViewToMapNoLock (DashboardDescriptor &vd)
 
void addLinkToMap (LinkDescriptor &ld)
 
void removeTableFromMap (const std::string &tableName, const int tableId, const bool is_on_error=false)
 
void doDropTable (const TableDescriptor *td)
 
void doTruncateTable (const TableDescriptor *td)
 
void renamePhysicalTable (const TableDescriptor *td, const std::string &newTableName)
 
void instantiateFragmenter (TableDescriptor *td) const
 
void getAllColumnMetadataForTable (const TableDescriptor *td, std::list< const ColumnDescriptor *> &colDescs, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
 
std::string calculateSHA1 (const std::string &data)
 
std::string generatePhysicalTableName (const std::string &logicalTableName, const int32_t &shardNumber)
 
std::vector< DBObjectparseDashboardObjects (const std::string &view_meta, const int &user_id)
 
void createOrUpdateDashboardSystemRole (const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
 
const int getColumnIdBySpiUnlocked (const int table_id, const size_t spi) const
 
void serializeTableJsonUnlocked (const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
 
void dropTableFromJsonUnlocked (const std::string &table_name) const
 

Protected Attributes

std::string basePath_
 
TableDescriptorMap tableDescriptorMap_
 
TableDescriptorMapById tableDescriptorMapById_
 
ColumnDescriptorMap columnDescriptorMap_
 
ColumnDescriptorMapById columnDescriptorMapById_
 
DictDescriptorMapById dictDescriptorMapByRef_
 
DashboardDescriptorMap dashboardDescriptorMap_
 
LinkDescriptorMap linkDescriptorMap_
 
LinkDescriptorMapById linkDescriptorMapById_
 
SqliteConnector sqliteConnector_
 
DBMetadata currentDB_
 
std::shared_ptr< Data_Namespace::DataMgrdataMgr_
 
const std::vector< LeafHostInfostring_dict_hosts_
 
std::shared_ptr< CalcitecalciteMgr_
 
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
 
int nextTempTableId_
 
int nextTempDictId_
 
ColumnDescriptorsForRoll columnDescriptorsForRoll
 

Static Protected Attributes

static const std::string physicalTableNameTag_
 

Private Member Functions

void adjustAlteredTableFiles (const std::string &temp_data_dir, const std::unordered_map< int, int > &all_column_ids_map) const
 
void renameTableDirectories (const std::string &temp_data_dir, const std::vector< std::string > &target_paths, const std::string &name_prefix) const
 

Private Attributes

DeletedColumnPerTableMap deletedColumnPerTable_
 

Static Private Attributes

static std::map< std::string, std::shared_ptr< Catalog > > mapd_cat_map_
 

Detailed Description

class for a per-database catalog. also includes metadata for the current database and the current user.

Catalog

Definition at line 81 of file Catalog.h.

Member Typedef Documentation

◆ ColumnDescriptorMap

Definition at line 247 of file Catalog.h.

◆ ColumnDescriptorMapById

Definition at line 249 of file Catalog.h.

◆ ColumnDescriptorsForRoll

Definition at line 349 of file Catalog.h.

◆ ColumnIdKey

typedef std::tuple<int, int> Catalog_Namespace::Catalog::ColumnIdKey
protected

Definition at line 248 of file Catalog.h.

◆ ColumnKey

typedef std::tuple<int, std::string> Catalog_Namespace::Catalog::ColumnKey
protected

Definition at line 246 of file Catalog.h.

◆ DashboardDescriptorMap

typedef std::map<std::string, std::shared_ptr<DashboardDescriptor> > Catalog_Namespace::Catalog::DashboardDescriptorMap
protected

Definition at line 252 of file Catalog.h.

◆ DeletedColumnPerTableMap

typedef std::unordered_map<const TableDescriptor*, const ColumnDescriptor*> Catalog_Namespace::Catalog::DeletedColumnPerTableMap
protected

Definition at line 256 of file Catalog.h.

◆ DictDescriptorMapById

typedef std::map<DictRef, std::unique_ptr<DictDescriptor> > Catalog_Namespace::Catalog::DictDescriptorMapById
protected

Definition at line 250 of file Catalog.h.

◆ LinkDescriptorMap

typedef std::map<std::string, LinkDescriptor*> Catalog_Namespace::Catalog::LinkDescriptorMap
protected

Definition at line 253 of file Catalog.h.

◆ LinkDescriptorMapById

Definition at line 254 of file Catalog.h.

◆ LogicalToPhysicalTableMapById

typedef std::map<int32_t, std::vector<int32_t> > Catalog_Namespace::Catalog::LogicalToPhysicalTableMapById
protected

Definition at line 245 of file Catalog.h.

◆ TableDescriptorMap

typedef std::map<std::string, TableDescriptor*> Catalog_Namespace::Catalog::TableDescriptorMap
protected

Definition at line 243 of file Catalog.h.

◆ TableDescriptorMapById

Definition at line 244 of file Catalog.h.

Constructor & Destructor Documentation

◆ Catalog()

Catalog_Namespace::Catalog::Catalog ( const std::string &  basePath,
const DBMetadata curDB,
std::shared_ptr< Data_Namespace::DataMgr dataMgr,
const std::vector< LeafHostInfo > &  string_dict_hosts,
std::shared_ptr< Calcite calcite,
bool  is_new_db 
)

Constructor - takes basePath to already extant data directory for writing.

Parameters
basePathdirectory path for writing catalog
dbNamename of the database
fragmenterFragmenter object metadata - expects for this directory to already exist

Definition at line 149 of file Catalog.cpp.

References basePath_, buildMaps(), CheckAndExecuteMigrations(), CheckAndExecuteMigrationsPostBuildMaps(), currentDB_, Catalog_Namespace::DBMetadata::dbName, g_serialize_temp_tables, and Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath().

155  : basePath_(basePath)
156  , sqliteConnector_(curDB.dbName, basePath + "/mapd_catalogs/")
157  , currentDB_(curDB)
158  , dataMgr_(dataMgr)
159  , string_dict_hosts_(string_dict_hosts)
160  , calciteMgr_(calcite)
163  , sqliteMutex_()
164  , sharedMutex_()
167  if (!is_new_db) {
169  }
170  buildMaps();
171  if (!is_new_db) {
173  }
175  boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
176  }
177 }
const int MAPD_TEMP_TABLE_START_ID
Definition: Catalog.cpp:90
mapd_shared_mutex sharedMutex_
Definition: Catalog.h:364
SqliteConnector sqliteConnector_
Definition: Catalog.h:331
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:335
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:333
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:141
const int MAPD_TEMP_DICT_START_ID
Definition: Catalog.cpp:92
std::atomic< std::thread::id > thread_holding_sqlite_lock
Definition: Catalog.h:365
bool g_serialize_temp_tables
Definition: Catalog.cpp:85
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:336
std::atomic< std::thread::id > thread_holding_write_lock
Definition: Catalog.h:366
void CheckAndExecuteMigrationsPostBuildMaps()
Definition: Catalog.cpp:876
+ Here is the call graph for this function:

◆ ~Catalog()

Catalog_Namespace::Catalog::~Catalog ( )

Destructor - deletes all ColumnDescriptor and TableDescriptor structures which were allocated on the heap and writes Catalog to Sqlite.

Definition at line 179 of file Catalog.cpp.

References basePath_, columnDescriptorMap_, currentDB_, Catalog_Namespace::DBMetadata::dbName, g_serialize_temp_tables, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and tableDescriptorMap_.

179  {
180  cat_write_lock write_lock(this);
181  // must clean up heap-allocated TableDescriptor and ColumnDescriptor structs
182  for (TableDescriptorMap::iterator tableDescIt = tableDescriptorMap_.begin();
183  tableDescIt != tableDescriptorMap_.end();
184  ++tableDescIt) {
185  if (tableDescIt->second->fragmenter != nullptr) {
186  delete tableDescIt->second->fragmenter;
187  }
188  delete tableDescIt->second;
189  }
190 
191  // TableDescriptorMapById points to the same descriptors. No need to delete
192 
193  for (ColumnDescriptorMap::iterator columnDescIt = columnDescriptorMap_.begin();
194  columnDescIt != columnDescriptorMap_.end();
195  ++columnDescIt) {
196  delete columnDescIt->second;
197  }
198 
199  // ColumnDescriptorMapById points to the same descriptors. No need to delete
200 
202  boost::filesystem::remove(table_json_filepath(basePath_, currentDB_.dbName));
203  }
204 }
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:325
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:141
bool g_serialize_temp_tables
Definition: Catalog.cpp:85
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:101
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:323
+ Here is the call graph for this function:

Member Function Documentation

◆ addColumn()

void Catalog_Namespace::Catalog::addColumn ( const TableDescriptor td,
ColumnDescriptor cd 
)

Definition at line 1729 of file Catalog.cpp.

References addDictionary(), columnDescriptorMap_, columnDescriptorMapById_, columnDescriptorsForRoll, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfoCore< TYPE_FACET_PACK >::get_comp_param(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_compression(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_dimension(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_notnull(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_scale(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_size(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_subtype(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), SqliteConnector::getData(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isSystemCol, ColumnDescriptor::isVirtualCol, kENCODING_DICT, SqliteConnector::query_with_text_params(), sqliteConnector_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, to_string(), to_upper(), and ColumnDescriptor::virtualExpr.

1729  {
1730  // caller must handle sqlite/chunk transaction TOGETHER
1731  cd.tableId = td.tableId;
1733  addDictionary(cd);
1734  }
1735 
1737  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, coldim, "
1738  "colscale, is_notnull, "
1739  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, virtual_expr, "
1740  "is_deletedcol) "
1741  "VALUES (?, "
1742  "(SELECT max(columnid) + 1 FROM mapd_columns WHERE tableid = ?), "
1743  "?, ?, ?, "
1744  "?, "
1745  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1746  std::vector<std::string>{std::to_string(td.tableId),
1747  std::to_string(td.tableId),
1748  cd.columnName,
1757  "",
1760  cd.virtualExpr,
1762 
1764  "UPDATE mapd_tables SET ncolumns = ncolumns + 1 WHERE tableid = ?",
1765  std::vector<std::string>{std::to_string(td.tableId)});
1766 
1768  "SELECT columnid FROM mapd_columns WHERE tableid = ? AND name = ?",
1769  std::vector<std::string>{std::to_string(td.tableId), cd.columnName});
1770  cd.columnId = sqliteConnector_.getData<int>(0, 0);
1771 
1772  ++tableDescriptorMapById_[td.tableId]->nColumns;
1773  auto ncd = new ColumnDescriptor(cd);
1776  columnDescriptorsForRoll.emplace_back(nullptr, ncd);
1777 }
std::string virtualExpr
HOST DEVICE int get_size() const
Definition: sqltypes.h:336
T getData(const int row, const int col)
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:328
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:325
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:333
SqliteConnector sqliteConnector_
Definition: Catalog.h:331
void query_with_text_params(std::string const &query_only)
HOST DEVICE int get_scale() const
Definition: sqltypes.h:331
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:326
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:334
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:326
std::string to_string(char const *&&v)
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:327
std::tuple< int, int > ColumnIdKey
Definition: Catalog.h:248
std::tuple< int, std::string > ColumnKey
Definition: Catalog.h:246
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:324
specifies the content in-memory of a row in the column metadata table
std::string to_upper(const std::string &str)
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:335
SQLTypeInfo columnType
DictRef addDictionary(ColumnDescriptor &cd)
Definition: Catalog.cpp:1642
ColumnDescriptorsForRoll columnDescriptorsForRoll
Definition: Catalog.h:350
std::string columnName
+ Here is the call graph for this function:

◆ addDictionary()

DictRef Catalog_Namespace::Catalog::addDictionary ( ColumnDescriptor cd)

Definition at line 1642 of file Catalog.cpp.

References CHECK, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, DictDescriptor::dictFolderPath, setColumnDictionary(), string_dict_hosts_, tableDescriptorMapById_, and ColumnDescriptor::tableId.

Referenced by addColumn().

1642  {
1643  const auto& td = *tableDescriptorMapById_[cd.tableId];
1644  list<DictDescriptor> dds;
1645  setColumnDictionary(cd, dds, td, true);
1646  auto& dd = dds.back();
1647  CHECK(dd.dictRef.dictId);
1648 
1649  std::unique_ptr<StringDictionaryClient> client;
1650  if (!string_dict_hosts_.empty()) {
1651  client.reset(new StringDictionaryClient(
1652  string_dict_hosts_.front(), DictRef(currentDB_.dbId, -1), true));
1653  }
1654  if (client) {
1655  client->create(dd.dictRef, dd.dictIsTemp);
1656  }
1657 
1658  DictDescriptor* new_dd = new DictDescriptor(dd);
1659  dictDescriptorMapByRef_[dd.dictRef].reset(new_dd);
1660  if (!dd.dictIsTemp) {
1661  boost::filesystem::create_directory(new_dd->dictFolderPath);
1662  }
1663  return dd.dictRef;
1664 }
std::string dictFolderPath
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:335
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:327
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:324
struct dict_ref_t DictRef
Definition: DictRef.h:37
#define CHECK(condition)
Definition: Logger.h:193
Descriptor for a dictionary for a string columne.
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:2487
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ addFrontendViewToMap()

void Catalog_Namespace::Catalog::addFrontendViewToMap ( DashboardDescriptor vd)
protected

Definition at line 1210 of file Catalog.cpp.

References addFrontendViewToMapNoLock().

Referenced by createDashboard().

1210  {
1211  cat_write_lock write_lock(this);
1213 }
void addFrontendViewToMapNoLock(DashboardDescriptor &vd)
Definition: Catalog.cpp:1215
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:101
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ addFrontendViewToMapNoLock()

void Catalog_Namespace::Catalog::addFrontendViewToMapNoLock ( DashboardDescriptor vd)
protected

Definition at line 1215 of file Catalog.cpp.

References dashboardDescriptorMap_, DashboardDescriptor::dashboardName, to_string(), and DashboardDescriptor::userId.

Referenced by addFrontendViewToMap(), and replaceDashboard().

1215  {
1216  cat_write_lock write_lock(this);
1218  std::make_shared<DashboardDescriptor>(vd);
1219 }
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:328
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:101
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ addLinkToMap()

void Catalog_Namespace::Catalog::addLinkToMap ( LinkDescriptor ld)
protected

Definition at line 1292 of file Catalog.cpp.

References currentDB_, Catalog_Namespace::DBMetadata::dbId, LinkDescriptor::link, linkDescriptorMap_, linkDescriptorMapById_, LinkDescriptor::linkId, and to_string().

Referenced by createLink().

1292  {
1293  cat_write_lock write_lock(this);
1294  LinkDescriptor* new_ld = new LinkDescriptor();
1295  *new_ld = ld;
1297  linkDescriptorMapById_[ld.linkId] = new_ld;
1298 }
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:101
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:330
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:329
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ addReferenceToForeignDict()

void Catalog_Namespace::Catalog::addReferenceToForeignDict ( ColumnDescriptor referencing_column,
Parser::SharedDictionaryDef  shared_dict_def,
const bool  persist_reference 
)
protected

Definition at line 2396 of file Catalog.cpp.

References CHECK, CHECK_GE, ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, SQLTypeInfoCore< TYPE_FACET_PACK >::get_comp_param(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_foreign_col(), SqliteConnector::query_with_text_params(), sqliteConnector_, and to_string().

Referenced by setColumnSharedDictionary().

2398  {
2399  cat_write_lock write_lock(this);
2400  const auto foreign_ref_col = get_foreign_col(*this, shared_dict_def);
2401  CHECK(foreign_ref_col);
2402  referencing_column.columnType = foreign_ref_col->columnType;
2403  const int dict_id = referencing_column.columnType.get_comp_param();
2404  const DictRef dict_ref(currentDB_.dbId, dict_id);
2405  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
2406  CHECK(dictIt != dictDescriptorMapByRef_.end());
2407  const auto& dd = dictIt->second;
2408  CHECK_GE(dd->refcount, 1);
2409  ++dd->refcount;
2410  if (persist_reference) {
2411  cat_sqlite_lock sqlite_lock(this);
2413  "UPDATE mapd_dictionaries SET refcount = refcount + 1 WHERE dictid = ?",
2414  {std::to_string(dict_id)});
2415  }
2416 }
SqliteConnector sqliteConnector_
Definition: Catalog.h:331
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:327
void query_with_text_params(std::string const &query_only)
#define CHECK_GE(x, y)
Definition: Logger.h:206
const ColumnDescriptor * get_foreign_col(const Catalog &cat, const Parser::SharedDictionaryDef &shared_dict_def)
Definition: Catalog.cpp:2384
std::string to_string(char const *&&v)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:101
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:335
#define CHECK(condition)
Definition: Logger.h:193
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:102
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ addTableToMap()

void Catalog_Namespace::Catalog::addTableToMap ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::list< DictDescriptor > &  dicts 
)
protected

Definition at line 1078 of file Catalog.cpp.

References CHECK, columnDescriptorMap_, columnDescriptorMapById_, ColumnDescriptor::columnId, TableDescriptor::columnIdBySpi_, ColumnDescriptor::columnName, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, DictDescriptor::dictFolderPath, dict_ref_t::dictId, TableDescriptor::hasDeletedCol, TableDescriptor::mutex_, setDeletedColumnUnlocked(), string_dict_hosts_, tableDescriptorMap_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, and to_upper().

Referenced by createTable().

1080  {
1081  cat_write_lock write_lock(this);
1082  TableDescriptor* new_td = new TableDescriptor();
1083  *new_td = td;
1084  new_td->mutex_ = std::make_shared<std::mutex>();
1085  tableDescriptorMap_[to_upper(td.tableName)] = new_td;
1086  tableDescriptorMapById_[td.tableId] = new_td;
1087  for (auto cd : columns) {
1088  ColumnDescriptor* new_cd = new ColumnDescriptor();
1089  *new_cd = cd;
1090  ColumnKey columnKey(new_cd->tableId, to_upper(new_cd->columnName));
1091  columnDescriptorMap_[columnKey] = new_cd;
1092  ColumnIdKey columnIdKey(new_cd->tableId, new_cd->columnId);
1093  columnDescriptorMapById_[columnIdKey] = new_cd;
1094 
1095  // Add deleted column to the map
1096  if (cd.isDeletedCol) {
1097  CHECK(new_td->hasDeletedCol);
1098  setDeletedColumnUnlocked(new_td, new_cd);
1099  }
1100  }
1101 
1102  std::sort(new_td->columnIdBySpi_.begin(),
1103  new_td->columnIdBySpi_.end(),
1104  [](const size_t a, const size_t b) -> bool { return a < b; });
1105 
1106  std::unique_ptr<StringDictionaryClient> client;
1107  DictRef dict_ref(currentDB_.dbId, -1);
1108  if (!string_dict_hosts_.empty()) {
1109  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
1110  }
1111  for (auto dd : dicts) {
1112  if (!dd.dictRef.dictId) {
1113  // Dummy entry created for a shard of a logical table, nothing to do.
1114  continue;
1115  }
1116  dict_ref.dictId = dd.dictRef.dictId;
1117  if (client) {
1118  client->create(dict_ref, dd.dictIsTemp);
1119  }
1120  DictDescriptor* new_dd = new DictDescriptor(dd);
1121  dictDescriptorMapByRef_[dict_ref].reset(new_dd);
1122  if (!dd.dictIsTemp) {
1123  boost::filesystem::create_directory(new_dd->dictFolderPath);
1124  }
1125  }
1126 }
std::string dictFolderPath
std::string tableName
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:325
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:335
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:327
std::vector< int > columnIdBySpi_
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:326
std::shared_ptr< std::mutex > mutex_
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:2375
std::tuple< int, int > ColumnIdKey
Definition: Catalog.h:248
std::tuple< int, std::string > ColumnKey
Definition: Catalog.h:246
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:324
specifies the content in-memory of a row in the column metadata table
std::string to_upper(const std::string &str)
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:101
#define CHECK(condition)
Definition: Logger.h:193
Descriptor for a dictionary for a string columne.
specifies the content in-memory of a row in the table metadata table
std::string columnName
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:323
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ adjustAlteredTableFiles()

void Catalog_Namespace::Catalog::adjustAlteredTableFiles ( const std::string &  temp_data_dir,
const std::unordered_map< int, int > &  all_column_ids_map 
) const
private

Definition at line 360 of file DumpRestore.cpp.

References CHECK, ThreadController_NS::SimpleThreadController< FutureReturnType >::checkThreadsStatus(), cpu_threads(), anonymous_namespace{StringDictionary.cpp}::file_size(), ThreadController_NS::SimpleThreadController< FutureReturnType >::finish(), MAPD_FILE_EXT, Catalog_Namespace::simple_file_closer, split(), ThreadController_NS::SimpleThreadController< FutureReturnType >::startThread(), and to_string().

362  {
363  boost::filesystem::path base_path(temp_data_dir);
364  boost::filesystem::recursive_directory_iterator end_it;
366  for (boost::filesystem::recursive_directory_iterator fit(base_path); fit != end_it;
367  ++fit) {
368  if (boost::filesystem::is_regular_file(fit->status())) {
369  const std::string file_path = fit->path().string();
370  const std::string file_name = fit->path().filename().string();
371  std::vector<std::string> tokens;
372  boost::split(tokens, file_name, boost::is_any_of("."));
373  // ref. FileMgr::init for hint of data file name layout
374  if (tokens.size() > 2 && MAPD_FILE_EXT == "." + tokens[2]) {
375  thread_controller.startThread([file_name, file_path, tokens, &column_ids_map] {
376  const auto page_size = boost::lexical_cast<int64_t>(tokens[1]);
377  const auto file_size = boost::filesystem::file_size(file_path);
378  std::unique_ptr<FILE, decltype(simple_file_closer)> fp(
379  std::fopen(file_path.c_str(), "r+"), simple_file_closer);
380  if (!fp) {
381  throw std::runtime_error("Failed to open " + file_path +
382  " for update: " + std::strerror(errno));
383  }
384  // ref. FileInfo::openExistingFile for hint of chunk header layout
385  for (size_t page = 0; page < file_size / page_size; ++page) {
386  int ints[8];
387  if (0 != std::fseek(fp.get(), page * page_size, SEEK_SET)) {
388  throw std::runtime_error("Failed to seek to page# " + std::to_string(page) +
389  file_path + " for read: " + std::strerror(errno));
390  }
391  if (1 != fread(ints, sizeof ints, 1, fp.get())) {
392  throw std::runtime_error("Failed to read " + file_path + ": " +
393  std::strerror(errno));
394  }
395  if (ints[0] > 0) { // header size
396  auto cit = column_ids_map.find(ints[3]);
397  CHECK(cit != column_ids_map.end());
398  if (ints[3] != cit->second) {
399  ints[3] = cit->second;
400  if (0 != std::fseek(fp.get(), page * page_size, SEEK_SET)) {
401  throw std::runtime_error("Failed to seek to page# " +
402  std::to_string(page) + file_path +
403  " for write: " + std::strerror(errno));
404  }
405  if (1 != fwrite(ints, sizeof ints, 1, fp.get())) {
406  throw std::runtime_error("Failed to write " + file_path + ": " +
407  std::strerror(errno));
408  }
409  }
410  }
411  }
412  });
413  thread_controller.checkThreadsStatus();
414  }
415  }
416  }
417  thread_controller.finish();
418 }
#define MAPD_FILE_EXT
Definition: File.h:26
std::string to_string(char const *&&v)
std::vector< std::string > split(const std::string &str, const std::string &delim)
split apart a string into a vector of substrings
#define CHECK(condition)
Definition: Logger.h:193
int cpu_threads()
Definition: thread_count.h:25
+ Here is the call graph for this function:

◆ buildMaps()

void Catalog_Namespace::Catalog::buildMaps ( )
protected

Definition at line 892 of file Catalog.cpp.

References basePath_, CHECK, ColumnDescriptor::chunks, columnDescriptorMap_, columnDescriptorMapById_, ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, currentDB_, dashboardDescriptorMap_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, TableDescriptor::fragmenter, TableDescriptor::fragments, TableDescriptor::fragPageSize, TableDescriptor::fragType, generate_dashboard_system_rolename(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_physical_cols(), SqliteConnector::getData(), SqliteConnector::getNumRows(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::getUserFromId(), TableDescriptor::hasDeletedCol, SQLTypeInfoCore< TYPE_FACET_PACK >::is_geometry(), ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, SqliteConnector::isNull(), ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, TableDescriptor::keyMetainfo, LinkDescriptor::link, linkDescriptorMap_, linkDescriptorMapById_, LinkDescriptor::linkId, logicalToPhysicalTableMapById_, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRows, TableDescriptor::nColumns, TableDescriptor::nShards, TableDescriptor::partitions, SqliteConnector::query(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_comp_param(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_compression(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_dimension(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_notnull(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_scale(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_size(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_subtype(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_type(), setDeletedColumnUnlocked(), TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, sqliteConnector_, tableDescriptorMap_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), to_upper(), LinkDescriptor::updateTime, LinkDescriptor::userId, TableDescriptor::userId, LinkDescriptor::viewMetadata, TableDescriptor::viewSQL, LinkDescriptor::viewState, and ColumnDescriptor::virtualExpr.

Referenced by Catalog().

892  {
893  cat_write_lock write_lock(this);
894  cat_sqlite_lock sqlite_lock(this);
895 
896  string dictQuery(
897  "SELECT dictid, name, nbits, is_shared, refcount from mapd_dictionaries");
898  sqliteConnector_.query(dictQuery);
899  size_t numRows = sqliteConnector_.getNumRows();
900  for (size_t r = 0; r < numRows; ++r) {
901  int dictId = sqliteConnector_.getData<int>(r, 0);
902  std::string dictName = sqliteConnector_.getData<string>(r, 1);
903  int dictNBits = sqliteConnector_.getData<int>(r, 2);
904  bool is_shared = sqliteConnector_.getData<bool>(r, 3);
905  int refcount = sqliteConnector_.getData<int>(r, 4);
906  std::string fname = basePath_ + "/mapd_data/DB_" + std::to_string(currentDB_.dbId) +
907  "_DICT_" + std::to_string(dictId);
908  DictRef dict_ref(currentDB_.dbId, dictId);
909  DictDescriptor* dd = new DictDescriptor(
910  dict_ref, dictName, dictNBits, is_shared, refcount, fname, false);
911  dictDescriptorMapByRef_[dict_ref].reset(dd);
912  }
913 
914  string tableQuery(
915  "SELECT tableid, name, ncolumns, isview, fragments, frag_type, max_frag_rows, "
916  "max_chunk_size, frag_page_size, "
917  "max_rows, partitions, shard_column_id, shard, num_shards, key_metainfo, userid, "
918  "sort_column_id "
919  "from mapd_tables");
920  sqliteConnector_.query(tableQuery);
921  numRows = sqliteConnector_.getNumRows();
922  for (size_t r = 0; r < numRows; ++r) {
923  TableDescriptor* td = new TableDescriptor();
924  td->tableId = sqliteConnector_.getData<int>(r, 0);
925  td->tableName = sqliteConnector_.getData<string>(r, 1);
926  td->nColumns = sqliteConnector_.getData<int>(r, 2);
927  td->isView = sqliteConnector_.getData<bool>(r, 3);
928  td->fragments = sqliteConnector_.getData<string>(r, 4);
929  td->fragType =
931  td->maxFragRows = sqliteConnector_.getData<int>(r, 6);
932  td->maxChunkSize = sqliteConnector_.getData<int>(r, 7);
933  td->fragPageSize = sqliteConnector_.getData<int>(r, 8);
934  td->maxRows = sqliteConnector_.getData<int64_t>(r, 9);
935  td->partitions = sqliteConnector_.getData<string>(r, 10);
936  td->shardedColumnId = sqliteConnector_.getData<int>(r, 11);
937  td->shard = sqliteConnector_.getData<int>(r, 12);
938  td->nShards = sqliteConnector_.getData<int>(r, 13);
939  td->keyMetainfo = sqliteConnector_.getData<string>(r, 14);
940  td->userId = sqliteConnector_.getData<int>(r, 15);
941  td->sortedColumnId =
942  sqliteConnector_.isNull(r, 16) ? 0 : sqliteConnector_.getData<int>(r, 16);
943  if (!td->isView) {
944  td->fragmenter = nullptr;
945  }
946  td->hasDeletedCol = false;
949  }
950  string columnQuery(
951  "SELECT tableid, columnid, name, coltype, colsubtype, coldim, colscale, "
952  "is_notnull, compression, comp_param, "
953  "size, chunks, is_systemcol, is_virtualcol, virtual_expr, is_deletedcol from "
954  "mapd_columns ORDER BY tableid, "
955  "columnid");
956  sqliteConnector_.query(columnQuery);
957  numRows = sqliteConnector_.getNumRows();
958  int32_t skip_physical_cols = 0;
959  for (size_t r = 0; r < numRows; ++r) {
961  cd->tableId = sqliteConnector_.getData<int>(r, 0);
962  cd->columnId = sqliteConnector_.getData<int>(r, 1);
963  cd->columnName = sqliteConnector_.getData<string>(r, 2);
971  cd->columnType.set_size(sqliteConnector_.getData<int>(r, 10));
972  cd->chunks = sqliteConnector_.getData<string>(r, 11);
973  cd->isSystemCol = sqliteConnector_.getData<bool>(r, 12);
974  cd->isVirtualCol = sqliteConnector_.getData<bool>(r, 13);
975  cd->virtualExpr = sqliteConnector_.getData<string>(r, 14);
976  cd->isDeletedCol = sqliteConnector_.getData<bool>(r, 15);
977  cd->isGeoPhyCol = skip_physical_cols > 0;
978  ColumnKey columnKey(cd->tableId, to_upper(cd->columnName));
979  columnDescriptorMap_[columnKey] = cd;
980  ColumnIdKey columnIdKey(cd->tableId, cd->columnId);
981  columnDescriptorMapById_[columnIdKey] = cd;
982 
983  if (skip_physical_cols <= 0) {
984  skip_physical_cols = cd->columnType.get_physical_cols();
985  }
986 
987  auto td_itr = tableDescriptorMapById_.find(cd->tableId);
988  CHECK(td_itr != tableDescriptorMapById_.end());
989 
990  if (cd->isDeletedCol) {
991  td_itr->second->hasDeletedCol = true;
992  setDeletedColumnUnlocked(td_itr->second, cd);
993  } else if (cd->columnType.is_geometry() || skip_physical_cols-- <= 0) {
994  tableDescriptorMapById_[cd->tableId]->columnIdBySpi_.push_back(cd->columnId);
995  }
996  }
997  // sort columnIdBySpi_ based on columnId
998  for (auto& tit : tableDescriptorMapById_) {
999  std::sort(tit.second->columnIdBySpi_.begin(),
1000  tit.second->columnIdBySpi_.end(),
1001  [](const size_t a, const size_t b) -> bool { return a < b; });
1002  }
1003 
1004  string viewQuery("SELECT tableid, sql FROM mapd_views");
1005  sqliteConnector_.query(viewQuery);
1006  numRows = sqliteConnector_.getNumRows();
1007  for (size_t r = 0; r < numRows; ++r) {
1008  int32_t tableId = sqliteConnector_.getData<int>(r, 0);
1009  TableDescriptor* td = tableDescriptorMapById_[tableId];
1010  td->viewSQL = sqliteConnector_.getData<string>(r, 1);
1011  td->fragmenter = nullptr;
1012  }
1013 
1014  string frontendViewQuery(
1015  "SELECT id, state, name, image_hash, strftime('%Y-%m-%dT%H:%M:%SZ', update_time), "
1016  "userid, "
1017  "metadata "
1018  "FROM mapd_dashboards");
1019  sqliteConnector_.query(frontendViewQuery);
1020  numRows = sqliteConnector_.getNumRows();
1021  for (size_t r = 0; r < numRows; ++r) {
1022  std::shared_ptr<DashboardDescriptor> vd = std::make_shared<DashboardDescriptor>();
1023  vd->dashboardId = sqliteConnector_.getData<int>(r, 0);
1024  vd->dashboardState = sqliteConnector_.getData<string>(r, 1);
1025  vd->dashboardName = sqliteConnector_.getData<string>(r, 2);
1026  vd->imageHash = sqliteConnector_.getData<string>(r, 3);
1027  vd->updateTime = sqliteConnector_.getData<string>(r, 4);
1028  vd->userId = sqliteConnector_.getData<int>(r, 5);
1029  vd->dashboardMetadata = sqliteConnector_.getData<string>(r, 6);
1030  vd->user = getUserFromId(vd->userId);
1031  vd->dashboardSystemRoleName = generate_dashboard_system_rolename(
1033  dashboardDescriptorMap_[std::to_string(vd->userId) + ":" + vd->dashboardName] = vd;
1034  }
1035 
1036  string linkQuery(
1037  "SELECT linkid, userid, link, view_state, strftime('%Y-%m-%dT%H:%M:%SZ', "
1038  "update_time), view_metadata "
1039  "FROM mapd_links");
1040  sqliteConnector_.query(linkQuery);
1041  numRows = sqliteConnector_.getNumRows();
1042  for (size_t r = 0; r < numRows; ++r) {
1043  LinkDescriptor* ld = new LinkDescriptor();
1044  ld->linkId = sqliteConnector_.getData<int>(r, 0);
1045  ld->userId = sqliteConnector_.getData<int>(r, 1);
1046  ld->link = sqliteConnector_.getData<string>(r, 2);
1047  ld->viewState = sqliteConnector_.getData<string>(r, 3);
1048  ld->updateTime = sqliteConnector_.getData<string>(r, 4);
1049  ld->viewMetadata = sqliteConnector_.getData<string>(r, 5);
1051  linkDescriptorMapById_[ld->linkId] = ld;
1052  }
1053 
1054  /* rebuild map linking logical tables to corresponding physical ones */
1055  string logicalToPhysicalTableMapQuery(
1056  "SELECT logical_table_id, physical_table_id "
1057  "FROM mapd_logical_to_physical");
1058  sqliteConnector_.query(logicalToPhysicalTableMapQuery);
1059  numRows = sqliteConnector_.getNumRows();
1060  for (size_t r = 0; r < numRows; ++r) {
1061  int32_t logical_tb_id = sqliteConnector_.getData<int>(r, 0);
1062  int32_t physical_tb_id = sqliteConnector_.getData<int>(r, 1);
1063  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(logical_tb_id);
1064  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
1065  /* add new entity to the map logicalToPhysicalTableMapById_ */
1066  std::vector<int32_t> physicalTables;
1067  physicalTables.push_back(physical_tb_id);
1068  const auto it_ok =
1069  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
1070  CHECK(it_ok.second);
1071  } else {
1072  /* update map logicalToPhysicalTableMapById_ */
1073  physicalTableIt->second.push_back(physical_tb_id);
1074  }
1075  }
1076 }
std::string virtualExpr
std::string partitions
T getData(const int row, const int col)
SQLTypes
Definition: sqltypes.h:41
std::string tableName
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:325
EncodingType
Definition: encodetypes.h:22
SqliteConnector sqliteConnector_
Definition: Catalog.h:331
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:327
void set_size(int s)
Definition: sqltypes.h:424
void query(const std::string &queryString)
std::string fragments
HOST DEVICE void set_type(SQLTypes t)
Definition: sqltypes.h:416
void set_dimension(int d)
Definition: sqltypes.h:418
void set_scale(int s)
Definition: sqltypes.h:421
void set_compression(EncodingType c)
Definition: sqltypes.h:426
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:326
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:328
std::string to_string(char const *&&v)
void set_notnull(bool n)
Definition: sqltypes.h:423
std::string chunks
size_t getNumRows() const
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:417
std::string getUserFromId(const int32_t id)
Definition: Catalog.cpp:882
void setDeletedColumnUnlocked(const TableDescriptor *td, const ColumnDescriptor *cd)
Definition: Catalog.cpp:2375
std::tuple< int, int > ColumnIdKey
Definition: Catalog.h:248
std::tuple< int, std::string > ColumnKey
Definition: Catalog.h:246
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:324
specifies the content in-memory of a row in the column metadata table
std::string keyMetainfo
std::string to_upper(const std::string &str)
Fragmenter_Namespace::FragmenterType fragType
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:101
bool isNull(const int row, const int col) const
void set_comp_param(int p)
Definition: sqltypes.h:427
int get_physical_cols() const
Definition: sqltypes.h:347
#define CHECK(condition)
Definition: Logger.h:193
Descriptor for a dictionary for a string columne.
bool is_geometry() const
Definition: sqltypes.h:489
std::string viewSQL
SQLTypeInfo columnType
specifies the content in-memory of a row in the table metadata table
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:102
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:330
Fragmenter_Namespace::AbstractFragmenter * fragmenter
std::string columnName
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:338
TableDescriptorMap tableDescriptorMap_
Definition: Catalog.h:323
LinkDescriptorMap linkDescriptorMap_
Definition: Catalog.h:329
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ calculateSHA1()

std::string Catalog_Namespace::Catalog::calculateSHA1 ( const std::string &  data)
protected

Definition at line 2975 of file Catalog.cpp.

Referenced by createLink().

2975  {
2976  boost::uuids::detail::sha1 sha1;
2977  unsigned int digest[5];
2978  sha1.process_bytes(data.c_str(), data.length());
2979  sha1.get_digest(digest);
2980  std::stringstream ss;
2981  for (size_t i = 0; i < 5; i++) {
2982  ss << std::hex << digest[i];
2983  }
2984  return ss.str();
2985 }
+ Here is the caller graph for this function:

◆ CheckAndExecuteMigrations()

void Catalog_Namespace::Catalog::CheckAndExecuteMigrations ( )
protected

Definition at line 861 of file Catalog.cpp.

References recordOwnershipOfObjectsInObjectPermissions(), updateDeletedColumnIndicator(), updateDictionaryNames(), updateDictionarySchema(), updateFixlenArrayColumns(), updateFrontendViewAndLinkUsers(), updateFrontendViewSchema(), updateFrontendViewsToDashboards(), updateLinkSchema(), updateLogicalToPhysicalTableLinkSchema(), updatePageSize(), and updateTableDescriptorSchema().

Referenced by Catalog().

861  {
870  updatePageSize();
874 }
void updateFrontendViewAndLinkUsers()
Definition: Catalog.cpp:354
void recordOwnershipOfObjectsInObjectPermissions()
Definition: Catalog.cpp:555
void updateFrontendViewsToDashboards()
Definition: Catalog.cpp:108
void updateLogicalToPhysicalTableLinkSchema()
Definition: Catalog.cpp:494
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ CheckAndExecuteMigrationsPostBuildMaps()

void Catalog_Namespace::Catalog::CheckAndExecuteMigrationsPostBuildMaps ( )
protected

Definition at line 876 of file Catalog.cpp.

References checkDateInDaysColumnMigration(), and createDashboardSystemRoles().

Referenced by Catalog().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ checkDateInDaysColumnMigration()

void Catalog_Namespace::Catalog::checkDateInDaysColumnMigration ( )
protected

Definition at line 682 of file Catalog.cpp.

References logger::ERROR, getCurrentDB(), SqliteConnector::getData(), Executor::getExecutor(), SqliteConnector::getNumRows(), logger::INFO, kDATE, kENCODING_DATE_IN_DAYS, LOG, MAPD_VERSION, SqliteConnector::query(), SqliteConnector::query_with_text_params(), TableOptimizer::recomputeMetadata(), sqliteConnector_, tableDescriptorMapById_, and to_string().

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

682  {
683  cat_sqlite_lock sqlite_lock(this);
684  std::vector<int> tables_migrated = {};
685  std::unordered_map<int, std::vector<std::string>> tables_to_migrate;
686  sqliteConnector_.query("BEGIN TRANSACTION");
687  try {
689  "select name from sqlite_master WHERE type='table' AND "
690  "name='mapd_version_history'");
691  if (sqliteConnector_.getNumRows() == 0) {
693  "CREATE TABLE mapd_version_history(version integer, migration_history text "
694  "unique)");
696  "CREATE TABLE mapd_date_in_days_column_migration_tmp(table_id integer primary "
697  "key)");
698  } else {
700  "select * from mapd_version_history where migration_history = "
701  "'date_in_days_column'");
702  if (sqliteConnector_.getNumRows() != 0) {
703  // no need for further execution
704  sqliteConnector_.query("END TRANSACTION");
705  return;
706  }
707  LOG(INFO) << "Performing Date in days columns migration.";
709  "select name from sqlite_master where type='table' AND "
710  "name='mapd_date_in_days_column_migration_tmp'");
711  if (sqliteConnector_.getNumRows() != 0) {
713  "select table_id from mapd_date_in_days_column_migration_tmp");
714  if (sqliteConnector_.getNumRows() != 0) {
715  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
716  tables_migrated.push_back(sqliteConnector_.getData<int>(i, 0));
717  }
718  }
719  } else {
721  "CREATE TABLE mapd_date_in_days_column_migration_tmp(table_id integer "
722  "primary key)");
723  }
724  }
726  "SELECT tables.tableid, tables.name, columns.name FROM mapd_tables tables, "
727  "mapd_columns columns where tables.tableid = columns.tableid AND "
728  "columns.coltype = ?1 AND columns.compression = ?2",
729  std::vector<std::string>{
730  std::to_string(static_cast<int>(SQLTypes::kDATE)),
732  if (sqliteConnector_.getNumRows() != 0) {
733  for (size_t i = 0; i < sqliteConnector_.getNumRows(); i++) {
734  tables_to_migrate[sqliteConnector_.getData<int>(i, 0)] = {
735  sqliteConnector_.getData<std::string>(i, 1),
736  sqliteConnector_.getData<std::string>(i, 2)};
737  }
738  }
739  } catch (const std::exception& e) {
740  LOG(ERROR) << "Failed to complete migraion on date in days column: " << e.what();
741  sqliteConnector_.query("ROLLBACK");
742  throw;
743  }
744  sqliteConnector_.query("END TRANSACTION");
745 
746  for (auto& id_names : tables_to_migrate) {
747  if (std::find(tables_migrated.begin(), tables_migrated.end(), id_names.first) ==
748  tables_migrated.end()) {
749  sqliteConnector_.query("BEGIN TRANSACTION");
750  try {
751  LOG(INFO) << "Table: " << id_names.second[0]
752  << " may suffer from issues with DATE column: " << id_names.second[1]
753  << ". Running an OPTIMIZE command to solve any issues with metadata.";
754 
755  auto executor = Executor::getExecutor(getCurrentDB().dbId);
756  TableDescriptorMapById::iterator tableDescIt =
757  tableDescriptorMapById_.find(id_names.first);
758  if (tableDescIt == tableDescriptorMapById_.end()) {
759  throw runtime_error("Table descriptor does not exist for table " +
760  id_names.second[0] + " does not exist.");
761  }
762  auto td = tableDescIt->second;
763  TableOptimizer optimizer(td, executor.get(), *this);
764  optimizer.recomputeMetadata();
765 
767  "INSERT INTO mapd_date_in_days_column_migration_tmp VALUES(?)",
768  std::vector<std::string>{std::to_string(id_names.first)});
769  } catch (const std::exception& e) {
770  LOG(ERROR) << "Failed to complete migraion on date in days column: " << e.what();
771  sqliteConnector_.query("ROLLBACK");
772  throw;
773  }
774  sqliteConnector_.query("COMMIT");
775  }
776  }
777 
778  sqliteConnector_.query("BEGIN TRANSACTION");
779  try {
780  sqliteConnector_.query("DROP TABLE mapd_date_in_days_column_migration_tmp");
782  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
783  std::vector<std::string>{std::to_string(MAPD_VERSION), "date_in_days_column"});
784  } catch (const std::exception& e) {
785  LOG(ERROR) << "Failed to complete migraion on date in days column: " << e.what();
786  sqliteConnector_.query("ROLLBACK");
787  throw;
788  }
789  sqliteConnector_.query("END TRANSACTION");
790  LOG(INFO) << "Migration successfull on Date in days columns";
791 }
T getData(const int row, const int col)
static std::shared_ptr< Executor > getExecutor(const int db_id, const std::string &debug_dir="", const std::string &debug_file="", const MapDParameters mapd_parameters=MapDParameters())
Definition: Execute.cpp:127
#define LOG(tag)
Definition: Logger.h:188
SqliteConnector sqliteConnector_
Definition: Catalog.h:331
void query_with_text_params(std::string const &query_only)
void query(const std::string &queryString)
Driver for running cleanup processes on a table. TableOptimizer provides functions for various cleanu...
std::string to_string(char const *&&v)
size_t getNumRows() const
static const int32_t MAPD_VERSION
Definition: release.h:33
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:324
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:176
Definition: sqltypes.h:56
void recomputeMetadata() const
Recomputes per-chunk metadata for each fragment in the table. Updates and deletes can cause chunk met...
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:102
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ checkMetadataForDeletedRecs()

const bool Catalog_Namespace::Catalog::checkMetadataForDeletedRecs ( int  dbId,
int  tableId,
int  columnId 
) const

Definition at line 2317 of file Catalog.cpp.

References dataMgr_.

Referenced by getDeletedColumnIfRowsDeleted().

2319  {
2320  // check if there are rows deleted by examining metadata for the deletedColumn metadata
2321  ChunkKey chunkKeyPrefix = {dbId, tableId, columnId};
2322  std::vector<std::pair<ChunkKey, ChunkMetadata>> chunkMetadataVec;
2323  dataMgr_->getChunkMetadataVecForKeyPrefix(chunkMetadataVec, chunkKeyPrefix);
2324  int64_t chunk_max{0};
2325 
2326  for (auto cm : chunkMetadataVec) {
2327  chunk_max = cm.second.chunkStats.max.tinyintval;
2328  // delete has occured
2329  if (chunk_max == 1) {
2330  return true;
2331  }
2332  }
2333  return false;
2334 }
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:333
std::vector< int > ChunkKey
Definition: types.h:35
+ Here is the caller graph for this function:

◆ checkpoint()

void Catalog_Namespace::Catalog::checkpoint ( const int  logicalTableId) const

Definition at line 3074 of file Catalog.cpp.

References Data_Namespace::DataMgr::checkpoint(), getCurrentDB(), getDataMgr(), getMetadataForTable(), and getPhysicalTablesDescriptors().

Referenced by Importer_NS::Loader::checkpoint(), and TableOptimizer::vacuumDeletedRows().

3074  {
3075  const auto td = getMetadataForTable(logicalTableId);
3076  const auto shards = getPhysicalTablesDescriptors(td);
3077  for (const auto shard : shards) {
3078  getDataMgr().checkpoint(getCurrentDB().dbId, shard->tableId);
3079  }
3080 }
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:177
void checkpoint(const int db_id, const int tb_id)
Definition: DataMgr.cpp:406
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logicalTableDesc) const
Definition: Catalog.cpp:3042
const DBMetadata & getCurrentDB() const
Definition: Catalog.h:176
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ createDashboard()

int32_t Catalog_Namespace::Catalog::createDashboard ( DashboardDescriptor vd)

Definition at line 2841 of file Catalog.cpp.

References addFrontendViewToMap(), createOrUpdateDashboardSystemRole(), currentDB_, DashboardDescriptor::dashboardId, DashboardDescriptor::dashboardMetadata, DashboardDescriptor::dashboardName, DashboardDescriptor::dashboardState, DashboardDescriptor::dashboardSystemRoleName, Catalog_Namespace::DBMetadata::dbId, generate_dashboard_system_rolename(), SqliteConnector::getData(), SqliteConnector::getNumRows(), DashboardDescriptor::imageHash, SqliteConnector::query(), SqliteConnector::query_with_text_params(), sqliteConnector_, to_string(), DashboardDescriptor::updateTime, and DashboardDescriptor::userId.

2841  {
2842  {
2843  cat_write_lock write_lock(this);
2844  cat_sqlite_lock sqlite_lock(this);
2845  sqliteConnector_.query("BEGIN TRANSACTION");
2846  try {
2847  // TODO(andrew): this should be an upsert
2849  "SELECT id FROM mapd_dashboards WHERE name = ? and userid = ?",
2850  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
2851  if (sqliteConnector_.getNumRows() > 0) {
2853  "UPDATE mapd_dashboards SET state = ?, image_hash = ?, metadata = ?, "
2854  "update_time = "
2855  "datetime('now') where name = ? "
2856  "and userid = ?",
2857  std::vector<std::string>{vd.dashboardState,
2858  vd.imageHash,
2859  vd.dashboardMetadata,
2860  vd.dashboardName,
2861  std::to_string(vd.userId)});
2862  } else {
2864  "INSERT INTO mapd_dashboards (name, state, image_hash, metadata, "
2865  "update_time, "
2866  "userid) "
2867  "VALUES "
2868  "(?,?,?,?, "
2869  "datetime('now'), ?)",
2870  std::vector<std::string>{vd.dashboardName,
2871  vd.dashboardState,
2872  vd.imageHash,
2873  vd.dashboardMetadata,
2874  std::to_string(vd.userId)});
2875  }
2876  } catch (std::exception& e) {
2877  sqliteConnector_.query("ROLLBACK TRANSACTION");
2878  throw;
2879  }
2880  sqliteConnector_.query("END TRANSACTION");
2881 
2882  // now get the auto generated dashboardId
2883  try {
2885  "SELECT id, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_dashboards "
2886  "WHERE name = ? and userid = ?",
2887  std::vector<std::string>{vd.dashboardName, std::to_string(vd.userId)});
2888  vd.dashboardId = sqliteConnector_.getData<int>(0, 0);
2889  vd.updateTime = sqliteConnector_.getData<std::string>(0, 1);
2890  } catch (std::exception& e) {
2891  throw;
2892  }
2896  }
2897  // NOTE(wamsi): Transactionally unsafe
2900  return vd.dashboardId;
2901 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:331
void query_with_text_params(std::string const &query_only)
void query(const std::string &queryString)
void addFrontendViewToMap(DashboardDescriptor &vd)
Definition: Catalog.cpp:1210
std::string to_string(char const *&&v)
size_t getNumRows() const
std::string dashboardSystemRoleName
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1251
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:101
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:102
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)
+ Here is the call graph for this function:

◆ createDashboardSystemRoles()

void Catalog_Namespace::Catalog::createDashboardSystemRoles ( )
protected

Definition at line 793 of file Catalog.cpp.

References createOrUpdateDashboardSystemRole(), currentDB_, Catalog_Namespace::DBMetadata::dbId, logger::ERROR, generate_dashboard_system_rolename(), SqliteConnector::getData(), Catalog_Namespace::SysCatalog::getGranteesOfSharedDashboards(), SqliteConnector::getNumRows(), Catalog_Namespace::SysCatalog::getRoleGrantee(), Catalog_Namespace::SysCatalog::grantRoleBatch(), logger::INFO, Catalog_Namespace::SysCatalog::instance(), LOG, MAPD_VERSION, SqliteConnector::query(), SqliteConnector::query_with_text_params(), run_benchmark_import::result, sqliteConnector_, and to_string().

Referenced by CheckAndExecuteMigrationsPostBuildMaps().

793  {
794  std::unordered_map<std::string, std::pair<int, std::string>> dashboards;
795  std::vector<std::string> dashboard_ids;
796  static const std::string migration_name{"dashboard_roles_migration"};
797  {
798  cat_sqlite_lock sqlite_lock(this);
799  sqliteConnector_.query("BEGIN TRANSACTION");
800  try {
801  // migration_history should be present in all catalogs by now
802  // if not then would be created before this migration
804  "select * from mapd_version_history where migration_history = '" +
805  migration_name + "'");
806  if (sqliteConnector_.getNumRows() != 0) {
807  // no need for further execution
808  sqliteConnector_.query("END TRANSACTION");
809  return;
810  }
811  LOG(INFO) << "Performing dashboard internal roles Migration.";
812  sqliteConnector_.query("select id, userid, metadata from mapd_dashboards");
813  for (size_t i = 0; i < sqliteConnector_.getNumRows(); ++i) {
816  sqliteConnector_.getData<string>(i, 0)))) {
817  // Successfully created roles during previous migration/crash
818  // No need to include them
819  continue;
820  }
821  dashboards[sqliteConnector_.getData<string>(i, 0)] = std::make_pair(
822  sqliteConnector_.getData<int>(i, 1), sqliteConnector_.getData<string>(i, 2));
823  dashboard_ids.push_back(sqliteConnector_.getData<string>(i, 0));
824  }
825  } catch (const std::exception& e) {
826  sqliteConnector_.query("ROLLBACK TRANSACTION");
827  throw;
828  }
829  sqliteConnector_.query("END TRANSACTION");
830  }
831  // All current grantees with shared dashboards.
832  const auto active_grantees =
834 
835  try {
836  // NOTE(wamsi): Transactionally unsafe
837  for (auto dash : dashboards) {
838  createOrUpdateDashboardSystemRole(dash.second.second,
839  dash.second.first,
841  std::to_string(currentDB_.dbId), dash.first));
842  auto result = active_grantees.find(dash.first);
843  if (result != active_grantees.end()) {
846  dash.first)},
847  result->second);
848  }
849  }
851  "INSERT INTO mapd_version_history(version, migration_history) values(?,?)",
852  std::vector<std::string>{std::to_string(MAPD_VERSION), migration_name});
853  } catch (const std::exception& e) {
854  LOG(ERROR) << "Failed to create dashboard system roles during migration: "
855  << e.what();
856  throw;
857  }
858  LOG(INFO) << "Successfully created dashboard system roles during migration.";
859 }
T getData(const int row, const int col)
#define LOG(tag)
Definition: Logger.h:188
SqliteConnector sqliteConnector_
Definition: Catalog.h:331
void query_with_text_params(std::string const &query_only)
void query(const std::string &queryString)
std::string to_string(char const *&&v)
void grantRoleBatch(const std::vector< std::string > &roles, const std::vector< std::string > &grantees)
size_t getNumRows() const
static SysCatalog & instance()
Definition: SysCatalog.h:257
static const int32_t MAPD_VERSION
Definition: release.h:33
void createOrUpdateDashboardSystemRole(const std::string &view_meta, const int32_t &user_id, const std::string &dash_role_name)
Definition: Catalog.cpp:1251
std::unordered_map< std::string, std::vector< std::string > > getGranteesOfSharedDashboards(const std::vector< std::string > &dashboard_ids)
Role * getRoleGrantee(const std::string &name) const
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:102
std::string generate_dashboard_system_rolename(const std::string &db_id, const std::string &dash_id)
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ createLink()

std::string Catalog_Namespace::Catalog::createLink ( LinkDescriptor ld,
size_t  min_length 
)

Definition at line 2987 of file Catalog.cpp.

References addLinkToMap(), calculateSHA1(), SqliteConnector::getData(), SqliteConnector::getNumRows(), LinkDescriptor::link, LinkDescriptor::linkId, SqliteConnector::query(), SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), sqliteConnector_, to_string(), LinkDescriptor::updateTime, LinkDescriptor::userId, LinkDescriptor::viewMetadata, and LinkDescriptor::viewState.

2987  {
2988  cat_write_lock write_lock(this);
2989  cat_sqlite_lock sqlite_lock(this);
2990  sqliteConnector_.query("BEGIN TRANSACTION");
2991  try {
2993  .substr(0, 8);
2995  "SELECT linkid FROM mapd_links WHERE link = ? and userid = ?",
2996  std::vector<std::string>{ld.link, std::to_string(ld.userId)});
2997  if (sqliteConnector_.getNumRows() > 0) {
2999  "UPDATE mapd_links SET update_time = datetime('now') WHERE userid = ? AND link "
3000  "= ?",
3001  std::vector<std::string>{std::to_string(ld.userId), ld.link});
3002  } else {
3004  "INSERT INTO mapd_links (userid, link, view_state, view_metadata, update_time) "
3005  "VALUES (?,?,?,?, "
3006  "datetime('now'))",
3007  std::vector<std::string>{
3008  std::to_string(ld.userId), ld.link, ld.viewState, ld.viewMetadata});
3009  }
3010  // now get the auto generated dashid
3012  "SELECT linkid, strftime('%Y-%m-%dT%H:%M:%SZ', update_time) FROM mapd_links "
3013  "WHERE link = ?",
3014  ld.link);
3015  ld.linkId = sqliteConnector_.getData<int>(0, 0);
3016  ld.updateTime = sqliteConnector_.getData<std::string>(0, 1);
3017  } catch (std::exception& e) {
3018  sqliteConnector_.query("ROLLBACK TRANSACTION");
3019  throw;
3020  }
3021  sqliteConnector_.query("END TRANSACTION");
3022  addLinkToMap(ld);
3023  return ld.link;
3024 }
T getData(const int row, const int col)
SqliteConnector sqliteConnector_
Definition: Catalog.h:331
void query_with_text_params(std::string const &query_only)
void query(const std::string &queryString)
std::string to_string(char const *&&v)
size_t getNumRows() const
std::string calculateSHA1(const std::string &data)
Definition: Catalog.cpp:2975
void addLinkToMap(LinkDescriptor &ld)
Definition: Catalog.cpp:1292
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:101
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:102
void query_with_text_param(const std::string &queryString, const std::string &text_param)
+ Here is the call graph for this function:

◆ createOrUpdateDashboardSystemRole()

void Catalog_Namespace::Catalog::createOrUpdateDashboardSystemRole ( const std::string &  view_meta,
const int32_t &  user_id,
const std::string &  dash_role_name 
)
protected

Definition at line 1251 of file Catalog.cpp.

References Catalog_Namespace::SysCatalog::createRole(), Grantee::findDbObject(), Grantee::getDbObjects(), Catalog_Namespace::SysCatalog::getRoleGrantee(), Catalog_Namespace::SysCatalog::grantDBObjectPrivilegesBatch(), Catalog_Namespace::SysCatalog::instance(), parseDashboardObjects(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivileges(), TableDBObjectType, and ViewDBObjectType.

Referenced by createDashboard(), createDashboardSystemRoles(), and replaceDashboard().

1253  {
1254  auto objects = parseDashboardObjects(view_meta, user_id);
1255  Role* rl = SysCatalog::instance().getRoleGrantee(dash_role_name);
1256  if (!rl) {
1257  // Dashboard role does not exist
1258  // create role and grant privileges
1259  // NOTE(wamsi): Transactionally unsafe
1260  SysCatalog::instance().createRole(dash_role_name, false);
1261  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1262  } else {
1263  // Dashboard system role already exists
1264  // Add/remove privileges on objects
1265  auto ex_objects = rl->getDbObjects(true);
1266  for (auto key : *ex_objects | boost::adaptors::map_keys) {
1267  if (key.permissionType != TableDBObjectType &&
1268  key.permissionType != ViewDBObjectType) {
1269  continue;
1270  }
1271  bool found = false;
1272  for (auto obj : objects) {
1273  found = key == obj.getObjectKey() ? true : false;
1274  if (found) {
1275  break;
1276  }
1277  }
1278  if (!found) {
1279  // revoke privs on object since the object is no
1280  // longer used by the dashboard as source
1281  // NOTE(wamsi): Transactionally unsafe
1283  dash_role_name, *rl->findDbObject(key, true), *this);
1284  }
1285  }
1286  // Update privileges on remaining objects
1287  // NOTE(wamsi): Transactionally unsafe
1288  SysCatalog::instance().grantDBObjectPrivilegesBatch({dash_role_name}, objects, *this);
1289  }
1290 }
void revokeDBObjectPrivileges(const std::string &grantee, const DBObject &object, const Catalog_Namespace::Catalog &catalog)
DBObject * findDbObject(const DBObjectKey &objectKey, bool only_direct) const
Definition: Grantee.cpp:72
Definition: Grantee.h:76
void createRole(const std::string &roleName, const bool &userPrivateRole=false)
static SysCatalog & instance()
Definition: SysCatalog.h:257
void grantDBObjectPrivilegesBatch(const std::vector< std::string > &grantees, const std::vector< DBObject > &objects, const Catalog_Namespace::Catalog &catalog)
const DBObjectMap * getDbObjects(bool only_direct) const
Definition: Grantee.h:56
Role * getRoleGrantee(const std::string &name) const
std::vector< DBObject > parseDashboardObjects(const std::string &view_meta, const int &user_id)
Definition: Catalog.cpp:1221
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ createShardedTable()

void Catalog_Namespace::Catalog::createShardedTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs 
)

Definition at line 2528 of file Catalog.cpp.

References CHECK, createTable(), generatePhysicalTableName(), logicalToPhysicalTableMapById_, TableDescriptor::nShards, TableDescriptor::shard, TableDescriptor::tableId, TableDescriptor::tableName, and updateLogicalToPhysicalTableMap().

2531  {
2532  cat_write_lock write_lock(this);
2533 
2534  /* create logical table */
2535  TableDescriptor tdl(td);
2536  createTable(tdl, cols, shared_dict_defs, true); // create logical table
2537  int32_t logical_tb_id = tdl.tableId;
2538 
2539  /* create physical tables and link them to the logical table */
2540  std::vector<int32_t> physicalTables;
2541  for (int32_t i = 1; i <= td.nShards; i++) {
2542  TableDescriptor tdp(td);
2543  tdp.tableName = generatePhysicalTableName(tdp.tableName, i);
2544  tdp.shard = i - 1;
2545  createTable(tdp, cols, shared_dict_defs, false); // create physical table
2546  int32_t physical_tb_id = tdp.tableId;
2547 
2548  /* add physical table to the vector of physical tables */
2549  physicalTables.push_back(physical_tb_id);
2550  }
2551 
2552  if (!physicalTables.empty()) {
2553  /* add logical to physical tables correspondence to the map */
2554  const auto it_ok =
2555  logicalToPhysicalTableMapById_.emplace(logical_tb_id, physicalTables);
2556  CHECK(it_ok.second);
2557  /* update sqlite mapd_logical_to_physical in sqlite database */
2558  updateLogicalToPhysicalTableMap(logical_tb_id);
2559  }
2560 }
void createTable(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs, bool isLogicalTable)
Definition: Catalog.cpp:1966
std::string generatePhysicalTableName(const std::string &logicalTableName, const int32_t &shardNumber)
Definition: Catalog.cpp:3124
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:101
#define CHECK(condition)
Definition: Logger.h:193
specifies the content in-memory of a row in the table metadata table
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:338
void updateLogicalToPhysicalTableMap(const int32_t logical_tb_id)
Definition: Catalog.cpp:508
+ Here is the call graph for this function:

◆ createTable()

void Catalog_Namespace::Catalog::createTable ( TableDescriptor td,
const std::list< ColumnDescriptor > &  columns,
const std::vector< Parser::SharedDictionaryDef > &  shared_dict_defs,
bool  isLogicalTable 
)

Definition at line 1966 of file Catalog.cpp.

References addTableToMap(), calciteMgr_, ColumnDescriptor::columnId, TableDescriptor::columnIdBySpi_, ColumnDescriptor::columnName, ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, dict_ref_t::dictId, Data_Namespace::DISK_LEVEL, expandGeoColumn(), TableDescriptor::fragPageSize, TableDescriptor::fragType, g_serialize_temp_tables, g_test_against_columnId_gap, SQLTypeInfoCore< TYPE_FACET_PACK >::get_comp_param(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_compression(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_dimension(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_notnull(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_scale(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_size(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_subtype(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_type(), SqliteConnector::getData(), TableDescriptor::hasDeletedCol, SQLTypeInfoCore< TYPE_FACET_PACK >::is_array(), IS_GEO, ColumnDescriptor::isDeletedCol, ColumnDescriptor::isGeoPhyCol, ColumnDescriptor::isSystemCol, TableDescriptor::isView, ColumnDescriptor::isVirtualCol, kBIGINT, kBOOLEAN, kENCODING_DICT, TableDescriptor::keyMetainfo, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRows, TableDescriptor::nColumns, nextTempDictId_, nextTempTableId_, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::persistenceLevel, SqliteConnector::query(), SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), removeTableFromMap(), serializeTableJsonUnlocked(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_comp_param(), SQLTypeInfoCore< TYPE_FACET_PACK >::set_size(), setColumnDictionary(), setColumnSharedDictionary(), TableDescriptor::shard, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, sqliteConnector_, ColumnDescriptor::tableId, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), TableDescriptor::userId, TableDescriptor::viewSQL, and ColumnDescriptor::virtualExpr.

Referenced by createShardedTable().

1970  {
1971  cat_write_lock write_lock(this);
1972  list<ColumnDescriptor> cds;
1973  list<DictDescriptor> dds;
1974  std::set<std::string> toplevel_column_names;
1975  list<ColumnDescriptor> columns;
1976  for (auto cd : cols) {
1977  if (cd.columnName == "rowid") {
1978  throw std::runtime_error(
1979  "Cannot create column with name rowid. rowid is a system defined column.");
1980  }
1981  columns.push_back(cd);
1982  toplevel_column_names.insert(cd.columnName);
1983  if (cd.columnType.is_geometry()) {
1984  expandGeoColumn(cd, columns);
1985  }
1986  }
1987 
1988  ColumnDescriptor cd;
1989  // add row_id column -- Must be last column in the table
1990  cd.columnName = "rowid";
1991  cd.isSystemCol = true;
1992  cd.columnType = SQLTypeInfo(kBIGINT, true);
1993 #ifdef MATERIALIZED_ROWID
1994  cd.isVirtualCol = false;
1995 #else
1996  cd.isVirtualCol = true;
1997  cd.virtualExpr = "MAPD_FRAG_ID * MAPD_ROWS_PER_FRAG + MAPD_FRAG_ROW_ID";
1998 #endif
1999  columns.push_back(cd);
2000  toplevel_column_names.insert(cd.columnName);
2001 
2002  if (td.hasDeletedCol) {
2003  ColumnDescriptor cd_del;
2004  cd_del.columnName = "$deleted$";
2005  cd_del.isSystemCol = true;
2006  cd_del.isVirtualCol = false;
2007  cd_del.columnType = SQLTypeInfo(kBOOLEAN, true);
2008  cd_del.isDeletedCol = true;
2009 
2010  columns.push_back(cd_del);
2011  }
2012 
2013  td.nColumns = columns.size();
2014  cat_sqlite_lock sqlite_lock(this);
2015  sqliteConnector_.query("BEGIN TRANSACTION");
2017  try {
2019  "INSERT INTO mapd_tables (name, userid, ncolumns, isview, fragments, "
2020  "frag_type, max_frag_rows, "
2021  "max_chunk_size, "
2022  "frag_page_size, max_rows, partitions, shard_column_id, shard, num_shards, "
2023  "sort_column_id, "
2024  "key_metainfo) VALUES (?, ?, ?, "
2025  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2026 
2027  std::vector<std::string>{td.tableName,
2028  std::to_string(td.userId),
2030  std::to_string(td.isView),
2031  "",
2036  std::to_string(td.maxRows),
2037  td.partitions,
2039  std::to_string(td.shard),
2040  std::to_string(td.nShards),
2042  td.keyMetainfo});
2043 
2044  // now get the auto generated tableid
2046  "SELECT tableid FROM mapd_tables WHERE name = ?", td.tableName);
2047  td.tableId = sqliteConnector_.getData<int>(0, 0);
2048  int colId = 1;
2049  for (auto cd : columns) {
2051  const bool is_foreign_col =
2052  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2053  if (!is_foreign_col) {
2054  setColumnDictionary(cd, dds, td, isLogicalTable);
2055  }
2056  }
2057 
2058  if (toplevel_column_names.count(cd.columnName)) {
2059  // make up colId gap for sanity test (begin with 1 bc much code depends on it!)
2060  if (colId > 1) {
2061  colId += g_test_against_columnId_gap;
2062  }
2063  if (!cd.isGeoPhyCol) {
2064  td.columnIdBySpi_.push_back(colId);
2065  }
2066  }
2067 
2069  "INSERT INTO mapd_columns (tableid, columnid, name, coltype, colsubtype, "
2070  "coldim, colscale, is_notnull, "
2071  "compression, comp_param, size, chunks, is_systemcol, is_virtualcol, "
2072  "virtual_expr, is_deletedcol) "
2073  "VALUES (?, ?, ?, ?, ?, "
2074  "?, "
2075  "?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
2076  std::vector<std::string>{std::to_string(td.tableId),
2077  std::to_string(colId),
2078  cd.columnName,
2087  "",
2090  cd.virtualExpr,
2092  cd.tableId = td.tableId;
2093  cd.columnId = colId++;
2094  cds.push_back(cd);
2095  }
2096  if (td.isView) {
2098  "INSERT INTO mapd_views (tableid, sql) VALUES (?,?)",
2099  std::vector<std::string>{std::to_string(td.tableId), td.viewSQL});
2100  }
2101  } catch (std::exception& e) {
2102  sqliteConnector_.query("ROLLBACK TRANSACTION");
2103  throw;
2104  }
2105 
2106  } else { // Temporary table
2107  td.tableId = nextTempTableId_++;
2108  int colId = 1;
2109  for (auto cd : columns) {
2110  auto col_ti = cd.columnType;
2111  if (IS_GEO(col_ti.get_type())) {
2112  throw runtime_error("Geometry types in temporary tables are not supported.");
2113  }
2114 
2116  const bool is_foreign_col =
2117  setColumnSharedDictionary(cd, cds, dds, td, shared_dict_defs);
2118 
2119  if (!is_foreign_col) {
2120  // Create a new temporary dictionary
2121  std::string fileName("");
2122  std::string folderPath("");
2124  nextTempDictId_++;
2125  DictDescriptor dd(dict_ref,
2126  fileName,
2128  false,
2129  1,
2130  folderPath,
2131  true); // Is dictName (2nd argument) used?
2132  dds.push_back(dd);
2133  if (!cd.columnType.is_array()) {
2135  }
2136  cd.columnType.set_comp_param(dict_ref.dictId);
2137  }
2138  }
2139  cd.tableId = td.tableId;
2140  cd.columnId = colId++;
2141  td.columnIdBySpi_.push_back(cd.columnId);
2142  cds.push_back(cd);
2143  }
2144 
2146  serializeTableJsonUnlocked(&td, cds);
2147  }
2148  }
2149 
2150  try {
2151  addTableToMap(td, cds, dds);
2152  calciteMgr_->updateMetadata(currentDB_.dbName, td.tableName);
2153  } catch (std::exception& e) {
2154  sqliteConnector_.query("ROLLBACK TRANSACTION");
2155  removeTableFromMap(td.tableName, td.tableId, true);
2156  throw;
2157  }
2158 
2159  sqliteConnector_.query("END TRANSACTION");
2160 }
void serializeTableJsonUnlocked(const TableDescriptor *td, const std::list< ColumnDescriptor > &cds) const
Definition: Catalog.cpp:2162
std::string virtualExpr
std::string partitions
HOST DEVICE int get_size() const
Definition: sqltypes.h:336
T getData(const int row, const int col)
HOST DEVICE int get_dimension() const
Definition: sqltypes.h:328
std::string tableName
HOST DEVICE bool get_notnull() const
Definition: sqltypes.h:333
SqliteConnector sqliteConnector_
Definition: Catalog.h:331
void set_size(int s)
Definition: sqltypes.h:424
void query_with_text_params(std::string const &query_only)
HOST DEVICE int get_scale() const
Definition: sqltypes.h:331
HOST DEVICE SQLTypes get_type() const
Definition: sqltypes.h:326
void query(const std::string &queryString)
void addTableToMap(TableDescriptor &td, const std::list< ColumnDescriptor > &columns, const std::list< DictDescriptor > &dicts)
Definition: Catalog.cpp:1078
std::vector< int > columnIdBySpi_
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:334
std::string to_string(char const *&&v)
bool is_array() const
Definition: sqltypes.h:485
HOST DEVICE SQLTypes get_subtype() const
Definition: sqltypes.h:327
specifies the content in-memory of a row in the column metadata table
static void expandGeoColumn(const ColumnDescriptor &cd, std::list< ColumnDescriptor > &columns)
Definition: Catalog.cpp:1836
SQLTypeInfoCore< ArrayContextTypeSizer, ExecutorTypePackaging, DateTimeFacilities > SQLTypeInfo
Definition: sqltypes.h:852
std::string keyMetainfo
int g_test_against_columnId_gap
Definition: Catalog.cpp:80
bool g_serialize_temp_tables
Definition: Catalog.cpp:85
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:336
Fragmenter_Namespace::FragmenterType fragType
Data_Namespace::MemoryLevel persistenceLevel
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1128
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:101
void set_comp_param(int p)
Definition: sqltypes.h:427
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:335
bool setColumnSharedDictionary(ColumnDescriptor &cd, std::list< ColumnDescriptor > &cdd, std::list< DictDescriptor > &dds, const TableDescriptor td, const std::vector< Parser::SharedDictionaryDef > &shared_dict_defs)
Definition: Catalog.cpp:2418
Descriptor for a dictionary for a string columne.
std::string viewSQL
SQLTypeInfo columnType
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:102
void setColumnDictionary(ColumnDescriptor &cd, std::list< DictDescriptor > &dds, const TableDescriptor &td, const bool isLogicalTable)
Definition: Catalog.cpp:2487
std::string columnName
void query_with_text_param(const std::string &queryString, const std::string &text_param)
#define IS_GEO(T)
Definition: sqltypes.h:167
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ delDictionary()

void Catalog_Namespace::Catalog::delDictionary ( const ColumnDescriptor cd)

Definition at line 1668 of file Catalog.cpp.

References basePath_, ColumnDescriptor::columnName, ColumnDescriptor::columnType, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, SQLTypeInfoCore< TYPE_FACET_PACK >::get_comp_param(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_compression(), SQLTypeInfoCore< TYPE_FACET_PACK >::is_string(), SQLTypeInfoCore< TYPE_FACET_PACK >::is_string_array(), kENCODING_DICT, SqliteConnector::query_with_text_param(), File_Namespace::renameForDelete(), sqliteConnector_, string_dict_hosts_, tableDescriptorMapById_, ColumnDescriptor::tableId, TableDescriptor::tableName, and to_string().

Referenced by roll().

1668  {
1669  if (!(cd.columnType.is_string() || cd.columnType.is_string_array())) {
1670  return;
1671  }
1672  if (!(cd.columnType.get_compression() == kENCODING_DICT)) {
1673  return;
1674  }
1675  if (!(cd.columnType.get_comp_param() > 0)) {
1676  return;
1677  }
1678 
1679  const auto& td = *tableDescriptorMapById_[cd.tableId];
1680  const auto dictId = cd.columnType.get_comp_param();
1681  const DictRef dictRef(currentDB_.dbId, dictId);
1682  const auto dictName =
1683  td.tableName + "_" + cd.columnName + "_dict" + std::to_string(dictId);
1684  sqliteConnector_.query_with_text_param("DELETE FROM mapd_dictionaries WHERE name = ?",
1685  dictName);
1686  File_Namespace::renameForDelete(basePath_ + "/mapd_data/DB_" +
1687  std::to_string(currentDB_.dbId) + "_DICT_" +
1688  std::to_string(dictId));
1689 
1690  std::unique_ptr<StringDictionaryClient> client;
1691  if (!string_dict_hosts_.empty()) {
1692  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dictRef, true));
1693  }
1694  if (client) {
1695  client->drop(dictRef);
1696  }
1697 
1698  dictDescriptorMapByRef_.erase(dictRef);
1699 }
bool is_string_array() const
Definition: sqltypes.h:478
SqliteConnector sqliteConnector_
Definition: Catalog.h:331
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:335
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:327
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:334
std::string to_string(char const *&&v)
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:324
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:335
SQLTypeInfo columnType
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_<EPOCH>_<oldname>.
Definition: File.cpp:182
std::string columnName
void query_with_text_param(const std::string &queryString, const std::string &text_param)
bool is_string() const
Definition: sqltypes.h:477
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ deleteMetadataForDashboard() [1/2]

void Catalog_Namespace::Catalog::deleteMetadataForDashboard ( const std::string &  userId,
const std::string &  dashName 
)

Definition at line 1474 of file Catalog.cpp.

References dashboardDescriptorMap_, logger::ERROR, getMetadataForDashboard(), LOG, SqliteConnector::query(), SqliteConnector::query_with_text_params(), and sqliteConnector_.

Referenced by deleteMetadataForDashboard().

1475  {
1476  cat_write_lock write_lock(this);
1477 
1478  auto viewDescIt = dashboardDescriptorMap_.find(userId + ":" + dashName);
1479  if (viewDescIt == dashboardDescriptorMap_.end()) { // check to make sure view exists
1480  LOG(ERROR) << "No metadata for dashboard for user " << userId << " dashboard "
1481  << dashName << " does not exist in map";
1482  throw runtime_error("No metadata for dashboard for user " + userId + " dashboard " +
1483  dashName + " does not exist in map");
1484  }
1485  // found view in Map now remove it
1486  dashboardDescriptorMap_.erase(viewDescIt);
1487  // remove from DB
1488  cat_sqlite_lock sqlite_lock(this);
1489  sqliteConnector_.query("BEGIN TRANSACTION");
1490  try {
1492  "DELETE FROM mapd_dashboards WHERE name = ? and userid = ?",
1493  std::vector<std::string>{dashName, userId});
1494  } catch (std::exception& e) {
1495  sqliteConnector_.query("ROLLBACK TRANSACTION");
1496  throw;
1497  }
1498  sqliteConnector_.query("END TRANSACTION");
1499 }
#define LOG(tag)
Definition: Logger.h:188
SqliteConnector sqliteConnector_
Definition: Catalog.h:331
void query_with_text_params(std::string const &query_only)
void query(const std::string &queryString)
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:328
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:101
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:102
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ deleteMetadataForDashboard() [2/2]

void Catalog_Namespace::Catalog::deleteMetadataForDashboard ( const int32_t  dashboard_id)

Definition at line 1535 of file Catalog.cpp.

References DashboardDBObjectType, dashboardDescriptorMap_, deleteMetadataForDashboard(), getMetadataForLink(), Catalog_Namespace::SysCatalog::instance(), linkDescriptorMap_, name(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivilegesFromAll(), and to_string().

1535  {
1536  std::string userId;
1537  std::string name;
1538  bool found{false};
1539  {
1540  cat_read_lock read_lock(this);
1541  for (auto descp : dashboardDescriptorMap_) {
1542  auto dash = descp.second.get();
1543  if (dash->dashboardId == id) {
1544  userId = std::to_string(dash->userId);
1545  name = dash->dashboardName;
1546  found = true;
1547  break;
1548  }
1549  }
1550  }
1551  if (found) {
1552  // TODO: transactionally unsafe
1554  DBObject(id, DashboardDBObjectType), this);
1555  deleteMetadataForDashboard(userId, name);
1556  }
1557 }
void revokeDBObjectPrivilegesFromAll(DBObject object, Catalog *catalog)
void deleteMetadataForDashboard(const std::string &userId, const std::string &dashName)
Definition: Catalog.cpp:1474
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:328
std::string to_string(char const *&&v)
static SysCatalog & instance()
Definition: SysCatalog.h:257
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:100
std::string name() const
Definition: Catalog.h:219
+ Here is the call graph for this function:

◆ doDropTable()

void Catalog_Namespace::Catalog::doDropTable ( const TableDescriptor td)
protected

Definition at line 2702 of file Catalog.cpp.

References dropTableFromJsonUnlocked(), eraseTablePhysicalData(), g_serialize_temp_tables, SqliteConnector::getData(), SqliteConnector::getNumRows(), TableDescriptor::isView, kENCODING_DICT, SqliteConnector::query_with_text_param(), SqliteConnector::query_with_text_params(), sqliteConnector_, table_is_temporary(), TableDescriptor::tableId, TableDescriptor::tableName, and to_string().

Referenced by dropTable().

2702  {
2703  const int tableId = td->tableId;
2704  sqliteConnector_.query_with_text_param("DELETE FROM mapd_tables WHERE tableid = ?",
2705  std::to_string(tableId));
2707  "select comp_param from mapd_columns where compression = ? and tableid = ?",
2708  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
2709  int numRows = sqliteConnector_.getNumRows();
2710  std::vector<int> dict_id_list;
2711  for (int r = 0; r < numRows; ++r) {
2712  dict_id_list.push_back(sqliteConnector_.getData<int>(r, 0));
2713  }
2714  for (auto dict_id : dict_id_list) {
2716  "UPDATE mapd_dictionaries SET refcount = refcount - 1 WHERE dictid = ?",
2717  std::vector<std::string>{std::to_string(dict_id)});
2718  }
2720  "DELETE FROM mapd_dictionaries WHERE dictid in (select comp_param from "
2721  "mapd_columns where compression = ? "
2722  "and tableid = ?) and refcount = 0",
2723  std::vector<std::string>{std::to_string(kENCODING_DICT), std::to_string(tableId)});
2724  sqliteConnector_.query_with_text_param("DELETE FROM mapd_columns WHERE tableid = ?",
2725  std::to_string(tableId));
2726  if (td->isView) {
2727  sqliteConnector_.query_with_text_param("DELETE FROM mapd_views WHERE tableid = ?",
2728  std::to_string(tableId));
2729  }
2732  }
2734 }
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:3094
T getData(const int row, const int col)
std::string tableName
SqliteConnector sqliteConnector_
Definition: Catalog.h:331
void query_with_text_params(std::string const &query_only)
std::string to_string(char const *&&v)
size_t getNumRows() const
bool table_is_temporary(const TableDescriptor *td)
bool g_serialize_temp_tables
Definition: Catalog.cpp:85
void dropTableFromJsonUnlocked(const std::string &table_name) const
Definition: Catalog.cpp:2224
void query_with_text_param(const std::string &queryString, const std::string &text_param)
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ doTruncateTable()

void Catalog_Namespace::Catalog::doTruncateTable ( const TableDescriptor td)
protected

Definition at line 2580 of file Catalog.cpp.

References CHECK, CHECK_GE, columnDescriptorMapById_, Data_Namespace::CPU_LEVEL, currentDB_, dataMgr_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, dict_ref_t::dictId, DictDescriptor::dictIsTemp, DictDescriptor::dictRef, TableDescriptor::fragmenter, getMetadataForDict(), Data_Namespace::GPU_LEVEL, Catalog_Namespace::SysCatalog::instance(), kENCODING_DICT, File_Namespace::renameForDelete(), string_dict_hosts_, tableDescriptorMapById_, and TableDescriptor::tableId.

Referenced by truncateTable().

2580  {
2581  cat_write_lock write_lock(this);
2582 
2583  const int tableId = td->tableId;
2584  // must destroy fragmenter before deleteChunks is called.
2585  if (td->fragmenter != nullptr) {
2586  auto tableDescIt = tableDescriptorMapById_.find(tableId);
2587  delete td->fragmenter;
2588  tableDescIt->second->fragmenter = nullptr; // get around const-ness
2589  }
2590  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
2591  // assuming deleteChunksWithPrefix is atomic
2592  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
2593  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
2594 
2595  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
2596 
2597  std::unique_ptr<StringDictionaryClient> client;
2598  if (SysCatalog::instance().isAggregator()) {
2599  CHECK(!string_dict_hosts_.empty());
2600  DictRef dict_ref(currentDB_.dbId, -1);
2601  client.reset(new StringDictionaryClient(string_dict_hosts_.front(), dict_ref, true));
2602  }
2603  // clean up any dictionaries
2604  // delete all column descriptors for the table
2605  for (const auto& columnDescriptor : columnDescriptorMapById_) {
2606  auto cd = columnDescriptor.second;
2607  if (cd->tableId != td->tableId) {
2608  continue;
2609  }
2610  const int dict_id = cd->columnType.get_comp_param();
2611  // Dummy dictionaries created for a shard of a logical table have the id set to zero.
2612  if (cd->columnType.get_compression() == kENCODING_DICT && dict_id) {
2613  const DictRef dict_ref(currentDB_.dbId, dict_id);
2614  const auto dictIt = dictDescriptorMapByRef_.find(dict_ref);
2615  CHECK(dictIt != dictDescriptorMapByRef_.end());
2616  const auto& dd = dictIt->second;
2617  CHECK_GE(dd->refcount, 1);
2618  // if this is the only table using this dict reset the dict
2619  if (dd->refcount == 1) {
2620  // close the dictionary
2621  dd->stringDict.reset();
2622  File_Namespace::renameForDelete(dd->dictFolderPath);
2623  if (client) {
2624  client->drop(dd->dictRef);
2625  }
2626  if (!dd->dictIsTemp) {
2627  boost::filesystem::create_directory(dd->dictFolderPath);
2628  }
2629  }
2630 
2631  DictDescriptor* new_dd = new DictDescriptor(dd->dictRef,
2632  dd->dictName,
2633  dd->dictNBits,
2634  dd->dictIsShared,
2635  dd->refcount,
2636  dd->dictFolderPath,
2637  dd->dictIsTemp);
2638  dictDescriptorMapByRef_.erase(dictIt);
2639  // now create new Dict -- need to figure out what to do here for temp tables
2640  if (client) {
2641  client->create(new_dd->dictRef, new_dd->dictIsTemp);
2642  }
2643  dictDescriptorMapByRef_[new_dd->dictRef].reset(new_dd);
2645  }
2646  }
2647 }
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:335
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:327
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:333
#define CHECK_GE(x, y)
Definition: Logger.h:206
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:326
static SysCatalog & instance()
Definition: SysCatalog.h:257
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:324
int32_t dictId
Definition: DictRef.h:10
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1377
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:101
#define CHECK(condition)
Definition: Logger.h:193
std::vector< int > ChunkKey
Definition: types.h:35
Descriptor for a dictionary for a string columne.
void renameForDelete(const std::string directoryName)
Renames a directory to DELETE_ME_<EPOCH>_<oldname>.
Definition: File.cpp:182
Fragmenter_Namespace::AbstractFragmenter * fragmenter
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ dropTable()

void Catalog_Namespace::Catalog::dropTable ( const TableDescriptor td)

Definition at line 2669 of file Catalog.cpp.

References CHECK, doDropTable(), getMetadataForTable(), Catalog_Namespace::SysCatalog::instance(), TableDescriptor::isView, logicalToPhysicalTableMapById_, SqliteConnector::query(), SqliteConnector::query_with_text_param(), Catalog_Namespace::SysCatalog::revokeDBObjectPrivilegesFromAll(), sqliteConnector_, TableDBObjectType, TableDescriptor::tableId, TableDescriptor::tableName, to_string(), and ViewDBObjectType.

2669  {
2672  cat_write_lock write_lock(this);
2673  cat_sqlite_lock sqlite_lock(this);
2674  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
2675  sqliteConnector_.query("BEGIN TRANSACTION");
2676  try {
2677  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2678  // remove all corresponding physical tables if this is a logical table
2679  const auto physicalTables = physicalTableIt->second;
2680  CHECK(!physicalTables.empty());
2681  for (size_t i = 0; i < physicalTables.size(); i++) {
2682  int32_t physical_tb_id = physicalTables[i];
2683  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2684  CHECK(phys_td);
2685  doDropTable(phys_td);
2686  }
2687 
2688  // remove corresponding record from the logicalToPhysicalTableMap in sqlite database
2690  "DELETE FROM mapd_logical_to_physical WHERE logical_table_id = ?",
2691  std::to_string(td->tableId));
2693  }
2694  doDropTable(td);
2695  } catch (std::exception& e) {
2696  sqliteConnector_.query("ROLLBACK TRANSACTION");
2697  throw;
2698  }
2699  sqliteConnector_.query("END TRANSACTION");
2700 }
void doDropTable(const TableDescriptor *td)
Definition: Catalog.cpp:2702
std::string tableName
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
SqliteConnector sqliteConnector_
Definition: Catalog.h:331
void revokeDBObjectPrivilegesFromAll(DBObject object, Catalog *catalog)
void query(const std::string &queryString)
std::string to_string(char const *&&v)
static SysCatalog & instance()
Definition: SysCatalog.h:257
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:101
#define CHECK(condition)
Definition: Logger.h:193
specifies the content in-memory of a row in the table metadata table
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:102
void query_with_text_param(const std::string &queryString, const std::string &text_param)
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:338
+ Here is the call graph for this function:

◆ dropTableFromJsonUnlocked()

void Catalog_Namespace::Catalog::dropTableFromJsonUnlocked ( const std::string &  table_name) const
protected

Definition at line 2224 of file Catalog.cpp.

References basePath_, CHECK, currentDB_, Catalog_Namespace::DBMetadata::dbName, Catalog_Namespace::anonymous_namespace{Catalog.cpp}::table_json_filepath(), and VLOG.

Referenced by doDropTable().

2224  {
2225  // relies on the catalog write lock
2226  using namespace rapidjson;
2227 
2228  VLOG(1) << "Dropping temporary table " << table_name << " to JSON for Calcite.";
2229 
2230  const auto db_name = currentDB_.dbName;
2231  const auto file_path = table_json_filepath(basePath_, db_name);
2232 
2233  CHECK(boost::filesystem::exists(file_path));
2234  Document d;
2235 
2236  std::ifstream reader(file_path.string());
2237  CHECK(reader.is_open());
2238  IStreamWrapper json_read_wrapper(reader);
2239  d.ParseStream(json_read_wrapper);
2240 
2241  CHECK(d.IsObject());
2242  auto table_name_ref = StringRef(table_name.c_str());
2243  CHECK(d.HasMember(table_name_ref));
2244  CHECK(d.RemoveMember(table_name_ref));
2245 
2246  // Overwrite the existing file
2247  std::ofstream writer(file_path.string(), writer.trunc | writer.out);
2248  CHECK(writer.is_open());
2249  OStreamWrapper json_wrapper(writer);
2250 
2251  Writer<OStreamWrapper> json_writer(json_wrapper);
2252  d.Accept(json_writer);
2253  writer.close();
2254 }
auto table_json_filepath(const std::string &base_path, const std::string &db_name)
Definition: Catalog.cpp:141
#define CHECK(condition)
Definition: Logger.h:193
#define VLOG(n)
Definition: Logger.h:283
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ dumpSchema()

std::string Catalog_Namespace::Catalog::dumpSchema ( const TableDescriptor td) const

Definition at line 274 of file DumpRestore.cpp.

References CHECK, TableDescriptor::fragPageSize, TableDescriptor::hasDeletedCol, join(), kCHAR, kENCODING_DICT, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRows, TableDescriptor::nShards, TableDescriptor::partitions, TableDescriptor::shardedColumnId, TableDescriptor::sortedColumnId, TableDescriptor::tableId, and to_string().

274  {
275  std::ostringstream os;
276  os << "CREATE TABLE @T (";
277  // gather column defines
278  const auto cds = getAllColumnMetadataForTable(td->tableId, false, false, false);
279  std::string comma;
280  std::vector<std::string> shared_dicts;
281  std::map<const std::string, const ColumnDescriptor*> dict_root_cds;
282  for (const auto cd : cds) {
283  if (!(cd->isSystemCol || cd->isVirtualCol)) {
284  const auto& ti = cd->columnType;
285  os << comma << cd->columnName;
286  // CHAR is perculiar... better dump it as TEXT(32) like \d does
287  if (ti.get_type() == SQLTypes::kCHAR) {
288  os << " "
289  << "TEXT";
290  } else if (ti.get_subtype() == SQLTypes::kCHAR) {
291  os << " "
292  << "TEXT[]";
293  } else {
294  os << " " << ti.get_type_name();
295  }
296  os << (ti.get_notnull() ? " NOT NULL" : "");
297  if (ti.is_string()) {
298  if (ti.get_compression() == kENCODING_DICT) {
299  // if foreign reference, get referenced tab.col
300  const auto dict_id = ti.get_comp_param();
301  const DictRef dict_ref(currentDB_.dbId, dict_id);
302  const auto dict_it = dictDescriptorMapByRef_.find(dict_ref);
303  CHECK(dict_it != dictDescriptorMapByRef_.end());
304  const auto dict_name = dict_it->second->dictName;
305  // when migrating a table, any foreign dict ref will be dropped
306  // and the first cd of a dict will become root of the dict
307  if (dict_root_cds.end() == dict_root_cds.find(dict_name)) {
308  dict_root_cds[dict_name] = cd;
309  os << " ENCODING " << ti.get_compression_name() << "(" << (ti.get_size() * 8)
310  << ")";
311  } else {
312  const auto dict_root_cd = dict_root_cds[dict_name];
313  shared_dicts.push_back("SHARED DICTIONARY (" + cd->columnName +
314  ") REFERENCES @T(" + dict_root_cd->columnName + ")");
315  // "... shouldn't specify an encoding, it borrows from the referenced column"
316  }
317  } else {
318  os << " ENCODING NONE";
319  }
320  } else if (ti.get_size() > 0 && ti.get_size() != ti.get_logical_size()) {
321  const auto comp_param = ti.get_comp_param() ? ti.get_comp_param() : 32;
322  os << " ENCODING " << ti.get_compression_name() << "(" << comp_param << ")";
323  }
324  comma = ", ";
325  }
326  }
327  // gather SHARED DICTIONARYs
328  if (shared_dicts.size()) {
329  os << ", " << boost::algorithm::join(shared_dicts, ", ");
330  }
331  // gather WITH options ...
332  std::vector<std::string> with_options;
333  with_options.push_back("FRAGMENT_SIZE=" + std::to_string(td->maxFragRows));
334  with_options.push_back("MAX_CHUNK_SIZE=" + std::to_string(td->maxChunkSize));
335  with_options.push_back("PAGE_SIZE=" + std::to_string(td->fragPageSize));
336  with_options.push_back("MAX_ROWS=" + std::to_string(td->maxRows));
337  with_options.emplace_back(td->hasDeletedCol ? "VACUUM='DELAYED'"
338  : "VACUUM='IMMEDIATE'");
339  if (!td->partitions.empty()) {
340  with_options.push_back("PARTITIONS='" + td->partitions + "'");
341  }
342  if (td->nShards > 0) {
343  const auto shard_cd = getMetadataForColumn(td->tableId, td->shardedColumnId);
344  CHECK(shard_cd);
345  os << ", SHARD KEY(" << shard_cd->columnName << ")";
346  with_options.push_back("SHARD_COUNT=" + std::to_string(td->nShards));
347  }
348  if (td->sortedColumnId > 0) {
349  const auto sort_cd = getMetadataForColumn(td->tableId, td->sortedColumnId);
350  CHECK(sort_cd);
351  with_options.push_back("SORT_COLUMN='" + sort_cd->columnName + "'");
352  }
353  os << ") WITH (" + boost::algorithm::join(with_options, ", ") + ");";
354  return os.str();
355 }
std::string partitions
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:327
std::string join(T const &container, std::string const &delim)
const ColumnDescriptor * getMetadataForColumn(int tableId, const std::string &colName) const
std::string to_string(char const *&&v)
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1608
Definition: sqltypes.h:44
#define CHECK(condition)
Definition: Logger.h:193
+ Here is the call graph for this function:

◆ dumpTable()

void Catalog_Namespace::Catalog::dumpTable ( const TableDescriptor td,
const std::string &  path,
const std::string &  compression 
) const

Definition at line 201 of file DumpRestore.cpp.

References Catalog_Namespace::abs_path(), Data_Namespace::DISK_LEVEL, g_cluster, Lock_Namespace::TableLockMgrImpl< TableLockMgr >::getReadLockForTable(), TableDescriptor::isView, join(), TableDescriptor::persistenceLevel, run, Catalog_Namespace::simple_file_closer, TableDescriptor::tableId, TableDescriptor::tableName, and to_string().

203  {
204  if (g_cluster) {
205  throw std::runtime_error("DUMP/RESTORE is not supported yet on distributed setup.");
206  }
207  if (boost::filesystem::exists(archive_path)) {
208  throw std::runtime_error("Archive " + archive_path + " already exists.");
209  }
211  throw std::runtime_error("Dumping view or temporary table is not supported.");
212  }
213  // collect paths of files to archive
214  const auto global_file_mgr = getDataMgr().getGlobalFileMgr();
215  std::vector<std::string> file_paths;
216  auto file_writer = [&file_paths, global_file_mgr](const std::string& file_name,
217  const std::string& file_type,
218  const std::string& file_data) {
219  const auto file_path = abs_path(global_file_mgr) + "/" + file_name;
220  std::unique_ptr<FILE, decltype(simple_file_closer)> fp(
221  std::fopen(file_path.c_str(), "w"), simple_file_closer);
222  if (!fp) {
223  throw std::runtime_error("Failed to create " + file_type + " file '" + file_path +
224  "': " + std::strerror(errno));
225  }
226  if (std::fwrite(file_data.data(), 1, file_data.size(), fp.get()) < file_data.size()) {
227  throw std::runtime_error("Failed to write " + file_type + " file '" + file_path +
228  "': " + std::strerror(errno));
229  }
230  file_paths.push_back(file_name);
231  };
232  // grab table read lock for concurrent SELECT (but would block capped COPY or INSERT)
233  auto table_read_lock =
235 
236  // grab catalog read lock only. no need table read or checkpoint lock
237  // because want to allow concurrent inserts while this dump proceeds.
238  const auto table_name = td->tableName;
239  {
240  cat_read_lock read_lock(this);
241  // - gen schema file
242  const auto schema_str = dumpSchema(td);
243  file_writer(table_schema_filename, "table schema", schema_str);
244  // - gen column-old-info file
245  const auto cds = getAllColumnMetadataForTable(td->tableId, true, true, true);
246  std::vector<std::string> column_oldinfo;
247  std::transform(cds.begin(),
248  cds.end(),
249  std::back_inserter(column_oldinfo),
250  [&](const auto cd) -> std::string {
251  return cd->columnName + ":" + std::to_string(cd->columnId) + ":" +
253  });
254  const auto column_oldinfo_str = boost::algorithm::join(column_oldinfo, " ");
255  file_writer(table_oldinfo_filename, "table old info", column_oldinfo_str);
256  // - gen table epoch
257  const auto epoch = getTableEpoch(currentDB_.dbId, td->tableId);
258  file_writer(table_epoch_filename, "table epoch", std::to_string(epoch));
259  // - collect table data file paths ...
260  const auto data_file_dirs = getTableDataDirectories(td);
261  file_paths.insert(file_paths.end(), data_file_dirs.begin(), data_file_dirs.end());
262  // - collect table dict file paths ...
263  const auto dict_file_dirs = getTableDictDirectories(td);
264  file_paths.insert(file_paths.end(), dict_file_dirs.begin(), dict_file_dirs.end());
265  // tar takes time. release cat lock to yield the cat to concurrent CREATE statements.
266  }
267  // run tar to archive the files ... this may take a while !!
268  run("tar " + compression + " -cvf \"" + archive_path + "\" " +
269  boost::algorithm::join(file_paths, " "),
270  abs_path(global_file_mgr));
271 }
static ReadLock getReadLockForTable(const Catalog_Namespace::Catalog &cat, const std::string &table_name)
Definition: TableLockMgr.h:95
std::string run(const std::string &cmd, const std::string &chdir="")
Definition: DumpRestore.cpp:75
std::string tableName
bool g_cluster
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:177
std::string join(T const &container, std::string const &delim)
int32_t getTableEpoch(const int32_t db_id, const int32_t table_id) const
Definition: Catalog.cpp:2257
static constexpr char const * table_schema_filename
Definition: DumpRestore.cpp:65
std::string to_string(char const *&&v)
std::string dumpSchema(const TableDescriptor *td) const
std::string abs_path(const File_Namespace::GlobalFileMgr *global_file_mgr)
Definition: DumpRestore.cpp:71
std::vector< std::string > getTableDictDirectories(const TableDescriptor *td) const
std::string getColumnDictDirectory(const ColumnDescriptor *cd) const
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:100
static constexpr char const * table_oldinfo_filename
Definition: DumpRestore.cpp:66
static constexpr char const * table_epoch_filename
Definition: DumpRestore.cpp:67
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1608
Data_Namespace::MemoryLevel persistenceLevel
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
Definition: DataMgr.cpp:437
std::vector< std::string > getTableDataDirectories(const TableDescriptor *td) const
+ Here is the call graph for this function:

◆ eraseDBData()

void Catalog_Namespace::Catalog::eraseDBData ( )

Definition at line 3082 of file Catalog.cpp.

References basePath_, calciteMgr_, currentDB_, Catalog_Namespace::DBMetadata::dbName, eraseTablePhysicalData(), getAllTableMetadata(), and run_benchmark_import::tables.

3082  {
3083  cat_write_lock write_lock(this);
3084  // Physically erase all tables and dictionaries from disc and memory
3085  const auto tables = getAllTableMetadata();
3086  for (const auto table : tables) {
3087  eraseTablePhysicalData(table);
3088  }
3089  // Physically erase database metadata
3090  boost::filesystem::remove(basePath_ + "/mapd_catalogs/" + currentDB_.dbName);
3091  calciteMgr_->updateMetadata(currentDB_.dbName, "");
3092 }
std::list< const TableDescriptor * > getAllTableMetadata() const
Definition: Catalog.cpp:1625
void eraseTablePhysicalData(const TableDescriptor *td)
Definition: Catalog.cpp:3094
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:336
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:101
+ Here is the call graph for this function:

◆ eraseTablePhysicalData()

void Catalog_Namespace::Catalog::eraseTablePhysicalData ( const TableDescriptor td)

Definition at line 3094 of file Catalog.cpp.

References calciteMgr_, Data_Namespace::CPU_LEVEL, currentDB_, dataMgr_, Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::DBMetadata::dbName, TableDescriptor::fragmenter, Data_Namespace::GPU_LEVEL, INJECT_TIMER, TableDescriptor::isView, removeTableFromMap(), tableDescriptorMapById_, TableDescriptor::tableId, and TableDescriptor::tableName.

Referenced by doDropTable(), and eraseDBData().

3094  {
3095  cat_write_lock write_lock(this);
3096  const int tableId = td->tableId;
3097  // must destroy fragmenter before deleteChunks is called.
3098  if (td->fragmenter != nullptr) {
3099  auto tableDescIt = tableDescriptorMapById_.find(tableId);
3100  {
3101  INJECT_TIMER(deleting_fragmenter);
3102  delete td->fragmenter;
3103  }
3104  tableDescIt->second->fragmenter = nullptr; // get around const-ness
3105  }
3106  ChunkKey chunkKeyPrefix = {currentDB_.dbId, tableId};
3107  {
3108  INJECT_TIMER(deleteChunksWithPrefix);
3109  // assuming deleteChunksWithPrefix is atomic
3110  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::CPU_LEVEL);
3111  dataMgr_->deleteChunksWithPrefix(chunkKeyPrefix, MemoryLevel::GPU_LEVEL);
3112  }
3113  if (!td->isView) {
3114  INJECT_TIMER(Remove_Table);
3115  dataMgr_->removeTableRelatedDS(currentDB_.dbId, tableId);
3116  }
3117  calciteMgr_->updateMetadata(currentDB_.dbName, td->tableName);
3118  {
3119  INJECT_TIMER(removeTableFromMap_);
3120  removeTableFromMap(td->tableName, tableId);
3121  }
3122 }
std::string tableName
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:333
#define INJECT_TIMER(DESC)
Definition: measure.h:91
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:324
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:336
void removeTableFromMap(const std::string &tableName, const int tableId, const bool is_on_error=false)
Definition: Catalog.cpp:1128
write_lock< Catalog > cat_write_lock
Definition: Catalog.cpp:101
std::vector< int > ChunkKey
Definition: types.h:35
Fragmenter_Namespace::AbstractFragmenter * fragmenter
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ expandGeoColumn()

void Catalog_Namespace::Catalog::expandGeoColumn ( const ColumnDescriptor cd,
std::list< ColumnDescriptor > &  columns 
)
static

Definition at line 1836 of file Catalog.cpp.

References CHECK, ColumnDescriptor::columnName, ColumnDescriptor::columnType, IS_GEO, kARRAY, kDOUBLE, kENCODING_GEOINT, kENCODING_NONE, kINT, kLINESTRING, kMULTIPOLYGON, kPOINT, kPOLYGON, kTINYINT, SQLTypeInfoCore< TYPE_FACET_PACK >::set_size(), and SQLTypeInfoCore< TYPE_FACET_PACK >::set_subtype().

Referenced by createTable().

1837  {
1838  const auto& col_ti = cd.columnType;
1839  if (IS_GEO(col_ti.get_type())) {
1840  switch (col_ti.get_type()) {
1841  case kPOINT: {
1842  ColumnDescriptor physical_cd_coords(true);
1843  physical_cd_coords.columnName = cd.columnName + "_coords";
1844  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, true);
1845  // Raw data: compressed/uncompressed coords
1846  coords_ti.set_subtype(kTINYINT);
1847  size_t unit_size;
1848  if (col_ti.get_compression() == kENCODING_GEOINT &&
1849  col_ti.get_comp_param() == 32) {
1850  unit_size = 4 * sizeof(int8_t);
1851  } else {
1852  CHECK(col_ti.get_compression() == kENCODING_NONE);
1853  unit_size = 8 * sizeof(int8_t);
1854  }
1855  coords_ti.set_size(2 * unit_size);
1856  physical_cd_coords.columnType = coords_ti;
1857  columns.push_back(physical_cd_coords);
1858 
1859  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1860 
1861  break;
1862  }
1863  case kLINESTRING: {
1864  ColumnDescriptor physical_cd_coords(true);
1865  physical_cd_coords.columnName = cd.columnName + "_coords";
1866  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, true);
1867  // Raw data: compressed/uncompressed coords
1868  coords_ti.set_subtype(kTINYINT);
1869  physical_cd_coords.columnType = coords_ti;
1870  columns.push_back(physical_cd_coords);
1871 
1872  ColumnDescriptor physical_cd_bounds(true);
1873  physical_cd_bounds.columnName = cd.columnName + "_bounds";
1874  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, true);
1875  bounds_ti.set_subtype(kDOUBLE);
1876  bounds_ti.set_size(4 * sizeof(double));
1877  physical_cd_bounds.columnType = bounds_ti;
1878  columns.push_back(physical_cd_bounds);
1879 
1880  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1881 
1882  break;
1883  }
1884  case kPOLYGON: {
1885  ColumnDescriptor physical_cd_coords(true);
1886  physical_cd_coords.columnName = cd.columnName + "_coords";
1887  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, true);
1888  // Raw data: compressed/uncompressed coords
1889  coords_ti.set_subtype(kTINYINT);
1890  physical_cd_coords.columnType = coords_ti;
1891  columns.push_back(physical_cd_coords);
1892 
1893  ColumnDescriptor physical_cd_ring_sizes(true);
1894  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
1895  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, true);
1896  ring_sizes_ti.set_subtype(kINT);
1897  physical_cd_ring_sizes.columnType = ring_sizes_ti;
1898  columns.push_back(physical_cd_ring_sizes);
1899 
1900  ColumnDescriptor physical_cd_bounds(true);
1901  physical_cd_bounds.columnName = cd.columnName + "_bounds";
1902  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, true);
1903  bounds_ti.set_subtype(kDOUBLE);
1904  bounds_ti.set_size(4 * sizeof(double));
1905  physical_cd_bounds.columnType = bounds_ti;
1906  columns.push_back(physical_cd_bounds);
1907 
1908  ColumnDescriptor physical_cd_render_group(true);
1909  physical_cd_render_group.columnName = cd.columnName + "_render_group";
1910  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, true);
1911  physical_cd_render_group.columnType = render_group_ti;
1912  columns.push_back(physical_cd_render_group);
1913 
1914  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1915 
1916  break;
1917  }
1918  case kMULTIPOLYGON: {
1919  ColumnDescriptor physical_cd_coords(true);
1920  physical_cd_coords.columnName = cd.columnName + "_coords";
1921  SQLTypeInfo coords_ti = SQLTypeInfo(kARRAY, true);
1922  // Raw data: compressed/uncompressed coords
1923  coords_ti.set_subtype(kTINYINT);
1924  physical_cd_coords.columnType = coords_ti;
1925  columns.push_back(physical_cd_coords);
1926 
1927  ColumnDescriptor physical_cd_ring_sizes(true);
1928  physical_cd_ring_sizes.columnName = cd.columnName + "_ring_sizes";
1929  SQLTypeInfo ring_sizes_ti = SQLTypeInfo(kARRAY, true);
1930  ring_sizes_ti.set_subtype(kINT);
1931  physical_cd_ring_sizes.columnType = ring_sizes_ti;
1932  columns.push_back(physical_cd_ring_sizes);
1933 
1934  ColumnDescriptor physical_cd_poly_rings(true);
1935  physical_cd_poly_rings.columnName = cd.columnName + "_poly_rings";
1936  SQLTypeInfo poly_rings_ti = SQLTypeInfo(kARRAY, true);
1937  poly_rings_ti.set_subtype(kINT);
1938  physical_cd_poly_rings.columnType = poly_rings_ti;
1939  columns.push_back(physical_cd_poly_rings);
1940 
1941  ColumnDescriptor physical_cd_bounds(true);
1942  physical_cd_bounds.columnName = cd.columnName + "_bounds";
1943  SQLTypeInfo bounds_ti = SQLTypeInfo(kARRAY, true);
1944  bounds_ti.set_subtype(kDOUBLE);
1945  bounds_ti.set_size(4 * sizeof(double));
1946  physical_cd_bounds.columnType = bounds_ti;
1947  columns.push_back(physical_cd_bounds);
1948 
1949  ColumnDescriptor physical_cd_render_group(true);
1950  physical_cd_render_group.columnName = cd.columnName + "_render_group";
1951  SQLTypeInfo render_group_ti = SQLTypeInfo(kINT, true);
1952  physical_cd_render_group.columnType = render_group_ti;
1953  columns.push_back(physical_cd_render_group);
1954 
1955  // If adding more physical columns - update SQLTypeInfo::get_physical_cols()
1956 
1957  break;
1958  }
1959  default:
1960  throw runtime_error("Unrecognized geometry type.");
1961  break;
1962  }
1963  }
1964 }
void set_size(int s)
Definition: sqltypes.h:424
HOST DEVICE void set_subtype(SQLTypes st)
Definition: sqltypes.h:417
specifies the content in-memory of a row in the column metadata table
SQLTypeInfoCore< ArrayContextTypeSizer, ExecutorTypePackaging, DateTimeFacilities > SQLTypeInfo
Definition: sqltypes.h:852
#define CHECK(condition)
Definition: Logger.h:193
Definition: sqltypes.h:48
SQLTypeInfo columnType
std::string columnName
#define IS_GEO(T)
Definition: sqltypes.h:167
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ generatePhysicalTableName()

std::string Catalog_Namespace::Catalog::generatePhysicalTableName ( const std::string &  logicalTableName,
const int32_t &  shardNumber 
)
protected

Definition at line 3124 of file Catalog.cpp.

References physicalTableNameTag_, and to_string().

Referenced by createShardedTable(), and renameTable().

3125  {
3126  std::string physicalTableName =
3127  logicalTableName + physicalTableNameTag_ + std::to_string(shardNumber);
3128  return (physicalTableName);
3129 }
std::string to_string(char const *&&v)
static const std::string physicalTableNameTag_
Definition: Catalog.h:340
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ get() [1/2]

std::shared_ptr< Catalog > Catalog_Namespace::Catalog::get ( const std::string &  dbName)
static

Definition at line 3135 of file Catalog.cpp.

References Catalog_Namespace::DBMetadata::dbName, mapd_cat_map_, and set().

Referenced by Catalog_Namespace::SysCatalog::createDatabase(), Catalog_Namespace::SysCatalog::dropDatabase(), Catalog_Namespace::SysCatalog::login(), main(), and Catalog_Namespace::SysCatalog::switchDatabase().

3135  {
3136  auto cat_it = mapd_cat_map_.find(dbName);
3137  if (cat_it != mapd_cat_map_.end()) {
3138  return cat_it->second;
3139  }
3140  return nullptr;
3141 }
static std::map< std::string, std::shared_ptr< Catalog > > mapd_cat_map_
Definition: Catalog.h:353
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ get() [2/2]

static std::shared_ptr<Catalog> Catalog_Namespace::Catalog::get ( const std::string &  basePath,
const DBMetadata curDB,
std::shared_ptr< Data_Namespace::DataMgr dataMgr,
const std::vector< LeafHostInfo > &  string_dict_hosts,
std::shared_ptr< Calcite calcite,
bool  is_new_db 
)
static

◆ getAllColumnMetadataForTable() [1/2]

list< const ColumnDescriptor * > Catalog_Namespace::Catalog::getAllColumnMetadataForTable ( const int  tableId,
const bool  fetchSystemColumns,
const bool  fetchVirtualColumns,
const bool  fetchPhysicalColumns 
) const

Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular table specified by table id.

Parameters
tableIdtable id we want the column metadata for
Returns
list of pointers to const ColumnDescriptor structs - one for each and every column in the table

Definition at line 1608 of file Catalog.cpp.

References getMetadataForTableImpl().

Referenced by Analyzer::RangeTableEntry::add_all_column_descs(), Parser::InsertStmt::analyze(), Parser::InsertValuesStmt::analyze(), Fragmenter_Namespace::copyDataOfShard(), Parser::InsertValuesStmt::determineLeafIndex(), Analyzer::RangeTableEntry::expand_star_in_targetlist(), getMetadataForLink(), getShardColumnMetadataForTable(), instantiateFragmenter(), TableOptimizer::recomputeMetadata(), and validate_shared_dictionary().

1612  {
1613  cat_read_lock read_lock(this);
1614  list<const ColumnDescriptor*> columnDescriptors;
1615  const TableDescriptor* td =
1616  getMetadataForTableImpl(tableId, false); // dont instantiate fragmenter
1618  columnDescriptors,
1619  fetchSystemColumns,
1620  fetchVirtualColumns,
1621  fetchPhysicalColumns);
1622  return columnDescriptors;
1623 }
const TableDescriptor * getMetadataForTableImpl(int tableId, const bool populateFragmenter) const
Definition: Catalog.cpp:1357
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:100
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1608
specifies the content in-memory of a row in the table metadata table
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getAllColumnMetadataForTable() [2/2]

void Catalog_Namespace::Catalog::getAllColumnMetadataForTable ( const TableDescriptor td,
std::list< const ColumnDescriptor *> &  colDescs,
const bool  fetchSystemColumns,
const bool  fetchVirtualColumns,
const bool  fetchPhysicalColumns 
) const
protected

◆ getAllDashboardsMetadata()

list< const DashboardDescriptor * > Catalog_Namespace::Catalog::getAllDashboardsMetadata ( ) const

Definition at line 1634 of file Catalog.cpp.

References dashboardDescriptorMap_.

1634  {
1635  list<const DashboardDescriptor*> view_list;
1636  for (auto p : dashboardDescriptorMap_) {
1637  view_list.push_back(p.second.get());
1638  }
1639  return view_list;
1640 }
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:328

◆ getAllTableMetadata()

list< const TableDescriptor * > Catalog_Namespace::Catalog::getAllTableMetadata ( ) const

Definition at line 1625 of file Catalog.cpp.

References tableDescriptorMapById_.

Referenced by eraseDBData().

1625  {
1626  cat_read_lock read_lock(this);
1627  list<const TableDescriptor*> table_list;
1628  for (auto p : tableDescriptorMapById_) {
1629  table_list.push_back(p.second);
1630  }
1631  return table_list;
1632 }
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:324
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:100
+ Here is the caller graph for this function:

◆ getBasePath()

const std::string& Catalog_Namespace::Catalog::getBasePath ( ) const
inline

Definition at line 179 of file Catalog.h.

179 { return basePath_; }

◆ getCalciteMgr()

std::shared_ptr<Calcite> Catalog_Namespace::Catalog::getCalciteMgr ( ) const
inline

Definition at line 178 of file Catalog.h.

178 { return calciteMgr_; }
std::shared_ptr< Calcite > calciteMgr_
Definition: Catalog.h:336

◆ getColumnDictDirectory()

std::string Catalog_Namespace::Catalog::getColumnDictDirectory ( const ColumnDescriptor cd) const

Definition at line 171 of file DumpRestore.cpp.

References CHECK, ColumnDescriptor::columnType, SQLTypeInfoCore< TYPE_FACET_PACK >::get_comp_param(), SQLTypeInfoCore< TYPE_FACET_PACK >::get_compression(), SQLTypeInfoCore< TYPE_FACET_PACK >::is_string(), SQLTypeInfoCore< TYPE_FACET_PACK >::is_string_array(), and kENCODING_DICT.

171  {
172  if ((cd->columnType.is_string() || cd->columnType.is_string_array()) &&
174  cd->columnType.get_comp_param() > 0) {
175  const auto dictId = cd->columnType.get_comp_param();
176  const DictRef dictRef(currentDB_.dbId, dictId);
177  const auto dit = dictDescriptorMapByRef_.find(dictRef);
178  CHECK(dit != dictDescriptorMapByRef_.end());
179  CHECK(dit->second);
180  boost::filesystem::path file_path(dit->second->dictFolderPath);
181  return file_path.filename().string();
182  }
183  return std::string();
184 }
bool is_string_array() const
Definition: sqltypes.h:478
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:327
HOST DEVICE EncodingType get_compression() const
Definition: sqltypes.h:334
HOST DEVICE int get_comp_param() const
Definition: sqltypes.h:335
#define CHECK(condition)
Definition: Logger.h:193
SQLTypeInfo columnType
bool is_string() const
Definition: sqltypes.h:477
+ Here is the call graph for this function:

◆ getColumnIdBySpi()

const int Catalog_Namespace::Catalog::getColumnIdBySpi ( const int  tableId,
const size_t  spi 
) const

Definition at line 1459 of file Catalog.cpp.

References getColumnIdBySpiUnlocked().

Referenced by anonymous_namespace{RelAlgExecutor.cpp}::collect_used_input_desc(), and anonymous_namespace{RelAlgExecutor.cpp}::get_physical_inputs().

1459  {
1460  cat_read_lock read_lock(this);
1461  return getColumnIdBySpiUnlocked(table_id, spi);
1462 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:100
const int getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const
Definition: Catalog.cpp:1442
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getColumnIdBySpiUnlocked()

const int Catalog_Namespace::Catalog::getColumnIdBySpiUnlocked ( const int  table_id,
const size_t  spi 
) const
protected

Definition at line 1442 of file Catalog.cpp.

References CHECK, SPIMAP_MAGIC1, SPIMAP_MAGIC2, and tableDescriptorMapById_.

Referenced by getColumnIdBySpi(), and getMetadataForColumnBySpi().

1442  {
1443  const auto tabDescIt = tableDescriptorMapById_.find(table_id);
1444  CHECK(tableDescriptorMapById_.end() != tabDescIt);
1445  const auto& columnIdBySpi = tabDescIt->second->columnIdBySpi_;
1446 
1447  auto spx = spi;
1448  int phi = 0;
1449  if (spx >= SPIMAP_MAGIC1) // see Catalog.h
1450  {
1451  phi = (spx - SPIMAP_MAGIC1) % SPIMAP_MAGIC2;
1452  spx = (spx - SPIMAP_MAGIC1) / SPIMAP_MAGIC2;
1453  }
1454 
1455  CHECK(0 < spx && spx <= columnIdBySpi.size());
1456  return columnIdBySpi[spx - 1] + phi;
1457 }
#define SPIMAP_MAGIC1
Definition: Catalog.h:68
#define SPIMAP_MAGIC2
Definition: Catalog.h:69
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:324
#define CHECK(condition)
Definition: Logger.h:193
+ Here is the caller graph for this function:

◆ getCurrentDB()

◆ getDatabaseId()

int Catalog_Namespace::Catalog::getDatabaseId ( ) const
inline

Definition at line 192 of file Catalog.h.

192 { return currentDB_.dbId; }

◆ getDataMgr()

◆ getDeletedColumn()

const ColumnDescriptor * Catalog_Namespace::Catalog::getDeletedColumn ( const TableDescriptor td) const

Definition at line 2311 of file Catalog.cpp.

References deletedColumnPerTable_.

Referenced by TableOptimizer::recomputeMetadata(), and vacuumDeletedRows().

2311  {
2312  cat_read_lock read_lock(this);
2313  const auto it = deletedColumnPerTable_.find(td);
2314  return it != deletedColumnPerTable_.end() ? it->second : nullptr;
2315 }
DeletedColumnPerTableMap deletedColumnPerTable_
Definition: Catalog.h:354
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:100
+ Here is the caller graph for this function:

◆ getDeletedColumnIfRowsDeleted()

const ColumnDescriptor * Catalog_Namespace::Catalog::getDeletedColumnIfRowsDeleted ( const TableDescriptor td) const

Definition at line 2336 of file Catalog.cpp.

References CHECK, checkMetadataForDeletedRecs(), ColumnDescriptor::columnId, currentDB_, Catalog_Namespace::DBMetadata::dbId, deletedColumnPerTable_, getMetadataForTable(), logicalToPhysicalTableMapById_, and TableDescriptor::tableId.

Referenced by Executor::addDeletedColumn().

2337  {
2338  cat_read_lock read_lock(this);
2339 
2340  const auto it = deletedColumnPerTable_.find(td);
2341  // if not a table that supports delete return nullptr, nothing more to do
2342  if (it == deletedColumnPerTable_.end()) {
2343  return nullptr;
2344  }
2345  const ColumnDescriptor* cd = it->second;
2346 
2347  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(td->tableId);
2348 
2349  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2350  // check all shards
2351  const auto physicalTables = physicalTableIt->second;
2352  CHECK(!physicalTables.empty());
2353  for (size_t i = 0; i < physicalTables.size(); i++) {
2354  int32_t physical_tb_id = physicalTables[i];
2355  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2356  CHECK(phys_td);
2358  return cd;
2359  }
2360  }
2361  } else {
2363  return cd;
2364  }
2365  }
2366  // no deletes so far recorded in metadata
2367  return nullptr;
2368 }
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
const bool checkMetadataForDeletedRecs(int dbId, int tableId, int columnId) const
Definition: Catalog.cpp:2317
DeletedColumnPerTableMap deletedColumnPerTable_
Definition: Catalog.h:354
specifies the content in-memory of a row in the column metadata table
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:100
#define CHECK(condition)
Definition: Logger.h:193
specifies the content in-memory of a row in the table metadata table
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:338
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getDictionary()

void Catalog_Namespace::Catalog::getDictionary ( const ColumnDescriptor cd,
std::map< int, StringDictionary *> &  stringDicts 
)

Definition at line 1701 of file Catalog.cpp.

References CHECK, columnDescriptorMap_, ColumnDescriptor::columnName, currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, getMetadataForDict(), kENCODING_DICT, ColumnDescriptor::tableId, and to_upper().

1702  {
1703  // learn 'committed' ColumnDescriptor of this column
1704  auto cit = columnDescriptorMap_.find(ColumnKey(cd.tableId, to_upper(cd.columnName)));
1705  CHECK(cit != columnDescriptorMap_.end());
1706  auto& ccd = *cit->second;
1707 
1708  if (!(ccd.columnType.is_string() || ccd.columnType.is_string_array())) {
1709  return;
1710  }
1711  if (!(ccd.columnType.get_compression() == kENCODING_DICT)) {
1712  return;
1713  }
1714  if (!(ccd.columnType.get_comp_param() > 0)) {
1715  return;
1716  }
1717 
1718  auto dictId = ccd.columnType.get_comp_param();
1719  getMetadataForDict(dictId);
1720 
1721  const DictRef dictRef(currentDB_.dbId, dictId);
1722  auto dit = dictDescriptorMapByRef_.find(dictRef);
1723  CHECK(dit != dictDescriptorMapByRef_.end());
1724  CHECK(dit->second);
1725  CHECK(dit->second.get()->stringDict);
1726  stringDicts[ccd.columnId] = dit->second.get()->stringDict.get();
1727 }
ColumnDescriptorMap columnDescriptorMap_
Definition: Catalog.h:325
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:327
std::tuple< int, std::string > ColumnKey
Definition: Catalog.h:246
const DictDescriptor * getMetadataForDict(int dict_ref, bool loadDict=true) const
Definition: Catalog.cpp:1377
std::string to_upper(const std::string &str)
#define CHECK(condition)
Definition: Logger.h:193
std::string columnName
+ Here is the call graph for this function:

◆ getLogicalTableId()

int Catalog_Namespace::Catalog::getLogicalTableId ( const int  physicalTableId) const

Definition at line 3061 of file Catalog.cpp.

References logicalToPhysicalTableMapById_.

Referenced by Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumns(), and vacuumDeletedRows().

3061  {
3062  for (const auto& l : logicalToPhysicalTableMapById_) {
3063  if (l.second.end() != std::find_if(l.second.begin(),
3064  l.second.end(),
3065  [&](decltype(*l.second.begin()) tid) -> bool {
3066  return physicalTableId == tid;
3067  })) {
3068  return l.first;
3069  }
3070  }
3071  return physicalTableId;
3072 }
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:338
+ Here is the caller graph for this function:

◆ getMetadataForColumn() [1/2]

◆ getMetadataForColumn() [2/2]

const ColumnDescriptor * Catalog_Namespace::Catalog::getMetadataForColumn ( int  tableId,
int  columnId 
) const

Definition at line 1430 of file Catalog.cpp.

References columnDescriptorMapById_.

1430  {
1431  cat_read_lock read_lock(this);
1432 
1433  ColumnIdKey columnIdKey(tableId, columnId);
1434  auto colDescIt = columnDescriptorMapById_.find(columnIdKey);
1435  if (colDescIt == columnDescriptorMapById_
1436  .end()) { // need to check to make sure column exists for table
1437  return nullptr;
1438  }
1439  return colDescIt->second;
1440 }
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:326
std::tuple< int, int > ColumnIdKey
Definition: Catalog.h:248
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:100

◆ getMetadataForColumnBySpi()

const ColumnDescriptor * Catalog_Namespace::Catalog::getMetadataForColumnBySpi ( const int  tableId,
const size_t  spi 
) const

Definition at line 1464 of file Catalog.cpp.

References columnDescriptorMapById_, and getColumnIdBySpiUnlocked().

Referenced by RelAlgTranslator::translateGeoColumn().

1465  {
1466  cat_read_lock read_lock(this);
1467 
1468  const auto columnId = getColumnIdBySpiUnlocked(tableId, spi);
1469  ColumnIdKey columnIdKey(tableId, columnId);
1470  const auto colDescIt = columnDescriptorMapById_.find(columnIdKey);
1471  return columnDescriptorMapById_.end() == colDescIt ? nullptr : colDescIt->second;
1472 }
ColumnDescriptorMapById columnDescriptorMapById_
Definition: Catalog.h:326
std::tuple< int, int > ColumnIdKey
Definition: Catalog.h:248
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:100
const int getColumnIdBySpiUnlocked(const int table_id, const size_t spi) const
Definition: Catalog.cpp:1442
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getMetadataForDashboard() [1/2]

const DashboardDescriptor* Catalog_Namespace::Catalog::getMetadataForDashboard ( const std::string &  userId,
const std::string &  dashName 
) const

Referenced by deleteMetadataForDashboard(), getMetadataForDashboard(), and DBObject::loadKey().

+ Here is the caller graph for this function:

◆ getMetadataForDashboard() [2/2]

const DashboardDescriptor * Catalog_Namespace::Catalog::getMetadataForDashboard ( const int32_t  dashboard_id) const

Definition at line 1513 of file Catalog.cpp.

References dashboardDescriptorMap_, getMetadataForDashboard(), name(), and to_string().

1513  {
1514  cat_read_lock read_lock(this);
1515  std::string userId;
1516  std::string name;
1517  bool found{false};
1518  {
1519  for (auto descp : dashboardDescriptorMap_) {
1520  auto dash = descp.second.get();
1521  if (dash->dashboardId == id) {
1522  userId = std::to_string(dash->userId);
1523  name = dash->dashboardName;
1524  found = true;
1525  break;
1526  }
1527  }
1528  }
1529  if (found) {
1530  return getMetadataForDashboard(userId, name);
1531  }
1532  return nullptr;
1533 }
DashboardDescriptorMap dashboardDescriptorMap_
Definition: Catalog.h:328
std::string to_string(char const *&&v)
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:100
std::string name() const
Definition: Catalog.h:219
const DashboardDescriptor * getMetadataForDashboard(const std::string &userId, const std::string &dashName) const
+ Here is the call graph for this function:

◆ getMetadataForDict()

const DictDescriptor * Catalog_Namespace::Catalog::getMetadataForDict ( int  dict_ref,
bool  loadDict = true 
) const

Definition at line 1377 of file Catalog.cpp.

References currentDB_, Catalog_Namespace::DBMetadata::dbId, dictDescriptorMapByRef_, measure< TimeT >::execution(), g_cache_string_hash, logger::INFO, LOG, and string_dict_hosts_.

Referenced by Executor::computeStringDictionaryGenerations(), Parser::InsertValuesStmt::determineLeafIndex(), DictionaryValueConverter< TARGET_TYPE >::DictionaryValueConverter(), doTruncateTable(), MapDHandler::fixup_row_descriptor(), getDictionary(), Executor::getStringDictionaryProxy(), anonymous_namespace{Execute.cpp}::insert_one_dict_str(), MapDHandler::populateThriftColumnType(), StringValueConverter::StringValueConverter(), and Fragmenter_Namespace::InsertOrderFragmenter::updateColumn().

1378  {
1379  const DictRef dictRef(currentDB_.dbId, dictId);
1380  cat_read_lock read_lock(this);
1381  auto dictDescIt = dictDescriptorMapByRef_.find(dictRef);
1382  if (dictDescIt ==
1383  dictDescriptorMapByRef_.end()) { // check to make sure dictionary exists
1384  return nullptr;
1385  }
1386  auto& dd = dictDescIt->second;
1387 
1388  if (loadDict) {
1389  cat_sqlite_lock sqlite_lock(this);
1390  if (!dd->stringDict) {
1391  auto time_ms = measure<>::execution([&]() {
1392  if (string_dict_hosts_.empty()) {
1393  if (dd->dictIsTemp) {
1394  dd->stringDict = std::make_shared<StringDictionary>(
1395  dd->dictFolderPath, true, true, g_cache_string_hash);
1396  } else {
1397  dd->stringDict = std::make_shared<StringDictionary>(
1398  dd->dictFolderPath, false, true, g_cache_string_hash);
1399  }
1400  } else {
1401  dd->stringDict =
1402  std::make_shared<StringDictionary>(string_dict_hosts_.front(), dd->dictRef);
1403  }
1404  });
1405  LOG(INFO) << "Time to load Dictionary " << dd->dictRef.dbId << "_"
1406  << dd->dictRef.dictId << " was " << time_ms << "ms";
1407  }
1408  }
1409 
1410  return dd.get();
1411 }
#define LOG(tag)
Definition: Logger.h:188
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:335
DictDescriptorMapById dictDescriptorMapByRef_
Definition: Catalog.h:327
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:100
bool g_cache_string_hash
Definition: Execute.cpp:88
static TimeT::rep execution(F func, Args &&... args)
Definition: sample.cpp:29
sqlite_lock< Catalog > cat_sqlite_lock
Definition: Catalog.cpp:102
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getMetadataForLink() [1/2]

const LinkDescriptor* Catalog_Namespace::Catalog::getMetadataForLink ( const std::string &  link) const

Referenced by deleteMetadataForDashboard().

+ Here is the caller graph for this function:

◆ getMetadataForLink() [2/2]

const LinkDescriptor * Catalog_Namespace::Catalog::getMetadataForLink ( int  linkId) const

Definition at line 1568 of file Catalog.cpp.

References columnDescriptorMapById_, getAllColumnMetadataForTable(), linkDescriptorMapById_, and TableDescriptor::tableId.

1568  {
1569  cat_read_lock read_lock(this);
1570  auto linkDescIt = linkDescriptorMapById_.find(linkId);
1571  if (linkDescIt == linkDescriptorMapById_.end()) { // check to make sure view exists
1572  return nullptr;
1573  }
1574  return linkDescIt->second;
1575 }
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:100
LinkDescriptorMapById linkDescriptorMapById_
Definition: Catalog.h:330
+ Here is the call graph for this function:

◆ getMetadataForTable() [1/2]

const TableDescriptor* Catalog_Namespace::Catalog::getMetadataForTable ( const std::string &  tableName,
const bool  populateFragmenter = true 
) const

Returns a pointer to a const TableDescriptor struct matching the provided tableName.

Parameters
tableNametable specified column belongs to
Returns
pointer to const TableDescriptor object queried for or nullptr if it does not exist.

Referenced by Executor::addDeletedColumn(), Parser::InsertStmt::analyze(), Parser::QuerySpec::analyze_from_clause(), UpdelRoll::cancelUpdate(), anonymous_namespace{MapDHandler.cpp}::check_table_not_sharded(), BaselineJoinHashTable::checkHashJoinReplicationConstraint(), checkPermissionForTables(), checkpoint(), Fragmenter_Namespace::computeRowIndicesOfShards(), Fragmenter_Namespace::copyDataOfShard(), Parser::InsertValuesStmt::determineLeafIndex(), dropTable(), Executor::executeSimpleInsert(), Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_foreign_col(), anonymous_namespace{Execute.cpp}::get_table_name(), getDeletedColumnIfRowsDeleted(), getPhysicalTablesDescriptors(), Lock_Namespace::getTableChunkKey(), getTableEpoch(), anonymous_namespace{RelAlgDagBuilder.cpp}::getTableFromScanNode(), Fragmenter_Namespace::InsertDataLoader::insertData(), Parser::InsertIntoTableAsSelectStmt::LocalConnector::insertDataToLeaf(), instantiateFragmenter(), DBObject::loadKey(), parseDashboardObjects(), removeChunks(), renameTable(), Fragmenter_Namespace::set_chunk_metadata(), setColumnSharedDictionary(), setTableEpoch(), GroupByAndAggregate::shard_count_for_top_groups(), Fragmenter_Namespace::SortedOrderFragmenter::sortData(), truncateTable(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumnMetadata(), vacuumDeletedRows(), and validate_shared_dictionary().

+ Here is the caller graph for this function:

◆ getMetadataForTable() [2/2]

const TableDescriptor * Catalog_Namespace::Catalog::getMetadataForTable ( int  tableId) const

Definition at line 1373 of file Catalog.cpp.

References getMetadataForTableImpl().

1373  {
1374  return getMetadataForTableImpl(tableId, true);
1375 }
const TableDescriptor * getMetadataForTableImpl(int tableId, const bool populateFragmenter) const
Definition: Catalog.cpp:1357
+ Here is the call graph for this function:

◆ getMetadataForTableImpl()

const TableDescriptor * Catalog_Namespace::Catalog::getMetadataForTableImpl ( int  tableId,
const bool  populateFragmenter 
) const

Definition at line 1357 of file Catalog.cpp.

References TableDescriptor::fragmenter, instantiateFragmenter(), TableDescriptor::isView, TableDescriptor::mutex_, and tableDescriptorMapById_.

Referenced by getAllColumnMetadataForTable(), and getMetadataForTable().

1359  {
1360  cat_read_lock read_lock(this);
1361  auto tableDescIt = tableDescriptorMapById_.find(tableId);
1362  if (tableDescIt == tableDescriptorMapById_.end()) { // check to make sure table exists
1363  return nullptr;
1364  }
1365  TableDescriptor* td = tableDescIt->second;
1366  std::unique_lock<std::mutex> td_lock(*td->mutex_.get());
1367  if (populateFragmenter && td->fragmenter == nullptr && !td->isView) {
1369  }
1370  return td; // returns pointer to table descriptor
1371 }
std::shared_ptr< std::mutex > mutex_
TableDescriptorMapById tableDescriptorMapById_
Definition: Catalog.h:324
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:100
specifies the content in-memory of a row in the table metadata table
Fragmenter_Namespace::AbstractFragmenter * fragmenter
void instantiateFragmenter(TableDescriptor *td) const
Definition: Catalog.cpp:1300
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getPhysicalTablesDescriptors()

std::vector< const TableDescriptor * > Catalog_Namespace::Catalog::getPhysicalTablesDescriptors ( const TableDescriptor logicalTableDesc) const

Definition at line 3042 of file Catalog.cpp.

References CHECK, getMetadataForTable(), logicalToPhysicalTableMapById_, and TableDescriptor::tableId.

Referenced by checkpoint(), Fragmenter_Namespace::copyDataOfShard(), TableOptimizer::recomputeMetadata(), and vacuumDeletedRows().

3043  {
3044  cat_read_lock read_lock(this);
3045  const auto physicalTableIt =
3046  logicalToPhysicalTableMapById_.find(logicalTableDesc->tableId);
3047  if (physicalTableIt == logicalToPhysicalTableMapById_.end()) {
3048  return {logicalTableDesc};
3049  }
3050 
3051  const auto physicalTablesIds = physicalTableIt->second;
3052  CHECK(!physicalTablesIds.empty());
3053  std::vector<const TableDescriptor*> physicalTables;
3054  for (size_t i = 0; i < physicalTablesIds.size(); i++) {
3055  physicalTables.push_back(getMetadataForTable(physicalTablesIds[i]));
3056  }
3057 
3058  return physicalTables;
3059 }
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:100
#define CHECK(condition)
Definition: Logger.h:193
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:338
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getShardColumnMetadataForTable()

const ColumnDescriptor * Catalog_Namespace::Catalog::getShardColumnMetadataForTable ( const TableDescriptor td) const

Definition at line 3026 of file Catalog.cpp.

References getAllColumnMetadataForTable(), TableDescriptor::shardedColumnId, and TableDescriptor::tableId.

Referenced by Fragmenter_Namespace::computeRowIndicesOfShards(), and Parser::InsertValuesStmt::determineLeafIndex().

3027  {
3028  const auto column_descriptors =
3029  getAllColumnMetadataForTable(td->tableId, false, true, true);
3030 
3031  const ColumnDescriptor* shard_cd{nullptr};
3032  int i = 1;
3033  for (auto cd_itr = column_descriptors.begin(); cd_itr != column_descriptors.end();
3034  ++cd_itr, ++i) {
3035  if (i == td->shardedColumnId) {
3036  shard_cd = *cd_itr;
3037  }
3038  }
3039  return shard_cd;
3040 }
specifies the content in-memory of a row in the column metadata table
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1608
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ getSqliteConnector()

SqliteConnector& Catalog_Namespace::Catalog::getSqliteConnector ( )
inline

Definition at line 194 of file Catalog.h.

194 { return sqliteConnector_; }
SqliteConnector sqliteConnector_
Definition: Catalog.h:331

◆ getStringDictionaryHosts()

const std::vector< LeafHostInfo > & Catalog_Namespace::Catalog::getStringDictionaryHosts ( ) const

Definition at line 1413 of file Catalog.cpp.

References columnDescriptorMap_, getMetadataForColumn(), string_dict_hosts_, and to_upper().

1413  {
1414  return string_dict_hosts_;
1415 }
const std::vector< LeafHostInfo > string_dict_hosts_
Definition: Catalog.h:335
+ Here is the call graph for this function:

◆ getTableDataDirectories()

std::vector< std::string > Catalog_Namespace::Catalog::getTableDataDirectories ( const TableDescriptor td) const

Definition at line 158 of file DumpRestore.cpp.

159  {
160  const auto global_file_mgr = getDataMgr().getGlobalFileMgr();
161  std::vector<std::string> file_paths;
162  for (auto shard : getPhysicalTablesDescriptors(td)) {
163  const auto file_mgr = global_file_mgr->getFileMgr(currentDB_.dbId, shard->tableId);
164  boost::filesystem::path file_path(file_mgr->getFileMgrBasePath());
165  file_paths.push_back(file_path.filename().string());
166  }
167  return file_paths;
168 }
Data_Namespace::DataMgr & getDataMgr() const
Definition: Catalog.h:177
FileMgr * getFileMgr(const int db_id, const int tb_id)
std::vector< const TableDescriptor * > getPhysicalTablesDescriptors(const TableDescriptor *logicalTableDesc) const
Definition: Catalog.cpp:3042
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
Definition: DataMgr.cpp:437

◆ getTableDictDirectories()

std::vector< std::string > Catalog_Namespace::Catalog::getTableDictDirectories ( const TableDescriptor td) const

Definition at line 187 of file DumpRestore.cpp.

References TableDescriptor::tableId.

188  {
189  std::vector<std::string> file_paths;
190  for (auto cd : getAllColumnMetadataForTable(td->tableId, false, false, true)) {
191  auto file_base = getColumnDictDirectory(cd);
192  if (!file_base.empty() &&
193  file_paths.end() == std::find(file_paths.begin(), file_paths.end(), file_base)) {
194  file_paths.push_back(file_base);
195  }
196  }
197  return file_paths;
198 }
std::string getColumnDictDirectory(const ColumnDescriptor *cd) const
std::list< const ColumnDescriptor * > getAllColumnMetadataForTable(const int tableId, const bool fetchSystemColumns, const bool fetchVirtualColumns, const bool fetchPhysicalColumns) const
Returns a list of pointers to constant ColumnDescriptor structs for all the columns from a particular...
Definition: Catalog.cpp:1608

◆ getTableEpoch()

int32_t Catalog_Namespace::Catalog::getTableEpoch ( const int32_t  db_id,
const int32_t  table_id 
) const

Definition at line 2257 of file Catalog.cpp.

References CHECK, dataMgr_, logger::ERROR, getMetadataForTable(), LOG, and logicalToPhysicalTableMapById_.

Referenced by Importer_NS::Loader::getTableEpoch().

2257  {
2258  cat_read_lock read_lock(this);
2259  const auto physicalTableIt = logicalToPhysicalTableMapById_.find(table_id);
2260  if (physicalTableIt != logicalToPhysicalTableMapById_.end()) {
2261  // check all shards have same checkpoint
2262  const auto physicalTables = physicalTableIt->second;
2263  CHECK(!physicalTables.empty());
2264  size_t curr_epoch = 0;
2265  for (size_t i = 0; i < physicalTables.size(); i++) {
2266  int32_t physical_tb_id = physicalTables[i];
2267  const TableDescriptor* phys_td = getMetadataForTable(physical_tb_id);
2268  CHECK(phys_td);
2269  if (i == 0) {
2270  curr_epoch = dataMgr_->getTableEpoch(db_id, physical_tb_id);
2271  } else {
2272  if (curr_epoch != dataMgr_->getTableEpoch(db_id, physical_tb_id)) {
2273  // oh dear the leaves do not agree on the epoch for this table
2274  LOG(ERROR) << "Epochs on shards do not all agree on table id " << table_id
2275  << " db id " << db_id << " epoch " << curr_epoch << " leaf_epoch "
2276  << dataMgr_->getTableEpoch(db_id, physical_tb_id);
2277  return -1;
2278  }
2279  }
2280  }
2281  return curr_epoch;
2282  } else {
2283  return dataMgr_->getTableEpoch(db_id, table_id);
2284  }
2285 }
const TableDescriptor * getMetadataForTable(const std::string &tableName, const bool populateFragmenter=true) const
Returns a pointer to a const TableDescriptor struct matching the provided tableName.
#define LOG(tag)
Definition: Logger.h:188
std::shared_ptr< Data_Namespace::DataMgr > dataMgr_
Definition: Catalog.h:333
read_lock< Catalog > cat_read_lock
Definition: Catalog.cpp:100
#define CHECK(condition)
Definition: Logger.h:193
specifies the content in-memory of a row in the table metadata table
LogicalToPhysicalTableMapById logicalToPhysicalTableMapById_
Definition: Catalog.h:338
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ instantiateFragmenter()

void Catalog_Namespace::Catalog::instantiateFragmenter ( TableDescriptor td) const
protected

Definition at line 1300 of file Catalog.cpp.

References CHECK_EQ, currentDB_, dataMgr_, Catalog_Namespace::DBMetadata::dbId, measure< TimeT >::execution(), TableDescriptor::fragmenter, TableDescriptor::fragPageSize, TableDescriptor::fragType, getAllColumnMetadataForTable(), getMetadataForTable(), logger::INFO, Fragmenter_Namespace::INSERT_ORDER, TableDescriptor::isView, LOG, TableDescriptor::maxChunkSize, TableDescriptor::maxFragRows, TableDescriptor::maxRows, TableDescriptor::mutex_, TableDescriptor::persistenceLevel, TableDescriptor::shard, TableDescriptor::sortedColumnId, tableDescriptorMap_, TableDescriptor::tableId, TableDescriptor::tableName, and to_upper().

Referenced by getMetadataForTableImpl().

1300  {
1301  auto time_ms = measure<>::execution([&]() {
1302  // instanciate table fragmenter upon first use
1303  // assume only insert order fragmenter is supported
1305  vector<Chunk> chunkVec;
1306  list<const ColumnDescriptor*> columnDescs;
1307  getAllColumnMetadataForTable(td, columnDescs, true, false, true);
1308  Chunk::translateColumnDescriptorsToChunkVec(columnDescs, chunkVec);
1309  ChunkKey chunkKeyPrefix = {currentDB_.dbId, td->tableId};
1310  if (td->sortedColumnId > 0) {
1311  td->fragmenter = new SortedOrderFragmenter(chunkKeyPrefix,
1312  chunkVec,
1313  dataMgr_.get(),
1314  const_cast<Catalog*>(this),
1315  td->tableId,
1316  td->shard,
1317  td->maxFragRows,
1318  td->maxChunkSize,
1319  td->fragPageSize,
1320  td->maxRows,
1321  td->persistenceLevel);
1322  } else {
1323  td->fragmenter = new InsertOrderFragmenter(chunkKeyPrefix,
1324  chunkVec,
1325  dataMgr_.get(),
1326  const_cast<Catalog*>(this),
1327  td->tableId,
1328  td->shard,
1329  td->maxFragRows,
1330  td->maxChunkSize,
1331  td->fragPageSize,
1332  td->maxRows,
1333  td->persistenceLevel);
1334  }
1335  });
1336  LOG(INFO) << "Instantiating Fragmenter for table " << td->tableName << " took "
1337  << time_ms << "ms";
1338 }
#define CHECK_EQ(x, y)
Definition: Logger.h:201
std::string tableName
#define LOG(tag)
Definition: Logger.h:188