OmniSciDB  c1a53651b2
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
InternalStorageStatsDataWrapper.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
18 
19 #include <filesystem>
20 
21 #include "Catalog/SysCatalog.h"
22 #include "ImportExport/Importer.h"
23 
24 namespace foreign_storage {
27 
29  const int db_id,
30  const ForeignTable* foreign_table)
31  : InternalSystemDataWrapper(db_id, foreign_table) {}
32 
33 namespace {
35  import_buffer->add_value(import_buffer->getColumnDesc(), "", true, {});
36 }
37 
39  const std::vector<StorageDetails>& storage_details,
40  std::map<std::string, import_export::TypedImportBuffer*>& import_buffers) {
41  for (const auto& storage_detail : storage_details) {
42  set_node_name(import_buffers);
43  if (import_buffers.find("database_id") != import_buffers.end()) {
44  import_buffers["database_id"]->addInt(storage_detail.database_id);
45  }
46  if (import_buffers.find("database_name") != import_buffers.end()) {
47  import_buffers["database_name"]->addString(get_db_name(storage_detail.database_id));
48  }
49  if (import_buffers.find("table_id") != import_buffers.end()) {
50  import_buffers["table_id"]->addInt(storage_detail.table_id);
51  }
52  if (import_buffers.find("table_name") != import_buffers.end()) {
53  import_buffers["table_name"]->addString(
54  get_table_name(storage_detail.database_id, storage_detail.table_id));
55  }
56  if (import_buffers.find("epoch") != import_buffers.end()) {
57  import_buffers["epoch"]->addInt(storage_detail.storage_stats.epoch);
58  }
59  if (import_buffers.find("epoch_floor") != import_buffers.end()) {
60  import_buffers["epoch_floor"]->addInt(storage_detail.storage_stats.epoch_floor);
61  }
62  if (import_buffers.find("fragment_count") != import_buffers.end()) {
63  auto import_buffer = import_buffers["fragment_count"];
64  if (storage_detail.storage_stats.fragment_count.has_value()) {
65  import_buffer->addInt(storage_detail.storage_stats.fragment_count.value());
66  } else {
67  set_null(import_buffer);
68  }
69  }
70  if (import_buffers.find("shard_id") != import_buffers.end()) {
71  import_buffers["shard_id"]->addInt(storage_detail.shard_id);
72  }
73  if (import_buffers.find("data_file_count") != import_buffers.end()) {
74  import_buffers["data_file_count"]->addInt(
75  storage_detail.storage_stats.data_file_count);
76  }
77  if (import_buffers.find("metadata_file_count") != import_buffers.end()) {
78  import_buffers["metadata_file_count"]->addInt(
79  storage_detail.storage_stats.metadata_file_count);
80  }
81  if (import_buffers.find("total_data_file_size") != import_buffers.end()) {
82  import_buffers["total_data_file_size"]->addBigint(
83  storage_detail.storage_stats.total_data_file_size);
84  }
85  if (import_buffers.find("total_data_page_count") != import_buffers.end()) {
86  import_buffers["total_data_page_count"]->addBigint(
87  storage_detail.storage_stats.total_data_page_count);
88  }
89  if (import_buffers.find("total_free_data_page_count") != import_buffers.end()) {
90  auto import_buffer = import_buffers["total_free_data_page_count"];
91  if (storage_detail.storage_stats.total_free_data_page_count.has_value()) {
92  import_buffer->addBigint(
93  storage_detail.storage_stats.total_free_data_page_count.value());
94  } else {
95  set_null(import_buffer);
96  }
97  }
98  if (import_buffers.find("total_metadata_file_size") != import_buffers.end()) {
99  import_buffers["total_metadata_file_size"]->addBigint(
100  storage_detail.storage_stats.total_metadata_file_size);
101  }
102  if (import_buffers.find("total_metadata_page_count") != import_buffers.end()) {
103  import_buffers["total_metadata_page_count"]->addBigint(
104  storage_detail.storage_stats.total_metadata_page_count);
105  }
106  if (import_buffers.find("total_free_metadata_page_count") != import_buffers.end()) {
107  auto import_buffer = import_buffers["total_free_metadata_page_count"];
108  if (storage_detail.storage_stats.total_free_metadata_page_count.has_value()) {
109  import_buffer->addBigint(
110  storage_detail.storage_stats.total_free_metadata_page_count.value());
111  } else {
112  set_null(import_buffer);
113  }
114  }
115  if (import_buffers.find("total_dictionary_data_file_size") != import_buffers.end()) {
116  import_buffers["total_dictionary_data_file_size"]->addBigint(
117  storage_detail.total_dictionary_data_file_size);
118  }
119  }
120 }
121 } // namespace
122 
124  const std::string& table_name) {
126  storage_details_.clear();
127  const auto global_file_mgr =
129  CHECK(global_file_mgr);
130  auto& sys_catalog = Catalog_Namespace::SysCatalog::instance();
131  for (const auto& catalog : sys_catalog.getCatalogsForAllDbs()) {
132  if (catalog->name() != shared::kInfoSchemaDbName) {
133  std::set<std::string> found_dict_paths;
134  for (const auto& [table_id, shard_id] :
135  catalog->getAllPersistedTableAndShardIds()) {
136  uint64_t total_dictionary_file_size{0};
137  auto logical_table_id = catalog->getLogicalTableId(table_id);
138  for (const auto& dict_path :
139  catalog->getTableDictDirectoryPaths(logical_table_id)) {
140  if (found_dict_paths.find(dict_path) == found_dict_paths.end()) {
141  found_dict_paths.emplace(dict_path);
142  } else {
143  // Skip shared dictionaries.
144  continue;
145  }
146  CHECK(std::filesystem::is_directory(dict_path));
147  for (const auto& file_entry : std::filesystem::directory_iterator(dict_path)) {
148  CHECK(file_entry.is_regular_file());
149  total_dictionary_file_size += static_cast<uint64_t>(file_entry.file_size());
150  }
151  }
152  auto db_id = catalog->getDatabaseId();
153  storage_details_.emplace_back(db_id,
154  logical_table_id,
155  shard_id,
156  total_dictionary_file_size,
157  global_file_mgr->getStorageStats(db_id, table_id));
158  }
159  }
160  }
161  row_count_ = storage_details_.size();
162 }
163 
165  const std::string& table_name,
166  std::map<std::string, import_export::TypedImportBuffer*>& import_buffers) {
169 }
170 } // namespace foreign_storage
#define CHECK_EQ(x, y)
Definition: Logger.h:301
void populateChunkBuffersForTable(const std::string &table_name, std::map< std::string, import_export::TypedImportBuffer * > &import_buffers) override
void populate_import_buffers_for_storage_details(const std::vector< StorageDetails > &storage_details, std::map< std::string, import_export::TypedImportBuffer * > &import_buffers)
const std::string kInfoSchemaDbName
Data_Namespace::DataMgr & getDataMgr() const
Definition: SysCatalog.h:234
static SysCatalog & instance()
Definition: SysCatalog.h:343
This file contains the class specification and related data structures for SysCatalog.
void set_node_name(std::map< std::string, import_export::TypedImportBuffer * > &import_buffers)
CONSTEXPR DEVICE void set_null(T &value)
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
Definition: DataMgr.cpp:606
static constexpr const char * STORAGE_DETAILS_SYS_TABLE_NAME
Definition: Catalog.h:113
#define CHECK(condition)
Definition: Logger.h:291
void initializeObjectsForTable(const std::string &table_name) override
std::string get_db_name(int32_t db_id)
const ColumnDescriptor * getColumnDesc() const
Definition: Importer.h:319
void add_value(const ColumnDescriptor *cd, const std::string_view val, const bool is_null, const CopyParams &copy_params, const bool check_not_null=true)
Definition: Importer.cpp:528
std::string get_table_name(int32_t db_id, int32_t table_id)