OmniSciDB  d2f719934e
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
InternalStorageStatsDataWrapper.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2021 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
18 
19 #include <filesystem>
20 
21 #include "Catalog/SysCatalog.h"
22 #include "ImportExport/Importer.h"
23 
24 namespace foreign_storage {
27 
29  const int db_id,
30  const ForeignTable* foreign_table)
31  : InternalSystemDataWrapper(db_id, foreign_table) {}
32 
33 namespace {
35  import_buffer->add_value(import_buffer->getColumnDesc(), "", true, {});
36 }
37 
39  const std::vector<StorageDetails>& storage_details,
40  std::map<std::string, import_export::TypedImportBuffer*>& import_buffers) {
41  for (const auto& storage_detail : storage_details) {
42  if (import_buffers.find("node") != import_buffers.end()) {
43  import_buffers["node"]->addString(storage_detail.node);
44  }
45  if (import_buffers.find("database_id") != import_buffers.end()) {
46  import_buffers["database_id"]->addInt(storage_detail.database_id);
47  }
48  if (import_buffers.find("table_id") != import_buffers.end()) {
49  import_buffers["table_id"]->addInt(storage_detail.table_id);
50  }
51  if (import_buffers.find("epoch") != import_buffers.end()) {
52  import_buffers["epoch"]->addInt(storage_detail.storage_stats.epoch);
53  }
54  if (import_buffers.find("epoch_floor") != import_buffers.end()) {
55  import_buffers["epoch_floor"]->addInt(storage_detail.storage_stats.epoch_floor);
56  }
57  if (import_buffers.find("fragment_count") != import_buffers.end()) {
58  auto import_buffer = import_buffers["fragment_count"];
59  if (storage_detail.storage_stats.fragment_count.has_value()) {
60  import_buffer->addInt(storage_detail.storage_stats.fragment_count.value());
61  } else {
62  set_null(import_buffer);
63  }
64  }
65  if (import_buffers.find("shard_id") != import_buffers.end()) {
66  import_buffers["shard_id"]->addInt(storage_detail.shard_id);
67  }
68  if (import_buffers.find("data_file_count") != import_buffers.end()) {
69  import_buffers["data_file_count"]->addInt(
70  storage_detail.storage_stats.data_file_count);
71  }
72  if (import_buffers.find("metadata_file_count") != import_buffers.end()) {
73  import_buffers["metadata_file_count"]->addInt(
74  storage_detail.storage_stats.metadata_file_count);
75  }
76  if (import_buffers.find("total_data_file_size") != import_buffers.end()) {
77  import_buffers["total_data_file_size"]->addBigint(
78  storage_detail.storage_stats.total_data_file_size);
79  }
80  if (import_buffers.find("total_data_page_count") != import_buffers.end()) {
81  import_buffers["total_data_page_count"]->addBigint(
82  storage_detail.storage_stats.total_data_page_count);
83  }
84  if (import_buffers.find("total_free_data_page_count") != import_buffers.end()) {
85  auto import_buffer = import_buffers["total_free_data_page_count"];
86  if (storage_detail.storage_stats.total_free_data_page_count.has_value()) {
87  import_buffer->addBigint(
88  storage_detail.storage_stats.total_free_data_page_count.value());
89  } else {
90  set_null(import_buffer);
91  }
92  }
93  if (import_buffers.find("total_metadata_file_size") != import_buffers.end()) {
94  import_buffers["total_metadata_file_size"]->addBigint(
95  storage_detail.storage_stats.total_metadata_file_size);
96  }
97  if (import_buffers.find("total_metadata_page_count") != import_buffers.end()) {
98  import_buffers["total_metadata_page_count"]->addBigint(
99  storage_detail.storage_stats.total_metadata_page_count);
100  }
101  if (import_buffers.find("total_free_metadata_page_count") != import_buffers.end()) {
102  auto import_buffer = import_buffers["total_free_metadata_page_count"];
103  if (storage_detail.storage_stats.total_free_metadata_page_count.has_value()) {
104  import_buffer->addBigint(
105  storage_detail.storage_stats.total_free_metadata_page_count.value());
106  } else {
107  set_null(import_buffer);
108  }
109  }
110  if (import_buffers.find("total_dictionary_data_file_size") != import_buffers.end()) {
111  import_buffers["total_dictionary_data_file_size"]->addBigint(
112  storage_detail.total_dictionary_data_file_size);
113  }
114  }
115 }
116 } // namespace
117 
119  const std::string& table_name) {
121  storage_details_.clear();
122  const auto global_file_mgr =
124  CHECK(global_file_mgr);
125  auto& sys_catalog = Catalog_Namespace::SysCatalog::instance();
126  for (const auto& catalog : sys_catalog.getCatalogsForAllDbs()) {
127  if (catalog->name() != INFORMATION_SCHEMA_DB) {
128  for (const auto& [table_id, shard_id] :
129  catalog->getAllPersistedTableAndShardIds()) {
130  uint64_t total_dictionary_file_size{0};
131  auto logical_table_id = catalog->getLogicalTableId(table_id);
132  for (const auto& dict_path :
133  catalog->getTableDictDirectoryPaths(logical_table_id)) {
134  CHECK(std::filesystem::is_directory(dict_path));
135  for (const auto& file_entry : std::filesystem::directory_iterator(dict_path)) {
136  CHECK(file_entry.is_regular_file());
137  total_dictionary_file_size += static_cast<uint64_t>(file_entry.file_size());
138  }
139  }
140  auto db_id = catalog->getDatabaseId();
141  storage_details_.emplace_back(db_id,
142  logical_table_id,
143  shard_id,
144  total_dictionary_file_size,
145  global_file_mgr->getStorageStats(db_id, table_id));
146  }
147  }
148  }
149  row_count_ = storage_details_.size();
150 }
151 
153  const std::string& table_name,
154  std::map<std::string, import_export::TypedImportBuffer*>& import_buffers) {
157 }
158 } // namespace foreign_storage
#define CHECK_EQ(x, y)
Definition: Logger.h:219
void add_value(const ColumnDescriptor *cd, const std::string_view val, const bool is_null, const CopyParams &copy_params)
Definition: Importer.cpp:549
void populateChunkBuffersForTable(const std::string &table_name, std::map< std::string, import_export::TypedImportBuffer * > &import_buffers) override
void populate_import_buffers_for_storage_details(const std::vector< StorageDetails > &storage_details, std::map< std::string, import_export::TypedImportBuffer * > &import_buffers)
Data_Namespace::DataMgr & getDataMgr() const
Definition: SysCatalog.h:217
static SysCatalog & instance()
Definition: SysCatalog.h:326
This file contains the class specification and related data structures for SysCatalog.
CONSTEXPR DEVICE void set_null(T &value)
File_Namespace::GlobalFileMgr * getGlobalFileMgr() const
Definition: DataMgr.cpp:616
static constexpr const char * STORAGE_DETAILS_SYS_TABLE_NAME
Definition: Catalog.h:106
#define CHECK(condition)
Definition: Logger.h:211
void initializeObjectsForTable(const std::string &table_name) override
const ColumnDescriptor * getColumnDesc() const
Definition: Importer.h:311
const std::string INFORMATION_SCHEMA_DB
Definition: SysCatalog.h:65