OmniSciDB  c1a53651b2
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
InternalMemoryStatsDataWrapper.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
18 
19 #include "Catalog/SysCatalog.h"
20 #include "ImportExport/Importer.h"
21 
22 namespace foreign_storage {
25 
27  const int db_id,
28  const ForeignTable* foreign_table)
29  : InternalSystemDataWrapper(db_id, foreign_table) {}
30 
31 namespace {
33  const std::map<std::string, std::vector<MemoryInfo>>& memory_info_by_device_type,
34  std::map<std::string, import_export::TypedImportBuffer*>& import_buffers) {
35  for (const auto& [device_type, memory_info_vector] : memory_info_by_device_type) {
36  int32_t device_id{0};
37  for (const auto& memory_info : memory_info_vector) {
38  size_t used_page_count{0}, free_page_count{0};
39  for (const auto& memory_data : memory_info.nodeMemoryData) {
40  if (memory_data.memStatus == Buffer_Namespace::MemStatus::FREE) {
41  free_page_count += memory_data.numPages;
42  } else {
43  used_page_count += memory_data.numPages;
44  }
45  }
46  set_node_name(import_buffers);
47  if (import_buffers.find("device_id") != import_buffers.end()) {
48  import_buffers["device_id"]->addInt(device_id);
49  }
50  if (import_buffers.find("device_type") != import_buffers.end()) {
51  import_buffers["device_type"]->addString(device_type);
52  }
53  if (import_buffers.find("max_page_count") != import_buffers.end()) {
54  import_buffers["max_page_count"]->addBigint(memory_info.maxNumPages);
55  }
56  if (import_buffers.find("page_size") != import_buffers.end()) {
57  import_buffers["page_size"]->addBigint(memory_info.pageSize);
58  }
59  if (import_buffers.find("allocated_page_count") != import_buffers.end()) {
60  import_buffers["allocated_page_count"]->addBigint(memory_info.numPageAllocated);
61  }
62  if (import_buffers.find("used_page_count") != import_buffers.end()) {
63  import_buffers["used_page_count"]->addBigint(used_page_count);
64  }
65  if (import_buffers.find("free_page_count") != import_buffers.end()) {
66  import_buffers["free_page_count"]->addBigint(free_page_count);
67  }
68  device_id++;
69  }
70  }
71 }
72 
74  import_buffer->add_value(import_buffer->getColumnDesc(), "", true, {});
75 }
76 
77 bool is_table_chunk(const ChunkKey& chunk_key) {
78  // Non-table chunks (temporary buffers) use a negative id for the database id.
79  return (!chunk_key.empty() && chunk_key[CHUNK_KEY_DB_IDX] > 0);
80 }
81 
82 std::string get_column_name(int32_t db_id, int32_t table_id, int32_t column_id) {
84  CHECK(catalog);
85  auto column_name = catalog->getColumnName(table_id, column_id);
86  if (column_name.has_value()) {
87  return column_name.value();
88  } else {
89  // It is possible for the column to be concurrently deleted while querying the system
90  // table.
92  }
93 }
94 
96  const std::map<std::string, std::vector<MemoryInfo>>& memory_info_by_device_type,
97  std::map<std::string, import_export::TypedImportBuffer*>& import_buffers) {
98  for (const auto& [device_type, memory_info_vector] : memory_info_by_device_type) {
99  int32_t device_id{0};
100  for (const auto& memory_info : memory_info_vector) {
101  for (const auto& memory_data : memory_info.nodeMemoryData) {
102  set_node_name(import_buffers);
103  const auto& chunk_key = memory_data.chunk_key;
104  if (import_buffers.find("database_id") != import_buffers.end()) {
105  auto import_buffer = import_buffers["database_id"];
106  if (is_table_chunk(chunk_key)) {
107  import_buffer->addInt(chunk_key[CHUNK_KEY_DB_IDX]);
108  } else {
109  set_null(import_buffer);
110  }
111  }
112  if (import_buffers.find("database_name") != import_buffers.end()) {
113  auto import_buffer = import_buffers["database_name"];
114  if (is_table_chunk(chunk_key)) {
115  import_buffer->addString(get_db_name(chunk_key[CHUNK_KEY_DB_IDX]));
116  } else {
117  set_null(import_buffer);
118  }
119  }
120  if (import_buffers.find("table_id") != import_buffers.end()) {
121  auto import_buffer = import_buffers["table_id"];
122  if (is_table_chunk(chunk_key)) {
123  import_buffer->addInt(chunk_key[CHUNK_KEY_TABLE_IDX]);
124  } else {
125  set_null(import_buffer);
126  }
127  }
128  if (import_buffers.find("table_name") != import_buffers.end()) {
129  auto import_buffer = import_buffers["table_name"];
130  if (is_table_chunk(chunk_key)) {
131  import_buffer->addString(get_table_name(chunk_key[CHUNK_KEY_DB_IDX],
132  chunk_key[CHUNK_KEY_TABLE_IDX]));
133  } else {
134  set_null(import_buffer);
135  }
136  }
137  if (import_buffers.find("column_id") != import_buffers.end()) {
138  auto import_buffer = import_buffers["column_id"];
139  if (is_table_chunk(chunk_key)) {
140  import_buffer->addInt(chunk_key[CHUNK_KEY_COLUMN_IDX]);
141  } else {
142  set_null(import_buffer);
143  }
144  }
145  if (import_buffers.find("column_name") != import_buffers.end()) {
146  auto import_buffer = import_buffers["column_name"];
147  if (is_table_chunk(chunk_key)) {
148  import_buffer->addString(get_column_name(chunk_key[CHUNK_KEY_DB_IDX],
149  chunk_key[CHUNK_KEY_TABLE_IDX],
150  chunk_key[CHUNK_KEY_COLUMN_IDX]));
151  } else {
152  set_null(import_buffer);
153  }
154  }
155  if (import_buffers.find("chunk_key") != import_buffers.end()) {
156  import_buffers["chunk_key"]->addArray(ArrayDatum(
157  chunk_key.size() * sizeof(int32_t),
158  reinterpret_cast<int8_t*>(const_cast<int32_t*>(chunk_key.data())),
159  DoNothingDeleter()));
160  }
161  if (import_buffers.find("device_id") != import_buffers.end()) {
162  import_buffers["device_id"]->addInt(device_id);
163  }
164  if (import_buffers.find("device_type") != import_buffers.end()) {
165  import_buffers["device_type"]->addString(device_type);
166  }
167  if (import_buffers.find("memory_status") != import_buffers.end()) {
168  auto memory_status =
169  (memory_data.memStatus == Buffer_Namespace::MemStatus::FREE ? "FREE"
170  : "USED");
171  import_buffers["memory_status"]->addString(memory_status);
172  }
173  if (import_buffers.find("page_count") != import_buffers.end()) {
174  import_buffers["page_count"]->addBigint(memory_data.numPages);
175  }
176  if (import_buffers.find("page_size") != import_buffers.end()) {
177  import_buffers["page_size"]->addBigint(memory_info.pageSize);
178  }
179  if (import_buffers.find("slab_id") != import_buffers.end()) {
180  import_buffers["slab_id"]->addInt(memory_data.slabNum);
181  }
182  if (import_buffers.find("start_page") != import_buffers.end()) {
183  import_buffers["start_page"]->addBigint(memory_data.startPage);
184  }
185  if (import_buffers.find("last_touch_epoch") != import_buffers.end()) {
186  import_buffers["last_touch_epoch"]->addBigint(memory_data.touch);
187  }
188  }
189  device_id++;
190  }
191  }
192 }
193 } // namespace
194 
196  const std::string& table_name) {
198  row_count_ = 0;
199  const auto& data_mgr = Catalog_Namespace::SysCatalog::instance().getDataMgr();
200  // DataMgr::getMemoryInfoUnlocked() is used here because a lock on buffer_access_mutex_
201  // is already acquired in DataMgr::getChunkMetadataVecForKeyPrefix()
203  data_mgr.getMemoryInfoUnlocked(MemoryLevel::CPU_LEVEL);
205  data_mgr.getMemoryInfoUnlocked(MemoryLevel::GPU_LEVEL);
207  for (const auto& [device_type, memory_info_vector] : memory_info_by_device_type_) {
208  row_count_ += memory_info_vector.size();
209  }
210  } else if (foreign_table_->tableName ==
212  for (auto& [device_type, memory_info_vector] : memory_info_by_device_type_) {
213  for (auto& memory_info : memory_info_vector) {
214  for (auto& memory_data : memory_info.nodeMemoryData) {
215  if (memory_data.memStatus == Buffer_Namespace::MemStatus::FREE) {
216  memory_data.chunk_key.clear();
217  } else {
218  if (is_table_chunk(memory_data.chunk_key)) {
219  CHECK_GE(memory_data.chunk_key.size(), static_cast<size_t>(4));
220  }
221  }
222  }
223  row_count_ += memory_info.nodeMemoryData.size();
224  }
225  }
226  } else {
227  UNREACHABLE() << "Unexpected table name: " << table_name;
228  }
229 }
230 
232  const std::string& table_name,
233  std::map<std::string, import_export::TypedImportBuffer*>& import_buffers) {
236  import_buffers);
237  } else if (foreign_table_->tableName ==
240  import_buffers);
241  } else {
242  UNREACHABLE() << "Unexpected table name: " << foreign_table_->tableName;
243  }
244 }
245 } // namespace foreign_storage
static constexpr const char * MEMORY_DETAILS_SYS_TABLE_NAME
Definition: Catalog.h:112
std::vector< int > ChunkKey
Definition: types.h:36
std::string tableName
std::string get_column_name(int32_t db_id, int32_t table_id, int32_t column_id)
void initializeObjectsForTable(const std::string &table_name) override
constexpr const char * kDeletedValueIndicator
#define CHUNK_KEY_DB_IDX
Definition: types.h:38
#define UNREACHABLE()
Definition: Logger.h:337
#define CHECK_GE(x, y)
Definition: Logger.h:306
std::conditional_t< is_cuda_compiler(), DeviceArrayDatum, HostArrayDatum > ArrayDatum
Definition: sqltypes.h:219
Data_Namespace::DataMgr & getDataMgr() const
Definition: SysCatalog.h:234
static SysCatalog & instance()
Definition: SysCatalog.h:343
This file contains the class specification and related data structures for SysCatalog.
void set_node_name(std::map< std::string, import_export::TypedImportBuffer * > &import_buffers)
CONSTEXPR DEVICE void set_null(T &value)
#define CHUNK_KEY_TABLE_IDX
Definition: types.h:39
static constexpr const char * MEMORY_SUMMARY_SYS_TABLE_NAME
Definition: Catalog.h:111
std::map< std::string, std::vector< MemoryInfo > > memory_info_by_device_type_
std::shared_ptr< Catalog > getCatalog(const std::string &dbName)
void populate_import_buffers_for_memory_details(const std::map< std::string, std::vector< MemoryInfo >> &memory_info_by_device_type, std::map< std::string, import_export::TypedImportBuffer * > &import_buffers)
#define CHECK(condition)
Definition: Logger.h:291
std::string get_db_name(int32_t db_id)
const ColumnDescriptor * getColumnDesc() const
Definition: Importer.h:319
void populateChunkBuffersForTable(const std::string &table_name, std::map< std::string, import_export::TypedImportBuffer * > &import_buffers) override
#define CHUNK_KEY_COLUMN_IDX
Definition: types.h:40
void add_value(const ColumnDescriptor *cd, const std::string_view val, const bool is_null, const CopyParams &copy_params, const bool check_not_null=true)
Definition: Importer.cpp:528
void populate_import_buffers_for_memory_summary(const std::map< std::string, std::vector< MemoryInfo >> &memory_info_by_device_type, std::map< std::string, import_export::TypedImportBuffer * > &import_buffers)
std::string get_table_name(int32_t db_id, int32_t table_id)