OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
foreign_storage::InternalSystemDataWrapper Class Referenceabstract

#include <InternalSystemDataWrapper.h>

+ Inheritance diagram for foreign_storage::InternalSystemDataWrapper:
+ Collaboration diagram for foreign_storage::InternalSystemDataWrapper:

Public Member Functions

 InternalSystemDataWrapper ()
 
 InternalSystemDataWrapper (const int db_id, const ForeignTable *foreign_table)
 
void populateChunkMetadata (ChunkMetadataVector &chunk_metadata_vector) override
 
void populateChunkBuffers (const ChunkToBufferMap &required_buffers, const ChunkToBufferMap &optional_buffers, AbstractBuffer *delete_buffer) override
 
void validateServerOptions (const ForeignServer *foreign_server) const override
 
void validateTableOptions (const ForeignTable *foreign_table) const override
 
const std::set
< std::string_view > & 
getSupportedTableOptions () const override
 
void validateUserMappingOptions (const UserMapping *user_mapping, const ForeignServer *foreign_server) const override
 
const std::set
< std::string_view > & 
getSupportedUserMappingOptions () const override
 
std::string getSerializedDataWrapper () const override
 
void restoreDataWrapperInternals (const std::string &file_path, const ChunkMetadataVector &chunk_metadata) override
 
bool isRestored () const override
 
- Public Member Functions inherited from foreign_storage::ForeignDataWrapper
 ForeignDataWrapper ()=default
 
virtual ~ForeignDataWrapper ()=default
 
virtual const std::set
< std::string > 
getAlterableTableOptions () const
 
virtual void validateSchema (const std::list< ColumnDescriptor > &columns) const
 
virtual ParallelismLevel getCachedParallelismLevel () const
 
virtual ParallelismLevel getNonCachedParallelismLevel () const
 
virtual bool isLazyFragmentFetchingEnabled () const
 

Protected Member Functions

virtual void initializeObjectsForTable (const std::string &table_name)=0
 
virtual void populateChunkBuffersForTable (const std::string &table_name, std::map< std::string, import_export::TypedImportBuffer * > &import_buffers)=0
 

Protected Attributes

const int db_id_
 
const ForeignTableforeign_table_
 
size_t row_count_ {0}
 

Additional Inherited Members

- Public Types inherited from foreign_storage::ForeignDataWrapper
enum  ParallelismLevel { NONE, INTRA_FRAGMENT, INTER_FRAGMENT }
 

Detailed Description

Definition at line 34 of file InternalSystemDataWrapper.h.

Constructor & Destructor Documentation

foreign_storage::InternalSystemDataWrapper::InternalSystemDataWrapper ( )
foreign_storage::InternalSystemDataWrapper::InternalSystemDataWrapper ( const int  db_id,
const ForeignTable foreign_table 
)

Member Function Documentation

std::string foreign_storage::InternalSystemDataWrapper::getSerializedDataWrapper ( ) const
overridevirtual

Serialize internal state of wrapper into file at given path if implemented

Implements foreign_storage::ForeignDataWrapper.

Definition at line 226 of file InternalSystemDataWrapper.cpp.

226  {
227  return {};
228 }
const std::set< std::string_view > & foreign_storage::InternalSystemDataWrapper::getSupportedTableOptions ( ) const
overridevirtual

Gets the set of supported table options for the data wrapper.

Implements foreign_storage::ForeignDataWrapper.

Definition at line 82 of file InternalSystemDataWrapper.cpp.

83  {
84  static const std::set<std::string_view> supported_table_options{};
85  return supported_table_options;
86 }
const std::set< std::string_view > & foreign_storage::InternalSystemDataWrapper::getSupportedUserMappingOptions ( ) const
overridevirtual

Gets the set of supported user mapping options for the data wrapper.

Implements foreign_storage::ForeignDataWrapper.

Definition at line 95 of file InternalSystemDataWrapper.cpp.

95  {
96  static const std::set<std::string_view> supported_user_mapping_options{};
97  return supported_user_mapping_options;
98 }
virtual void foreign_storage::InternalSystemDataWrapper::initializeObjectsForTable ( const std::string &  table_name)
protectedpure virtual

Implemented in foreign_storage::InternalStorageStatsDataWrapper, foreign_storage::InternalExecutorStatsDataWrapper, foreign_storage::InternalMLModelMetadataDataWrapper, foreign_storage::InternalMemoryStatsDataWrapper, and foreign_storage::InternalCatalogDataWrapper.

Referenced by populateChunkMetadata().

+ Here is the caller graph for this function:

bool foreign_storage::InternalSystemDataWrapper::isRestored ( ) const
overridevirtual

Implements foreign_storage::ForeignDataWrapper.

Definition at line 234 of file InternalSystemDataWrapper.cpp.

234  {
235  return false;
236 }
void foreign_storage::InternalSystemDataWrapper::populateChunkBuffers ( const ChunkToBufferMap required_buffers,
const ChunkToBufferMap optional_buffers,
AbstractBuffer delete_buffer 
)
overridevirtual

Populates given chunk buffers identified by chunk keys. All provided chunk buffers are expected to be for the same fragment.

Parameters
required_buffers- chunk buffers that must always be populated
optional_buffers- chunk buffers that can be optionally populated, if the data wrapper has to scan through chunk data anyways (typically for row wise data formats)
delete_buffer- chunk buffer for fragment's delete column, if non-null data wrapper is expected to mark deleted rows in buffer and continue processing

Implements foreign_storage::ForeignDataWrapper.

Definition at line 186 of file InternalSystemDataWrapper.cpp.

References CHECK, CHECK_EQ, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_FRAGMENT_IDX, foreign_storage::TextFileBufferParser::convertImportBuffersToDataBlocks(), db_id_, DEBUG_TIMER, foreign_table_, foreign_storage::anonymous_namespace{InternalSystemDataWrapper.cpp}::initialize_chunks(), foreign_storage::anonymous_namespace{InternalSystemDataWrapper.cpp}::initialize_import_buffers(), Catalog_Namespace::SysCatalog::instance(), shared::kInfoSchemaDbName, populateChunkBuffersForTable(), row_count_, and TableDescriptor::tableName.

189  {
190  auto timer = DEBUG_TIMER(__func__);
191  CHECK(optional_buffers.empty());
192 
193  auto& sys_catalog = Catalog_Namespace::SysCatalog::instance();
194  auto catalog = sys_catalog.getCatalog(db_id_);
195  CHECK(catalog);
196  CHECK_EQ(catalog->name(), shared::kInfoSchemaDbName);
197 
198  auto fragment_id = required_buffers.begin()->first[CHUNK_KEY_FRAGMENT_IDX];
199  CHECK_EQ(fragment_id, 0)
200  << "In-memory system tables are expected to have a single fragment.";
201 
202  std::map<ChunkKey, Chunk_NS::Chunk> chunks;
203  std::set<const ColumnDescriptor*> columns_to_parse;
205  chunks, required_buffers, row_count_, columns_to_parse, fragment_id, *catalog);
206 
207  // initialize import buffers from columns.
208  std::vector<std::unique_ptr<import_export::TypedImportBuffer>> import_buffers;
209  std::map<std::string, import_export::TypedImportBuffer*> import_buffers_map;
211  import_buffers, import_buffers_map, columns_to_parse, *catalog);
213 
214  auto column_id_to_data_blocks_map =
216  for (auto& [chunk_key, chunk] : chunks) {
217  auto data_block_entry =
218  column_id_to_data_blocks_map.find(chunk_key[CHUNK_KEY_COLUMN_IDX]);
219  CHECK(data_block_entry != column_id_to_data_blocks_map.end());
220  chunk.appendData(data_block_entry->second, row_count_, 0);
221  chunk.setBuffer(nullptr);
222  chunk.setIndexBuffer(nullptr);
223  }
224 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
std::string tableName
static std::map< int, DataBlockPtr > convertImportBuffersToDataBlocks(const std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &import_buffers, const bool skip_dict_encoding=false)
#define CHUNK_KEY_FRAGMENT_IDX
Definition: types.h:41
const std::string kInfoSchemaDbName
static SysCatalog & instance()
Definition: SysCatalog.h:343
void initialize_chunks(std::map< ChunkKey, Chunk_NS::Chunk > &chunks, const ChunkToBufferMap &buffers, size_t row_count, std::set< const ColumnDescriptor * > &columns_to_parse, int32_t fragment_id, const Catalog_Namespace::Catalog &catalog)
virtual void populateChunkBuffersForTable(const std::string &table_name, std::map< std::string, import_export::TypedImportBuffer * > &import_buffers)=0
#define CHECK(condition)
Definition: Logger.h:291
#define DEBUG_TIMER(name)
Definition: Logger.h:412
#define CHUNK_KEY_COLUMN_IDX
Definition: types.h:40
void initialize_import_buffers(std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &import_buffers, std::map< std::string, import_export::TypedImportBuffer * > &import_buffers_map, const std::set< const ColumnDescriptor * > &columns_to_parse, const Catalog_Namespace::Catalog &catalog)

+ Here is the call graph for this function:

virtual void foreign_storage::InternalSystemDataWrapper::populateChunkBuffersForTable ( const std::string &  table_name,
std::map< std::string, import_export::TypedImportBuffer * > &  import_buffers 
)
protectedpure virtual

Implemented in foreign_storage::InternalStorageStatsDataWrapper, foreign_storage::InternalExecutorStatsDataWrapper, foreign_storage::InternalMLModelMetadataDataWrapper, foreign_storage::InternalMemoryStatsDataWrapper, and foreign_storage::InternalCatalogDataWrapper.

Referenced by populateChunkBuffers().

+ Here is the caller graph for this function:

void foreign_storage::InternalSystemDataWrapper::populateChunkMetadata ( ChunkMetadataVector chunk_metadata_vector)
overridevirtual

Populates given chunk metadata vector with metadata for all chunks in related foreign table.

Parameters
chunk_metadata_vector- vector that will be populated with chunk metadata

Implements foreign_storage::ForeignDataWrapper.

Definition at line 163 of file InternalSystemDataWrapper.cpp.

References CHECK, CHECK_EQ, db_id_, foreign_table_, foreign_storage::get_placeholder_metadata(), foreign_storage::ForeignTableSchema::getLogicalColumns(), initializeObjectsForTable(), Catalog_Namespace::SysCatalog::instance(), shared::kInfoSchemaDbName, TableDescriptor::maxFragRows, row_count_, TableDescriptor::tableId, and TableDescriptor::tableName.

164  {
165  auto& sys_catalog = Catalog_Namespace::SysCatalog::instance();
166  auto catalog = sys_catalog.getCatalog(db_id_);
167  CHECK(catalog);
168  CHECK_EQ(catalog->name(), shared::kInfoSchemaDbName);
169 
171  if (row_count_ > static_cast<size_t>(foreign_table_->maxFragRows)) {
172  throw ForeignStorageException{
173  "System table size exceeds the maximum supported size."};
174  }
176  for (auto column : schema.getLogicalColumns()) {
177  ChunkKey chunk_key = {db_id_, foreign_table_->tableId, column->columnId, 0};
178  if (column->columnType.is_varlen_indeed()) {
179  chunk_key.emplace_back(1);
180  }
181  chunk_metadata_vector.emplace_back(
182  chunk_key, get_placeholder_metadata(column->columnType, row_count_));
183  }
184 }
#define CHECK_EQ(x, y)
Definition: Logger.h:301
std::vector< int > ChunkKey
Definition: types.h:36
std::string tableName
virtual void initializeObjectsForTable(const std::string &table_name)=0
const std::string kInfoSchemaDbName
std::shared_ptr< ChunkMetadata > get_placeholder_metadata(const SQLTypeInfo &type, size_t num_elements)
static SysCatalog & instance()
Definition: SysCatalog.h:343
#define CHECK(condition)
Definition: Logger.h:291

+ Here is the call graph for this function:

void foreign_storage::InternalSystemDataWrapper::restoreDataWrapperInternals ( const std::string &  file_path,
const ChunkMetadataVector chunk_metadata 
)
overridevirtual

Restore internal state of datawrapper

Parameters
file_path- location of file created by serializeMetadata
chunk_metadata_vector- vector of chunk metadata recovered from disk

Implements foreign_storage::ForeignDataWrapper.

Definition at line 230 of file InternalSystemDataWrapper.cpp.

232  {}
void foreign_storage::InternalSystemDataWrapper::validateServerOptions ( const ForeignServer foreign_server) const
overridevirtual

Checks that the options for the given foreign server object are valid.

Parameters
foreign_server- foreign server object containing options to be validated

Implements foreign_storage::ForeignDataWrapper.

Definition at line 74 of file InternalSystemDataWrapper.cpp.

References CHECK, and foreign_storage::OptionsContainer::options.

75  {
76  CHECK(foreign_server->options.empty());
77 }
#define CHECK(condition)
Definition: Logger.h:291
void foreign_storage::InternalSystemDataWrapper::validateTableOptions ( const ForeignTable foreign_table) const
overridevirtual

Checks that the options for the given foreign table object are valid.

Parameters
foreign_table- foreign table object containing options to be validated

Implements foreign_storage::ForeignDataWrapper.

Definition at line 79 of file InternalSystemDataWrapper.cpp.

80  {}
void foreign_storage::InternalSystemDataWrapper::validateUserMappingOptions ( const UserMapping user_mapping,
const ForeignServer foreign_server 
) const
overridevirtual

Checks that the options for the given user mapping object are valid.

Parameters
user_mapping- user mapping object containing options to be validated

Implements foreign_storage::ForeignDataWrapper.

Definition at line 88 of file InternalSystemDataWrapper.cpp.

References CHECK, and foreign_storage::UserMapping::options.

90  {
91  CHECK(user_mapping->options.empty());
92 }
#define CHECK(condition)
Definition: Logger.h:291

Member Data Documentation

const int foreign_storage::InternalSystemDataWrapper::db_id_
protected

Definition at line 70 of file InternalSystemDataWrapper.h.

Referenced by populateChunkBuffers(), and populateChunkMetadata().


The documentation for this class was generated from the following files: