OmniSciDB  d2f719934e
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
foreign_storage::InternalSystemDataWrapper Class Referenceabstract

#include <InternalSystemDataWrapper.h>

+ Inheritance diagram for foreign_storage::InternalSystemDataWrapper:
+ Collaboration diagram for foreign_storage::InternalSystemDataWrapper:

Public Member Functions

 InternalSystemDataWrapper ()
 
 InternalSystemDataWrapper (const int db_id, const ForeignTable *foreign_table)
 
void populateChunkMetadata (ChunkMetadataVector &chunk_metadata_vector) override
 
void populateChunkBuffers (const ChunkToBufferMap &required_buffers, const ChunkToBufferMap &optional_buffers) override
 
void validateServerOptions (const ForeignServer *foreign_server) const override
 
void validateTableOptions (const ForeignTable *foreign_table) const override
 
const std::set
< std::string_view > & 
getSupportedTableOptions () const override
 
void validateUserMappingOptions (const UserMapping *user_mapping, const ForeignServer *foreign_server) const override
 
const std::set
< std::string_view > & 
getSupportedUserMappingOptions () const override
 
std::string getSerializedDataWrapper () const override
 
void restoreDataWrapperInternals (const std::string &file_path, const ChunkMetadataVector &chunk_metadata) override
 
bool isRestored () const override
 
- Public Member Functions inherited from foreign_storage::ForeignDataWrapper
 ForeignDataWrapper ()=default
 
virtual ~ForeignDataWrapper ()=default
 
virtual void validateSchema (const std::list< ColumnDescriptor > &columns) const
 
virtual ParallelismLevel getCachedParallelismLevel () const
 
virtual ParallelismLevel getNonCachedParallelismLevel () const
 

Protected Member Functions

virtual void initializeObjectsForTable (const std::string &table_name)=0
 
virtual void populateChunkBuffersForTable (const std::string &table_name, std::map< std::string, import_export::TypedImportBuffer * > &import_buffers)=0
 

Protected Attributes

const int db_id_
 
const ForeignTableforeign_table_
 
size_t row_count_ {0}
 

Additional Inherited Members

- Public Types inherited from foreign_storage::ForeignDataWrapper
enum  ParallelismLevel { NONE, INTRA_FRAGMENT, INTER_FRAGMENT }
 

Detailed Description

Definition at line 27 of file InternalSystemDataWrapper.h.

Constructor & Destructor Documentation

foreign_storage::InternalSystemDataWrapper::InternalSystemDataWrapper ( )
foreign_storage::InternalSystemDataWrapper::InternalSystemDataWrapper ( const int  db_id,
const ForeignTable foreign_table 
)

Member Function Documentation

std::string foreign_storage::InternalSystemDataWrapper::getSerializedDataWrapper ( ) const
overridevirtual

Serialize internal state of wrapper into file at given path if implemented

Implements foreign_storage::ForeignDataWrapper.

Definition at line 203 of file InternalSystemDataWrapper.cpp.

203  {
204  return {};
205 }
const std::set< std::string_view > & foreign_storage::InternalSystemDataWrapper::getSupportedTableOptions ( ) const
overridevirtual

Gets the set of supported table options for the data wrapper.

Implements foreign_storage::ForeignDataWrapper.

Definition at line 43 of file InternalSystemDataWrapper.cpp.

44  {
45  static const std::set<std::string_view> supported_table_options{};
46  return supported_table_options;
47 }
const std::set< std::string_view > & foreign_storage::InternalSystemDataWrapper::getSupportedUserMappingOptions ( ) const
overridevirtual

Gets the set of supported user mapping options for the data wrapper.

Implements foreign_storage::ForeignDataWrapper.

Definition at line 56 of file InternalSystemDataWrapper.cpp.

56  {
57  static const std::set<std::string_view> supported_user_mapping_options{};
58  return supported_user_mapping_options;
59 }
virtual void foreign_storage::InternalSystemDataWrapper::initializeObjectsForTable ( const std::string &  table_name)
protectedpure virtual

Implemented in foreign_storage::InternalStorageStatsDataWrapper, foreign_storage::InternalMemoryStatsDataWrapper, and foreign_storage::InternalCatalogDataWrapper.

Referenced by populateChunkMetadata().

+ Here is the caller graph for this function:

bool foreign_storage::InternalSystemDataWrapper::isRestored ( ) const
overridevirtual

Implements foreign_storage::ForeignDataWrapper.

Definition at line 211 of file InternalSystemDataWrapper.cpp.

211  {
212  return false;
213 }
void foreign_storage::InternalSystemDataWrapper::populateChunkBuffers ( const ChunkToBufferMap required_buffers,
const ChunkToBufferMap optional_buffers 
)
overridevirtual

Populates given chunk buffers identified by chunk keys. All provided chunk buffers are expected to be for the same fragment.

Parameters
required_buffers- chunk buffers that must always be populated
optional_buffers- chunk buffers that can be optionally populated, if the data wrapper has to scan through chunk data anyways (typically for row wise data formats)

Implements foreign_storage::ForeignDataWrapper.

Definition at line 159 of file InternalSystemDataWrapper.cpp.

References CHECK, CHECK_EQ, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_FRAGMENT_IDX, foreign_storage::TextFileBufferParser::convertImportBuffersToDataBlocks(), db_id_, DEBUG_TIMER, foreign_table_, TableDescriptor::fragmenter, INFORMATION_SCHEMA_DB, foreign_storage::anonymous_namespace{InternalSystemDataWrapper.cpp}::initialize_chunks(), foreign_storage::anonymous_namespace{InternalSystemDataWrapper.cpp}::initialize_import_buffers(), Catalog_Namespace::SysCatalog::instance(), populateChunkBuffersForTable(), row_count_, and TableDescriptor::tableName.

161  {
162  auto timer = DEBUG_TIMER(__func__);
163  CHECK(optional_buffers.empty());
164 
165  auto& sys_catalog = Catalog_Namespace::SysCatalog::instance();
166  auto catalog = sys_catalog.getCatalog(db_id_);
167  CHECK(catalog);
168  CHECK_EQ(catalog->name(), INFORMATION_SCHEMA_DB);
169 
170  auto fragment_id = required_buffers.begin()->first[CHUNK_KEY_FRAGMENT_IDX];
171  CHECK_EQ(fragment_id, 0);
172 
173  std::map<ChunkKey, Chunk_NS::Chunk> chunks;
174  std::set<const ColumnDescriptor*> columns_to_parse;
176  chunks, required_buffers, row_count_, columns_to_parse, fragment_id, *catalog);
177 
178  // initialize import buffers from columns.
179  std::vector<std::unique_ptr<import_export::TypedImportBuffer>> import_buffers;
180  std::map<std::string, import_export::TypedImportBuffer*> import_buffers_map;
182  import_buffers, import_buffers_map, columns_to_parse, *catalog);
184 
185  auto column_id_to_data_blocks_map =
187  for (auto& [chunk_key, chunk] : chunks) {
188  auto data_block_entry =
189  column_id_to_data_blocks_map.find(chunk_key[CHUNK_KEY_COLUMN_IDX]);
190  CHECK(data_block_entry != column_id_to_data_blocks_map.end());
191  chunk.appendData(data_block_entry->second, row_count_, 0);
192  auto cd = chunk.getColumnDesc();
193  if (!cd->columnType.is_varlen_indeed()) {
195  auto metadata = chunk.getBuffer()->getEncoder()->getMetadata(cd->columnType);
196  foreign_table_->fragmenter->updateColumnChunkMetadata(cd, fragment_id, metadata);
197  }
198  chunk.setBuffer(nullptr);
199  chunk.setIndexBuffer(nullptr);
200  }
201 }
#define CHECK_EQ(x, y)
Definition: Logger.h:219
static std::map< int, DataBlockPtr > convertImportBuffersToDataBlocks(const std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &import_buffers)
std::string tableName
#define CHUNK_KEY_FRAGMENT_IDX
Definition: types.h:42
static SysCatalog & instance()
Definition: SysCatalog.h:326
void initialize_chunks(std::map< ChunkKey, Chunk_NS::Chunk > &chunks, const ChunkToBufferMap &buffers, size_t row_count, std::set< const ColumnDescriptor * > &columns_to_parse, int32_t fragment_id, const Catalog_Namespace::Catalog &catalog)
virtual void populateChunkBuffersForTable(const std::string &table_name, std::map< std::string, import_export::TypedImportBuffer * > &import_buffers)=0
std::shared_ptr< Fragmenter_Namespace::AbstractFragmenter > fragmenter
#define CHECK(condition)
Definition: Logger.h:211
#define DEBUG_TIMER(name)
Definition: Logger.h:358
#define CHUNK_KEY_COLUMN_IDX
Definition: types.h:41
void initialize_import_buffers(std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &import_buffers, std::map< std::string, import_export::TypedImportBuffer * > &import_buffers_map, const std::set< const ColumnDescriptor * > &columns_to_parse, const Catalog_Namespace::Catalog &catalog)
const std::string INFORMATION_SCHEMA_DB
Definition: SysCatalog.h:65

+ Here is the call graph for this function:

virtual void foreign_storage::InternalSystemDataWrapper::populateChunkBuffersForTable ( const std::string &  table_name,
std::map< std::string, import_export::TypedImportBuffer * > &  import_buffers 
)
protectedpure virtual

Implemented in foreign_storage::InternalStorageStatsDataWrapper, foreign_storage::InternalMemoryStatsDataWrapper, and foreign_storage::InternalCatalogDataWrapper.

Referenced by populateChunkBuffers().

+ Here is the caller graph for this function:

void foreign_storage::InternalSystemDataWrapper::populateChunkMetadata ( ChunkMetadataVector chunk_metadata_vector)
overridevirtual

Populates given chunk metadata vector with metadata for all chunks in related foreign table.

Parameters
chunk_metadata_vector- vector that will be populated with chunk metadata

Implements foreign_storage::ForeignDataWrapper.

Definition at line 124 of file InternalSystemDataWrapper.cpp.

References CHECK, CHECK_EQ, db_id_, foreign_table_, Data_Namespace::AbstractBuffer::getEncoder(), foreign_storage::ForeignTableSchema::getLogicalColumns(), Encoder::getMetadata(), INFORMATION_SCHEMA_DB, Data_Namespace::AbstractBuffer::initEncoder(), initializeObjectsForTable(), Catalog_Namespace::SysCatalog::instance(), row_count_, TableDescriptor::tableId, and TableDescriptor::tableName.

125  {
126  auto& sys_catalog = Catalog_Namespace::SysCatalog::instance();
127  auto catalog = sys_catalog.getCatalog(db_id_);
128  CHECK(catalog);
129  CHECK_EQ(catalog->name(), INFORMATION_SCHEMA_DB);
130 
133  for (auto column : schema.getLogicalColumns()) {
134  ChunkKey chunk_key = {db_id_, foreign_table_->tableId, column->columnId, 0};
135  if (column->columnType.is_varlen_indeed()) {
136  chunk_key.emplace_back(1);
137  }
138  ForeignStorageBuffer empty_buffer;
139  // Use default encoder metadata
140  empty_buffer.initEncoder(column->columnType);
141  auto chunk_metadata = empty_buffer.getEncoder()->getMetadata(column->columnType);
142  chunk_metadata->numElements = row_count_;
143  if (!column->columnType.is_varlen_indeed()) {
144  chunk_metadata->numBytes = column->columnType.get_size() * row_count_;
145  }
146  if (column->columnType.is_array()) {
147  ForeignStorageBuffer scalar_buffer;
148  scalar_buffer.initEncoder(column->columnType.get_elem_type());
149  auto scalar_metadata =
150  scalar_buffer.getEncoder()->getMetadata(column->columnType.get_elem_type());
151  chunk_metadata->chunkStats.min = scalar_metadata->chunkStats.min;
152  chunk_metadata->chunkStats.max = scalar_metadata->chunkStats.max;
153  }
154  chunk_metadata->chunkStats.has_nulls = true;
155  chunk_metadata_vector.emplace_back(chunk_key, chunk_metadata);
156  }
157 }
#define CHECK_EQ(x, y)
Definition: Logger.h:219
std::vector< int > ChunkKey
Definition: types.h:37
std::string tableName
virtual void initializeObjectsForTable(const std::string &table_name)=0
void initEncoder(const SQLTypeInfo &tmp_sql_type)
virtual void getMetadata(const std::shared_ptr< ChunkMetadata > &chunkMetadata)
Definition: Encoder.cpp:227
static SysCatalog & instance()
Definition: SysCatalog.h:326
#define CHECK(condition)
Definition: Logger.h:211
const std::string INFORMATION_SCHEMA_DB
Definition: SysCatalog.h:65

+ Here is the call graph for this function:

void foreign_storage::InternalSystemDataWrapper::restoreDataWrapperInternals ( const std::string &  file_path,
const ChunkMetadataVector chunk_metadata 
)
overridevirtual

Restore internal state of datawrapper

Parameters
file_path- location of file created by serializeMetadata
chunk_metadata_vector- vector of chunk metadata recovered from disk

Implements foreign_storage::ForeignDataWrapper.

Definition at line 207 of file InternalSystemDataWrapper.cpp.

209  {}
void foreign_storage::InternalSystemDataWrapper::validateServerOptions ( const ForeignServer foreign_server) const
overridevirtual

Checks that the options for the given foreign server object are valid.

Parameters
foreign_server- foreign server object containing options to be validated

Implements foreign_storage::ForeignDataWrapper.

Definition at line 35 of file InternalSystemDataWrapper.cpp.

References CHECK, and foreign_storage::OptionsContainer::options.

36  {
37  CHECK(foreign_server->options.empty());
38 }
#define CHECK(condition)
Definition: Logger.h:211
void foreign_storage::InternalSystemDataWrapper::validateTableOptions ( const ForeignTable foreign_table) const
overridevirtual

Checks that the options for the given foreign table object are valid.

Parameters
foreign_table- foreign table object containing options to be validated

Implements foreign_storage::ForeignDataWrapper.

Definition at line 40 of file InternalSystemDataWrapper.cpp.

41  {}
void foreign_storage::InternalSystemDataWrapper::validateUserMappingOptions ( const UserMapping user_mapping,
const ForeignServer foreign_server 
) const
overridevirtual

Checks that the options for the given user mapping object are valid.

Parameters
user_mapping- user mapping object containing options to be validated

Implements foreign_storage::ForeignDataWrapper.

Definition at line 49 of file InternalSystemDataWrapper.cpp.

References CHECK, and foreign_storage::UserMapping::options.

51  {
52  CHECK(user_mapping->options.empty());
53 }
#define CHECK(condition)
Definition: Logger.h:211

Member Data Documentation

const int foreign_storage::InternalSystemDataWrapper::db_id_
protected

Definition at line 62 of file InternalSystemDataWrapper.h.

Referenced by populateChunkBuffers(), and populateChunkMetadata().


The documentation for this class was generated from the following files: