Inheritance diagram for foreign_storage::ParquetImportBatchResult:

Collaboration diagram for foreign_storage::ParquetImportBatchResult:

Public Member Functions
	ParquetImportBatchResult ()=default

	ParquetImportBatchResult (const ForeignTable foreign_table, const int db_id, const ForeignTableSchema schema)

	ParquetImportBatchResult (ParquetImportBatchResult &&other)=default

std::optional < Fragmenter_Namespace::InsertData >	getInsertData () const override

import_export::ImportStatus	getImportStatus () const override

std::pair< std::map< int, Chunk_NS::Chunk >, std::map < int, StringDictionary * > >	getChunksAndDictionaries () const

void	populateInsertData (const std::map< int, Chunk_NS::Chunk > &chunks)

void	populateImportStatus (const size_t num_rows_completed, const size_t num_rows_rejected)

Public Member Functions inherited from import_export::ImportBatchResult
virtual	~ImportBatchResult ()=default

Private Attributes
std::optional < Fragmenter_Namespace::InsertData >	insert_data_

std::map< int, std::unique_ptr < AbstractBuffer > >	import_buffers_

const ForeignTable *	foreign_table_

int	db_id_

const ForeignTableSchema *	schema_

import_export::ImportStatus	import_status_

Detailed Description

Definition at line 96 of file ParquetImporter.cpp.

Constructor & Destructor Documentation

foreign_storage::ParquetImportBatchResult::ParquetImportBatchResult ( )

default

foreign_storage::ParquetImportBatchResult::ParquetImportBatchResult	(	const ForeignTable *	foreign_table,
		const int	db_id,
		const ForeignTableSchema *	schema
	)

Definition at line 190 of file ParquetImporter.cpp.

References foreign_storage::ForeignTableSchema::getLogicalAndPhysicalColumns(), import_buffers_, and schema_.

     : foreign_table_(foreign_table), db_id_(db_id), schema_(schema) {
   for (const auto column_descriptor : schema_->getLogicalAndPhysicalColumns()) {
     if (column_descriptor->columnType.is_array()) {
       import_buffers_[column_descriptor->columnId] =
           std::make_unique<TypedParquetStorageBuffer<ArrayDatum>>();
     } else if ((column_descriptor->columnType.is_string() &&
                 !column_descriptor->columnType.is_dict_encoded_string()) ||
                column_descriptor->columnType.is_geometry()) {
       import_buffers_[column_descriptor->columnId] =
           std::make_unique<TypedParquetStorageBuffer<std::string>>();
     } else {
       import_buffers_[column_descriptor->columnId] =
           std::make_unique<ForeignStorageBuffer>();
     }
   }
 }

Here is the call graph for this function:

foreign_storage::ParquetImportBatchResult::ParquetImportBatchResult ( ParquetImportBatchResult && other )

default

Member Function Documentation

std::pair< std::map< int, Chunk_NS::Chunk >, std::map< int, StringDictionary * > > foreign_storage::ParquetImportBatchResult::getChunksAndDictionaries ( ) const

Definition at line 160 of file ParquetImporter.cpp.

References CHECK, db_id_, Catalog_Namespace::SysCatalog::getCatalog(), foreign_storage::ForeignTableSchema::getLogicalAndPhysicalColumns(), import_buffers_, Catalog_Namespace::SysCatalog::instance(), schema_, and Chunk_NS::Chunk::setBuffer().

                                                          {
   std::map<int, Chunk_NS::Chunk> chunks;
   std::map<int, StringDictionary*> string_dictionaries;
   const auto catalog = Catalog_Namespace::SysCatalog::instance().getCatalog(db_id_);
 
   for (const auto column_descriptor : schema_->getLogicalAndPhysicalColumns()) {
     const bool is_dictionary_encoded_string_column =
         column_descriptor->columnType.is_dict_encoded_string() ||
         (column_descriptor->columnType.is_array() &&
          column_descriptor->columnType.get_elem_type().is_dict_encoded_string());
 
     if (is_dictionary_encoded_string_column) {
       auto dict_descriptor = catalog->getMetadataForDict(
           column_descriptor->columnType.get_comp_param(), true);
       CHECK(dict_descriptor);
       auto string_dictionary = dict_descriptor->stringDict.get();
       string_dictionaries[column_descriptor->columnId] = string_dictionary;
     }
 
     Chunk_NS::Chunk chunk{column_descriptor};
     chunk.setBuffer(import_buffers_.at(column_descriptor->columnId).get());
     if (column_descriptor->columnType.is_varlen_indeed()) {
       chunk.setIndexBuffer(nullptr);  // index buffers are unused
     }
     chunk.initEncoder();
     chunks[column_descriptor->columnId] = chunk;
   }
   return {chunks, string_dictionaries};
 }

Here is the call graph for this function:

import_export::ImportStatus foreign_storage::ParquetImportBatchResult::getImportStatus ( ) const

overridevirtual

Implements import_export::ImportBatchResult.

Definition at line 215 of file ParquetImporter.cpp.

References import_status_.

                                                                           {
   return import_status_;
 }

std::optional< Fragmenter_Namespace::InsertData > foreign_storage::ParquetImportBatchResult::getInsertData ( ) const

overridevirtual

Implements import_export::ImportBatchResult.

Definition at line 210 of file ParquetImporter.cpp.

References insert_data_.

           {
   return insert_data_;
 }

void foreign_storage::ParquetImportBatchResult::populateImportStatus	(	const size_t	num_rows_completed,
		const size_t	num_rows_rejected
	)

Definition at line 124 of file ParquetImporter.cpp.

References import_status_, import_export::ImportStatus::rows_completed, and import_export::ImportStatus::rows_rejected.

                                                                                     {
   import_status_.rows_completed = num_rows_completed;
   import_status_.rows_rejected = num_rows_rejected;
 }

void foreign_storage::ParquetImportBatchResult::populateInsertData ( const std::map< int, Chunk_NS::Chunk > & chunks )

Definition at line 130 of file ParquetImporter.cpp.

References DataBlockPtr::arraysPtr, CHECK, db_id_, foreign_table_, insert_data_, DataBlockPtr::numbersPtr, DataBlockPtr::stringsPtr, and TableDescriptor::tableId.

                                               {
   insert_data_ = Fragmenter_Namespace::InsertData{};
   size_t num_rows = chunks.begin()->second.getBuffer()->getEncoder()->getNumElems();
   for (const auto& [column_id, chunk] : chunks) {
     auto column_descriptor = chunk.getColumnDesc();
     CHECK(chunk.getBuffer()->getEncoder()->getNumElems() == num_rows);
     insert_data_->columnIds.emplace_back(column_id);
     auto buffer = chunk.getBuffer();
     DataBlockPtr block_ptr;
     if (column_descriptor->columnType.is_array()) {
       auto array_buffer = dynamic_cast<TypedParquetStorageBuffer<ArrayDatum>*>(buffer);
       block_ptr.arraysPtr = array_buffer->getBufferPtr();
     } else if ((column_descriptor->columnType.is_string() &&
                 !column_descriptor->columnType.is_dict_encoded_string()) ||
                column_descriptor->columnType.is_geometry()) {
       auto string_buffer = dynamic_cast<TypedParquetStorageBuffer<std::string>*>(buffer);
       block_ptr.stringsPtr = string_buffer->getBufferPtr();
     } else {
       block_ptr.numbersPtr = buffer->getMemoryPtr();
     }
     insert_data_->data.emplace_back(block_ptr);
   }
   insert_data_->databaseId = db_id_;
   insert_data_->tableId = foreign_table_->tableId;
   insert_data_->is_default.assign(insert_data_->columnIds.size(), false);
   insert_data_->numRows = num_rows;
 }