AbstractTextFileDataWrapper.cpp File Reference

#include "AbstractTextFileDataWrapper.h"
#include <algorithm>
#include <condition_variable>
#include <mutex>
#include <queue>
#include <rapidjson/document.h>
#include <boost/filesystem.hpp>
#include "Catalog/ColumnDescriptor.h"
#include "DataMgr/ForeignStorage/FileReader.h"
#include "DataMgr/ForeignStorage/ForeignTableSchema.h"
#include "DataMgr/ForeignStorage/FsiChunkUtils.h"
#include "ForeignStorageException.h"
#include "Shared/JsonUtils.h"
#include "Shared/misc.h"

Include dependency graph for AbstractTextFileDataWrapper.cpp:

Go to the source code of this file.

Classes
struct	foreign_storage::ParseFileRegionResult

Namespaces
	foreign_storage
	foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}

Functions
void	foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::throw_fragment_id_out_of_bounds_error (const TableDescriptor *table, const int32_t fragment_id, const int32_t max_fragment_id)
std::set< const ColumnDescriptor * >	foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::get_columns (const ChunkToBufferMap &buffers, const Catalog_Namespace::Catalog &catalog, const int32_t table_id, const int fragment_id)
bool	foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::skip_metadata_scan (const ColumnDescriptor *column)
void	foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::throw_unexpected_number_of_items (const size_t num_expected, const size_t num_loaded, const std::string &item_type, const std::string &foreign_table_name)
ParseFileRegionResult	foreign_storage::parse_file_regions (const FileRegions &file_regions, const size_t start_index, const size_t end_index, FileReader &file_reader, ParseBufferRequest &parse_file_request, const std::map< int, Chunk_NS::Chunk > &column_id_to_chunk_map, const TextFileBufferParser &parser)
size_t	foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::get_buffer_size (const import_export::CopyParams &copy_params, const bool size_known, const size_t file_size)
size_t	foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::get_buffer_size (const FileRegions &file_regions)
size_t	foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::get_thread_count (const import_export::CopyParams &copy_params, const bool size_known, const size_t file_size, const size_t buffer_size)
size_t	foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::get_thread_count (const import_export::CopyParams &copy_params, const FileRegions &file_regions)
void	foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::resize_delete_buffer (AbstractBuffer *delete_buffer, const size_t chunk_element_count)
bool	foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::no_deferred_requests (MetadataScanMultiThreadingParams &multi_threading_params)
bool	foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::is_file_scan_finished (const FileReader *file_reader, MetadataScanMultiThreadingParams &multi_threading_params)
size_t	foreign_storage::num_rows_to_process (const size_t start_row_index, const size_t max_fragment_size, const size_t rows_remaining)
std::vector< size_t >	foreign_storage::partition_by_fragment (const size_t start_row_index, const size_t max_fragment_size, const size_t buffer_row_count)
std::optional< ParseBufferRequest >	foreign_storage::get_next_scan_request (MetadataScanMultiThreadingParams &multi_threading_params)
void	foreign_storage::add_file_region (std::map< int, FileRegions > &fragment_id_to_file_regions_map, int fragment_id, size_t first_row_index, const ParseBufferResult &result, const std::string &file_path)
void	foreign_storage::update_stats (Encoder *encoder, const SQLTypeInfo &column_type, DataBlockPtr data_block, const size_t row_count)
foreign_storage::ForeignStorageCache *	foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::get_cache_if_enabled (std::shared_ptr< Catalog_Namespace::Catalog > &catalog, const bool disable_cache)
void	foreign_storage::cache_blocks (std::map< ChunkKey, Chunk_NS::Chunk > &cached_chunks, DataBlockPtr data_block, size_t row_count, ChunkKey &chunk_key, const ColumnDescriptor *column, bool is_first_block, bool disable_cache)
void	foreign_storage::append_data_block_to_chunk (const foreign_storage::IterativeFileScanParameters &file_scan_param, DataBlockPtr data_block, size_t row_count, const int column_id, const ColumnDescriptor *column, const size_t element_count_required)
std::pair< std::map< int, DataBlockPtr >, std::map< int, DataBlockPtr > >	foreign_storage::partition_data_blocks (const std::map< int, const ColumnDescriptor * > &column_by_id, const std::map< int, DataBlockPtr > &data_blocks)
void	foreign_storage::update_delete_buffer (const ParseBufferRequest &request, const ParseBufferResult &result, const foreign_storage::IterativeFileScanParameters &file_scan_param, const size_t start_position_in_fragment)
void	foreign_storage::populate_chunks_using_data_blocks (MetadataScanMultiThreadingParams &multi_threading_params, int fragment_id, const ParseBufferRequest &request, ParseBufferResult &result, std::map< int, const ColumnDescriptor * > &column_by_id, std::map< int, FileRegions > &fragment_id_to_file_regions_map, const foreign_storage::IterativeFileScanParameters &file_scan_param, const size_t expected_current_element_count)
void	foreign_storage::process_data_blocks (MetadataScanMultiThreadingParams &multi_threading_params, int fragment_id, const ParseBufferRequest &request, ParseBufferResult &result, std::map< int, const ColumnDescriptor * > &column_by_id, std::map< int, FileRegions > &fragment_id_to_file_regions_map)
void	foreign_storage::add_request_to_pool (MetadataScanMultiThreadingParams &multi_threading_params, ParseBufferRequest &request)
void	foreign_storage::scan_metadata (MetadataScanMultiThreadingParams &multi_threading_params, std::map< int, FileRegions > &fragment_id_to_file_regions_map, const TextFileBufferParser &parser)
ParseBufferRequest	foreign_storage::get_request_from_pool (MetadataScanMultiThreadingParams &multi_threading_params)
bool	foreign_storage::request_pool_non_empty (MetadataScanMultiThreadingParams &multi_threading_params)
void	foreign_storage::defer_scan_request (MetadataScanMultiThreadingParams &multi_threading_params, ParseBufferRequest &request)
void	foreign_storage::dispatch_all_deferred_requests (MetadataScanMultiThreadingParams &multi_threading_params)
void	foreign_storage::dispatch_scan_request (MetadataScanMultiThreadingParams &multi_threading_params, ParseBufferRequest &request)
void	foreign_storage::populate_chunks (MetadataScanMultiThreadingParams &multi_threading_params, std::map< int, FileRegions > &fragment_id_to_file_regions_map, const TextFileBufferParser &parser, foreign_storage::IterativeFileScanParameters &file_scan_param)
void	foreign_storage::resize_buffer_if_needed (std::unique_ptr< char[]> &buffer, size_t &buffer_size, const size_t alloc_size)
void	foreign_storage::reset_multithreading_params (foreign_storage::MetadataScanMultiThreadingParams &multi_threading_params)
void	foreign_storage::dispatch_scan_requests (const foreign_storage::ForeignTable *table, const size_t &buffer_size, const std::string &file_path, FileReader &file_reader, const import_export::CopyParams &copy_params, MetadataScanMultiThreadingParams &multi_threading_params, size_t &first_row_index_in_buffer, size_t &current_file_offset, const TextFileBufferParser &parser, const foreign_storage::IterativeFileScanParameters *file_scan_param, foreign_storage::AbstractTextFileDataWrapper::ResidualBuffer &iterative_residual_buffer, const bool is_first_file_scan_call, int &iterative_scan_last_fragment_id)
void	foreign_storage::dispatch_scan_requests_with_exception_handling (const foreign_storage::ForeignTable *table, const size_t &buffer_size, const std::string &file_path, FileReader &file_reader, const import_export::CopyParams &copy_params, MetadataScanMultiThreadingParams &multi_threading_params, size_t &first_row_index_in_buffer, size_t &current_file_offset, const TextFileBufferParser &parser, const foreign_storage::IterativeFileScanParameters *file_scan_param, foreign_storage::AbstractTextFileDataWrapper::ResidualBuffer &iterative_residual_buffer, const bool is_first_file_scan_call, int &iterative_scan_last_fragment_id)
void	foreign_storage::dispatch_scan_requests_with_exception_handling (const foreign_storage::ForeignTable *table, const size_t &buffer_size, const std::string &file_path, FileReader &file_reader, const import_export::CopyParams &copy_params, MetadataScanMultiThreadingParams &multi_threading_params, size_t &first_row_index_in_buffer, size_t &current_file_offset, const TextFileBufferParser &parser, const foreign_storage::IterativeFileScanParameters *file_scan_param, foreign_storage::AbstractTextFileDataWrapper::ResidualBuffer &iterative_residual_buffer, const bool is_first_file_scan_call)
void	foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::add_placeholder_metadata (const ColumnDescriptor *column, const ForeignTable *foreign_table, const int db_id, const size_t start_row, const size_t total_num_rows, std::map< ChunkKey, std::shared_ptr< ChunkMetadata >> &chunk_metadata_map)
void	foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::initialize_non_append_mode_scan (const std::map< ChunkKey, std::shared_ptr< ChunkMetadata >> &chunk_metadata_map, const std::map< int, FileRegions > &fragment_id_to_file_regions_map, const foreign_storage::OptionsMap &server_options, std::unique_ptr< FileReader > &file_reader, const std::string &file_path, const import_export::CopyParams &copy_params, const shared::FilePathOptions &file_path_options, const std::optional< size_t > &max_file_count, const foreign_storage::ForeignTable *foreign_table, const foreign_storage::UserMapping *user_mapping, const foreign_storage::TextFileBufferParser &parser, std::function< std::string()> get_s3_key, size_t &num_rows, size_t &append_start_offset)