OmniSciDB  6686921089
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
AbstractTextFileDataWrapper.cpp File Reference
#include "AbstractTextFileDataWrapper.h"
#include <algorithm>
#include <condition_variable>
#include <mutex>
#include <queue>
#include <rapidjson/document.h>
#include <boost/filesystem.hpp>
#include "DataMgr/ForeignStorage/FileReader.h"
#include "DataMgr/ForeignStorage/ForeignTableSchema.h"
#include "DataMgr/ForeignStorage/FsiChunkUtils.h"
#include "FsiJsonUtils.h"
#include "Shared/misc.h"
+ Include dependency graph for AbstractTextFileDataWrapper.cpp:

Go to the source code of this file.

Classes

struct  foreign_storage::ParseFileRegionResult
 
struct  foreign_storage::MetadataScanMultiThreadingParams
 

Namespaces

 foreign_storage
 
 foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}
 

Functions

std::set< const
ColumnDescriptor * > 
foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::get_columns (const ChunkToBufferMap &buffers, const Catalog_Namespace::Catalog &catalog, const int32_t table_id, const int fragment_id)
 
bool foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::skip_metadata_scan (const ColumnDescriptor *column)
 
ParseFileRegionResult foreign_storage::parse_file_regions (const FileRegions &file_regions, const size_t start_index, const size_t end_index, FileReader &file_reader, ParseBufferRequest &parse_file_request, const std::map< int, Chunk_NS::Chunk > &column_id_to_chunk_map, const TextFileBufferParser &parser)
 
size_t foreign_storage::get_buffer_size (const import_export::CopyParams &copy_params, const bool size_known, const size_t file_size)
 
size_t foreign_storage::get_buffer_size (const FileRegions &file_regions)
 
size_t foreign_storage::get_thread_count (const import_export::CopyParams &copy_params, const bool size_known, const size_t file_size, const size_t buffer_size)
 
size_t foreign_storage::get_thread_count (const import_export::CopyParams &copy_params, const FileRegions &file_regions)
 
std::vector< size_t > foreign_storage::partition_by_fragment (const size_t start_row_index, const size_t max_fragment_size, const size_t buffer_row_count)
 
std::optional< ParseBufferRequest > foreign_storage::get_next_metadata_scan_request (MetadataScanMultiThreadingParams &multi_threading_params)
 
void foreign_storage::add_file_region (std::map< int, FileRegions > &fragment_id_to_file_regions_map, int fragment_id, size_t first_row_index, const ParseBufferResult &result, const std::string &file_path)
 
void foreign_storage::update_stats (Encoder *encoder, const SQLTypeInfo &column_type, DataBlockPtr data_block, const size_t row_count)
 
foreign_storage::ForeignStorageCacheforeign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::get_cache_if_enabled (std::shared_ptr< Catalog_Namespace::Catalog > &catalog)
 
void foreign_storage::cache_blocks (std::map< ChunkKey, Chunk_NS::Chunk > &cached_chunks, DataBlockPtr data_block, size_t row_count, ChunkKey &chunk_key, const ColumnDescriptor *column, bool is_first_block, bool is_last_block)
 
void foreign_storage::process_data_blocks (MetadataScanMultiThreadingParams &multi_threading_params, int fragment_id, const ParseBufferRequest &request, ParseBufferResult &result, std::map< int, const ColumnDescriptor * > &column_by_id, std::map< int, FileRegions > &fragment_id_to_file_regions_map)
 
void foreign_storage::add_request_to_pool (MetadataScanMultiThreadingParams &multi_threading_params, ParseBufferRequest &request)
 
void foreign_storage::scan_metadata (MetadataScanMultiThreadingParams &multi_threading_params, std::map< int, FileRegions > &fragment_id_to_file_regions_map, const TextFileBufferParser &parser)
 
ParseBufferRequest foreign_storage::get_request_from_pool (MetadataScanMultiThreadingParams &multi_threading_params)
 
void foreign_storage::dispatch_metadata_scan_request (MetadataScanMultiThreadingParams &multi_threading_params, ParseBufferRequest &request)
 
void foreign_storage::resize_buffer_if_needed (std::unique_ptr< char[]> &buffer, size_t &buffer_size, const size_t alloc_size)
 
void foreign_storage::dispatch_metadata_scan_requests (const size_t &buffer_size, const std::string &file_path, FileReader &file_reader, const import_export::CopyParams &copy_params, MetadataScanMultiThreadingParams &multi_threading_params, size_t &first_row_index_in_buffer, size_t &current_file_offset, const TextFileBufferParser &parser)
 
void foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::add_placeholder_metadata (const ColumnDescriptor *column, const ForeignTable *foreign_table, const int db_id, const size_t start_row, const size_t total_num_rows, std::map< ChunkKey, std::shared_ptr< ChunkMetadata >> &chunk_metadata_map)