OmniSciDB  85c2d10cdc
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
CsvDataWrapper.cpp File Reference
#include "CsvDataWrapper.h"
#include <algorithm>
#include <condition_variable>
#include <mutex>
#include <queue>
#include <rapidjson/document.h>
#include <boost/filesystem.hpp>
#include "CsvShared.h"
#include "DataMgr/ForeignStorage/CsvFileBufferParser.h"
#include "DataMgr/ForeignStorage/CsvReader.h"
#include "DataMgr/ForeignStorage/ForeignTableSchema.h"
#include "FsiJsonUtils.h"
#include "ImportExport/DelimitedParserUtils.h"
#include "ImportExport/Importer.h"
#include "Shared/sqltypes.h"
#include "Utils/DdlUtils.h"
+ Include dependency graph for CsvDataWrapper.cpp:

Go to the source code of this file.

Classes

struct  foreign_storage::ParseFileRegionResult
 
struct  foreign_storage::MetadataScanMultiThreadingParams
 

Namespaces

 foreign_storage
 
 foreign_storage::anonymous_namespace{CsvDataWrapper.cpp}
 

Functions

std::set< const
ColumnDescriptor * > 
foreign_storage::anonymous_namespace{CsvDataWrapper.cpp}::get_columns (const ChunkToBufferMap &buffers, std::shared_ptr< Catalog_Namespace::Catalog > catalog, const int32_t table_id, const int fragment_id)
 
bool foreign_storage::anonymous_namespace{CsvDataWrapper.cpp}::skip_metadata_scan (const ColumnDescriptor *column)
 
ParseFileRegionResult foreign_storage::parse_file_regions (const FileRegions &file_regions, const size_t start_index, const size_t end_index, CsvReader &csv_reader, csv_file_buffer_parser::ParseBufferRequest &parse_file_request, const std::map< int, Chunk_NS::Chunk > &column_id_to_chunk_map)
 
size_t foreign_storage::get_buffer_size (const import_export::CopyParams &copy_params, const bool size_known, const size_t file_size)
 
size_t foreign_storage::get_buffer_size (const FileRegions &file_regions)
 
size_t foreign_storage::get_thread_count (const import_export::CopyParams &copy_params, const bool size_known, const size_t file_size, const size_t buffer_size)
 
size_t foreign_storage::get_thread_count (const import_export::CopyParams &copy_params, const FileRegions &file_regions)
 
std::vector< size_t > foreign_storage::partition_by_fragment (const size_t start_row_index, const size_t max_fragment_size, const size_t buffer_row_count)
 
std::optional
< csv_file_buffer_parser::ParseBufferRequest > 
foreign_storage::get_next_metadata_scan_request (MetadataScanMultiThreadingParams &multi_threading_params)
 
void foreign_storage::add_file_region (std::map< int, FileRegions > &fragment_id_to_file_regions_map, int fragment_id, size_t first_row_index, const csv_file_buffer_parser::ParseBufferResult &result, const std::string &file_path)
 
size_t foreign_storage::get_var_length_data_block_size (DataBlockPtr data_block, SQLTypeInfo sql_type_info)
 
void foreign_storage::update_stats (Encoder *encoder, const SQLTypeInfo &column_type, DataBlockPtr data_block, const size_t row_count)
 
foreign_storage::ForeignStorageCacheforeign_storage::anonymous_namespace{CsvDataWrapper.cpp}::get_cache_if_enabled (std::shared_ptr< Catalog_Namespace::Catalog > &catalog)
 
void foreign_storage::cache_blocks (std::map< ChunkKey, Chunk_NS::Chunk > &cached_chunks, DataBlockPtr data_block, size_t row_count, ChunkKey &chunk_key, const ColumnDescriptor *column, bool is_first_block, bool is_last_block)
 
void foreign_storage::process_data_blocks (MetadataScanMultiThreadingParams &multi_threading_params, int fragment_id, const csv_file_buffer_parser::ParseBufferRequest &request, csv_file_buffer_parser::ParseBufferResult &result, std::map< int, const ColumnDescriptor * > &column_by_id, std::map< int, FileRegions > &fragment_id_to_file_regions_map)
 
void foreign_storage::add_request_to_pool (MetadataScanMultiThreadingParams &multi_threading_params, csv_file_buffer_parser::ParseBufferRequest &request)
 
void foreign_storage::scan_metadata (MetadataScanMultiThreadingParams &multi_threading_params, std::map< int, FileRegions > &fragment_id_to_file_regions_map)
 
csv_file_buffer_parser::ParseBufferRequest foreign_storage::get_request_from_pool (MetadataScanMultiThreadingParams &multi_threading_params)
 
void foreign_storage::dispatch_metadata_scan_request (MetadataScanMultiThreadingParams &multi_threading_params, csv_file_buffer_parser::ParseBufferRequest &request)
 
void foreign_storage::resize_buffer_if_needed (std::unique_ptr< char[]> &buffer, size_t &buffer_size, const size_t alloc_size)
 
void foreign_storage::dispatch_metadata_scan_requests (const size_t &buffer_size, const std::string &file_path, CsvReader &csv_reader, const import_export::CopyParams &copy_params, MetadataScanMultiThreadingParams &multi_threading_params, size_t &first_row_index_in_buffer, size_t &current_file_offset)
 
void foreign_storage::anonymous_namespace{CsvDataWrapper.cpp}::add_placeholder_metadata (const ColumnDescriptor *column, const ForeignTable *foreign_table, const int db_id, const size_t start_row, const size_t total_num_rows, std::map< ChunkKey, std::shared_ptr< ChunkMetadata >> &chunk_metadata_map)