OmniSciDB  2e3a973ef4
CsvDataWrapper.cpp File Reference
#include "CsvDataWrapper.h"
#include <algorithm>
#include <condition_variable>
#include <mutex>
#include <regex>
#include <rapidjson/document.h>
#include <boost/filesystem.hpp>
#include "DataMgr/ForeignStorage/CsvFileBufferParser.h"
#include "DataMgr/ForeignStorage/CsvReader.h"
#include "FsiJsonUtils.h"
#include "ImportExport/DelimitedParserUtils.h"
#include "ImportExport/Importer.h"
#include "Utils/DdlUtils.h"
+ Include dependency graph for CsvDataWrapper.cpp:

Go to the source code of this file.

Classes

struct  foreign_storage::ParseFileRegionResult
 
struct  foreign_storage::MetadataScanMultiThreadingParams
 

Namespaces

 foreign_storage
 
 foreign_storage::anonymous_namespace{CsvDataWrapper.cpp}
 

Functions

std::set< const ColumnDescriptor * > foreign_storage::anonymous_namespace{CsvDataWrapper.cpp}::get_columns (const std::map< ChunkKey, AbstractBuffer *> &buffers, std::shared_ptr< Catalog_Namespace::Catalog > catalog, const int32_t table_id, const int fragment_id)
 
ParseFileRegionResult foreign_storage::parse_file_regions (const FileRegions &file_regions, const size_t start_index, const size_t end_index, CsvReader &csv_reader, std::mutex &file_access_mutex, csv_file_buffer_parser::ParseBufferRequest &parse_file_request, const std::map< int, Chunk_NS::Chunk > &column_id_to_chunk_map)
 
size_t foreign_storage::get_buffer_size (const import_export::CopyParams &copy_params, const bool size_known, const size_t file_size)
 
size_t foreign_storage::get_buffer_size (const FileRegions &file_regions)
 
size_t foreign_storage::get_thread_count (const import_export::CopyParams &copy_params, const bool size_known, const size_t file_size, const size_t buffer_size)
 
size_t foreign_storage::get_thread_count (const import_export::CopyParams &copy_params, const FileRegions &file_regions)
 
void foreign_storage::initialize_import_buffers (const std::list< const ColumnDescriptor *> &columns, std::shared_ptr< Catalog_Namespace::Catalog > catalog, std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &import_buffers, const std::set< int > &column_filter_set={})
 
std::vector< size_t > foreign_storage::partition_by_fragment (const size_t start_row_index, const size_t max_fragment_size, const size_t buffer_row_count)
 
std::optional< csv_file_buffer_parser::ParseBufferRequest > foreign_storage::get_next_metadata_scan_request (MetadataScanMultiThreadingParams &multi_threading_params)
 
void foreign_storage::add_file_region (std::map< int, FileRegions > &fragment_id_to_file_regions_map, std::mutex &file_region_mutex, int fragment_id, size_t first_row_index, const csv_file_buffer_parser::ParseBufferResult &result)
 
size_t foreign_storage::get_var_length_data_block_size (DataBlockPtr data_block, SQLTypeInfo sql_type_info)
 
void foreign_storage::update_stats (Encoder *encoder, const SQLTypeInfo &column_type, DataBlockPtr data_block, const size_t row_count)
 
void foreign_storage::update_metadata (MetadataScanMultiThreadingParams &multi_threading_params, int fragment_id, const csv_file_buffer_parser::ParseBufferRequest &request, const csv_file_buffer_parser::ParseBufferResult &result, std::map< int, const ColumnDescriptor *> &column_by_id)
 
void foreign_storage::add_request_to_pool (MetadataScanMultiThreadingParams &multi_threading_params, csv_file_buffer_parser::ParseBufferRequest &request)
 
void foreign_storage::scan_metadata (MetadataScanMultiThreadingParams &multi_threading_params, std::map< int, FileRegions > &fragment_id_to_file_regions_map, std::mutex &file_region_mutex)
 
csv_file_buffer_parser::ParseBufferRequest foreign_storage::get_request_from_pool (MetadataScanMultiThreadingParams &multi_threading_params)
 
void foreign_storage::dispatch_metadata_scan_request (MetadataScanMultiThreadingParams &multi_threading_params, csv_file_buffer_parser::ParseBufferRequest &request)
 
void foreign_storage::resize_buffer_if_needed (std::unique_ptr< char[]> &buffer, size_t &buffer_size, const size_t alloc_size)
 
void foreign_storage::dispatch_metadata_scan_requests (const size_t &buffer_size, const std::string &file_path, CsvReader &csv_reader, const import_export::CopyParams &copy_params, MetadataScanMultiThreadingParams &multi_threading_params, size_t &first_row_index_in_buffer, size_t &current_file_offset)
 
void foreign_storage::set_value (rapidjson::Value &json_val, const FileRegion &file_region, rapidjson::Document::AllocatorType &allocator)
 
void foreign_storage::get_value (const rapidjson::Value &json_val, FileRegion &file_region)