OmniSciDB
bf83d84833
|
Classes | |
struct | ParseBufferRequest |
struct | ParseBufferResult |
Functions | |
void | set_array_flags_and_geo_columns_count (std::unique_ptr< bool[]> &array_flags, int &phys_cols, int &point_cols, const std::list< const ColumnDescriptor * > &columns) |
void | validate_expected_column_count (std::vector< std::string_view > &row, size_t num_cols, int point_cols, const std::string &file_name) |
bool | is_coordinate_scalar (const std::string_view datum) |
bool | set_coordinates_from_separate_lon_lat_columns (const std::string_view lon_str, const std::string_view lat_str, std::vector< double > &coords, const bool is_lon_lat_order) |
bool | is_null_datum (const std::string_view datum, const ColumnDescriptor *column, const std::string &null_indicator) |
void | process_geo_column (std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &import_buffers, size_t &col_idx, const import_export::CopyParams ©_params, std::list< const ColumnDescriptor * >::iterator &cd_it, std::vector< std::string_view > &row, size_t &import_idx, bool is_null, size_t first_row_index, size_t row_index_plus_one, std::shared_ptr< Catalog_Namespace::Catalog > catalog) |
std::map< int, DataBlockPtr > | convert_import_buffers_to_data_blocks (const std::vector< std::unique_ptr< import_export::TypedImportBuffer >> &import_buffers) |
bool | skip_column_import (ParseBufferRequest &request, int column_idx) |
ParseBufferResult | parse_buffer (ParseBufferRequest &request, bool convert_data_blocks) |
Variables | |
static constexpr bool | PROMOTE_POLYGON_TO_MULTIPOLYGON = true |
std::map<int, DataBlockPtr> foreign_storage::csv_file_buffer_parser::convert_import_buffers_to_data_blocks | ( | const std::vector< std::unique_ptr< import_export::TypedImportBuffer >> & | import_buffers | ) |
Definition at line 199 of file CsvFileBufferParser.h.
References DataBlockPtr::arraysPtr, CHECK, CHECK_EQ, IS_STRING, kARRAY, kBOOLEAN, kENCODING_DICT, kENCODING_NONE, DataBlockPtr::numbersPtr, run_benchmark_import::result, and DataBlockPtr::stringsPtr.
Referenced by parse_buffer().
bool foreign_storage::csv_file_buffer_parser::is_coordinate_scalar | ( | const std::string_view | datum | ) |
Definition at line 62 of file CsvFileBufferParser.h.
Referenced by parse_buffer(), process_geo_column(), and set_coordinates_from_separate_lon_lat_columns().
bool foreign_storage::csv_file_buffer_parser::is_null_datum | ( | const std::string_view | datum, |
const ColumnDescriptor * | column, | ||
const std::string & | null_indicator | ||
) |
Definition at line 98 of file CsvFileBufferParser.h.
References ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::get_notnull(), is_null(), and SQLTypeInfo::is_string().
Referenced by parse_buffer().
ParseBufferResult foreign_storage::csv_file_buffer_parser::parse_buffer | ( | ParseBufferRequest & | request, |
bool | convert_data_blocks | ||
) |
Parses a given CSV file buffer and returns data blocks for each column in the file along with metadata related to rows and row offsets within the buffer.
Definition at line 343 of file CsvFileBufferParser.h.
References foreign_storage::csv_file_buffer_parser::ParseBufferRequest::begin_pos, foreign_storage::csv_file_buffer_parser::ParseBufferRequest::buffer, foreign_storage::csv_file_buffer_parser::ParseBufferRequest::buffer_size, CHECK, convert_import_buffers_to_data_blocks(), foreign_storage::csv_file_buffer_parser::ParseBufferRequest::copy_params, foreign_storage::csv_file_buffer_parser::ParseBufferRequest::end_pos, logger::ERROR, foreign_storage::csv_file_buffer_parser::ParseBufferRequest::file_offset, import_export::delimited_parser::find_beginning(), foreign_storage::csv_file_buffer_parser::ParseBufferRequest::first_row_index, import_export::delimited_parser::get_row(), foreign_storage::csv_file_buffer_parser::ParseBufferRequest::getCatalog(), foreign_storage::csv_file_buffer_parser::ParseBufferRequest::getColumns(), foreign_storage::csv_file_buffer_parser::ParseBufferRequest::getFilePath(), foreign_storage::csv_file_buffer_parser::ParseBufferRequest::import_buffers, is_coordinate_scalar(), is_null(), is_null_datum(), kPOINT, LOG, import_export::CopyParams::null_str, shared::printContainer(), process_geo_column(), foreign_storage::csv_file_buffer_parser::ParseBufferRequest::process_row_count, run_benchmark_import::result, set_array_flags_and_geo_columns_count(), skip_column_import(), and validate_expected_column_count().
Referenced by foreign_storage::parse_file_regions(), and foreign_storage::scan_metadata().
void foreign_storage::csv_file_buffer_parser::process_geo_column | ( | std::vector< std::unique_ptr< import_export::TypedImportBuffer >> & | import_buffers, |
size_t & | col_idx, | ||
const import_export::CopyParams & | copy_params, | ||
std::list< const ColumnDescriptor * >::iterator & | cd_it, | ||
std::vector< std::string_view > & | row, | ||
size_t & | import_idx, | ||
bool | is_null, | ||
size_t | first_row_index, | ||
size_t | row_index_plus_one, | ||
std::shared_ptr< Catalog_Namespace::Catalog > | catalog | ||
) |
Definition at line 115 of file CsvFileBufferParser.h.
References CHECK, Geospatial::GeoTypesFactory::getGeoColumns(), Geospatial::GeoTypesFactory::getNullGeoColumns(), is_coordinate_scalar(), IS_GEO, kMULTIPOLYGON, kPOINT, kPOLYGON, import_export::CopyParams::lonlat, import_export::CopyParams::null_str, PROMOTE_POLYGON_TO_MULTIPOLYGON, set_coordinates_from_separate_lon_lat_columns(), import_export::Importer::set_geo_physical_import_buffer(), and to_string().
Referenced by parse_buffer().
void foreign_storage::csv_file_buffer_parser::set_array_flags_and_geo_columns_count | ( | std::unique_ptr< bool[]> & | array_flags, |
int & | phys_cols, | ||
int & | point_cols, | ||
const std::list< const ColumnDescriptor * > & | columns | ||
) |
Definition at line 29 of file CsvFileBufferParser.h.
References kARRAY, and kPOINT.
Referenced by parse_buffer().
bool foreign_storage::csv_file_buffer_parser::set_coordinates_from_separate_lon_lat_columns | ( | const std::string_view | lon_str, |
const std::string_view | lat_str, | ||
std::vector< double > & | coords, | ||
const bool | is_lon_lat_order | ||
) |
Definition at line 68 of file CsvFileBufferParser.h.
References is_coordinate_scalar(), and gpu_enabled::swap().
Referenced by process_geo_column().
bool foreign_storage::csv_file_buffer_parser::skip_column_import | ( | ParseBufferRequest & | request, |
int | column_idx | ||
) |
Definition at line 335 of file CsvFileBufferParser.h.
References foreign_storage::csv_file_buffer_parser::ParseBufferRequest::import_buffers.
Referenced by parse_buffer().
void foreign_storage::csv_file_buffer_parser::validate_expected_column_count | ( | std::vector< std::string_view > & | row, |
size_t | num_cols, | ||
int | point_cols, | ||
const std::string & | file_name | ||
) |
Definition at line 52 of file CsvFileBufferParser.h.
References foreign_storage::throw_number_of_columns_mismatch_error().
Referenced by parse_buffer().
|
static |
Definition at line 27 of file CsvFileBufferParser.h.
Referenced by process_geo_column().