Namespaces
	anonymous_namespace{AbstractFileStorageDataWrapper.cpp}

	anonymous_namespace{CachingForeignStorageMgr.cpp}

	anonymous_namespace{CsvDataWrapper.cpp}

	anonymous_namespace{CsvReader.cpp}

	anonymous_namespace{ForeignStorageCache.cpp}

	anonymous_namespace{LazyParquetChunkLoader.cpp}

	anonymous_namespace{ParquetDataWrapper.cpp}

	Csv

	csv_file_buffer_parser

	json_utils

Classes
struct	ForeignServer

struct	ForeignTable

struct	OptionsContainer

class	AbstractFileStorageDataWrapper

class	CachingForeignStorageMgr

struct	ParseFileRegionResult

struct	MetadataScanMultiThreadingParams

class	CsvDataWrapper

class	CsvReader

class	SingleFileReader

class	ArchiveWrapper

class	CompressedFileReader

class	MultiFileReader

class	LocalMultiFileReader

struct	FileRegion

class	ForeignDataWrapper

struct	DataWrapperType
	Encapsulates an enumeration of foreign data wrapper type strings. More...

class	ForeignDataWrapperFactory

class	ForeignStorageBuffer

class	ForeignStorageCache

class	ForeignStorageException

class	MockForeignDataWrapper

class	ForeignStorageMgr

class	ForeignTableSchema

struct	ColumnType

struct	FragmentType

struct	Interval

class	LazyParquetChunkLoader

class	ParquetArrayEncoder

class	ParquetDataWrapper

class	ParquetDateFromTimestampEncoder

class	ParquetDateInSecondsEncoder

class	ParquetDecimalEncoder

class	ParquetEncoder

class	ParquetScalarEncoder

class	ParquetFixedLengthArrayEncoder

class	ParquetFixedLengthEncoder

class	ParquetUnsignedFixedLengthEncoder

class	ParquetGeospatialEncoder

class	ParquetInPlaceEncoder

class	TypedParquetInPlaceEncoder

class	ParquetMetadataValidator

class	TimestampBoundsValidator

class	IntegralFixedLengthBoundsValidator

class	DateInSecondsBoundsValidator

class	FloatPointValidator

struct	RowGroupInterval

struct	RowGroupMetadata

class	FileReaderMap

class	ParquetStringEncoder

class	ParquetStringNoneEncoder

class	ParquetTimeEncoder

class	ParquetTimestampEncoder

class	ParquetVariableLengthArrayEncoder

class	ForeignTableRefreshScheduler

Typedefs
using	OptionsMap = std::map< std::string, std::string, std::less<>>

using	FileRegions = std::vector< FileRegion >

using	ChunkToBufferMap = std::map< ChunkKey, AbstractBuffer * >

using	read_lock = mapd_shared_lock< mapd_shared_mutex >

using	write_lock = mapd_unique_lock< mapd_shared_mutex >

using	UniqueReaderPtr = std::unique_ptr< parquet::arrow::FileReader >

using	ReaderPtr = parquet::arrow::FileReader *

Functions
int64_t	get_interval_duration (const std::string &interval)

int64_t	get_next_refresh_time (const std::map< std::string, std::string, std::less<>> &foreign_table_options)

ParseFileRegionResult	parse_file_regions (const FileRegions &file_regions, const size_t start_index, const size_t end_index, CsvReader &csv_reader, csv_file_buffer_parser::ParseBufferRequest &parse_file_request, const std::map< int, Chunk_NS::Chunk > &column_id_to_chunk_map)

size_t	get_buffer_size (const import_export::CopyParams &copy_params, const bool size_known, const size_t file_size)

size_t	get_buffer_size (const FileRegions &file_regions)

size_t	get_thread_count (const import_export::CopyParams &copy_params, const bool size_known, const size_t file_size, const size_t buffer_size)

size_t	get_thread_count (const import_export::CopyParams &copy_params, const FileRegions &file_regions)

std::vector< size_t >	partition_by_fragment (const size_t start_row_index, const size_t max_fragment_size, const size_t buffer_row_count)

std::optional < csv_file_buffer_parser::ParseBufferRequest >	get_next_metadata_scan_request (MetadataScanMultiThreadingParams &multi_threading_params)

void	add_file_region (std::map< int, FileRegions > &fragment_id_to_file_regions_map, int fragment_id, size_t first_row_index, const csv_file_buffer_parser::ParseBufferResult &result, const std::string &file_path)

void	update_stats (Encoder *encoder, const SQLTypeInfo &column_type, DataBlockPtr data_block, const size_t row_count)

void	cache_blocks (std::map< ChunkKey, Chunk_NS::Chunk > &cached_chunks, DataBlockPtr data_block, size_t row_count, ChunkKey &chunk_key, const ColumnDescriptor *column, bool is_first_block, bool is_last_block)

void	process_data_blocks (MetadataScanMultiThreadingParams &multi_threading_params, int fragment_id, const csv_file_buffer_parser::ParseBufferRequest &request, csv_file_buffer_parser::ParseBufferResult &result, std::map< int, const ColumnDescriptor * > &column_by_id, std::map< int, FileRegions > &fragment_id_to_file_regions_map)

void	add_request_to_pool (MetadataScanMultiThreadingParams &multi_threading_params, csv_file_buffer_parser::ParseBufferRequest &request)

void	scan_metadata (MetadataScanMultiThreadingParams &multi_threading_params, std::map< int, FileRegions > &fragment_id_to_file_regions_map)

csv_file_buffer_parser::ParseBufferRequest	get_request_from_pool (MetadataScanMultiThreadingParams &multi_threading_params)

void	dispatch_metadata_scan_request (MetadataScanMultiThreadingParams &multi_threading_params, csv_file_buffer_parser::ParseBufferRequest &request)

void	resize_buffer_if_needed (std::unique_ptr< char[]> &buffer, size_t &buffer_size, const size_t alloc_size)

void	dispatch_metadata_scan_requests (const size_t &buffer_size, const std::string &file_path, CsvReader &csv_reader, const import_export::CopyParams &copy_params, MetadataScanMultiThreadingParams &multi_threading_params, size_t &first_row_index_in_buffer, size_t &current_file_offset)

void	set_value (rapidjson::Value &json_val, const FileRegion &file_region, rapidjson::Document::AllocatorType &allocator)

void	get_value (const rapidjson::Value &json_val, FileRegion &file_region)

void	throw_removed_row_error (const std::string &file_path)

void	throw_removed_file_error (const std::string &file_path)

void	throw_number_of_columns_mismatch_error (size_t num_table_cols, size_t num_file_cols, const std::string &file_path)

std::vector< ChunkKey >	get_keys_vec_from_table (const ChunkKey &destination_chunk_key)

std::set< ChunkKey >	get_keys_set_from_table (const ChunkKey &destination_chunk_key)

void	refresh_foreign_table (Catalog_Namespace::Catalog &catalog, const std::string &table_name, const bool evict_cached_entries)

bool	is_metadata_placeholder (const ChunkMetadata &metadata)

void	set_value (rapidjson::Value &json_val, const RowGroupInterval &value, rapidjson::Document::AllocatorType &allocator)

void	get_value (const rapidjson::Value &json_val, RowGroupInterval &value)

template<typename T >
ArrayDatum	encode_as_array_datum (const std::vector< T > &data)

template<typename V , std::enable_if_t< std::is_integral< V >::value, int > = 0>
V	get_null_value ()

template<typename D , std::enable_if_t< std::is_integral< D >::value, int > = 0>
std::pair< D, D >	get_min_max_bounds ()

template<typename D , typename T >
bool	check_bounds (const T &value)

template<typename D >
std::string	datetime_to_string (const D &timestamp, const SQLTypeInfo &column_type)

void	throw_parquet_metadata_out_of_bounds_error (const std::string &min_value, const std::string &max_value, const std::string &encountered_value)

UniqueReaderPtr	open_parquet_table (const std::string &file_path, std::shared_ptr< arrow::fs::FileSystem > &file_system)

std::pair< int, int >	get_parquet_table_size (const ReaderPtr &reader)

const parquet::ColumnDescriptor *	get_column_descriptor (const parquet::arrow::FileReader *reader, const int logical_column_index)

parquet::Type::type	get_physical_type (ReaderPtr &reader, const int logical_column_index)

void	validate_equal_column_descriptor (const parquet::ColumnDescriptor reference_descriptor, const parquet::ColumnDescriptor new_descriptor, const std::string &reference_file_path, const std::string &new_file_path)

std::unique_ptr< ColumnDescriptor >	get_sub_type_column_descriptor (const ColumnDescriptor *column)

std::shared_ptr < parquet::Statistics >	validate_and_get_column_metadata_statistics (const parquet::ColumnChunkMetaData *column_metadata)

template<typename T >
auto	partition_for_threads (const std::set< T > &items, size_t max_threads)

void	validate_non_foreign_table_write (const TableDescriptor *table_descriptor)

Variables
const std::string	wrapper_file_name = "wrapper_metadata.json"

Typedef Documentation

using foreign_storage::ChunkToBufferMap = typedef std::map<ChunkKey, AbstractBuffer*>

Definition at line 28 of file ForeignDataWrapper.h.

using foreign_storage::FileRegions = typedef std::vector<FileRegion>

Definition at line 71 of file CsvShared.h.

using foreign_storage::OptionsMap = typedef std::map<std::string, std::string, std::less<>>

Definition at line 30 of file OptionsContainer.h.

using foreign_storage::read_lock = typedef mapd_shared_lock<mapd_shared_mutex>

Definition at line 28 of file ForeignStorageCache.cpp.

using foreign_storage::ReaderPtr = typedef parquet::arrow::FileReader*

Definition at line 33 of file ParquetShared.h.

using foreign_storage::UniqueReaderPtr = typedef std::unique_ptr<parquet::arrow::FileReader>

Definition at line 32 of file ParquetShared.h.

using foreign_storage::write_lock = typedef mapd_unique_lock<mapd_shared_mutex>

Definition at line 29 of file ForeignStorageCache.cpp.

Function Documentation

void foreign_storage::add_file_region	(	std::map< int, FileRegions > &	fragment_id_to_file_regions_map,
		int	fragment_id,
		size_t	first_row_index,
		const csv_file_buffer_parser::ParseBufferResult &	result,
		const std::string &	file_path
	)

Creates a new file region based on metadata from parsed CSV file buffers and adds the new region to the fragment id to file regions map.

Definition at line 418 of file CsvDataWrapper.cpp.

References foreign_storage::csv_file_buffer_parser::ParseBufferResult::row_count, and foreign_storage::csv_file_buffer_parser::ParseBufferResult::row_offsets.

Referenced by process_data_blocks().

                                                  {
   fragment_id_to_file_regions_map[fragment_id].emplace_back(
       // file naming is handled by CsvReader
       FileRegion(result.row_offsets.front(),
                  first_row_index,
                  result.row_count,
                  result.row_offsets.back() - result.row_offsets.front()));
 }

Here is the caller graph for this function:

void foreign_storage::add_request_to_pool	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		csv_file_buffer_parser::ParseBufferRequest &	request
	)

Adds the request object for a processed request back to the request pool for reuse in subsequent requests.

Definition at line 562 of file CsvDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::request_pool, foreign_storage::MetadataScanMultiThreadingParams::request_pool_condition, and foreign_storage::MetadataScanMultiThreadingParams::request_pool_mutex.

Referenced by scan_metadata().

                                                                             {
   std::unique_lock<std::mutex> completed_requests_queue_lock(
       multi_threading_params.request_pool_mutex);
   multi_threading_params.request_pool.emplace(std::move(request));
   completed_requests_queue_lock.unlock();
   multi_threading_params.request_pool_condition.notify_all();
 }

Here is the caller graph for this function:

void foreign_storage::cache_blocks	(	std::map< ChunkKey, Chunk_NS::Chunk > &	cached_chunks,
		DataBlockPtr	data_block,
		size_t	row_count,
		ChunkKey &	chunk_key,
		const ColumnDescriptor *	column,
		bool	is_first_block,
		bool	is_last_block
	)

Definition at line 462 of file CsvDataWrapper.cpp.

References CHECK, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, ColumnDescriptor::columnType, foreign_storage::anonymous_namespace{CsvDataWrapper.cpp}::get_cache_if_enabled(), Catalog_Namespace::SysCatalog::getCatalog(), Catalog_Namespace::SysCatalog::instance(), and SQLTypeInfo::is_varlen_indeed().

Referenced by process_data_blocks().

                                       {
   auto catalog =
       Catalog_Namespace::SysCatalog::instance().getCatalog(chunk_key[CHUNK_KEY_DB_IDX]);
   CHECK(catalog);
   auto cache = get_cache_if_enabled(catalog);
   if (cache) {
     ChunkKey index_key = {chunk_key[CHUNK_KEY_DB_IDX],
                           chunk_key[CHUNK_KEY_TABLE_IDX],
                           chunk_key[CHUNK_KEY_COLUMN_IDX],
                           chunk_key[CHUNK_KEY_FRAGMENT_IDX],
                           2};
     // Create actual data chunks to prepopulate cache
     if (cached_chunks.find(chunk_key) == cached_chunks.end()) {
       cached_chunks[chunk_key] = Chunk_NS::Chunk{column};
       cached_chunks[chunk_key].setBuffer(
           cache->getChunkBufferForPrecaching(chunk_key, is_first_block));
       if (column->columnType.is_varlen_indeed()) {
         cached_chunks[chunk_key].setIndexBuffer(
             cache->getChunkBufferForPrecaching(index_key, is_first_block));
       }
       if (is_first_block) {
         cached_chunks[chunk_key].initEncoder();
       }
     }
     cached_chunks[chunk_key].appendData(data_block, row_count, 0);
     if (is_last_block) {
       // cache the chunks now so they are tracked by eviction algorithm
       std::vector<ChunkKey> key_to_cache{chunk_key};
       if (column->columnType.is_varlen_indeed()) {
         key_to_cache.push_back(index_key);
       }
       cache->cacheTableChunks(key_to_cache);
     }
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename D , typename T >

bool foreign_storage::check_bounds ( const T & value )

inline

Definition at line 53 of file ParquetMetadataValidator.h.

                                          {
   auto [min_value, max_value] = get_min_max_bounds<D>();
   return value >= min_value && value <= max_value;
 }

template<typename D >

std::string foreign_storage::datetime_to_string	(	const D &	timestamp,
		const SQLTypeInfo &	column_type
	)

inline

Definition at line 59 of file ParquetMetadataValidator.h.

References CHECK, test_fsi::d, DatumToString(), SQLTypeInfo::is_date(), and SQLTypeInfo::is_timestamp().

Referenced by foreign_storage::TimestampBoundsValidator< T >::getMinMaxBoundsAsStrings(), foreign_storage::DateInSecondsBoundsValidator< T >::getMinMaxBoundsAsStrings(), foreign_storage::TimestampBoundsValidator< T >::validateValue(), and foreign_storage::DateInSecondsBoundsValidator< T >::validateValue().

                                                                       {
   CHECK(column_type.is_timestamp() || column_type.is_date());
   Datum d;
   d.bigintval = timestamp;
   return DatumToString(d, column_type);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::dispatch_metadata_scan_request	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		csv_file_buffer_parser::ParseBufferRequest &	request
	)

Dispatches a new metadata scan request by adding the request to the pending requests queue to be consumed by a worker thread.

Definition at line 647 of file CsvDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::pending_requests, foreign_storage::MetadataScanMultiThreadingParams::pending_requests_condition, and foreign_storage::MetadataScanMultiThreadingParams::pending_requests_mutex.

Referenced by dispatch_metadata_scan_requests().

                                                        {
   {
     std::unique_lock<std::mutex> pending_requests_lock(
         multi_threading_params.pending_requests_mutex);
     multi_threading_params.pending_requests.emplace(std::move(request));
   }
   multi_threading_params.pending_requests_condition.notify_all();
 }

Here is the caller graph for this function:

void foreign_storage::dispatch_metadata_scan_requests	(	const size_t &	buffer_size,
		const std::string &	file_path,
		CsvReader &	csv_reader,
		const import_export::CopyParams &	copy_params,
		MetadataScanMultiThreadingParams &	multi_threading_params,
		size_t &	first_row_index_in_buffer,
		size_t &	current_file_offset
	)

Reads from a CSV file iteratively and dispatches metadata scan requests that are processed by worker threads.

Definition at line 676 of file CsvDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::continue_processing, dispatch_metadata_scan_request(), import_export::delimited_parser::find_row_end_pos(), get_request_from_pool(), foreign_storage::CsvReader::isScanFinished(), import_export::CopyParams::line_delim, foreign_storage::MetadataScanMultiThreadingParams::pending_requests, foreign_storage::MetadataScanMultiThreadingParams::pending_requests_condition, foreign_storage::MetadataScanMultiThreadingParams::pending_requests_mutex, foreign_storage::CsvReader::read(), and resize_buffer_if_needed().

Referenced by foreign_storage::CsvDataWrapper::populateChunkMetadata().

                                  {
   auto alloc_size = buffer_size;
   auto residual_buffer = std::make_unique<char[]>(alloc_size);
   size_t residual_buffer_size = 0;
   size_t residual_buffer_alloc_size = alloc_size;
 
   while (!csv_reader.isScanFinished()) {
     {
       std::lock_guard<std::mutex> pending_requests_lock(
           multi_threading_params.pending_requests_mutex);
       if (!multi_threading_params.continue_processing) {
         break;
       }
     }
     auto request = get_request_from_pool(multi_threading_params);
     resize_buffer_if_needed(request.buffer, request.buffer_alloc_size, alloc_size);
 
     if (residual_buffer_size > 0) {
       memcpy(request.buffer.get(), residual_buffer.get(), residual_buffer_size);
     }
     size_t size = residual_buffer_size;
     size += csv_reader.read(request.buffer.get() + residual_buffer_size,
                             alloc_size - residual_buffer_size);
 
     if (size == 0) {
       // In some cases at the end of a file we will read 0 bytes even when
       // csv_reader.isScanFinished() is false
       continue;
     } else if (size == 1 && request.buffer[0] == copy_params.line_delim) {
       // In some cases files with newlines at the end will be encoded with a second
       // newline that can end up being the only thing in the buffer
       current_file_offset++;
       continue;
     }
     unsigned int num_rows_in_buffer = 0;
     request.end_pos =
         import_export::delimited_parser::find_row_end_pos(alloc_size,
                                                           request.buffer,
                                                           size,
                                                           copy_params,
                                                           first_row_index_in_buffer,
                                                           num_rows_in_buffer,
                                                           nullptr,
                                                           &csv_reader);
     request.buffer_size = size;
     request.buffer_alloc_size = alloc_size;
     request.first_row_index = first_row_index_in_buffer;
     request.file_offset = current_file_offset;
     request.buffer_row_count = num_rows_in_buffer;
 
     residual_buffer_size = size - request.end_pos;
     if (residual_buffer_size > 0) {
       resize_buffer_if_needed(residual_buffer, residual_buffer_alloc_size, alloc_size);
       memcpy(residual_buffer.get(),
              request.buffer.get() + request.end_pos,
              residual_buffer_size);
     }
 
     current_file_offset += request.end_pos;
     first_row_index_in_buffer += num_rows_in_buffer;
 
     dispatch_metadata_scan_request(multi_threading_params, request);
   }
 
   std::unique_lock<std::mutex> pending_requests_queue_lock(
       multi_threading_params.pending_requests_mutex);
   multi_threading_params.pending_requests_condition.wait(
       pending_requests_queue_lock, [&multi_threading_params] {
         return multi_threading_params.pending_requests.empty() ||
                (multi_threading_params.continue_processing == false);
       });
   multi_threading_params.continue_processing = false;
   pending_requests_queue_lock.unlock();
   multi_threading_params.pending_requests_condition.notify_all();
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >

ArrayDatum foreign_storage::encode_as_array_datum ( const std::vector< T > & data )

inline

Definition at line 39 of file ParquetGeospatialEncoder.h.

References omnisci.dtypes::T.

Referenced by foreign_storage::ParquetGeospatialEncoder::processGeoElement(), and foreign_storage::ParquetGeospatialEncoder::processNullGeoElement().

                                                                   {
   const size_t num_bytes = data.size() * sizeof(T);
   std::shared_ptr<int8_t> buffer(new int8_t[num_bytes], std::default_delete<int8_t[]>());
   memcpy(buffer.get(), data.data(), num_bytes);
   return ArrayDatum(num_bytes, buffer, false);
 }

Here is the caller graph for this function:

size_t foreign_storage::get_buffer_size	(	const import_export::CopyParams &	copy_params,
		const bool	size_known,
		const size_t	file_size
	)

Gets the appropriate buffer size to be used when processing CSV file(s).

Definition at line 215 of file CsvDataWrapper.cpp.

References import_export::CopyParams::buffer_size.

Referenced by foreign_storage::CsvDataWrapper::populateChunkMetadata(), and foreign_storage::CsvDataWrapper::populateChunks().

                                                {
   size_t buffer_size = copy_params.buffer_size;
   if (size_known && file_size < buffer_size) {
     buffer_size = file_size + 1;  // +1 for end of line character, if missing
   }
   return buffer_size;
 }

Here is the caller graph for this function:

size_t foreign_storage::get_buffer_size ( const FileRegions & file_regions )

Definition at line 225 of file CsvDataWrapper.cpp.

References CHECK.

                                                         {
   size_t buffer_size = 0;
   for (const auto& file_region : file_regions) {
     buffer_size = std::max(buffer_size, file_region.region_size);
   }
   CHECK(buffer_size);
   return buffer_size;
 }

const parquet::ColumnDescriptor * foreign_storage::get_column_descriptor	(	const parquet::arrow::FileReader *	reader,
		const int	logical_column_index
	)

Definition at line 45 of file ParquetShared.cpp.

Referenced by foreign_storage::LazyParquetChunkLoader::appendRowGroups(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_equal_schema().

                                     {
   return reader->parquet_reader()->metadata()->schema()->Column(logical_column_index);
 }

Here is the caller graph for this function:

int64_t foreign_storage::get_interval_duration ( const std::string & interval )

inline

Gets the interval duration in seconds.

Parameters

interval - interval string with format of {interval_count}{interval_type} (e.g. 5H for "every 5 hours")

Returns: internal duration in seconds

Definition at line 16 of file RefreshTimeCalculator.h.

References UNREACHABLE.

Referenced by get_next_refresh_time().

                                                                 {
   int interval_count = std::stoi(interval.substr(0, interval.length() - 1));
   auto interval_type = std::tolower(interval[interval.length() - 1]);
   int64_t duration{0};
   if (interval_type == 's') {
     duration = interval_count;
   } else if (interval_type == 'h') {
     duration = interval_count * 60 * 60;
   } else if (interval_type == 'd') {
     duration = interval_count * 60 * 60 * 24;
   } else {
     UNREACHABLE();
   }
   return duration;
 }

Here is the caller graph for this function:

std::set<ChunkKey> foreign_storage::get_keys_set_from_table ( const ChunkKey & destination_chunk_key )

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer().

Here is the caller graph for this function:

std::vector<ChunkKey> foreign_storage::get_keys_vec_from_table ( const ChunkKey & destination_chunk_key )

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer().

Here is the caller graph for this function:

template<typename D , std::enable_if_t< std::is_integral< D >::value, int > = 0>

std::pair< D, D > foreign_storage::get_min_max_bounds ( )

inline

Definition at line 41 of file ParquetMetadataValidator.h.

                                           {
   static_assert(std::is_signed<D>::value,
                 "'get_min_max_bounds' is only valid for signed types");
   return {get_null_value<D>() + 1, std::numeric_limits<D>::max()};
 }

std::optional<csv_file_buffer_parser::ParseBufferRequest> foreign_storage::get_next_metadata_scan_request ( MetadataScanMultiThreadingParams & multi_threading_params )

Gets the next metadata scan request object from the pending requests queue. A null optional is returned if there are no further requests to be processed.

Definition at line 395 of file CsvDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::continue_processing, foreign_storage::MetadataScanMultiThreadingParams::pending_requests, foreign_storage::MetadataScanMultiThreadingParams::pending_requests_condition, and foreign_storage::MetadataScanMultiThreadingParams::pending_requests_mutex.

Referenced by scan_metadata().

                                                               {
   std::unique_lock<std::mutex> pending_requests_lock(
       multi_threading_params.pending_requests_mutex);
   multi_threading_params.pending_requests_condition.wait(
       pending_requests_lock, [&multi_threading_params] {
         return !multi_threading_params.pending_requests.empty() ||
                !multi_threading_params.continue_processing;
       });
   if (multi_threading_params.pending_requests.empty()) {
     return {};
   }
   auto request = std::move(multi_threading_params.pending_requests.front());
   multi_threading_params.pending_requests.pop();
   pending_requests_lock.unlock();
   multi_threading_params.pending_requests_condition.notify_all();
   return std::move(request);
 }

Here is the caller graph for this function:

int64_t foreign_storage::get_next_refresh_time ( const std::map< std::string, std::string, std::less<>> & foreign_table_options )

inline

Definition at line 33 of file RefreshTimeCalculator.h.

References CHECK, count, get_interval_duration(), foreign_storage::ForeignTable::NULL_REFRESH_TIME, foreign_storage::ForeignTable::REFRESH_INTERVAL_KEY, and foreign_storage::ForeignTable::REFRESH_START_DATE_TIME_KEY.

Referenced by Catalog_Namespace::anonymous_namespace{Catalog.cpp}::get_next_refresh_time().

                                                                           {
   int64_t current_time = std::chrono::duration_cast<std::chrono::seconds>(
                              std::chrono::system_clock::now().time_since_epoch())
                              .count();
   auto start_date_entry = foreign_table_options.find(
       foreign_storage::ForeignTable::REFRESH_START_DATE_TIME_KEY);
   CHECK(start_date_entry != foreign_table_options.end());
   auto start_date_time = dateTimeParse<kTIMESTAMP>(start_date_entry->second, 0);
 
   // If start date time is current or in the future, then that is the next refresh time
   if (start_date_time >= current_time) {
     return start_date_time;
   }
   auto interval_entry =
       foreign_table_options.find(foreign_storage::ForeignTable::REFRESH_INTERVAL_KEY);
   if (interval_entry != foreign_table_options.end()) {
     auto interval_duration = get_interval_duration(interval_entry->second);
     auto num_intervals =
         (current_time - start_date_time + interval_duration - 1) / interval_duration;
     return start_date_time + (num_intervals * interval_duration);
   } else {
     // If this was a one time refresh, then there is no next refresh time
     return foreign_storage::ForeignTable::NULL_REFRESH_TIME;
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename V , std::enable_if_t< std::is_integral< V >::value, int > = 0>

V foreign_storage::get_null_value ( )

inline

Definition at line 31 of file ParquetMetadataValidator.h.

                           {
   return inline_int_null_value<V>();
 }

std::pair< int, int > foreign_storage::get_parquet_table_size ( const ReaderPtr & reader )

Definition at line 38 of file ParquetShared.cpp.

Referenced by foreign_storage::LazyParquetChunkLoader::appendRowGroups(), and foreign_storage::LazyParquetChunkLoader::metadataScan().

                                                                   {
   auto file_metadata = reader->parquet_reader()->metadata();
   const auto num_row_groups = file_metadata->num_row_groups();
   const auto num_columns = file_metadata->num_columns();
   return std::make_pair(num_row_groups, num_columns);
 }

Here is the caller graph for this function:

parquet::Type::type foreign_storage::get_physical_type	(	ReaderPtr &	reader,
		const int	logical_column_index
	)

Definition at line 51 of file ParquetShared.cpp.

Referenced by anonymous_namespace{ArrowResultSetConverter.cpp}::get_arrow_type(), and ArrowResultSetConverter::initializeColumnBuilder().

                                                                                      {
   return reader->parquet_reader()
       ->metadata()
       ->schema()
       ->Column(logical_column_index)
       ->physical_type();
 }

Here is the caller graph for this function:

csv_file_buffer_parser::ParseBufferRequest foreign_storage::get_request_from_pool ( MetadataScanMultiThreadingParams & multi_threading_params )

Gets a request from the metadata scan request pool.

Definition at line 629 of file CsvDataWrapper.cpp.

References CHECK, foreign_storage::MetadataScanMultiThreadingParams::request_pool, foreign_storage::MetadataScanMultiThreadingParams::request_pool_condition, and foreign_storage::MetadataScanMultiThreadingParams::request_pool_mutex.

Referenced by dispatch_metadata_scan_requests().

                                                               {
   std::unique_lock<std::mutex> request_pool_lock(
       multi_threading_params.request_pool_mutex);
   multi_threading_params.request_pool_condition.wait(
       request_pool_lock,
       [&multi_threading_params] { return !multi_threading_params.request_pool.empty(); });
   auto request = std::move(multi_threading_params.request_pool.front());
   multi_threading_params.request_pool.pop();
   request_pool_lock.unlock();
   CHECK(request.buffer);
   return request;
 }

Here is the caller graph for this function:

std::unique_ptr< ColumnDescriptor > foreign_storage::get_sub_type_column_descriptor ( const ColumnDescriptor * column )

Definition at line 75 of file ParquetShared.cpp.

References ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::get_elem_type(), SQLTypeInfo::set_size(), and ColumnDescriptor::tableId.

Referenced by foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_array_encoder(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_array_mapping().

                                     {
   auto column_type = column->columnType.get_elem_type();
   if (column_type.get_size() == -1 && column_type.is_dict_encoded_string()) {
     column_type.set_size(4);  // override default size of -1
   }
   return std::make_unique<ColumnDescriptor>(
       column->tableId, column->columnId, column->columnName, column_type);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

size_t foreign_storage::get_thread_count	(	const import_export::CopyParams &	copy_params,
		const bool	size_known,
		const size_t	file_size,
		const size_t	buffer_size
	)

Gets the appropriate number of threads to be used for concurrent processing within the data wrapper.

Definition at line 238 of file CsvDataWrapper.cpp.

References CHECK, and import_export::CopyParams::threads.

Referenced by foreign_storage::CsvDataWrapper::populateChunkMetadata(), and foreign_storage::CsvDataWrapper::populateChunks().

                                                   {
   size_t thread_count = copy_params.threads;
   if (thread_count == 0) {
     thread_count = std::thread::hardware_concurrency();
   }
   if (size_known) {
     size_t num_buffers_in_file = (file_size + buffer_size - 1) / buffer_size;
     if (num_buffers_in_file < thread_count) {
       thread_count = num_buffers_in_file;
     }
   }
   CHECK(thread_count);
   return thread_count;
 }

Here is the caller graph for this function:

size_t foreign_storage::get_thread_count	(	const import_export::CopyParams &	copy_params,
		const FileRegions &	file_regions
	)

Definition at line 256 of file CsvDataWrapper.cpp.

References CHECK, and import_export::CopyParams::threads.

                                                          {
   size_t thread_count = copy_params.threads;
   if (thread_count == 0) {
     thread_count =
         std::min<size_t>(std::thread::hardware_concurrency(), file_regions.size());
   }
   CHECK(thread_count);
   return thread_count;
 }

void foreign_storage::get_value	(	const rapidjson::Value &	json_val,
		FileRegion &	file_region
	)

Definition at line 44 of file CsvShared.cpp.

References CHECK, foreign_storage::FileRegion::filename, foreign_storage::FileRegion::first_row_file_offset, foreign_storage::FileRegion::first_row_index, foreign_storage::json_utils::get_value_from_object(), foreign_storage::FileRegion::region_size, and foreign_storage::FileRegion::row_count.

                                                                         {
   CHECK(json_val.IsObject());
   json_utils::get_value_from_object(
       json_val, file_region.first_row_file_offset, "first_row_file_offset");
   json_utils::get_value_from_object(
       json_val, file_region.first_row_index, "first_row_index");
   json_utils::get_value_from_object(json_val, file_region.region_size, "region_size");
   json_utils::get_value_from_object(json_val, file_region.row_count, "row_count");
   if (json_val.HasMember("filename")) {
     json_utils::get_value_from_object(json_val, file_region.filename, "filename");
   }
 }

Here is the call graph for this function:

void foreign_storage::get_value	(	const rapidjson::Value &	json_val,
		RowGroupInterval &	value
	)

Definition at line 505 of file ParquetDataWrapper.cpp.

References CHECK, foreign_storage::RowGroupInterval::end_index, foreign_storage::RowGroupInterval::file_path, foreign_storage::json_utils::get_value_from_object(), and foreign_storage::RowGroupInterval::start_index.

                                                                         {
   CHECK(json_val.IsObject());
   json_utils::get_value_from_object(json_val, value.file_path, "file_path");
   json_utils::get_value_from_object(json_val, value.start_index, "start_index");
   json_utils::get_value_from_object(json_val, value.end_index, "end_index");
 }

Here is the call graph for this function:

bool foreign_storage::is_metadata_placeholder ( const ChunkMetadata & metadata )

inline

Check if ChunkMetadata corresponds to a chunk for which metadata must be populated.

Definition at line 27 of file MetadataPlaceholder.h.

References ChunkMetadata::chunkStats, Datum::intval, SQLTypeInfo::is_dict_encoded_type(), ChunkStats::max, ChunkStats::min, and ChunkMetadata::sqlType.

Referenced by anonymous_namespace{RelAlgExecutor.cpp}::prepare_string_dictionaries().

                                                                    {
   return metadata.chunkStats.min.intval > metadata.chunkStats.max.intval &&
          metadata.sqlType.is_dict_encoded_type();  // Only supported type for now
 }

Here is the call graph for this function:

Here is the caller graph for this function:

UniqueReaderPtr foreign_storage::open_parquet_table	(	const std::string &	file_path,
		std::shared_ptr< arrow::fs::FileSystem > &	file_system
	)

Definition at line 25 of file ParquetShared.cpp.

Referenced by foreign_storage::FileReaderMap::getOrInsert(), and foreign_storage::FileReaderMap::insert().

                                                                                   {
   UniqueReaderPtr reader;
   auto file_result = file_system->OpenInputFile(file_path);
   if (!file_result.ok()) {
     throw std::runtime_error{"Unable to access " + file_system->type_name() + " file: " +
                              file_path + ". " + file_result.status().message()};
   }
   auto infile = file_result.ValueOrDie();
   PARQUET_THROW_NOT_OK(OpenFile(infile, arrow::default_memory_pool(), &reader));
   return reader;
 }

Here is the caller graph for this function:

ParseFileRegionResult foreign_storage::parse_file_regions	(	const FileRegions &	file_regions,
		const size_t	start_index,
		const size_t	end_index,
		CsvReader &	csv_reader,
		csv_file_buffer_parser::ParseBufferRequest &	parse_file_request,
		const std::map< int, Chunk_NS::Chunk > &	column_id_to_chunk_map
	)

Parses a set of file regions given a handle to the file and range of indexes for the file regions to be parsed.

Definition at line 175 of file CsvDataWrapper.cpp.

Referenced by foreign_storage::CsvDataWrapper::populateChunks().

                                                               {
   ParseFileRegionResult load_file_region_result{};
   load_file_region_result.file_offset = file_regions[start_index].first_row_file_offset;
   load_file_region_result.row_count = 0;
 
   csv_file_buffer_parser::ParseBufferResult result;
   for (size_t i = start_index; i <= end_index; i++) {
     CHECK(file_regions[i].region_size <= parse_file_request.buffer_size);
     size_t read_size;
     {
       read_size = csv_reader.readRegion(parse_file_request.buffer.get(),
                                         file_regions[i].first_row_file_offset,
                                         file_regions[i].region_size);
     }
 
     CHECK_EQ(file_regions[i].region_size, read_size);
     parse_file_request.begin_pos = 0;
     parse_file_request.end_pos = file_regions[i].region_size;
     parse_file_request.first_row_index = file_regions[i].first_row_index;
     parse_file_request.file_offset = file_regions[i].first_row_file_offset;
     parse_file_request.process_row_count = file_regions[i].row_count;
 
     result = parse_buffer(parse_file_request, i == end_index);
     CHECK_EQ(file_regions[i].row_count, result.row_count);
     load_file_region_result.row_count += result.row_count;
   }
   load_file_region_result.column_id_to_data_blocks_map =
       result.column_id_to_data_blocks_map;
   return load_file_region_result;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::vector<size_t> foreign_storage::partition_by_fragment	(	const size_t	start_row_index,
		const size_t	max_fragment_size,
		const size_t	buffer_row_count
	)

Given a start row index, maximum fragment size, and number of rows in a buffer, this function returns a vector indicating how the rows in the buffer should be partitioned in order to fill up available fragment slots while staying within the capacity of fragments.

Definition at line 344 of file CsvDataWrapper.cpp.

References CHECK.

Referenced by scan_metadata().

                                                                          {
   CHECK(buffer_row_count > 0);
   std::vector<size_t> partitions{};
   size_t remaining_rows_in_last_fragment;
   if (start_row_index % max_fragment_size == 0) {
     remaining_rows_in_last_fragment = 0;
   } else {
     remaining_rows_in_last_fragment =
         max_fragment_size - (start_row_index % max_fragment_size);
   }
   if (buffer_row_count <= remaining_rows_in_last_fragment) {
     partitions.emplace_back(buffer_row_count);
   } else {
     if (remaining_rows_in_last_fragment > 0) {
       partitions.emplace_back(remaining_rows_in_last_fragment);
     }
     size_t remaining_buffer_row_count =
         buffer_row_count - remaining_rows_in_last_fragment;
     while (remaining_buffer_row_count > 0) {
       partitions.emplace_back(
           std::min<size_t>(remaining_buffer_row_count, max_fragment_size));
       remaining_buffer_row_count -= partitions.back();
     }
   }
   return partitions;
 }

Here is the caller graph for this function:

template<typename T >

auto foreign_storage::partition_for_threads	(	const std::set< T > &	items,
		size_t	max_threads
	)

Definition at line 41 of file ParquetShared.h.

References i.

Referenced by foreign_storage::LazyParquetChunkLoader::metadataScan(), and foreign_storage::ParquetDataWrapper::populateChunkBuffers().

                                                                        {
   const size_t items_per_thread = (items.size() + (max_threads - 1)) / max_threads;
   std::list<std::set<T>> items_by_thread;
   auto i = 0U;
   for (auto item : items) {
     if (i++ % items_per_thread == 0) {
       items_by_thread.emplace_back(std::set<T>{});
     }
     items_by_thread.back().emplace(item);
   }
   return items_by_thread;
 }

Here is the caller graph for this function:

void foreign_storage::process_data_blocks	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		int	fragment_id,
		const csv_file_buffer_parser::ParseBufferRequest &	request,
		csv_file_buffer_parser::ParseBufferResult &	result,
		std::map< int, const ColumnDescriptor * > &	column_by_id,
		std::map< int, FileRegions > &	fragment_id_to_file_regions_map
	)

Updates metadata encapsulated in encoders for all table columns given new data blocks gotten from parsing a new set of rows in a CSV file buffer. If cache is available, also append the data_blocks to chunks in the cache

Definition at line 509 of file CsvDataWrapper.cpp.

References add_file_region(), cache_blocks(), foreign_storage::MetadataScanMultiThreadingParams::cached_chunks, foreign_storage::MetadataScanMultiThreadingParams::chunk_encoder_buffers, foreign_storage::MetadataScanMultiThreadingParams::chunk_encoder_buffers_mutex, foreign_storage::csv_file_buffer_parser::ParseBufferResult::column_id_to_data_blocks_map, foreign_storage::csv_file_buffer_parser::ParseBufferRequest::db_id, foreign_storage::csv_file_buffer_parser::ParseBufferRequest::first_row_index, foreign_storage::csv_file_buffer_parser::ParseBufferRequest::getFilePath(), foreign_storage::csv_file_buffer_parser::ParseBufferRequest::getMaxFragRows(), foreign_storage::csv_file_buffer_parser::ParseBufferRequest::getTableId(), foreign_storage::csv_file_buffer_parser::ParseBufferResult::row_count, and update_stats().

Referenced by scan_metadata().

                                                                                     {
   std::lock_guard<std::mutex> lock(multi_threading_params.chunk_encoder_buffers_mutex);
   // File regions should be added in same order as appendData
   add_file_region(fragment_id_to_file_regions_map,
                   fragment_id,
                   request.first_row_index,
                   result,
                   request.getFilePath());
 
   for (auto& [column_id, data_block] : result.column_id_to_data_blocks_map) {
     ChunkKey chunk_key{request.db_id, request.getTableId(), column_id, fragment_id};
     const auto column = column_by_id[column_id];
     if (column->columnType.is_varlen_indeed()) {
       chunk_key.emplace_back(1);
     }
     if (multi_threading_params.chunk_encoder_buffers.find(chunk_key) ==
         multi_threading_params.chunk_encoder_buffers.end()) {
       multi_threading_params.chunk_encoder_buffers[chunk_key] =
           std::make_unique<ForeignStorageBuffer>();
       multi_threading_params.chunk_encoder_buffers[chunk_key]->initEncoder(
           column->columnType);
     }
     update_stats(multi_threading_params.chunk_encoder_buffers[chunk_key]->getEncoder(),
                  column->columnType,
                  data_block,
                  result.row_count);
     size_t num_elements = multi_threading_params.chunk_encoder_buffers[chunk_key]
                               ->getEncoder()
                               ->getNumElems() +
                           result.row_count;
     multi_threading_params.chunk_encoder_buffers[chunk_key]->getEncoder()->setNumElems(
         num_elements);
     cache_blocks(
         multi_threading_params.cached_chunks,
         data_block,
         result.row_count,
         chunk_key,
         column,
         (num_elements - result.row_count) == 0,  // Is the first block added to this chunk
         num_elements == request.getMaxFragRows()  // Is the last block for this chunk
     );
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::refresh_foreign_table	(	Catalog_Namespace::Catalog &	catalog,
		const std::string &	table_name,
		const bool	evict_cached_entries
	)

Definition at line 22 of file ForeignTableRefresh.cpp.

References CHUNK_KEY_FRAGMENT_IDX, Data_Namespace::CPU_LEVEL, Catalog_Namespace::DBMetadata::dbId, StorageType::FOREIGN_TABLE, Catalog_Namespace::Catalog::getCurrentDB(), Catalog_Namespace::Catalog::getDataMgr(), Catalog_Namespace::Catalog::getForeignTableUnlocked(), PostEvictionRefreshException::getOriginalException(), Data_Namespace::GPU_LEVEL, foreign_storage::ForeignTable::isAppendMode(), Catalog_Namespace::Catalog::removeFragmenterForTable(), and Catalog_Namespace::Catalog::updateForeignTableRefreshTimes().

Referenced by RefreshForeignTablesCommand::execute(), and foreign_storage::ForeignTableRefreshScheduler::start().

                                                             {
   auto& data_mgr = catalog.getDataMgr();
   auto table_lock =
       std::make_unique<lockmgr::TableSchemaLockContainer<lockmgr::WriteLock>>(
           lockmgr::TableSchemaLockContainer<lockmgr::WriteLock>::acquireTableDescriptor(
               catalog, table_name, false));
 
   const TableDescriptor* td = (*table_lock)();
   if (td->storageType != StorageType::FOREIGN_TABLE) {
     throw std::runtime_error{
         table_name +
         " is not a foreign table. Refreshes are applicable to only foreign tables."};
   }
 
   catalog.removeFragmenterForTable(td->tableId);
   ChunkKey table_key{catalog.getCurrentDB().dbId, td->tableId};
 
   if (catalog.getForeignTableUnlocked(td->tableId)->isAppendMode() &&
       !evict_cached_entries) {
     ChunkMetadataVector metadata_vec;
     data_mgr.getChunkMetadataVecForKeyPrefix(metadata_vec, table_key);
     int last_fragment_id = 0;
     for (const auto& [key, metadata] : metadata_vec) {
       if (key[CHUNK_KEY_FRAGMENT_IDX] > last_fragment_id) {
         last_fragment_id = key[CHUNK_KEY_FRAGMENT_IDX];
       }
     }
     for (const auto& [key, metadata] : metadata_vec) {
       if (key[CHUNK_KEY_FRAGMENT_IDX] == last_fragment_id) {
         data_mgr.deleteChunksWithPrefix(key, MemoryLevel::CPU_LEVEL);
         data_mgr.deleteChunksWithPrefix(key, MemoryLevel::GPU_LEVEL);
       }
     }
   } else {
     data_mgr.deleteChunksWithPrefix(table_key, MemoryLevel::CPU_LEVEL);
     data_mgr.deleteChunksWithPrefix(table_key, MemoryLevel::GPU_LEVEL);
   }
 
   try {
     data_mgr.getPersistentStorageMgr()->getForeignStorageMgr()->refreshTable(
         table_key, evict_cached_entries);
     catalog.updateForeignTableRefreshTimes(td->tableId);
   } catch (PostEvictionRefreshException& e) {
     catalog.updateForeignTableRefreshTimes(td->tableId);
     throw e.getOriginalException();
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::resize_buffer_if_needed	(	std::unique_ptr< char[]> &	buffer,
		size_t &	buffer_size,
		const size_t	alloc_size
	)

Optionally resizes the given buffer if the buffer size is less than the current buffer allocation size.

Definition at line 662 of file CsvDataWrapper.cpp.

References CHECK_LE.

Referenced by dispatch_metadata_scan_requests().

                                                       {
   CHECK_LE(buffer_size, alloc_size);
   if (buffer_size < alloc_size) {
     buffer = std::make_unique<char[]>(alloc_size);
     buffer_size = alloc_size;
   }
 }

Here is the caller graph for this function:

void foreign_storage::scan_metadata	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		std::map< int, FileRegions > &	fragment_id_to_file_regions_map
	)

Consumes and processes metadata scan requests from a pending requests queue and updates existing metadata objects based on newly scanned metadata.

Definition at line 575 of file CsvDataWrapper.cpp.

References add_request_to_pool(), foreign_storage::MetadataScanMultiThreadingParams::continue_processing, get_next_metadata_scan_request(), foreign_storage::csv_file_buffer_parser::parse_buffer(), partition_by_fragment(), foreign_storage::MetadataScanMultiThreadingParams::pending_requests_mutex, process_data_blocks(), and run_benchmark_import::result.

Referenced by foreign_storage::CsvDataWrapper::populateChunkMetadata().

                                                                               {
   std::map<int, const ColumnDescriptor*> column_by_id{};
   while (true) {
     auto request_opt = get_next_metadata_scan_request(multi_threading_params);
     if (!request_opt.has_value()) {
       break;
     }
     auto& request = request_opt.value();
     try {
       if (column_by_id.empty()) {
         for (const auto column : request.getColumns()) {
           column_by_id[column->columnId] = column;
         }
       }
       auto partitions = partition_by_fragment(
           request.first_row_index, request.getMaxFragRows(), request.buffer_row_count);
       request.begin_pos = 0;
       size_t row_index = request.first_row_index;
       for (const auto partition : partitions) {
         request.process_row_count = partition;
         for (const auto& import_buffer : request.import_buffers) {
           if (import_buffer != nullptr) {
             import_buffer->clear();
           }
         }
         auto result = parse_buffer(request, true);
         int fragment_id = row_index / request.getMaxFragRows();
         process_data_blocks(multi_threading_params,
                             fragment_id,
                             request,
                             result,
                             column_by_id,
                             fragment_id_to_file_regions_map);
         row_index += result.row_count;
         request.begin_pos = result.row_offsets.back() - request.file_offset;
       }
     } catch (...) {
       // Re-add request to pool so we dont block any other threads
       {
         std::lock_guard<std::mutex> pending_requests_lock(
             multi_threading_params.pending_requests_mutex);
         multi_threading_params.continue_processing = false;
       }
       add_request_to_pool(multi_threading_params, request);
       throw;
     }
     add_request_to_pool(multi_threading_params, request);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::set_value	(	rapidjson::Value &	json_val,
		const FileRegion &	file_region,
		rapidjson::Document::AllocatorType &	allocator
	)

Definition at line 26 of file CsvShared.cpp.

References foreign_storage::json_utils::add_value_to_object(), foreign_storage::FileRegion::filename, foreign_storage::FileRegion::first_row_file_offset, foreign_storage::FileRegion::first_row_index, foreign_storage::FileRegion::region_size, and foreign_storage::FileRegion::row_count.

                                                           {
   json_val.SetObject();
   json_utils::add_value_to_object(
       json_val, file_region.first_row_file_offset, "first_row_file_offset", allocator);
   json_utils::add_value_to_object(
       json_val, file_region.first_row_index, "first_row_index", allocator);
   json_utils::add_value_to_object(
       json_val, file_region.region_size, "region_size", allocator);
   json_utils::add_value_to_object(
       json_val, file_region.row_count, "row_count", allocator);
   if (file_region.filename.size()) {
     json_utils::add_value_to_object(
         json_val, file_region.filename, "filename", allocator);
   }
 }

Here is the call graph for this function:

void foreign_storage::set_value	(	rapidjson::Value &	json_val,
		const RowGroupInterval &	value,
		rapidjson::Document::AllocatorType &	allocator
	)

Definition at line 496 of file ParquetDataWrapper.cpp.

References foreign_storage::json_utils::add_value_to_object(), foreign_storage::RowGroupInterval::end_index, foreign_storage::RowGroupInterval::file_path, and foreign_storage::RowGroupInterval::start_index.

                                                           {
   json_val.SetObject();
   json_utils::add_value_to_object(json_val, value.file_path, "file_path", allocator);
   json_utils::add_value_to_object(json_val, value.start_index, "start_index", allocator);
   json_utils::add_value_to_object(json_val, value.end_index, "end_index", allocator);
 }

Here is the call graph for this function:

void foreign_storage::throw_number_of_columns_mismatch_error	(	size_t	num_table_cols,
		size_t	num_file_cols,
		const std::string &	file_path
	)

inline

Definition at line 36 of file ForeignDataWrapperShared.h.

References to_string().

Referenced by foreign_storage::csv_file_buffer_parser::validate_expected_column_count(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_number_of_columns().

                                                                                {
   throw std::runtime_error{"Mismatched number of logical columns: (expected " +
                            std::to_string(num_table_cols) + " columns, has " +
                            std::to_string(num_file_cols) + "): in file '" + file_path +
                            "'"};
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::throw_parquet_metadata_out_of_bounds_error	(	const std::string &	min_value,
		const std::string &	max_value,
		const std::string &	encountered_value
	)

inline

Definition at line 67 of file ParquetMetadataValidator.h.

Referenced by foreign_storage::TimestampBoundsValidator< T >::validateValue(), foreign_storage::IntegralFixedLengthBoundsValidator< T >::validateValue(), foreign_storage::DateInSecondsBoundsValidator< T >::validateValue(), and foreign_storage::FloatPointValidator< T >::validateValue().

                                         {
   std::stringstream error_message;
   error_message << "Parquet column contains values that are outside the range of the "
                    "OmniSci column "
                    "type. Consider using a wider column type. Min allowed value: "
                 << min_value << ". Max allowed value: " << max_value
                 << ". Encountered value: " << encountered_value << ".";
   throw std::runtime_error(error_message.str());
 }

Here is the caller graph for this function:

void foreign_storage::throw_removed_file_error ( const std::string & file_path )

inline

Definition at line 30 of file ForeignDataWrapperShared.h.

Referenced by foreign_storage::LocalMultiFileReader::checkForMoreRows(), and foreign_storage::ParquetDataWrapper::fetchChunkMetadata().

                                                                  {
   throw std::runtime_error{
       "Refresh of foreign table created with \"APPEND\" update type failed as "
       "file \"" +
       file_path + "\" was removed."};
 }

Here is the caller graph for this function:

void foreign_storage::throw_removed_row_error ( const std::string & file_path )

inline

Definition at line 23 of file ForeignDataWrapperShared.h.

Referenced by foreign_storage::SingleFileReader::checkForMoreRows(), and foreign_storage::ParquetDataWrapper::fetchChunkMetadata().

                                                                 {
   throw std::runtime_error{
       "Refresh of foreign table created with \"APPEND\" update type failed as file "
       "reduced in size: " +
       file_path};
 }

Here is the caller graph for this function:

void foreign_storage::update_stats	(	Encoder *	encoder,
		const SQLTypeInfo &	column_type,
		DataBlockPtr	data_block,
		const size_t	row_count
	)

Updates the statistics metadata encapsulated in the encoder given new data in a data block.

Definition at line 435 of file CsvDataWrapper.cpp.

References DataBlockPtr::arraysPtr, SQLTypeInfo::is_array(), SQLTypeInfo::is_varlen(), DataBlockPtr::numbersPtr, DataBlockPtr::stringsPtr, and Encoder::updateStats().

Referenced by process_data_blocks(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumnMetadata(), and StorageIOFacility::yieldUpdateCallback().

                                           {
   if (column_type.is_array()) {
     encoder->updateStats(data_block.arraysPtr, 0, row_count);
   } else if (!column_type.is_varlen()) {
     encoder->updateStats(data_block.numbersPtr, row_count);
   } else {
     encoder->updateStats(data_block.stringsPtr, 0, row_count);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::shared_ptr< parquet::Statistics > foreign_storage::validate_and_get_column_metadata_statistics ( const parquet::ColumnChunkMetaData * column_metadata )

Definition at line 85 of file ParquetShared.cpp.

References CHECK.

Referenced by foreign_storage::ParquetEncoder::getRowGroupMetadata(), and foreign_storage::TypedParquetInPlaceEncoder< int64_t, int32_t >::getRowGroupMetadata().

                                                        {
   CHECK(column_metadata->is_stats_set());
   std::shared_ptr<parquet::Statistics> stats = column_metadata->statistics();
   bool is_all_nulls = stats->null_count() == column_metadata->num_values();
   CHECK(is_all_nulls || stats->HasMinMax());
   return stats;
 }

Here is the caller graph for this function:

void foreign_storage::validate_equal_column_descriptor	(	const parquet::ColumnDescriptor *	reference_descriptor,
		const parquet::ColumnDescriptor *	new_descriptor,
		const std::string &	reference_file_path,
		const std::string &	new_file_path
	)

Definition at line 59 of file ParquetShared.cpp.

Referenced by foreign_storage::LazyParquetChunkLoader::appendRowGroups(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_equal_schema().

                                     {
   if (!reference_descriptor->Equals(*new_descriptor)) {
     throw std::runtime_error{"Parquet file \"" + new_file_path +
                              "\" has a different schema. Please ensure that all Parquet "
                              "files use the same schema. Reference Parquet file: " +
                              reference_file_path +
                              ", column name: " + reference_descriptor->name() +
                              ". New Parquet file: " + new_file_path +
                              ", column name: " + new_descriptor->name() + "."};
   }
 }

Here is the caller graph for this function:

void foreign_storage::validate_non_foreign_table_write ( const TableDescriptor * table_descriptor )

inline

Definition at line 22 of file FsiUtils.h.

References StorageType::FOREIGN_TABLE, and TableDescriptor::storageType.

Referenced by Parser::InsertStmt::analyze(), Parser::InsertValuesStmt::determineLeafIndex(), Parser::TruncateTableStmt::execute(), Parser::InsertValuesStmt::execute(), Parser::InsertIntoTableAsSelectStmt::populateData(), and RelModify::RelModify().

                                                                                       {
   if (table_descriptor && table_descriptor->storageType == StorageType::FOREIGN_TABLE) {
     throw std::runtime_error{
         "DELETE, INSERT, TRUNCATE, OR UPDATE commands are not supported for foreign "
         "tables."};
   }
 }

Here is the caller graph for this function:

Variable Documentation

const std::string foreign_storage::wrapper_file_name = "wrapper_metadata.json"

Definition at line 58 of file ForeignStorageCache.h.

Referenced by foreign_storage::CachingForeignStorageMgr::clearTable(), foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), and foreign_storage::CachingForeignStorageMgr::recoverDataWrapperFromDisk().

Namespaces

Classes

Typedefs

Functions

Variables

Typedef Documentation

Function Documentation

Variable Documentation