Namespaces
	anonymous_namespace{AbstractFileStorageDataWrapper.cpp}

	anonymous_namespace{AbstractTextFileDataWrapper.cpp}

	anonymous_namespace{CachingForeignStorageMgr.cpp}

	anonymous_namespace{CsvFileBufferParser.cpp}

	anonymous_namespace{FileReader.cpp}

	anonymous_namespace{ForeignStorageCache.cpp}

	anonymous_namespace{InternalCatalogDataWrapper.cpp}

	anonymous_namespace{InternalMemoryStatsDataWrapper.cpp}

	anonymous_namespace{InternalStorageStatsDataWrapper.cpp}

	anonymous_namespace{InternalSystemDataWrapper.cpp}

	anonymous_namespace{LazyParquetChunkLoader.cpp}

	anonymous_namespace{ParquetDataWrapper.cpp}

	anonymous_namespace{RefreshTimeCalculator.cpp}

	anonymous_namespace{RegexFileBufferParser.cpp}

	anonymous_namespace{RegexParserDataWrapper.cpp}

	anonymous_namespace{S3FilePathUtil.cpp}

	anonymous_namespace{TextFileBufferParser.cpp}

	Csv

	json_utils

Classes
struct	ForeignServer

struct	ForeignTable

struct	OptionsContainer

struct	UserMappingType

struct	UserMapping

class	RefreshTimeCalculator

class	AbstractFileStorageDataWrapper

struct	ParseFileRegionResult

struct	MetadataScanMultiThreadingParams

class	AbstractTextFileDataWrapper

class	CachingForeignStorageMgr

class	CsvDataWrapper

class	CsvFileBufferParser

class	FileReader

class	SingleFileReader

class	SingleTextFileReader

class	ArchiveWrapper

class	CompressedFileReader

class	MultiFileReader

class	LocalMultiFileReader

struct	FileRegion

class	ForeignDataWrapper

struct	DataWrapperType
	Encapsulates an enumeration of foreign data wrapper type strings. More...

class	ForeignDataWrapperFactory

class	ForeignStorageBuffer

class	ForeignStorageCache

class	ForeignStorageException

class	MetadataScanInfeasibleFragmentSizeException

class	ChunkSizeValidator

class	MockForeignDataWrapper

class	ForeignStorageMgr

class	ForeignTableSchema

class	GeospatialEncoder

class	InternalCatalogDataWrapper

class	InternalMemoryStatsDataWrapper

struct	StorageDetails

class	InternalStorageStatsDataWrapper

class	InternalSystemDataWrapper

struct	ColumnType

struct	FragmentType

struct	Interval

struct	ParquetBatchData

class	ParquetRowGroupReader

class	LazyParquetChunkLoader

class	OdbcGeospatialEncoder

class	ParquetArrayEncoder

class	ParquetArrayImportEncoder

class	ParquetDataWrapper

class	ParquetDateFromTimestampEncoder

class	ParquetDateInSecondsEncoder

class	ParquetDecimalEncoder

class	ParquetEncoder

class	ParquetImportEncoder

class	ParquetScalarEncoder

class	ParquetFixedLengthArrayEncoder

class	ParquetFixedLengthEncoder

class	ParquetUnsignedFixedLengthEncoder

class	ParquetGeospatialEncoder

class	ParquetGeospatialImportEncoder

class	RowGroupIntervalTracker

class	ParquetImportBatchResult

class	AbstractRowGroupIntervalTracker

class	ParquetImporter

class	ParquetInPlaceEncoder

class	TypedParquetInPlaceEncoder

class	ParquetMetadataValidator

class	TimestampBoundsValidator

class	IntegralFixedLengthBoundsValidator

class	DateInSecondsBoundsValidator

class	FloatPointValidator

struct	RowGroupInterval

struct	RowGroupMetadata

class	FileReaderMap

class	ParquetStringEncoder

class	ParquetStringImportEncoder

class	ParquetStringNoneEncoder

class	ParquetTimeEncoder

class	ParquetTimestampEncoder

class	ParquetVariableLengthArrayEncoder

class	RegexFileBufferParser

class	RegexParserDataWrapper

class	FileOrderS3

struct	ParseBufferRequest

struct	ParseBufferResult

class	TextFileBufferParser

class	TypedParquetStorageBuffer

class	ForeignTableRefreshScheduler

Typedefs
using	OptionsMap = std::map< std::string, std::string, std::less<>>

using	FirstLineByFilePath = std::map< std::string, std::string >

using	FileRegions = std::vector< FileRegion >

using	ChunkToBufferMap = std::map< ChunkKey, AbstractBuffer * >

using	read_lock = mapd_shared_lock< mapd_shared_mutex >

using	write_lock = mapd_unique_lock< mapd_shared_mutex >

using	InvalidRowGroupIndices = std::set< int64_t >

using	UniqueReaderPtr = std::unique_ptr< parquet::arrow::FileReader >

using	ReaderPtr = parquet::arrow::FileReader *

using	S3ObjectComparator = std::function< bool(const Aws::S3::Model::Object &, const Aws::S3::Model::Object &)>

Functions
ParseFileRegionResult	parse_file_regions (const FileRegions &file_regions, const size_t start_index, const size_t end_index, FileReader &file_reader, ParseBufferRequest &parse_file_request, const std::map< int, Chunk_NS::Chunk > &column_id_to_chunk_map, const TextFileBufferParser &parser)

size_t	get_buffer_size (const import_export::CopyParams &copy_params, const bool size_known, const size_t file_size)

size_t	get_buffer_size (const FileRegions &file_regions)

size_t	get_thread_count (const import_export::CopyParams &copy_params, const bool size_known, const size_t file_size, const size_t buffer_size)

size_t	get_thread_count (const import_export::CopyParams &copy_params, const FileRegions &file_regions)

std::vector< size_t >	partition_by_fragment (const size_t start_row_index, const size_t max_fragment_size, const size_t buffer_row_count)

std::optional< ParseBufferRequest >	get_next_metadata_scan_request (MetadataScanMultiThreadingParams &multi_threading_params)

void	add_file_region (std::map< int, FileRegions > &fragment_id_to_file_regions_map, int fragment_id, size_t first_row_index, const ParseBufferResult &result, const std::string &file_path)

void	update_stats (Encoder *encoder, const SQLTypeInfo &column_type, DataBlockPtr data_block, const size_t row_count)

void	cache_blocks (std::map< ChunkKey, Chunk_NS::Chunk > &cached_chunks, DataBlockPtr data_block, size_t row_count, ChunkKey &chunk_key, const ColumnDescriptor *column, bool is_first_block, bool is_last_block, bool disable_cache)

void	process_data_blocks (MetadataScanMultiThreadingParams &multi_threading_params, int fragment_id, const ParseBufferRequest &request, ParseBufferResult &result, std::map< int, const ColumnDescriptor * > &column_by_id, std::map< int, FileRegions > &fragment_id_to_file_regions_map)

void	add_request_to_pool (MetadataScanMultiThreadingParams &multi_threading_params, ParseBufferRequest &request)

void	scan_metadata (MetadataScanMultiThreadingParams &multi_threading_params, std::map< int, FileRegions > &fragment_id_to_file_regions_map, const TextFileBufferParser &parser)

ParseBufferRequest	get_request_from_pool (MetadataScanMultiThreadingParams &multi_threading_params)

void	dispatch_metadata_scan_request (MetadataScanMultiThreadingParams &multi_threading_params, ParseBufferRequest &request)

void	resize_buffer_if_needed (std::unique_ptr< char[]> &buffer, size_t &buffer_size, const size_t alloc_size)

void	dispatch_metadata_scan_requests (const size_t &buffer_size, const std::string &file_path, FileReader &file_reader, const import_export::CopyParams &copy_params, MetadataScanMultiThreadingParams &multi_threading_params, size_t &first_row_index_in_buffer, size_t &current_file_offset, const TextFileBufferParser &parser)

void	set_value (rapidjson::Value &json_val, const FileRegion &file_region, rapidjson::Document::AllocatorType &allocator)

void	get_value (const rapidjson::Value &json_val, FileRegion &file_region)

void	throw_removed_row_in_result_set_error (const std::string &select_statement)

void	throw_removed_row_in_file_error (const std::string &file_path)

void	throw_removed_file_error (const std::string &file_path)

void	throw_number_of_columns_mismatch_error (size_t num_table_cols, size_t num_file_cols, const std::string &file_path)

void	throw_file_access_error (const std::string &file_path, const std::string &message)

void	throw_file_not_found_error (const std::string &file_path)

void	throw_s3_compressed_mime_type (const std::string &file_path, const std::string &mime_type)

void	throw_s3_compressed_extension (const std::string &file_path, const std::string &ext_type)

bool	set_comp (const ChunkKey &left, const ChunkKey &right)

std::vector< ChunkKey >	get_column_key_vec (const ChunkKey &destination_chunk_key)

std::set< ChunkKey >	get_column_key_set (const ChunkKey &destination_chunk_key)

size_t	get_max_chunk_size (const ChunkKey &key)

bool	contains_fragment_key (const std::set< ChunkKey > &key_set, const ChunkKey &target_key)

void	refresh_foreign_table (Catalog_Namespace::Catalog &catalog, const std::string &table_name, const bool evict_cached_entries)

void	init_chunk_for_column (const ChunkKey &chunk_key, const std::map< ChunkKey, std::shared_ptr< ChunkMetadata >> &chunk_metadata_map, const std::map< ChunkKey, AbstractBuffer * > &buffers, Chunk_NS::Chunk &chunk)

std::shared_ptr< ChunkMetadata >	get_placeholder_metadata (const ColumnDescriptor *column, size_t num_elements)

template<typename T >
auto	partition_for_threads (const std::set< T > &items, size_t max_threads)

template<typename T >
auto	partition_for_threads (const std::vector< T > &items, size_t max_threads)

template<typename T >
ArrayDatum	encode_as_array_datum (const std::vector< T > &data)

bool	is_metadata_placeholder (const ChunkMetadata &metadata)

void	set_value (rapidjson::Value &json_val, const RowGroupInterval &value, rapidjson::Document::AllocatorType &allocator)

void	get_value (const rapidjson::Value &json_val, RowGroupInterval &value)

template<typename D , typename T >
bool	check_bounds (const T &value)

template<typename D >
std::string	datetime_to_string (const D &timestamp, const SQLTypeInfo &column_type)

void	throw_parquet_metadata_out_of_bounds_error (const std::string &min_value, const std::string &max_value, const std::string &encountered_value)

UniqueReaderPtr	open_parquet_table (const std::string &file_path, std::shared_ptr< arrow::fs::FileSystem > &file_system)

std::pair< int, int >	get_parquet_table_size (const ReaderPtr &reader)

const parquet::ColumnDescriptor *	get_column_descriptor (const parquet::arrow::FileReader *reader, const int logical_column_index)

parquet::Type::type	get_physical_type (ReaderPtr &reader, const int logical_column_index)

void	validate_equal_column_descriptor (const parquet::ColumnDescriptor reference_descriptor, const parquet::ColumnDescriptor new_descriptor, const std::string &reference_file_path, const std::string &new_file_path)

std::unique_ptr< ColumnDescriptor >	get_sub_type_column_descriptor (const ColumnDescriptor *column)

std::shared_ptr < parquet::Statistics >	validate_and_get_column_metadata_statistics (const parquet::ColumnChunkMetaData *column_metadata)

std::vector < Aws::S3::Model::Object >	s3_objects_filter_sort_files (const std::vector< Aws::S3::Model::Object > &file_paths, const std::optional< std::string > &filter_regex, const std::optional< std::string > &sort_by, const std::optional< std::string > &sort_regex)

template<typename V , std::enable_if_t< std::is_integral< V >::value, int > = 0>
V	get_null_value ()

template<typename D , std::enable_if_t< std::is_integral< D >::value, int > = 0>
std::pair< D, D >	get_min_max_bounds ()

void	validate_non_foreign_table_write (const TableDescriptor *table_descriptor)

Typedef Documentation

using foreign_storage::ChunkToBufferMap = typedef std::map<ChunkKey, AbstractBuffer*>

Definition at line 28 of file ForeignDataWrapper.h.

using foreign_storage::FileRegions = typedef std::vector<FileRegion>

Definition at line 67 of file FileRegions.h.

using foreign_storage::FirstLineByFilePath = typedef std::map<std::string, std::string>

Definition at line 33 of file FileReader.h.

using foreign_storage::InvalidRowGroupIndices = typedef std::set<int64_t>

Definition at line 89 of file ParquetEncoder.h.

using foreign_storage::OptionsMap = typedef std::map<std::string, std::string, std::less<>>

Definition at line 48 of file OptionsContainer.h.

using foreign_storage::read_lock = typedef mapd_shared_lock<mapd_shared_mutex>

Definition at line 29 of file ForeignStorageCache.cpp.

using foreign_storage::ReaderPtr = typedef parquet::arrow::FileReader*

Definition at line 33 of file ParquetShared.h.

using foreign_storage::S3ObjectComparator = typedef std::function<bool(const Aws::S3::Model::Object&, const Aws::S3::Model::Object&)>

Definition at line 19 of file S3FilePathUtil.h.

using foreign_storage::UniqueReaderPtr = typedef std::unique_ptr<parquet::arrow::FileReader>

Definition at line 32 of file ParquetShared.h.

using foreign_storage::write_lock = typedef mapd_unique_lock<mapd_shared_mutex>

Definition at line 30 of file ForeignStorageCache.cpp.

Function Documentation

void foreign_storage::add_file_region	(	std::map< int, FileRegions > &	fragment_id_to_file_regions_map,
		int	fragment_id,
		size_t	first_row_index,
		const ParseBufferResult &	result,
		const std::string &	file_path
	)

Creates a new file region based on metadata from parsed file buffers and adds the new region to the fragment id to file regions map.

Definition at line 416 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::ParseBufferResult::row_count, and foreign_storage::ParseBufferResult::row_offsets.

Referenced by process_data_blocks().

                                                  {
   fragment_id_to_file_regions_map[fragment_id].emplace_back(
       // file naming is handled by FileReader
       FileRegion(result.row_offsets.front(),
                  first_row_index,
                  result.row_count,
                  result.row_offsets.back() - result.row_offsets.front()));
 }

Here is the caller graph for this function:

void foreign_storage::add_request_to_pool	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		ParseBufferRequest &	request
	)

Adds the request object for a processed request back to the request pool for reuse in subsequent requests.

Definition at line 554 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::request_pool, foreign_storage::MetadataScanMultiThreadingParams::request_pool_condition, and foreign_storage::MetadataScanMultiThreadingParams::request_pool_mutex.

Referenced by scan_metadata().

                                                       {
   std::unique_lock<std::mutex> completed_requests_queue_lock(
       multi_threading_params.request_pool_mutex);
   multi_threading_params.request_pool.emplace(std::move(request));
   completed_requests_queue_lock.unlock();
   multi_threading_params.request_pool_condition.notify_all();
 }

Here is the caller graph for this function:

void foreign_storage::cache_blocks	(	std::map< ChunkKey, Chunk_NS::Chunk > &	cached_chunks,
		DataBlockPtr	data_block,
		size_t	row_count,
		ChunkKey &	chunk_key,
		const ColumnDescriptor *	column,
		bool	is_first_block,
		bool	is_last_block,
		bool	disable_cache
	)

Definition at line 461 of file AbstractTextFileDataWrapper.cpp.

References CHECK, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, ColumnDescriptor::columnType, foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::get_cache_if_enabled(), Catalog_Namespace::SysCatalog::getCatalog(), Catalog_Namespace::SysCatalog::instance(), and SQLTypeInfo::is_varlen_indeed().

Referenced by process_data_blocks().

                                       {
   auto catalog =
       Catalog_Namespace::SysCatalog::instance().getCatalog(chunk_key[CHUNK_KEY_DB_IDX]);
   CHECK(catalog);
   auto cache = get_cache_if_enabled(catalog, disable_cache);
   if (cache) {
     ChunkKey index_key = {chunk_key[CHUNK_KEY_DB_IDX],
                           chunk_key[CHUNK_KEY_TABLE_IDX],
                           chunk_key[CHUNK_KEY_COLUMN_IDX],
                           chunk_key[CHUNK_KEY_FRAGMENT_IDX],
                           2};
     // Create actual data chunks to prepopulate cache
     if (cached_chunks.find(chunk_key) == cached_chunks.end()) {
       cached_chunks[chunk_key] = Chunk_NS::Chunk{column, false};
       cached_chunks[chunk_key].setBuffer(
           cache->getChunkBufferForPrecaching(chunk_key, is_first_block));
       if (column->columnType.is_varlen_indeed()) {
         cached_chunks[chunk_key].setIndexBuffer(
             cache->getChunkBufferForPrecaching(index_key, is_first_block));
       }
       if (is_first_block) {
         cached_chunks[chunk_key].initEncoder();
       }
     }
     cached_chunks[chunk_key].appendData(data_block, row_count, 0);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename D , typename T >

bool foreign_storage::check_bounds ( const T & value )

inline

Definition at line 32 of file ParquetMetadataValidator.h.

                                          {
   auto [min_value, max_value] = get_min_max_bounds<D>();
   return value >= min_value && value <= max_value;
 }

bool foreign_storage::contains_fragment_key	(	const std::set< ChunkKey > &	key_set,
		const ChunkKey &	target_key
	)

template<typename D >

std::string foreign_storage::datetime_to_string	(	const D &	timestamp,
		const SQLTypeInfo &	column_type
	)

inline

Definition at line 38 of file ParquetMetadataValidator.h.

References Datum::bigintval, CHECK, DatumToString(), SQLTypeInfo::is_date(), and SQLTypeInfo::is_timestamp().

Referenced by foreign_storage::TimestampBoundsValidator< T >::getMinMaxBoundsAsStrings(), foreign_storage::DateInSecondsBoundsValidator< T >::getMinMaxBoundsAsStrings(), foreign_storage::TimestampBoundsValidator< T >::validateValue(), and foreign_storage::DateInSecondsBoundsValidator< T >::validateValue().

                                                                       {
   CHECK(column_type.is_timestamp() || column_type.is_date());
   Datum d;
   d.bigintval = timestamp;
   return DatumToString(d, column_type);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::dispatch_metadata_scan_request	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		ParseBufferRequest &	request
	)

Dispatches a new metadata scan request by adding the request to the pending requests queue to be consumed by a worker thread.

Definition at line 640 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::pending_requests, foreign_storage::MetadataScanMultiThreadingParams::pending_requests_condition, and foreign_storage::MetadataScanMultiThreadingParams::pending_requests_mutex.

Referenced by dispatch_metadata_scan_requests().

                                  {
   {
     std::unique_lock<std::mutex> pending_requests_lock(
         multi_threading_params.pending_requests_mutex);
     multi_threading_params.pending_requests.emplace(std::move(request));
   }
   multi_threading_params.pending_requests_condition.notify_all();
 }

Here is the caller graph for this function:

void foreign_storage::dispatch_metadata_scan_requests	(	const size_t &	buffer_size,
		const std::string &	file_path,
		FileReader &	file_reader,
		const import_export::CopyParams &	copy_params,
		MetadataScanMultiThreadingParams &	multi_threading_params,
		size_t &	first_row_index_in_buffer,
		size_t &	current_file_offset,
		const TextFileBufferParser &	parser
	)

Reads from a text file iteratively and dispatches metadata scan requests that are processed by worker threads.

Definition at line 669 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::continue_processing, dispatch_metadata_scan_request(), foreign_storage::TextFileBufferParser::findRowEndPosition(), get_request_from_pool(), foreign_storage::FileReader::isScanFinished(), import_export::CopyParams::line_delim, foreign_storage::MetadataScanMultiThreadingParams::pending_requests, foreign_storage::MetadataScanMultiThreadingParams::pending_requests_condition, foreign_storage::MetadataScanMultiThreadingParams::pending_requests_mutex, foreign_storage::FileReader::read(), and resize_buffer_if_needed().

Referenced by foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata().

                                         {
   auto alloc_size = buffer_size;
   auto residual_buffer = std::make_unique<char[]>(alloc_size);
   size_t residual_buffer_size = 0;
   size_t residual_buffer_alloc_size = alloc_size;
 
   while (!file_reader.isScanFinished()) {
     {
       std::lock_guard<std::mutex> pending_requests_lock(
           multi_threading_params.pending_requests_mutex);
       if (!multi_threading_params.continue_processing) {
         break;
       }
     }
     auto request = get_request_from_pool(multi_threading_params);
     resize_buffer_if_needed(request.buffer, request.buffer_alloc_size, alloc_size);
 
     if (residual_buffer_size > 0) {
       memcpy(request.buffer.get(), residual_buffer.get(), residual_buffer_size);
     }
     size_t size = residual_buffer_size;
     size += file_reader.read(request.buffer.get() + residual_buffer_size,
                              alloc_size - residual_buffer_size);
 
     if (size == 0) {
       // In some cases at the end of a file we will read 0 bytes even when
       // file_reader.isScanFinished() is false
       continue;
     } else if (size == 1 && request.buffer[0] == copy_params.line_delim) {
       // In some cases files with newlines at the end will be encoded with a second
       // newline that can end up being the only thing in the buffer
       current_file_offset++;
       continue;
     }
     unsigned int num_rows_in_buffer = 0;
     request.end_pos = parser.findRowEndPosition(alloc_size,
                                                 request.buffer,
                                                 size,
                                                 copy_params,
                                                 first_row_index_in_buffer,
                                                 num_rows_in_buffer,
                                                 &file_reader);
     request.buffer_size = size;
     request.buffer_alloc_size = alloc_size;
     request.first_row_index = first_row_index_in_buffer;
     request.file_offset = current_file_offset;
     request.buffer_row_count = num_rows_in_buffer;
 
     residual_buffer_size = size - request.end_pos;
     if (residual_buffer_size > 0) {
       resize_buffer_if_needed(residual_buffer, residual_buffer_alloc_size, alloc_size);
       memcpy(residual_buffer.get(),
              request.buffer.get() + request.end_pos,
              residual_buffer_size);
     }
 
     current_file_offset += request.end_pos;
     first_row_index_in_buffer += num_rows_in_buffer;
 
     dispatch_metadata_scan_request(multi_threading_params, request);
   }
 
   std::unique_lock<std::mutex> pending_requests_queue_lock(
       multi_threading_params.pending_requests_mutex);
   multi_threading_params.pending_requests_condition.wait(
       pending_requests_queue_lock, [&multi_threading_params] {
         return multi_threading_params.pending_requests.empty() ||
                (multi_threading_params.continue_processing == false);
       });
   multi_threading_params.continue_processing = false;
   pending_requests_queue_lock.unlock();
   multi_threading_params.pending_requests_condition.notify_all();
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >

ArrayDatum foreign_storage::encode_as_array_datum ( const std::vector< T > & data )

inline

Definition at line 32 of file GeospatialEncoder.h.

References omnisci.dtypes::T.

Referenced by foreign_storage::GeospatialEncoder::processGeoElement(), and foreign_storage::GeospatialEncoder::processNullGeoElement().

                                                                   {
   const size_t num_bytes = data.size() * sizeof(T);
   std::shared_ptr<int8_t> buffer(new int8_t[num_bytes], std::default_delete<int8_t[]>());
   memcpy(buffer.get(), data.data(), num_bytes);
   return ArrayDatum(num_bytes, buffer, false);
 }

Here is the caller graph for this function:

size_t foreign_storage::get_buffer_size	(	const import_export::CopyParams &	copy_params,
		const bool	size_known,
		const size_t	file_size
	)

Gets the appropriate buffer size to be used when processing file(s).

Definition at line 211 of file AbstractTextFileDataWrapper.cpp.

References import_export::CopyParams::buffer_size.

Referenced by foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata(), and foreign_storage::AbstractTextFileDataWrapper::populateChunks().

                                                {
   size_t buffer_size = copy_params.buffer_size;
   if (size_known && file_size < buffer_size) {
     buffer_size = file_size + 1;  // +1 for end of line character, if missing
   }
   return buffer_size;
 }

Here is the caller graph for this function:

size_t foreign_storage::get_buffer_size ( const FileRegions & file_regions )

Definition at line 221 of file AbstractTextFileDataWrapper.cpp.

References CHECK.

                                                         {
   size_t buffer_size = 0;
   for (const auto& file_region : file_regions) {
     buffer_size = std::max(buffer_size, file_region.region_size);
   }
   CHECK(buffer_size);
   return buffer_size;
 }

const parquet::ColumnDescriptor * foreign_storage::get_column_descriptor	(	const parquet::arrow::FileReader *	reader,
		const int	logical_column_index
	)

Definition at line 46 of file ParquetShared.cpp.

Referenced by foreign_storage::LazyParquetChunkLoader::appendRowGroups(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_equal_schema().

                                     {
   return reader->parquet_reader()->metadata()->schema()->Column(logical_column_index);
 }

Here is the caller graph for this function:

std::set<ChunkKey> foreign_storage::get_column_key_set ( const ChunkKey & destination_chunk_key )

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), foreign_storage::CachingForeignStorageMgr::getOptionalKeysWithinSizeLimit(), and foreign_storage::CachingForeignStorageMgr::getRequiredBuffersSize().

Here is the caller graph for this function:

std::vector<ChunkKey> foreign_storage::get_column_key_vec ( const ChunkKey & destination_chunk_key )

size_t foreign_storage::get_max_chunk_size ( const ChunkKey & key )

Referenced by foreign_storage::CachingForeignStorageMgr::getBufferSize().

Here is the caller graph for this function:

template<typename D , std::enable_if_t< std::is_integral< D >::value, int > = 0>

std::pair< D, D > foreign_storage::get_min_max_bounds ( )

inline

Definition at line 31 of file SharedMetadataValidator.h.

                                           {
   static_assert(std::is_signed<D>::value,
                 "'get_min_max_bounds' is only valid for signed types");
   return {get_null_value<D>() + 1, std::numeric_limits<D>::max()};
 }

std::optional<ParseBufferRequest> foreign_storage::get_next_metadata_scan_request ( MetadataScanMultiThreadingParams & multi_threading_params )

Gets the next metadata scan request object from the pending requests queue. A null optional is returned if there are no further requests to be processed.

Definition at line 393 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::continue_processing, foreign_storage::MetadataScanMultiThreadingParams::pending_requests, foreign_storage::MetadataScanMultiThreadingParams::pending_requests_condition, and foreign_storage::MetadataScanMultiThreadingParams::pending_requests_mutex.

Referenced by scan_metadata().

                                                               {
   std::unique_lock<std::mutex> pending_requests_lock(
       multi_threading_params.pending_requests_mutex);
   multi_threading_params.pending_requests_condition.wait(
       pending_requests_lock, [&multi_threading_params] {
         return !multi_threading_params.pending_requests.empty() ||
                !multi_threading_params.continue_processing;
       });
   if (multi_threading_params.pending_requests.empty()) {
     return {};
   }
   auto request = std::move(multi_threading_params.pending_requests.front());
   multi_threading_params.pending_requests.pop();
   pending_requests_lock.unlock();
   multi_threading_params.pending_requests_condition.notify_all();
   return std::move(request);
 }

Here is the caller graph for this function:

template<typename V , std::enable_if_t< std::is_integral< V >::value, int > = 0>

V foreign_storage::get_null_value ( )

inline

Definition at line 21 of file SharedMetadataValidator.h.

                           {
   return inline_int_null_value<V>();
 }

std::pair< int, int > foreign_storage::get_parquet_table_size ( const ReaderPtr & reader )

Definition at line 39 of file ParquetShared.cpp.

Referenced by foreign_storage::LazyParquetChunkLoader::appendRowGroups(), foreign_storage::LazyParquetChunkLoader::loadRowGroups(), and foreign_storage::LazyParquetChunkLoader::metadataScan().

                                                                   {
   auto file_metadata = reader->parquet_reader()->metadata();
   const auto num_row_groups = file_metadata->num_row_groups();
   const auto num_columns = file_metadata->num_columns();
   return std::make_pair(num_row_groups, num_columns);
 }

Here is the caller graph for this function:

parquet::Type::type foreign_storage::get_physical_type	(	ReaderPtr &	reader,
		const int	logical_column_index
	)

Definition at line 52 of file ParquetShared.cpp.

Referenced by anonymous_namespace{ArrowResultSetConverter.cpp}::get_arrow_type(), and ArrowResultSetConverter::initializeColumnBuilder().

                                                                                      {
   return reader->parquet_reader()
       ->metadata()
       ->schema()
       ->Column(logical_column_index)
       ->physical_type();
 }

Here is the caller graph for this function:

std::shared_ptr< ChunkMetadata > foreign_storage::get_placeholder_metadata	(	const ColumnDescriptor *	column,
		size_t	num_elements
	)

Definition at line 76 of file FsiChunkUtils.cpp.

References ColumnDescriptor::columnType, SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_size(), Data_Namespace::AbstractBuffer::getEncoder(), Encoder::getMetadata(), Data_Namespace::AbstractBuffer::initEncoder(), SQLTypeInfo::is_array(), and SQLTypeInfo::is_varlen_indeed().

Referenced by foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::add_placeholder_metadata().

                                                                              {
   ForeignStorageBuffer empty_buffer;
   // Use default encoder metadata as in parquet wrapper
   empty_buffer.initEncoder(column->columnType);
   auto chunk_metadata = empty_buffer.getEncoder()->getMetadata(column->columnType);
   chunk_metadata->numElements = num_elements;
 
   if (!column->columnType.is_varlen_indeed()) {
     chunk_metadata->numBytes = column->columnType.get_size() * num_elements;
   }
   // min/max not set by default for arrays, so get from elem type encoder
   if (column->columnType.is_array()) {
     ForeignStorageBuffer scalar_buffer;
     scalar_buffer.initEncoder(column->columnType.get_elem_type());
     auto scalar_metadata =
         scalar_buffer.getEncoder()->getMetadata(column->columnType.get_elem_type());
     chunk_metadata->chunkStats.min = scalar_metadata->chunkStats.min;
     chunk_metadata->chunkStats.max = scalar_metadata->chunkStats.max;
   }
   chunk_metadata->chunkStats.has_nulls = true;
   return chunk_metadata;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

ParseBufferRequest foreign_storage::get_request_from_pool ( MetadataScanMultiThreadingParams & multi_threading_params )

Gets a request from the metadata scan request pool.

Definition at line 622 of file AbstractTextFileDataWrapper.cpp.

References CHECK, foreign_storage::MetadataScanMultiThreadingParams::request_pool, foreign_storage::MetadataScanMultiThreadingParams::request_pool_condition, and foreign_storage::MetadataScanMultiThreadingParams::request_pool_mutex.

Referenced by dispatch_metadata_scan_requests().

                                                               {
   std::unique_lock<std::mutex> request_pool_lock(
       multi_threading_params.request_pool_mutex);
   multi_threading_params.request_pool_condition.wait(
       request_pool_lock,
       [&multi_threading_params] { return !multi_threading_params.request_pool.empty(); });
   auto request = std::move(multi_threading_params.request_pool.front());
   multi_threading_params.request_pool.pop();
   request_pool_lock.unlock();
   CHECK(request.buffer);
   return request;
 }

Here is the caller graph for this function:

std::unique_ptr< ColumnDescriptor > foreign_storage::get_sub_type_column_descriptor ( const ColumnDescriptor * column )

Definition at line 76 of file ParquetShared.cpp.

References ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::get_elem_type(), SQLTypeInfo::set_size(), and ColumnDescriptor::tableId.

Referenced by foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_array_encoder(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_array_mapping().

                                     {
   auto column_type = column->columnType.get_elem_type();
   if (column_type.get_size() == -1 && column_type.is_dict_encoded_string()) {
     column_type.set_size(4);  // override default size of -1
   }
   return std::make_unique<ColumnDescriptor>(
       column->tableId, column->columnId, column->columnName, column_type);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

size_t foreign_storage::get_thread_count	(	const import_export::CopyParams &	copy_params,
		const bool	size_known,
		const size_t	file_size,
		const size_t	buffer_size
	)

Gets the appropriate number of threads to be used for concurrent processing within the data wrapper.

Definition at line 234 of file AbstractTextFileDataWrapper.cpp.

References CHECK_GT, and import_export::CopyParams::threads.

Referenced by foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata(), and foreign_storage::AbstractTextFileDataWrapper::populateChunks().

                                                   {
   size_t thread_count = copy_params.threads;
   if (thread_count == 0) {
     thread_count = std::thread::hardware_concurrency();
   }
   if (size_known) {
     size_t num_buffers_in_file = (file_size + buffer_size - 1) / buffer_size;
     if (num_buffers_in_file < thread_count) {
       thread_count = num_buffers_in_file;
     }
   }
   CHECK_GT(thread_count, static_cast<size_t>(0));
   return thread_count;
 }

Here is the caller graph for this function:

size_t foreign_storage::get_thread_count	(	const import_export::CopyParams &	copy_params,
		const FileRegions &	file_regions
	)

Definition at line 252 of file AbstractTextFileDataWrapper.cpp.

References CHECK_GT, and import_export::CopyParams::threads.

                                                          {
   size_t thread_count = copy_params.threads;
   if (thread_count == 0) {
     thread_count =
         std::min<size_t>(std::thread::hardware_concurrency(), file_regions.size());
   }
   CHECK_GT(thread_count, static_cast<size_t>(0));
   return thread_count;
 }

void foreign_storage::get_value	(	const rapidjson::Value &	json_val,
		FileRegion &	file_region
	)

Definition at line 44 of file CsvShared.cpp.

References CHECK, foreign_storage::FileRegion::filename, foreign_storage::FileRegion::first_row_file_offset, foreign_storage::FileRegion::first_row_index, foreign_storage::json_utils::get_value_from_object(), foreign_storage::FileRegion::region_size, and foreign_storage::FileRegion::row_count.

                                                                         {
   CHECK(json_val.IsObject());
   json_utils::get_value_from_object(
       json_val, file_region.first_row_file_offset, "first_row_file_offset");
   json_utils::get_value_from_object(
       json_val, file_region.first_row_index, "first_row_index");
   json_utils::get_value_from_object(json_val, file_region.region_size, "region_size");
   json_utils::get_value_from_object(json_val, file_region.row_count, "row_count");
   if (json_val.HasMember("filename")) {
     json_utils::get_value_from_object(json_val, file_region.filename, "filename");
   }
 }

Here is the call graph for this function:

void foreign_storage::get_value	(	const rapidjson::Value &	json_val,
		RowGroupInterval &	value
	)

Definition at line 526 of file ParquetDataWrapper.cpp.

References CHECK, foreign_storage::RowGroupInterval::end_index, foreign_storage::RowGroupInterval::file_path, foreign_storage::json_utils::get_value_from_object(), and foreign_storage::RowGroupInterval::start_index.

                                                                         {
   CHECK(json_val.IsObject());
   json_utils::get_value_from_object(json_val, value.file_path, "file_path");
   json_utils::get_value_from_object(json_val, value.start_index, "start_index");
   json_utils::get_value_from_object(json_val, value.end_index, "end_index");
 }

Here is the call graph for this function:

void foreign_storage::init_chunk_for_column	(	const ChunkKey &	chunk_key,
		const std::map< ChunkKey, std::shared_ptr< ChunkMetadata >> &	chunk_metadata_map,
		const std::map< ChunkKey, AbstractBuffer * > &	buffers,
		Chunk_NS::Chunk &	chunk
	)

Definition at line 22 of file FsiChunkUtils.cpp.

References CHECK, CHECK_EQ, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, Catalog_Namespace::SysCatalog::getCatalog(), Chunk_NS::Chunk::initEncoder(), Catalog_Namespace::SysCatalog::instance(), Data_Namespace::AbstractBuffer::reserve(), Chunk_NS::Chunk::setBuffer(), Chunk_NS::Chunk::setColumnDesc(), Chunk_NS::Chunk::setIndexBuffer(), Chunk_NS::Chunk::setPinnable(), Data_Namespace::AbstractBuffer::size(), and UNREACHABLE.

Referenced by foreign_storage::AbstractTextFileDataWrapper::populateChunkMapForColumns().

                           {
   auto catalog =
       Catalog_Namespace::SysCatalog::instance().getCatalog(chunk_key[CHUNK_KEY_DB_IDX]);
   CHECK(catalog);
 
   ChunkKey data_chunk_key = chunk_key;
   AbstractBuffer* data_buffer = nullptr;
   AbstractBuffer* index_buffer = nullptr;
   const auto column = catalog->getMetadataForColumn(chunk_key[CHUNK_KEY_TABLE_IDX],
                                                     chunk_key[CHUNK_KEY_COLUMN_IDX]);
 
   if (column->columnType.is_varlen_indeed()) {
     data_chunk_key.push_back(1);
     ChunkKey index_chunk_key = chunk_key;
     index_chunk_key.push_back(2);
 
     CHECK(buffers.find(data_chunk_key) != buffers.end());
     CHECK(buffers.find(index_chunk_key) != buffers.end());
 
     data_buffer = buffers.find(data_chunk_key)->second;
     index_buffer = buffers.find(index_chunk_key)->second;
     CHECK_EQ(data_buffer->size(), static_cast<size_t>(0));
     CHECK_EQ(index_buffer->size(), static_cast<size_t>(0));
 
     size_t index_offset_size{0};
     if (column->columnType.is_string() || column->columnType.is_geometry()) {
       index_offset_size = sizeof(StringOffsetT);
     } else if (column->columnType.is_array()) {
       index_offset_size = sizeof(ArrayOffsetT);
     } else {
       UNREACHABLE();
     }
     CHECK(chunk_metadata_map.find(data_chunk_key) != chunk_metadata_map.end());
     index_buffer->reserve(index_offset_size *
                           (chunk_metadata_map.at(data_chunk_key)->numElements + 1));
   } else {
     data_chunk_key = chunk_key;
     CHECK(buffers.find(data_chunk_key) != buffers.end());
     data_buffer = buffers.find(data_chunk_key)->second;
   }
   CHECK(chunk_metadata_map.find(data_chunk_key) != chunk_metadata_map.end());
   data_buffer->reserve(chunk_metadata_map.at(data_chunk_key)->numBytes);
 
   chunk.setPinnable(false);
   chunk.setColumnDesc(column);
   chunk.setBuffer(data_buffer);
   chunk.setIndexBuffer(index_buffer);
   chunk.initEncoder();
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::is_metadata_placeholder ( const ChunkMetadata & metadata )

inline

Check if ChunkMetadata corresponds to a chunk for which metadata must be populated.

Definition at line 27 of file MetadataPlaceholder.h.

References ChunkMetadata::chunkStats, Datum::intval, SQLTypeInfo::is_dict_encoded_type(), ChunkStats::max, ChunkStats::min, and ChunkMetadata::sqlType.

Referenced by anonymous_namespace{RelAlgExecutor.cpp}::prepare_string_dictionaries().

                                                                    {
   return metadata.chunkStats.min.intval > metadata.chunkStats.max.intval &&
          metadata.sqlType.is_dict_encoded_type();  // Only supported type for now
 }

Here is the call graph for this function:

Here is the caller graph for this function:

UniqueReaderPtr foreign_storage::open_parquet_table	(	const std::string &	file_path,
		std::shared_ptr< arrow::fs::FileSystem > &	file_system
	)

Definition at line 26 of file ParquetShared.cpp.

Referenced by foreign_storage::FileReaderMap::getOrInsert(), foreign_storage::FileReaderMap::insert(), and foreign_storage::LazyParquetChunkLoader::loadRowGroups().

                                                                                   {
   UniqueReaderPtr reader;
   auto file_result = file_system->OpenInputFile(file_path);
   if (!file_result.ok()) {
     throw std::runtime_error{"Unable to access " + file_system->type_name() + " file: " +
                              file_path + ". " + file_result.status().message()};
   }
   auto infile = file_result.ValueOrDie();
   PARQUET_THROW_NOT_OK(OpenFile(infile, arrow::default_memory_pool(), &reader));
   return reader;
 }

Here is the caller graph for this function:

ParseFileRegionResult foreign_storage::parse_file_regions	(	const FileRegions &	file_regions,
		const size_t	start_index,
		const size_t	end_index,
		FileReader &	file_reader,
		ParseBufferRequest &	parse_file_request,
		const std::map< int, Chunk_NS::Chunk > &	column_id_to_chunk_map,
		const TextFileBufferParser &	parser
	)

Parses a set of file regions given a handle to the file and range of indexes for the file regions to be parsed.

Definition at line 173 of file AbstractTextFileDataWrapper.cpp.

Referenced by foreign_storage::AbstractTextFileDataWrapper::populateChunks().

                                         {
   auto timer = DEBUG_TIMER(__func__);
   ParseFileRegionResult load_file_region_result{};
   load_file_region_result.file_offset = file_regions[start_index].first_row_file_offset;
   load_file_region_result.row_count = 0;
 
   ParseBufferResult result;
   for (size_t i = start_index; i <= end_index; i++) {
     CHECK(file_regions[i].region_size <= parse_file_request.buffer_size);
     auto read_size = file_reader.readRegion(parse_file_request.buffer.get(),
                                             file_regions[i].first_row_file_offset,
                                             file_regions[i].region_size);
     CHECK_EQ(file_regions[i].region_size, read_size);
     parse_file_request.begin_pos = 0;
     parse_file_request.end_pos = file_regions[i].region_size;
     parse_file_request.first_row_index = file_regions[i].first_row_index;
     parse_file_request.file_offset = file_regions[i].first_row_file_offset;
     parse_file_request.process_row_count = file_regions[i].row_count;
 
     result = parser.parseBuffer(parse_file_request, i == end_index);
     CHECK_EQ(file_regions[i].row_count, result.row_count);
     load_file_region_result.row_count += result.row_count;
   }
   load_file_region_result.column_id_to_data_blocks_map =
       result.column_id_to_data_blocks_map;
   return load_file_region_result;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::vector<size_t> foreign_storage::partition_by_fragment	(	const size_t	start_row_index,
		const size_t	max_fragment_size,
		const size_t	buffer_row_count
	)

Given a start row index, maximum fragment size, and number of rows in a buffer, this function returns a vector indicating how the rows in the buffer should be partitioned in order to fill up available fragment slots while staying within the capacity of fragments.

Definition at line 341 of file AbstractTextFileDataWrapper.cpp.

References CHECK.

Referenced by scan_metadata().

                                                                          {
   CHECK(buffer_row_count > 0);
   std::vector<size_t> partitions{};
   size_t remaining_rows_in_last_fragment;
   if (start_row_index % max_fragment_size == 0) {
     remaining_rows_in_last_fragment = 0;
   } else {
     remaining_rows_in_last_fragment =
         max_fragment_size - (start_row_index % max_fragment_size);
   }
   if (buffer_row_count <= remaining_rows_in_last_fragment) {
     partitions.emplace_back(buffer_row_count);
   } else {
     if (remaining_rows_in_last_fragment > 0) {
       partitions.emplace_back(remaining_rows_in_last_fragment);
     }
     size_t remaining_buffer_row_count =
         buffer_row_count - remaining_rows_in_last_fragment;
     while (remaining_buffer_row_count > 0) {
       partitions.emplace_back(
           std::min<size_t>(remaining_buffer_row_count, max_fragment_size));
       remaining_buffer_row_count -= partitions.back();
     }
   }
   return partitions;
 }

Here is the caller graph for this function:

template<typename T >

auto foreign_storage::partition_for_threads	(	const std::set< T > &	items,
		size_t	max_threads
	)

Definition at line 40 of file FsiChunkUtils.h.

References i.

Referenced by foreign_storage::LazyParquetChunkLoader::metadataScan(), and foreign_storage::ParquetDataWrapper::populateChunkBuffers().

                                                                        {
   const size_t items_per_thread = (items.size() + (max_threads - 1)) / max_threads;
   std::list<std::set<T>> items_by_thread;
   auto i = 0U;
   for (auto item : items) {
     if (i++ % items_per_thread == 0) {
       items_by_thread.emplace_back(std::set<T>{});
     }
     items_by_thread.back().emplace(item);
   }
   return items_by_thread;
 }

Here is the caller graph for this function:

template<typename T >

auto foreign_storage::partition_for_threads	(	const std::vector< T > &	items,
		size_t	max_threads
	)

Definition at line 59 of file FsiChunkUtils.h.

References i.

                                                                           {
   const size_t items_per_thread = (items.size() + (max_threads - 1)) / max_threads;
   std::list<std::vector<T>> items_by_thread;
   auto i = 0U;
   for (auto item : items) {
     if (i++ % items_per_thread == 0) {
       items_by_thread.emplace_back(std::vector<T>{});
     }
     items_by_thread.back().emplace_back(item);
   }
   return items_by_thread;
 }

void foreign_storage::process_data_blocks	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		int	fragment_id,
		const ParseBufferRequest &	request,
		ParseBufferResult &	result,
		std::map< int, const ColumnDescriptor * > &	column_by_id,
		std::map< int, FileRegions > &	fragment_id_to_file_regions_map
	)

Updates metadata encapsulated in encoders for all table columns given new data blocks gotten from parsing a new set of rows in a file buffer. If cache is available, also append the data_blocks to chunks in the cache

Definition at line 501 of file AbstractTextFileDataWrapper.cpp.

References add_file_region(), cache_blocks(), foreign_storage::MetadataScanMultiThreadingParams::cached_chunks, foreign_storage::MetadataScanMultiThreadingParams::chunk_encoder_buffers, foreign_storage::MetadataScanMultiThreadingParams::chunk_encoder_buffers_mutex, foreign_storage::ParseBufferResult::column_id_to_data_blocks_map, foreign_storage::ParseBufferRequest::db_id, foreign_storage::MetadataScanMultiThreadingParams::disable_cache, foreign_storage::ParseBufferRequest::first_row_index, foreign_storage::ParseBufferRequest::getFilePath(), foreign_storage::ParseBufferRequest::getMaxFragRows(), foreign_storage::ParseBufferRequest::getTableId(), foreign_storage::ParseBufferResult::row_count, and update_stats().

Referenced by scan_metadata().

                                                                                     {
   std::lock_guard<std::mutex> lock(multi_threading_params.chunk_encoder_buffers_mutex);
   // File regions should be added in same order as appendData
   add_file_region(fragment_id_to_file_regions_map,
                   fragment_id,
                   request.first_row_index,
                   result,
                   request.getFilePath());
 
   for (auto& [column_id, data_block] : result.column_id_to_data_blocks_map) {
     ChunkKey chunk_key{request.db_id, request.getTableId(), column_id, fragment_id};
     const auto column = column_by_id[column_id];
     if (column->columnType.is_varlen_indeed()) {
       chunk_key.emplace_back(1);
     }
     if (multi_threading_params.chunk_encoder_buffers.find(chunk_key) ==
         multi_threading_params.chunk_encoder_buffers.end()) {
       multi_threading_params.chunk_encoder_buffers[chunk_key] =
           std::make_unique<ForeignStorageBuffer>();
       multi_threading_params.chunk_encoder_buffers[chunk_key]->initEncoder(
           column->columnType);
     }
     update_stats(multi_threading_params.chunk_encoder_buffers[chunk_key]->getEncoder(),
                  column->columnType,
                  data_block,
                  result.row_count);
     size_t num_elements = multi_threading_params.chunk_encoder_buffers[chunk_key]
                               ->getEncoder()
                               ->getNumElems() +
                           result.row_count;
     multi_threading_params.chunk_encoder_buffers[chunk_key]->getEncoder()->setNumElems(
         num_elements);
     cache_blocks(
         multi_threading_params.cached_chunks,
         data_block,
         result.row_count,
         chunk_key,
         column,
         (num_elements - result.row_count) == 0,  // Is the first block added to this chunk
         num_elements == request.getMaxFragRows(),  // Is the last block for this chunk
         multi_threading_params.disable_cache);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::refresh_foreign_table	(	Catalog_Namespace::Catalog &	catalog,
		const std::string &	table_name,
		const bool	evict_cached_entries
	)

Definition at line 22 of file ForeignTableRefresh.cpp.

References CHUNK_KEY_FRAGMENT_IDX, Data_Namespace::CPU_LEVEL, Catalog_Namespace::DBMetadata::dbId, StorageType::FOREIGN_TABLE, Catalog_Namespace::Catalog::getCurrentDB(), Catalog_Namespace::Catalog::getDataMgr(), Catalog_Namespace::Catalog::getForeignTable(), PostEvictionRefreshException::getOriginalException(), Data_Namespace::GPU_LEVEL, foreign_storage::ForeignTable::isAppendMode(), Catalog_Namespace::Catalog::removeFragmenterForTable(), and Catalog_Namespace::Catalog::updateForeignTableRefreshTimes().

Referenced by RefreshForeignTablesCommand::execute(), and foreign_storage::ForeignTableRefreshScheduler::start().

                                                             {
   auto& data_mgr = catalog.getDataMgr();
   auto table_lock =
       std::make_unique<lockmgr::TableSchemaLockContainer<lockmgr::WriteLock>>(
           lockmgr::TableSchemaLockContainer<lockmgr::WriteLock>::acquireTableDescriptor(
               catalog, table_name, false));
 
   const TableDescriptor* td = (*table_lock)();
   if (td->storageType != StorageType::FOREIGN_TABLE) {
     throw std::runtime_error{
         table_name +
         " is not a foreign table. Refreshes are applicable to only foreign tables."};
   }
 
   catalog.removeFragmenterForTable(td->tableId);
   ChunkKey table_key{catalog.getCurrentDB().dbId, td->tableId};
 
   if (catalog.getForeignTable(td->tableId)->isAppendMode() && !evict_cached_entries) {
     ChunkMetadataVector metadata_vec;
     data_mgr.getChunkMetadataVecForKeyPrefix(metadata_vec, table_key);
     int last_fragment_id = 0;
     for (const auto& [key, metadata] : metadata_vec) {
       if (key[CHUNK_KEY_FRAGMENT_IDX] > last_fragment_id) {
         last_fragment_id = key[CHUNK_KEY_FRAGMENT_IDX];
       }
     }
     for (const auto& [key, metadata] : metadata_vec) {
       if (key[CHUNK_KEY_FRAGMENT_IDX] == last_fragment_id) {
         data_mgr.deleteChunksWithPrefix(key, MemoryLevel::CPU_LEVEL);
         data_mgr.deleteChunksWithPrefix(key, MemoryLevel::GPU_LEVEL);
       }
     }
   } else {
     data_mgr.deleteChunksWithPrefix(table_key, MemoryLevel::CPU_LEVEL);
     data_mgr.deleteChunksWithPrefix(table_key, MemoryLevel::GPU_LEVEL);
   }
 
   try {
     data_mgr.getPersistentStorageMgr()->getForeignStorageMgr()->refreshTable(
         table_key, evict_cached_entries);
     catalog.updateForeignTableRefreshTimes(td->tableId);
   } catch (PostEvictionRefreshException& e) {
     catalog.updateForeignTableRefreshTimes(td->tableId);
     throw e.getOriginalException();
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::resize_buffer_if_needed	(	std::unique_ptr< char[]> &	buffer,
		size_t &	buffer_size,
		const size_t	alloc_size
	)

Optionally resizes the given buffer if the buffer size is less than the current buffer allocation size.

Definition at line 655 of file AbstractTextFileDataWrapper.cpp.

References CHECK_LE.

Referenced by dispatch_metadata_scan_requests().

                                                       {
   CHECK_LE(buffer_size, alloc_size);
   if (buffer_size < alloc_size) {
     buffer = std::make_unique<char[]>(alloc_size);
     buffer_size = alloc_size;
   }
 }

Here is the caller graph for this function:

std::vector< Aws::S3::Model::Object > foreign_storage::s3_objects_filter_sort_files	(	const std::vector< Aws::S3::Model::Object > &	file_paths,
		const std::optional< std::string > &	filter_regex,
		const std::optional< std::string > &	sort_by,
		const std::optional< std::string > &	sort_regex
	)

Definition at line 22 of file S3FilePathUtil.cpp.

References shared::PATHNAME_ORDER_TYPE, and foreign_storage::anonymous_namespace{S3FilePathUtil.cpp}::s3_objects_regex_file_filter().

                                               {
   auto result_files = filter_regex.has_value()
                           ? s3_objects_regex_file_filter(filter_regex.value(), file_paths)
                           : file_paths;
   // initial lexicographical order ensures a determinisitc ordering for files not matching
   // sort_regex
   auto initial_file_order = FileOrderS3(std::nullopt, shared::PATHNAME_ORDER_TYPE);
   auto lexi_comp = initial_file_order.getFileComparator();
   std::stable_sort(result_files.begin(), result_files.end(), lexi_comp);
 
   auto file_order = FileOrderS3(sort_regex, sort_by);
   auto comp = file_order.getFileComparator();
   std::stable_sort(result_files.begin(), result_files.end(), comp);
   return result_files;
 }

Here is the call graph for this function:

void foreign_storage::scan_metadata	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		std::map< int, FileRegions > &	fragment_id_to_file_regions_map,
		const TextFileBufferParser &	parser
	)

Consumes and processes metadata scan requests from a pending requests queue and updates existing metadata objects based on newly scanned metadata.

Definition at line 567 of file AbstractTextFileDataWrapper.cpp.

References add_request_to_pool(), foreign_storage::MetadataScanMultiThreadingParams::continue_processing, get_next_metadata_scan_request(), foreign_storage::TextFileBufferParser::parseBuffer(), partition_by_fragment(), foreign_storage::MetadataScanMultiThreadingParams::pending_requests_mutex, process_data_blocks(), and run_benchmark_import::result.

Referenced by foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata().

                                                        {
   std::map<int, const ColumnDescriptor*> column_by_id{};
   while (true) {
     auto request_opt = get_next_metadata_scan_request(multi_threading_params);
     if (!request_opt.has_value()) {
       break;
     }
     auto& request = request_opt.value();
     try {
       if (column_by_id.empty()) {
         for (const auto column : request.getColumns()) {
           column_by_id[column->columnId] = column;
         }
       }
       auto partitions = partition_by_fragment(
           request.first_row_index, request.getMaxFragRows(), request.buffer_row_count);
       request.begin_pos = 0;
       size_t row_index = request.first_row_index;
       for (const auto partition : partitions) {
         request.process_row_count = partition;
         for (const auto& import_buffer : request.import_buffers) {
           if (import_buffer != nullptr) {
             import_buffer->clear();
           }
         }
         auto result = parser.parseBuffer(request, true);
         int fragment_id = row_index / request.getMaxFragRows();
         process_data_blocks(multi_threading_params,
                             fragment_id,
                             request,
                             result,
                             column_by_id,
                             fragment_id_to_file_regions_map);
         row_index += result.row_count;
         request.begin_pos = result.row_offsets.back() - request.file_offset;
       }
     } catch (...) {
       // Re-add request to pool so we dont block any other threads
       {
         std::lock_guard<std::mutex> pending_requests_lock(
             multi_threading_params.pending_requests_mutex);
         multi_threading_params.continue_processing = false;
       }
       add_request_to_pool(multi_threading_params, request);
       throw;
     }
     add_request_to_pool(multi_threading_params, request);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::set_comp	(	const ChunkKey &	left,
		const ChunkKey &	right
	)

void foreign_storage::set_value	(	rapidjson::Value &	json_val,
		const FileRegion &	file_region,
		rapidjson::Document::AllocatorType &	allocator
	)

Definition at line 26 of file CsvShared.cpp.

References foreign_storage::json_utils::add_value_to_object(), foreign_storage::FileRegion::filename, foreign_storage::FileRegion::first_row_file_offset, foreign_storage::FileRegion::first_row_index, foreign_storage::FileRegion::region_size, and foreign_storage::FileRegion::row_count.

                                                           {
   json_val.SetObject();
   json_utils::add_value_to_object(
       json_val, file_region.first_row_file_offset, "first_row_file_offset", allocator);
   json_utils::add_value_to_object(
       json_val, file_region.first_row_index, "first_row_index", allocator);
   json_utils::add_value_to_object(
       json_val, file_region.region_size, "region_size", allocator);
   json_utils::add_value_to_object(
       json_val, file_region.row_count, "row_count", allocator);
   if (file_region.filename.size()) {
     json_utils::add_value_to_object(
         json_val, file_region.filename, "filename", allocator);
   }
 }

Here is the call graph for this function:

void foreign_storage::set_value	(	rapidjson::Value &	json_val,
		const RowGroupInterval &	value,
		rapidjson::Document::AllocatorType &	allocator
	)

Definition at line 517 of file ParquetDataWrapper.cpp.

References foreign_storage::json_utils::add_value_to_object(), foreign_storage::RowGroupInterval::end_index, foreign_storage::RowGroupInterval::file_path, and foreign_storage::RowGroupInterval::start_index.

                                                           {
   json_val.SetObject();
   json_utils::add_value_to_object(json_val, value.file_path, "file_path", allocator);
   json_utils::add_value_to_object(json_val, value.start_index, "start_index", allocator);
   json_utils::add_value_to_object(json_val, value.end_index, "end_index", allocator);
 }

Here is the call graph for this function:

void foreign_storage::throw_file_access_error	(	const std::string &	file_path,
		const std::string &	message
	)

inline

Definition at line 70 of file ForeignStorageException.h.

Referenced by foreign_storage::ParquetImporter::getAllFilePaths().

                                                               {
   std::string error_message{"Unable to access file \"" + file_path + "\". " + message};
   throw ForeignStorageException{error_message};
 }

Here is the caller graph for this function:

void foreign_storage::throw_file_not_found_error ( const std::string & file_path )

inline

Definition at line 76 of file ForeignStorageException.h.

Referenced by foreign_storage::ParquetImporter::getAllFilePaths().

                                                                    {
   throw ForeignStorageException{"File or directory \"" + file_path +
                                 "\" does not exist."};
 }

Here is the caller graph for this function:

void foreign_storage::throw_number_of_columns_mismatch_error	(	size_t	num_table_cols,
		size_t	num_file_cols,
		const std::string &	file_path
	)

inline

Definition at line 61 of file ForeignStorageException.h.

References to_string().

Referenced by foreign_storage::anonymous_namespace{RegexFileBufferParser.cpp}::regex_match_columns(), foreign_storage::anonymous_namespace{CsvFileBufferParser.cpp}::validate_expected_column_count(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_number_of_columns().

                                                                                {
   throw ForeignStorageException{"Mismatched number of logical columns: (expected " +
                                 std::to_string(num_table_cols) + " columns, has " +
                                 std::to_string(num_file_cols) + "): in file '" +
                                 file_path + "'"};
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::throw_parquet_metadata_out_of_bounds_error	(	const std::string &	min_value,
		const std::string &	max_value,
		const std::string &	encountered_value
	)

inline

Definition at line 46 of file ParquetMetadataValidator.h.

Referenced by foreign_storage::TimestampBoundsValidator< T >::validateValue(), foreign_storage::IntegralFixedLengthBoundsValidator< T >::validateValue(), foreign_storage::DateInSecondsBoundsValidator< T >::validateValue(), and foreign_storage::FloatPointValidator< T >::validateValue().

                                         {
   std::stringstream error_message;
   error_message << "Parquet column contains values that are outside the range of the "
                    "OmniSci column "
                    "type. Consider using a wider column type. Min allowed value: "
                 << min_value << ". Max allowed value: " << max_value
                 << ". Encountered value: " << encountered_value << ".";
   throw std::runtime_error(error_message.str());
 }

Here is the caller graph for this function:

void foreign_storage::throw_removed_file_error ( const std::string & file_path )

inline

Definition at line 54 of file ForeignStorageException.h.

Referenced by foreign_storage::LocalMultiFileReader::checkForMoreRows(), and foreign_storage::ParquetDataWrapper::fetchChunkMetadata().

                                                                  {
   throw ForeignStorageException{
       "Refresh of foreign table created with \"APPEND\" update type failed as "
       "file \"" +
       file_path + "\" was removed."};
 }

Here is the caller graph for this function:

void foreign_storage::throw_removed_row_in_file_error ( const std::string & file_path )

inline

Definition at line 47 of file ForeignStorageException.h.

Referenced by foreign_storage::SingleTextFileReader::checkForMoreRows(), and foreign_storage::ParquetDataWrapper::fetchChunkMetadata().

                                                                         {
   throw ForeignStorageException{
       "Refresh of foreign table created with \"APPEND\" update type failed as file "
       "reduced in size: \"" +
       file_path + "\""};
 }

Here is the caller graph for this function:

void foreign_storage::throw_removed_row_in_result_set_error ( const std::string & select_statement )

inline

Definition at line 39 of file ForeignStorageException.h.

                                                                                      {
   throw ForeignStorageException{
       "Refresh of foreign table created with \"APPEND\" update type failed as result set "
       "of select statement "
       "reduced in size: \"" +
       select_statement + "\""};
 }

void foreign_storage::throw_s3_compressed_extension	(	const std::string &	file_path,
		const std::string &	ext_type
	)

inline

Definition at line 88 of file ForeignStorageException.h.

                                                                      {
   throw ForeignStorageException{
       "File \"" + file_path + "\" has extension type \"" + ext_type +
       "\", compressed file formats are not supported by S3 Foreign Tables."};
 }

void foreign_storage::throw_s3_compressed_mime_type	(	const std::string &	file_path,
		const std::string &	mime_type
	)

inline

Definition at line 81 of file ForeignStorageException.h.

                                                                       {
   throw ForeignStorageException{
       "File \"" + file_path + "\" has mime type \"" + mime_type +
       "\", compressed file formats are not supported by S3 Foreign Tables."};
 }

void foreign_storage::update_stats	(	Encoder *	encoder,
		const SQLTypeInfo &	column_type,
		DataBlockPtr	data_block,
		const size_t	row_count
	)

Updates the statistics metadata encapsulated in the encoder given new data in a data block.

Definition at line 433 of file AbstractTextFileDataWrapper.cpp.

References DataBlockPtr::arraysPtr, SQLTypeInfo::is_array(), SQLTypeInfo::is_varlen(), DataBlockPtr::numbersPtr, DataBlockPtr::stringsPtr, and Encoder::updateStats().

Referenced by process_data_blocks(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumnMetadata(), and StorageIOFacility::yieldUpdateCallback().

                                           {
   if (column_type.is_array()) {
     encoder->updateStats(data_block.arraysPtr, 0, row_count);
   } else if (!column_type.is_varlen()) {
     encoder->updateStats(data_block.numbersPtr, row_count);
   } else {
     encoder->updateStats(data_block.stringsPtr, 0, row_count);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::shared_ptr< parquet::Statistics > foreign_storage::validate_and_get_column_metadata_statistics ( const parquet::ColumnChunkMetaData * column_metadata )

Definition at line 86 of file ParquetShared.cpp.

References CHECK.

Referenced by foreign_storage::ParquetEncoder::getRowGroupMetadata(), foreign_storage::TypedParquetInPlaceEncoder< V, V >::getRowGroupMetadata(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_definition_levels().

                                                        {
   CHECK(column_metadata->is_stats_set());
   std::shared_ptr<parquet::Statistics> stats = column_metadata->statistics();
   return stats;
 }

Here is the caller graph for this function:

void foreign_storage::validate_equal_column_descriptor	(	const parquet::ColumnDescriptor *	reference_descriptor,
		const parquet::ColumnDescriptor *	new_descriptor,
		const std::string &	reference_file_path,
		const std::string &	new_file_path
	)

Definition at line 60 of file ParquetShared.cpp.

Referenced by foreign_storage::LazyParquetChunkLoader::appendRowGroups(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_equal_schema().

                                     {
   if (!reference_descriptor->Equals(*new_descriptor)) {
     throw std::runtime_error{"Parquet file \"" + new_file_path +
                              "\" has a different schema. Please ensure that all Parquet "
                              "files use the same schema. Reference Parquet file: " +
                              reference_file_path +
                              ", column name: " + reference_descriptor->name() +
                              ". New Parquet file: " + new_file_path +
                              ", column name: " + new_descriptor->name() + "."};
   }
 }

Here is the caller graph for this function:

void foreign_storage::validate_non_foreign_table_write ( const TableDescriptor * table_descriptor )

inline

Definition at line 22 of file FsiUtils.h.

References StorageType::FOREIGN_TABLE, and TableDescriptor::storageType.

Referenced by Parser::InsertStmt::analyze(), Parser::InsertValuesStmt::determineLeafIndex(), Parser::TruncateTableStmt::execute(), Parser::InsertValuesStmt::execute(), Parser::InsertIntoTableAsSelectStmt::populateData(), and RelModify::RelModify().

                                                                                       {
   if (table_descriptor && table_descriptor->storageType == StorageType::FOREIGN_TABLE) {
     throw std::runtime_error{
         "DELETE, INSERT, TRUNCATE, OR UPDATE commands are not supported for foreign "
         "tables."};
   }
 }

Here is the caller graph for this function:

Namespaces

Classes

Typedefs

Functions

Typedef Documentation

Function Documentation