Namespaces
	anonymous_namespace{AbstractFileStorageDataWrapper.cpp}

	anonymous_namespace{AbstractTextFileDataWrapper.cpp}

	anonymous_namespace{CachingForeignStorageMgr.cpp}

	anonymous_namespace{CsvFileBufferParser.cpp}

	anonymous_namespace{FileReader.cpp}

	anonymous_namespace{ForeignDataWrapperFactory.cpp}

	anonymous_namespace{ForeignStorageCache.cpp}

	anonymous_namespace{ForeignTableRefresh.cpp}

	anonymous_namespace{InternalCatalogDataWrapper.cpp}

	anonymous_namespace{InternalMemoryStatsDataWrapper.cpp}

	anonymous_namespace{InternalStorageStatsDataWrapper.cpp}

	anonymous_namespace{InternalSystemDataWrapper.cpp}

	anonymous_namespace{LazyParquetChunkLoader.cpp}

	anonymous_namespace{LogFileBufferParser.cpp}

	anonymous_namespace{ParquetDataWrapper.cpp}

	anonymous_namespace{RefreshTimeCalculator.cpp}

	anonymous_namespace{RegexFileBufferParser.cpp}

	anonymous_namespace{RegexParserDataWrapper.cpp}

	anonymous_namespace{S3FilePathUtil.cpp}

	anonymous_namespace{TextFileBufferParser.cpp}

	Csv

	json_utils

Classes
struct	ForeignServer

struct	ForeignTable

struct	OptionsContainer

struct	UserMappingType

struct	UserMapping

class	RefreshTimeCalculator

class	AbstractFileStorageDataWrapper

struct	ParseFileRegionResult

struct	MetadataScanMultiThreadingParams

struct	IterativeFileScanParameters

class	AbstractTextFileDataWrapper

class	CachingForeignStorageMgr

class	CsvDataWrapper

class	CsvFileBufferParser

struct	DataPreview

class	FileReader

class	SingleFileReader

class	SingleTextFileReader

class	ArchiveWrapper

class	CompressedFileReader

class	MultiFileReader

class	LocalMultiFileReader

struct	FileRegion

class	ForeignDataWrapper

struct	DataWrapperType
	Encapsulates an enumeration of foreign data wrapper type strings. More...

class	ForeignDataWrapperFactory

class	ForeignStorageBuffer

class	ForeignStorageCache

class	ForeignStorageException

class	MetadataScanInfeasibleFragmentSizeException

class	RequestedFragmentIdOutOfBoundsException

class	ChunkSizeValidator

class	MockForeignDataWrapper

class	ForeignStorageMgr

class	ForeignTableSchema

class	GeospatialEncoder

class	InternalCatalogDataWrapper

class	InternalLogsDataWrapper

class	InternalMemoryStatsDataWrapper

struct	StorageDetails

class	InternalStorageStatsDataWrapper

class	InternalSystemDataWrapper

struct	ColumnType

struct	FragmentType

struct	Interval

struct	ParquetBatchData

class	ParquetRowGroupReader

struct	PreviewContext

class	LazyParquetChunkLoader

class	LogFileBufferParser

class	OdbcGeospatialEncoder

class	ParquetArrayDetectEncoder

class	ParquetArrayEncoder

class	ParquetArrayImportEncoder

class	ParquetDataWrapper

class	ParquetDateInDaysFromTimestampEncoder

class	ParquetDateInSecondsEncoder

class	ParquetDecimalEncoder

class	ParquetDetectStringEncoder

class	ParquetEncoder

class	ParquetImportEncoder

class	ParquetScalarEncoder

class	ParquetFixedLengthArrayEncoder

class	ParquetFixedLengthEncoder

class	ParquetUnsignedFixedLengthEncoder

class	ParquetGeospatialEncoder

class	ParquetGeospatialImportEncoder

class	RowGroupIntervalTracker

class	ParquetImportBatchResult

class	AbstractRowGroupIntervalTracker

class	ParquetImporter

class	ParquetInPlaceEncoder

class	TypedParquetInPlaceEncoder

class	ParquetMetadataValidator

class	TimestampBoundsValidator

class	IntegralFixedLengthBoundsValidator

class	BaseDateBoundsValidator

class	FloatPointValidator

struct	RowGroupInterval

struct	RowGroupMetadata

class	FileReaderMap

class	ParquetStringEncoder

class	ParquetStringImportEncoder

class	ParquetStringNoneEncoder

class	ParquetTimeEncoder

class	ParquetTimestampEncoder

class	ParquetVariableLengthArrayEncoder

class	PassThroughBuffer

class	RegexFileBufferParser

class	RegexParserDataWrapper

class	FileOrderS3

struct	ParseBufferRequest

struct	ParseBufferResult

class	TextFileBufferParser

class	TypedParquetDetectBuffer

class	TypedParquetStorageBuffer

class	ForeignTableRefreshScheduler

Typedefs
using	OptionsMap = std::map< std::string, std::string, std::less<>>

using	SampleRows = std::vector< std::vector< std::string >>

using	FirstLineByFilePath = std::map< std::string, std::string >

using	FileRegions = std::vector< FileRegion >

using	ChunkToBufferMap = std::map< ChunkKey, AbstractBuffer * >

using	RenderGroupAnalyzerMap = std::map< int, std::unique_ptr< import_export::RenderGroupAnalyzer >>

using	read_lock = heavyai::shared_lock< heavyai::shared_mutex >

using	write_lock = heavyai::unique_lock< heavyai::shared_mutex >

using	FilePathAndRowGroup = std::pair< std::string, int32_t >

using	RejectedRowIndices = std::set< int64_t >

using	InvalidRowGroupIndices = std::set< int64_t >

template<typename T >
using	DateInSecondsBoundsValidator = BaseDateBoundsValidator< T, true >

template<typename T >
using	DateInDaysBoundsValidator = BaseDateBoundsValidator< T, false >

using	UniqueReaderPtr = std::unique_ptr< parquet::arrow::FileReader >

using	ReaderPtr = parquet::arrow::FileReader *

template<typename V , typename T , T conversion_denominator, typename NullType = V>
using	ParquetDateInSecondsFromTimestampEncoder = ParquetTimestampEncoder< V, T, conversion_denominator, NullType >

using	S3ObjectComparator = std::function< bool(const Aws::S3::Model::Object &, const Aws::S3::Model::Object &)>

Functions
size_t	get_num_threads (const ForeignTable &table)

ParseFileRegionResult	parse_file_regions (const FileRegions &file_regions, const size_t start_index, const size_t end_index, FileReader &file_reader, ParseBufferRequest &parse_file_request, const std::map< int, Chunk_NS::Chunk > &column_id_to_chunk_map, const TextFileBufferParser &parser)

size_t	num_rows_to_process (const size_t start_row_index, const size_t max_fragment_size, const size_t rows_remaining)

std::vector< size_t >	partition_by_fragment (const size_t start_row_index, const size_t max_fragment_size, const size_t buffer_row_count)

std::optional< ParseBufferRequest >	get_next_scan_request (MetadataScanMultiThreadingParams &multi_threading_params)

void	add_file_region (std::map< int, FileRegions > &fragment_id_to_file_regions_map, int fragment_id, size_t first_row_index, const ParseBufferResult &result, const std::string &file_path)

void	update_stats (Encoder *encoder, const SQLTypeInfo &column_type, DataBlockPtr data_block, const size_t row_count)

void	cache_blocks (std::map< ChunkKey, Chunk_NS::Chunk > &cached_chunks, DataBlockPtr data_block, size_t row_count, ChunkKey &chunk_key, const ColumnDescriptor *column, bool is_first_block, bool disable_cache)

void	append_data_block_to_chunk (const foreign_storage::IterativeFileScanParameters &file_scan_param, DataBlockPtr data_block, size_t row_count, const int column_id, const ColumnDescriptor *column, const size_t element_count_required)

std::pair< std::map< int, DataBlockPtr >, std::map< int, DataBlockPtr > >	partition_data_blocks (const std::map< int, const ColumnDescriptor * > &column_by_id, const std::map< int, DataBlockPtr > &data_blocks)

void	update_delete_buffer (const ParseBufferRequest &request, const ParseBufferResult &result, const foreign_storage::IterativeFileScanParameters &file_scan_param, const size_t start_position_in_fragment)

void	populate_chunks_using_data_blocks (MetadataScanMultiThreadingParams &multi_threading_params, int fragment_id, const ParseBufferRequest &request, ParseBufferResult &result, std::map< int, const ColumnDescriptor * > &column_by_id, std::map< int, FileRegions > &fragment_id_to_file_regions_map, const foreign_storage::IterativeFileScanParameters &file_scan_param, const size_t expected_current_element_count)

void	process_data_blocks (MetadataScanMultiThreadingParams &multi_threading_params, int fragment_id, const ParseBufferRequest &request, ParseBufferResult &result, std::map< int, const ColumnDescriptor * > &column_by_id, std::map< int, FileRegions > &fragment_id_to_file_regions_map)

void	add_request_to_pool (MetadataScanMultiThreadingParams &multi_threading_params, ParseBufferRequest &request)

void	scan_metadata (MetadataScanMultiThreadingParams &multi_threading_params, std::map< int, FileRegions > &fragment_id_to_file_regions_map, const TextFileBufferParser &parser)

ParseBufferRequest	get_request_from_pool (MetadataScanMultiThreadingParams &multi_threading_params)

bool	request_pool_non_empty (MetadataScanMultiThreadingParams &multi_threading_params)

void	defer_scan_request (MetadataScanMultiThreadingParams &multi_threading_params, ParseBufferRequest &request)

void	dispatch_all_deferred_requests (MetadataScanMultiThreadingParams &multi_threading_params)

void	dispatch_scan_request (MetadataScanMultiThreadingParams &multi_threading_params, ParseBufferRequest &request)

void	populate_chunks (MetadataScanMultiThreadingParams &multi_threading_params, std::map< int, FileRegions > &fragment_id_to_file_regions_map, const TextFileBufferParser &parser, foreign_storage::IterativeFileScanParameters &file_scan_param)

void	resize_buffer_if_needed (std::unique_ptr< char[]> &buffer, size_t &buffer_size, const size_t alloc_size)

void	reset_multithreading_params (foreign_storage::MetadataScanMultiThreadingParams &multi_threading_params)

void	dispatch_scan_requests (const foreign_storage::ForeignTable table, const size_t &buffer_size, const std::string &file_path, FileReader &file_reader, const import_export::CopyParams &copy_params, MetadataScanMultiThreadingParams &multi_threading_params, size_t &first_row_index_in_buffer, size_t &current_file_offset, const TextFileBufferParser &parser, const foreign_storage::IterativeFileScanParameters file_scan_param, foreign_storage::AbstractTextFileDataWrapper::ResidualBuffer &iterative_residual_buffer, const bool is_first_file_scan_call, int &iterative_scan_last_fragment_id)

void	dispatch_scan_requests_with_exception_handling (const foreign_storage::ForeignTable table, const size_t &buffer_size, const std::string &file_path, FileReader &file_reader, const import_export::CopyParams &copy_params, MetadataScanMultiThreadingParams &multi_threading_params, size_t &first_row_index_in_buffer, size_t &current_file_offset, const TextFileBufferParser &parser, const foreign_storage::IterativeFileScanParameters file_scan_param, foreign_storage::AbstractTextFileDataWrapper::ResidualBuffer &iterative_residual_buffer, const bool is_first_file_scan_call, int &iterative_scan_last_fragment_id)

void	dispatch_scan_requests_with_exception_handling (const foreign_storage::ForeignTable table, const size_t &buffer_size, const std::string &file_path, FileReader &file_reader, const import_export::CopyParams &copy_params, MetadataScanMultiThreadingParams &multi_threading_params, size_t &first_row_index_in_buffer, size_t &current_file_offset, const TextFileBufferParser &parser, const foreign_storage::IterativeFileScanParameters file_scan_param, foreign_storage::AbstractTextFileDataWrapper::ResidualBuffer &iterative_residual_buffer, const bool is_first_file_scan_call)

void	set_value (rapidjson::Value &json_val, const FileRegion &file_region, rapidjson::Document::AllocatorType &allocator)

void	get_value (const rapidjson::Value &json_val, FileRegion &file_region)

std::optional< SQLTypes >	detect_geo_type (const SampleRows &sample_rows, size_t column_index)

std::tuple< std::unique_ptr < foreign_storage::ForeignServer > , std::unique_ptr < foreign_storage::UserMapping > , std::unique_ptr < foreign_storage::ForeignTable > >	create_proxy_fsi_objects (const std::string &copy_from_source, const import_export::CopyParams &copy_params, const int db_id, const TableDescriptor *table, const int32_t user_id)
	Create proxy fsi objects for use outside FSI. More...

std::tuple< std::unique_ptr < foreign_storage::ForeignServer > , std::unique_ptr < foreign_storage::UserMapping > , std::unique_ptr < foreign_storage::ForeignTable > >	create_proxy_fsi_objects (const std::string &copy_from_source, const import_export::CopyParams &copy_params, const TableDescriptor *table)
	Create proxy fsi objects for use outside FSI NOTE: parameters mirror function above. More...

void	validate_regex_parser_options (const import_export::CopyParams &copy_params)

bool	is_valid_source_type (const import_export::CopyParams &copy_params)

std::string	bool_to_option_value (const bool value)

void	throw_unexpected_number_of_items (const size_t &num_expected, const size_t &num_loaded, const std::string &item_type)

void	throw_removed_row_in_result_set_error (const std::string &select_statement)

void	throw_removed_row_in_file_error (const std::string &file_path)

void	throw_removed_file_error (const std::string &file_path)

void	throw_number_of_columns_mismatch_error (size_t num_table_cols, size_t num_file_cols, const std::string &file_path)

void	throw_file_access_error (const std::string &file_path, const std::string &message)

void	throw_file_not_found_error (const std::string &file_path)

void	throw_s3_compressed_mime_type (const std::string &file_path, const std::string &mime_type)

void	throw_s3_compressed_extension (const std::string &file_path, const std::string &ext_type)

bool	is_append_table_chunk_key (const ChunkKey &chunk_key)

bool	set_comp (const ChunkKey &left, const ChunkKey &right)

std::vector< ChunkKey >	get_column_key_vec (const ChunkKey &destination_chunk_key)

std::set< ChunkKey >	get_column_key_set (const ChunkKey &destination_chunk_key)

size_t	get_max_chunk_size (const ChunkKey &key)

bool	contains_fragment_key (const std::set< ChunkKey > &key_set, const ChunkKey &target_key)

bool	is_table_enabled_on_node (const ChunkKey &key)

void	refresh_foreign_table_unlocked (Catalog_Namespace::Catalog &catalog, const ForeignTable &td, const bool evict_cached_entries)

void	refresh_foreign_table (Catalog_Namespace::Catalog &catalog, const std::string &table_name, const bool evict_cached_entries)

void	init_chunk_for_column (const ChunkKey &chunk_key, const std::map< ChunkKey, std::shared_ptr< ChunkMetadata >> &chunk_metadata_map, const std::map< ChunkKey, AbstractBuffer * > &buffers, Chunk_NS::Chunk &chunk)

std::shared_ptr< ChunkMetadata >	get_placeholder_metadata (const SQLTypeInfo &type, size_t num_elements)

const foreign_storage::ForeignTable &	get_foreign_table_for_key (const ChunkKey &key)

bool	is_system_table_chunk_key (const ChunkKey &chunk_key)

bool	is_replicated_table_chunk_key (const ChunkKey &chunk_key)

bool	is_shardable_key (const ChunkKey &key)

bool	fragment_maps_to_leaf (const ChunkKey &key)

bool	key_does_not_shard_to_leaf (const ChunkKey &key)

template<typename T >
auto	partition_for_threads (const std::set< T > &items, size_t max_threads)

template<typename T >
auto	partition_for_threads (const std::vector< T > &items, size_t max_threads)

template<typename Container >
std::vector< std::future< void > >	create_futures_for_workers (const Container &items, size_t max_threads, std::function< void(const Container &)> lambda)

template<typename T >
ArrayDatum	encode_as_array_datum (const std::vector< T > &data)

std::string	get_db_name (int32_t db_id)

std::string	get_table_name (int32_t db_id, int32_t table_id)

void	set_node_name (std::map< std::string, import_export::TypedImportBuffer * > &import_buffers)

void	set_value (rapidjson::Value &json_val, const RowGroupInterval &value, rapidjson::Document::AllocatorType &allocator)

void	get_value (const rapidjson::Value &json_val, RowGroupInterval &value)

template<typename D , typename T >
bool	check_bounds (const T &value)

template<typename D >
std::string	datetime_to_string (const D &timestamp, const SQLTypeInfo &column_type)

void	throw_parquet_metadata_out_of_bounds_error (const std::string &min_value, const std::string &max_value, const std::string &encountered_value)

UniqueReaderPtr	open_parquet_table (const std::string &file_path, std::shared_ptr< arrow::fs::FileSystem > &file_system)

std::pair< int, int >	get_parquet_table_size (const ReaderPtr &reader)

const parquet::ColumnDescriptor *	get_column_descriptor (const parquet::arrow::FileReader *reader, const int logical_column_index)

parquet::Type::type	get_physical_type (ReaderPtr &reader, const int logical_column_index)

void	validate_equal_column_descriptor (const parquet::ColumnDescriptor reference_descriptor, const parquet::ColumnDescriptor new_descriptor, const std::string &reference_file_path, const std::string &new_file_path)

std::unique_ptr< ColumnDescriptor >	get_sub_type_column_descriptor (const ColumnDescriptor *column)

std::shared_ptr < parquet::Statistics >	validate_and_get_column_metadata_statistics (const parquet::ColumnChunkMetaData *column_metadata)

std::vector < Aws::S3::Model::Object >	s3_objects_filter_sort_files (const std::vector< Aws::S3::Model::Object > &file_paths, const shared::FilePathOptions &options)

template<typename V , std::enable_if_t< std::is_integral< V >::value, int > = 0>
V	get_null_value ()

template<typename D , std::enable_if_t< std::is_integral< D >::value, int > = 0>
std::pair< D, D >	get_min_max_bounds ()

void	populate_string_dictionary (int32_t table_id, int32_t col_id, int32_t db_id)

void	validate_non_foreign_table_write (const TableDescriptor *table_descriptor)

Variables
constexpr const char *	kDeletedValueIndicator {"<DELETED>"}

Typedef Documentation

using foreign_storage::ChunkToBufferMap = typedef std::map<ChunkKey, AbstractBuffer*>

Definition at line 34 of file ForeignDataWrapper.h.

template<typename T >

using foreign_storage::DateInDaysBoundsValidator = typedef BaseDateBoundsValidator<T, false>

Definition at line 277 of file ParquetMetadataValidator.h.

template<typename T >

using foreign_storage::DateInSecondsBoundsValidator = typedef BaseDateBoundsValidator<T, true>

Definition at line 274 of file ParquetMetadataValidator.h.

using foreign_storage::FilePathAndRowGroup = typedef std::pair<std::string, int32_t>

Definition at line 40 of file ParquetDataWrapper.h.

using foreign_storage::FileRegions = typedef std::vector<FileRegion>

Definition at line 60 of file FileRegions.h.

using foreign_storage::FirstLineByFilePath = typedef std::map<std::string, std::string>

Definition at line 37 of file FileReader.h.

using foreign_storage::InvalidRowGroupIndices = typedef std::set<int64_t>

Definition at line 123 of file ParquetEncoder.h.

using foreign_storage::OptionsMap = typedef std::map<std::string, std::string, std::less<>>

Definition at line 30 of file OptionsContainer.h.

template<typename V , typename T , T conversion_denominator, typename NullType = V>

using foreign_storage::ParquetDateInSecondsFromTimestampEncoder = typedef ParquetTimestampEncoder<V, T, conversion_denominator, NullType>

Definition at line 89 of file ParquetTimestampEncoder.h.

using foreign_storage::read_lock = typedef heavyai::shared_lock<heavyai::shared_mutex>

Definition at line 29 of file ForeignStorageCache.cpp.

using foreign_storage::ReaderPtr = typedef parquet::arrow::FileReader*

Definition at line 33 of file ParquetShared.h.

using foreign_storage::RejectedRowIndices = typedef std::set<int64_t>

Definition at line 28 of file ParquetEncoder.h.

using foreign_storage::RenderGroupAnalyzerMap = typedef std::map<int, std::unique_ptr<import_export::RenderGroupAnalyzer>>

Definition at line 37 of file ForeignDataWrapper.h.

using foreign_storage::S3ObjectComparator = typedef std::function<bool(const Aws::S3::Model::Object&, const Aws::S3::Model::Object&)>

Definition at line 17 of file S3FilePathUtil.h.

using foreign_storage::SampleRows = typedef std::vector<std::vector<std::string>>

Definition at line 26 of file DataPreview.h.

using foreign_storage::UniqueReaderPtr = typedef std::unique_ptr<parquet::arrow::FileReader>

Definition at line 32 of file ParquetShared.h.

using foreign_storage::write_lock = typedef heavyai::unique_lock<heavyai::shared_mutex>

Definition at line 30 of file ForeignStorageCache.cpp.

Function Documentation

void foreign_storage::add_file_region	(	std::map< int, FileRegions > &	fragment_id_to_file_regions_map,
		int	fragment_id,
		size_t	first_row_index,
		const ParseBufferResult &	result,
		const std::string &	file_path
	)

Creates a new file region based on metadata from parsed file buffers and adds the new region to the fragment id to file regions map.

Definition at line 552 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::ParseBufferResult::row_count, and foreign_storage::ParseBufferResult::row_offsets.

Referenced by populate_chunks_using_data_blocks(), and process_data_blocks().

                                                  {
   fragment_id_to_file_regions_map[fragment_id].emplace_back(
       // file naming is handled by FileReader
       FileRegion(file_path,
                  result.row_offsets.front(),
                  first_row_index,
                  result.row_count,
                  result.row_offsets.back() - result.row_offsets.front()));
 }

Here is the caller graph for this function:

void foreign_storage::add_request_to_pool	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		ParseBufferRequest &	request
	)

Adds the request object for a processed request back to the request pool for reuse in subsequent requests.

Definition at line 840 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::request_pool, foreign_storage::MetadataScanMultiThreadingParams::request_pool_condition, and foreign_storage::MetadataScanMultiThreadingParams::request_pool_mutex.

Referenced by dispatch_scan_requests(), populate_chunks(), and scan_metadata().

                                                       {
   std::unique_lock<std::mutex> completed_requests_queue_lock(
       multi_threading_params.request_pool_mutex);
   multi_threading_params.request_pool.emplace(std::move(request));
   completed_requests_queue_lock.unlock();
   multi_threading_params.request_pool_condition.notify_all();
 }

Here is the caller graph for this function:

void foreign_storage::append_data_block_to_chunk	(	const foreign_storage::IterativeFileScanParameters &	file_scan_param,
		DataBlockPtr	data_block,
		size_t	row_count,
		const int	column_id,
		const ColumnDescriptor *	column,
		const size_t	element_count_required
	)

Definition at line 639 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::IterativeFileScanParameters::column_id_to_chunk_map, shared::get_from_map(), foreign_storage::IterativeFileScanParameters::getChunkConditionalVariable(), and foreign_storage::IterativeFileScanParameters::getChunkMutex().

Referenced by populate_chunks_using_data_blocks().

                                          {
   auto chunk = shared::get_from_map(file_scan_param.column_id_to_chunk_map, column_id);
 
   auto& conditional_variable = file_scan_param.getChunkConditionalVariable(column_id);
   {
     std::unique_lock<std::mutex> chunk_lock(file_scan_param.getChunkMutex(column_id));
     conditional_variable.wait(chunk_lock, [element_count_required, &chunk]() {
       return chunk.getBuffer()->getEncoder()->getNumElems() == element_count_required;
     });
 
     chunk.appendData(data_block, row_count, 0);
   }
 
   conditional_variable
       .notify_all();  // notify any threads waiting on the correct element count
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::string foreign_storage::bool_to_option_value ( const bool value )

Definition at line 129 of file ForeignDataWrapperFactory.cpp.

Referenced by foreign_storage::ForeignDataWrapperFactory::createForeignTableProxy().

                                                  {
   return value ? "TRUE" : "FALSE";
 }

Here is the caller graph for this function:

void foreign_storage::cache_blocks	(	std::map< ChunkKey, Chunk_NS::Chunk > &	cached_chunks,
		DataBlockPtr	data_block,
		size_t	row_count,
		ChunkKey &	chunk_key,
		const ColumnDescriptor *	column,
		bool	is_first_block,
		bool	disable_cache
	)

Definition at line 598 of file AbstractTextFileDataWrapper.cpp.

References CHECK, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, ColumnDescriptor::columnType, foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::get_cache_if_enabled(), Catalog_Namespace::SysCatalog::getCatalog(), Catalog_Namespace::SysCatalog::instance(), SQLTypeInfo::is_varlen_indeed(), and key_does_not_shard_to_leaf().

Referenced by process_data_blocks().

                                       {
   auto catalog =
       Catalog_Namespace::SysCatalog::instance().getCatalog(chunk_key[CHUNK_KEY_DB_IDX]);
   CHECK(catalog);
   auto cache = get_cache_if_enabled(catalog, disable_cache);
   if (cache) {
     // This extra filter needs to be here because this wrapper is the only one that
     // accesses the cache directly and it should not be inserting chunks which are not
     // mapped to the current leaf (in distributed mode).
     if (key_does_not_shard_to_leaf(chunk_key)) {
       return;
     }
 
     ChunkKey index_key = {chunk_key[CHUNK_KEY_DB_IDX],
                           chunk_key[CHUNK_KEY_TABLE_IDX],
                           chunk_key[CHUNK_KEY_COLUMN_IDX],
                           chunk_key[CHUNK_KEY_FRAGMENT_IDX],
                           2};
     // Create actual data chunks to prepopulate cache
     if (cached_chunks.find(chunk_key) == cached_chunks.end()) {
       cached_chunks[chunk_key] = Chunk_NS::Chunk{column, false};
       cached_chunks[chunk_key].setBuffer(
           cache->getChunkBufferForPrecaching(chunk_key, is_first_block));
       if (column->columnType.is_varlen_indeed()) {
         cached_chunks[chunk_key].setIndexBuffer(
             cache->getChunkBufferForPrecaching(index_key, is_first_block));
       }
       if (is_first_block) {
         cached_chunks[chunk_key].initEncoder();
       }
     }
     cached_chunks[chunk_key].appendData(data_block, row_count, 0);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename D , typename T >

bool foreign_storage::check_bounds ( const T & value )

inline

Definition at line 32 of file ParquetMetadataValidator.h.

                                          {
   auto [min_value, max_value] = get_min_max_bounds<D>();
   return value >= min_value && value <= max_value;
 }

bool foreign_storage::contains_fragment_key	(	const std::set< ChunkKey > &	key_set,
		const ChunkKey &	target_key
	)

template<typename Container >

std::vector<std::future<void> > foreign_storage::create_futures_for_workers	(	const Container &	items,
		size_t	max_threads,
		std::function< void(const Container &)>	lambda
	)

Definition at line 74 of file FsiChunkUtils.h.

References threading_serial::async(), and partition_for_threads().

Referenced by import_export::ForeignDataImporter::importGeneralS3(), foreign_storage::ParquetDataWrapper::populateChunkBuffers(), and foreign_storage::LazyParquetChunkLoader::previewFiles().

                                                 {
   auto items_per_thread = partition_for_threads(items, max_threads);
   std::vector<std::future<void>> futures;
   for (const auto& items : items_per_thread) {
     futures.emplace_back(std::async(std::launch::async, lambda, items));
   }
 
   return futures;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::tuple< std::unique_ptr< foreign_storage::ForeignServer >, std::unique_ptr< foreign_storage::UserMapping >, std::unique_ptr< foreign_storage::ForeignTable > > foreign_storage::create_proxy_fsi_objects	(	const std::string &	copy_from_source,
		const import_export::CopyParams &	copy_params,
		const int	db_id,
		const TableDescriptor *	table,
		const int32_t	user_id
	)

Create proxy fsi objects for use outside FSI.

Parameters

copy_from_source	- the source that will be copied
copy_params	- CopyParams that specify parameters around use case
db_id	- db id of database in use case
table	- the table descriptor of the table in use case
user_id	- the user id of user in use case

Returns: tuple of FSI objects that can be used for particular use case (for example import or detect)

Definition at line 59 of file ForeignDataWrapperFactory.cpp.

References CHECK, foreign_storage::ForeignDataWrapperFactory::createForeignServerProxy(), foreign_storage::ForeignDataWrapperFactory::createForeignTableProxy(), and foreign_storage::ForeignDataWrapperFactory::createUserMappingProxyIfApplicable().

Referenced by create_proxy_fsi_objects(), and import_export::ForeignDataImporter::importGeneral().

                                                 {
   auto server = foreign_storage::ForeignDataWrapperFactory::createForeignServerProxy(
       db_id, user_id, copy_from_source, copy_params);
 
   CHECK(server);
   server->validate();
 
   auto user_mapping =
       foreign_storage::ForeignDataWrapperFactory::createUserMappingProxyIfApplicable(
           db_id, user_id, copy_from_source, copy_params, server.get());
 
   if (user_mapping) {
     user_mapping->validate(server.get());
   }
 
   auto foreign_table =
       foreign_storage::ForeignDataWrapperFactory::createForeignTableProxy(
           db_id, table, copy_from_source, copy_params, server.get());
 
   CHECK(foreign_table);
   foreign_table->validateOptionValues();
 
   return {std::move(server), std::move(user_mapping), std::move(foreign_table)};
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::tuple< std::unique_ptr< foreign_storage::ForeignServer >, std::unique_ptr< foreign_storage::UserMapping >, std::unique_ptr< foreign_storage::ForeignTable > > foreign_storage::create_proxy_fsi_objects	(	const std::string &	copy_from_source,
		const import_export::CopyParams &	copy_params,
		const TableDescriptor *	table
	)

Create proxy fsi objects for use outside FSI NOTE: parameters mirror function above.

Definition at line 91 of file ForeignDataWrapperFactory.cpp.

References create_proxy_fsi_objects().

                                                        {
   return create_proxy_fsi_objects(copy_from_source, copy_params, -1, table, -1);
 }

Here is the call graph for this function:

template<typename D >

std::string foreign_storage::datetime_to_string	(	const D &	timestamp,
		const SQLTypeInfo &	column_type
	)

inline

Definition at line 38 of file ParquetMetadataValidator.h.

References Datum::bigintval, CHECK, DatumToString(), SQLTypeInfo::is_date(), and SQLTypeInfo::is_timestamp().

Referenced by foreign_storage::TimestampBoundsValidator< T >::getMinMaxBoundsAsStrings(), foreign_storage::BaseDateBoundsValidator< T, is_in_seconds >::getMinMaxBoundsAsStrings(), foreign_storage::TimestampBoundsValidator< T >::validateValue(), and foreign_storage::BaseDateBoundsValidator< T, is_in_seconds >::validateValue().

                                                                       {
   CHECK(column_type.is_timestamp() || column_type.is_date());
   Datum d;
   d.bigintval = timestamp;
   return DatumToString(d, column_type);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::defer_scan_request	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		ParseBufferRequest &	request
	)

Definition at line 933 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::deferred_requests, and foreign_storage::MetadataScanMultiThreadingParams::deferred_requests_mutex.

Referenced by populate_chunks().

                                                      {
   std::unique_lock<std::mutex> deferred_requests_lock(
       multi_threading_params.deferred_requests_mutex);
   multi_threading_params.deferred_requests.emplace(std::move(request));
 }

Here is the caller graph for this function:

std::optional< SQLTypes > foreign_storage::detect_geo_type	(	const SampleRows &	sample_rows,
		size_t	column_index
	)

Definition at line 22 of file DataPreview.cpp.

References CHECK_EQ, CHECK_LT, kLINESTRING, kMULTIPOLYGON, kNULLT, kPOINT, kPOLYGON, and UNREACHABLE.

Referenced by foreign_storage::LazyParquetChunkLoader::previewFiles().

                                                              {
   std::optional<SQLTypes> tentative_geo_type{};
   for (const auto& row : sample_rows) {
     static std::regex geo_regex{
         "\\s*(POINT|LINESTRING|POLYGON|MULTIPOLYGON)\\s*\\(.+\\)\\s*"};
     std::smatch match;
     CHECK_LT(column_index, row.size());
     if (std::regex_match(row[column_index], match, geo_regex)) {
       CHECK_EQ(match.size(), static_cast<size_t>(2));
       SQLTypes geo_type{kNULLT};
       const auto& geo_type_str = match[1];
       if (geo_type_str == "POINT") {
         geo_type = kPOINT;
       } else if (geo_type_str == "LINESTRING") {
         geo_type = kLINESTRING;
       } else if (geo_type_str == "POLYGON") {
         geo_type = kPOLYGON;
       } else if (geo_type_str == "MULTIPOLYGON") {
         geo_type = kMULTIPOLYGON;
       } else {
         UNREACHABLE() << "Unexpected geo type match: " << geo_type_str;
       }
       if (tentative_geo_type.has_value()) {
         if (tentative_geo_type.value() != geo_type) {
           return {};  // geo type does not match between rows, can not be imported
         }
       } else {
         tentative_geo_type = geo_type;
       }
     }
   }
   return tentative_geo_type;
 }

Here is the caller graph for this function:

void foreign_storage::dispatch_all_deferred_requests ( MetadataScanMultiThreadingParams & multi_threading_params )

Definition at line 943 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::deferred_requests, foreign_storage::MetadataScanMultiThreadingParams::deferred_requests_mutex, foreign_storage::MetadataScanMultiThreadingParams::pending_requests, foreign_storage::MetadataScanMultiThreadingParams::pending_requests_condition, and foreign_storage::MetadataScanMultiThreadingParams::pending_requests_mutex.

Referenced by dispatch_scan_requests().

                                                               {
   std::unique_lock<std::mutex> deferred_requests_lock(
       multi_threading_params.deferred_requests_mutex);
   {
     std::unique_lock<std::mutex> pending_requests_lock(
         multi_threading_params.pending_requests_mutex);
 
     while (!multi_threading_params.deferred_requests.empty()) {
       auto& request = multi_threading_params.deferred_requests.front();
       multi_threading_params.pending_requests.emplace(std::move(request));
       multi_threading_params.deferred_requests.pop();
     }
     multi_threading_params.pending_requests_condition.notify_all();
   }
 }

Here is the caller graph for this function:

void foreign_storage::dispatch_scan_request	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		ParseBufferRequest &	request
	)

Dispatches a new metadata scan request by adding the request to the pending requests queue to be consumed by a worker thread.

Definition at line 964 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::pending_requests, foreign_storage::MetadataScanMultiThreadingParams::pending_requests_condition, and foreign_storage::MetadataScanMultiThreadingParams::pending_requests_mutex.

Referenced by dispatch_scan_requests().

                                                         {
   {
     std::unique_lock<std::mutex> pending_requests_lock(
         multi_threading_params.pending_requests_mutex);
     multi_threading_params.pending_requests.emplace(std::move(request));
   }
   multi_threading_params.pending_requests_condition.notify_all();
 }

Here is the caller graph for this function:

void foreign_storage::dispatch_scan_requests	(	const foreign_storage::ForeignTable *	table,
		const size_t &	buffer_size,
		const std::string &	file_path,
		FileReader &	file_reader,
		const import_export::CopyParams &	copy_params,
		MetadataScanMultiThreadingParams &	multi_threading_params,
		size_t &	first_row_index_in_buffer,
		size_t &	current_file_offset,
		const TextFileBufferParser &	parser,
		const foreign_storage::IterativeFileScanParameters *	file_scan_param,
		foreign_storage::AbstractTextFileDataWrapper::ResidualBuffer &	iterative_residual_buffer,
		const bool	is_first_file_scan_call,
		int &	iterative_scan_last_fragment_id
	)

Reads from a text file iteratively and dispatches metadata scan requests that are processed by worker threads.

Definition at line 1076 of file AbstractTextFileDataWrapper.cpp.

References add_request_to_pool(), foreign_storage::AbstractTextFileDataWrapper::ResidualBuffer::alloc_size, foreign_storage::MetadataScanMultiThreadingParams::continue_processing, dispatch_all_deferred_requests(), dispatch_scan_request(), foreign_storage::TextFileBufferParser::findRowEndPosition(), foreign_storage::IterativeFileScanParameters::fragment_id, get_request_from_pool(), foreign_storage::FileReader::getCurrentFilePath(), foreign_storage::FileReader::isScanFinished(), import_export::CopyParams::line_delim, TableDescriptor::maxFragRows, foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::no_deferred_requests(), foreign_storage::MetadataScanMultiThreadingParams::pending_requests, foreign_storage::MetadataScanMultiThreadingParams::pending_requests_condition, foreign_storage::MetadataScanMultiThreadingParams::pending_requests_mutex, foreign_storage::FileReader::read(), request_pool_non_empty(), foreign_storage::AbstractTextFileDataWrapper::ResidualBuffer::residual_buffer_alloc_size, foreign_storage::AbstractTextFileDataWrapper::ResidualBuffer::residual_buffer_size, foreign_storage::AbstractTextFileDataWrapper::ResidualBuffer::residual_data, and resize_buffer_if_needed().

Referenced by dispatch_scan_requests_with_exception_handling().

                                           {
   auto& alloc_size = iterative_residual_buffer.alloc_size;
   auto& residual_buffer = iterative_residual_buffer.residual_data;
   auto& residual_buffer_size = iterative_residual_buffer.residual_buffer_size;
   auto& residual_buffer_alloc_size = iterative_residual_buffer.residual_buffer_alloc_size;
 
   if (is_first_file_scan_call) {
     alloc_size = buffer_size;
     residual_buffer = std::make_unique<char[]>(alloc_size);
     residual_buffer_size = 0;
     residual_buffer_alloc_size = alloc_size;
   } else if (!no_deferred_requests(multi_threading_params)) {
     dispatch_all_deferred_requests(multi_threading_params);
   }
 
   // NOTE: During an interactive scan, it is possible for an entire fragment to
   // be parsed into requests, which sit in deferred requests; in order to avoid
   // stalling indefinitely while waiting on an available requests, the check
   // below determines if this is the case or not.
   bool current_fragment_fully_read_during_iterative_scan =
       file_scan_param && file_scan_param->fragment_id < iterative_scan_last_fragment_id;
 
   // NOTE: The conditional below behaves as follows:
   //
   // * for non-iterative scans,
   // current_fragment_fully_read_during_iterative_scan is false, and therefore
   // only the first clause of the conditional is relevant
   //
   // * for interactive scans, if
   // current_fragment_fully_read_during_iterative_scan is true, then the
   // conditional is skipped, unless it is determined there are still available
   // requests to work with, in which case the loop is entered; this is an
   // optimization that ensures maximal concurrency of loading while processing
   // requests
   while (!file_reader.isScanFinished() &&
          (request_pool_non_empty(multi_threading_params) ||
           !current_fragment_fully_read_during_iterative_scan)) {
     {
       std::lock_guard<std::mutex> pending_requests_lock(
           multi_threading_params.pending_requests_mutex);
       if (!multi_threading_params.continue_processing) {
         break;
       }
     }
     auto request = get_request_from_pool(multi_threading_params);
     request.full_path = file_reader.getCurrentFilePath();
     resize_buffer_if_needed(request.buffer, request.buffer_alloc_size, alloc_size);
 
     if (residual_buffer_size > 0) {
       memcpy(request.buffer.get(), residual_buffer.get(), residual_buffer_size);
     }
     size_t size = residual_buffer_size;
     size += file_reader.read(request.buffer.get() + residual_buffer_size,
                              alloc_size - residual_buffer_size);
 
     if (size == 0) {
       // In some cases at the end of a file we will read 0 bytes even when
       // file_reader.isScanFinished() is false. Also add request back to the pool to be
       // picked up again in the next iteration.
       add_request_to_pool(multi_threading_params, request);
       continue;
     } else if (size == 1 && request.buffer[0] == copy_params.line_delim) {
       // In some cases files with newlines at the end will be encoded with a second
       // newline that can end up being the only thing in the buffer. Also add request
       // back to the pool to be picked up again in the next iteration.
       current_file_offset++;
       add_request_to_pool(multi_threading_params, request);
       continue;
     }
     unsigned int num_rows_in_buffer = 0;
     request.end_pos = parser.findRowEndPosition(alloc_size,
                                                 request.buffer,
                                                 size,
                                                 copy_params,
                                                 first_row_index_in_buffer,
                                                 num_rows_in_buffer,
                                                 &file_reader);
     request.buffer_size = size;
     request.buffer_alloc_size = alloc_size;
     request.first_row_index = first_row_index_in_buffer;
     request.file_offset = current_file_offset;
     request.buffer_row_count = num_rows_in_buffer;
     request.processed_row_count = 0;
     request.begin_pos = 0;
 
     residual_buffer_size = size - request.end_pos;
     if (residual_buffer_size > 0) {
       resize_buffer_if_needed(residual_buffer, residual_buffer_alloc_size, alloc_size);
       memcpy(residual_buffer.get(),
              request.buffer.get() + request.end_pos,
              residual_buffer_size);
     }
 
     current_file_offset += request.end_pos;
     first_row_index_in_buffer += num_rows_in_buffer;
 
     if (num_rows_in_buffer > 0) {
       dispatch_scan_request(multi_threading_params, request);
     } else {
       add_request_to_pool(multi_threading_params, request);
     }
 
     if (file_scan_param) {
       const int32_t last_fragment_index =
           (first_row_index_in_buffer) / table->maxFragRows;
       if (last_fragment_index > file_scan_param->fragment_id) {
         iterative_scan_last_fragment_id = last_fragment_index;
         break;
       }
     }
   }
 
   std::unique_lock<std::mutex> pending_requests_queue_lock(
       multi_threading_params.pending_requests_mutex);
   multi_threading_params.pending_requests_condition.wait(
       pending_requests_queue_lock, [&multi_threading_params] {
         return multi_threading_params.pending_requests.empty() ||
                (multi_threading_params.continue_processing == false);
       });
   multi_threading_params.continue_processing = false;
   pending_requests_queue_lock.unlock();
   multi_threading_params.pending_requests_condition.notify_all();
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::dispatch_scan_requests_with_exception_handling	(	const foreign_storage::ForeignTable *	table,
		const size_t &	buffer_size,
		const std::string &	file_path,
		FileReader &	file_reader,
		const import_export::CopyParams &	copy_params,
		MetadataScanMultiThreadingParams &	multi_threading_params,
		size_t &	first_row_index_in_buffer,
		size_t &	current_file_offset,
		const TextFileBufferParser &	parser,
		const foreign_storage::IterativeFileScanParameters *	file_scan_param,
		foreign_storage::AbstractTextFileDataWrapper::ResidualBuffer &	iterative_residual_buffer,
		const bool	is_first_file_scan_call,
		int &	iterative_scan_last_fragment_id
	)

Definition at line 1214 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::continue_processing, dispatch_scan_requests(), foreign_storage::MetadataScanMultiThreadingParams::pending_requests_condition, and foreign_storage::MetadataScanMultiThreadingParams::pending_requests_mutex.

Referenced by dispatch_scan_requests_with_exception_handling(), foreign_storage::AbstractTextFileDataWrapper::iterativeFileScan(), and foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata().

                                           {
   try {
     dispatch_scan_requests(table,
                            buffer_size,
                            file_path,
                            file_reader,
                            copy_params,
                            multi_threading_params,
                            first_row_index_in_buffer,
                            current_file_offset,
                            parser,
                            file_scan_param,
                            iterative_residual_buffer,
                            is_first_file_scan_call,
                            iterative_scan_last_fragment_id);
   } catch (...) {
     {
       std::unique_lock<std::mutex> pending_requests_lock(
           multi_threading_params.pending_requests_mutex);
       multi_threading_params.continue_processing = false;
     }
     multi_threading_params.pending_requests_condition.notify_all();
     throw;
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::dispatch_scan_requests_with_exception_handling	(	const foreign_storage::ForeignTable *	table,
		const size_t &	buffer_size,
		const std::string &	file_path,
		FileReader &	file_reader,
		const import_export::CopyParams &	copy_params,
		MetadataScanMultiThreadingParams &	multi_threading_params,
		size_t &	first_row_index_in_buffer,
		size_t &	current_file_offset,
		const TextFileBufferParser &	parser,
		const foreign_storage::IterativeFileScanParameters *	file_scan_param,
		foreign_storage::AbstractTextFileDataWrapper::ResidualBuffer &	iterative_residual_buffer,
		const bool	is_first_file_scan_call
	)

Definition at line 1254 of file AbstractTextFileDataWrapper.cpp.

References dispatch_scan_requests_with_exception_handling().

                                         {
   int dummy;
   dispatch_scan_requests_with_exception_handling(table,
                                                  buffer_size,
                                                  file_path,
                                                  file_reader,
                                                  copy_params,
                                                  multi_threading_params,
                                                  first_row_index_in_buffer,
                                                  current_file_offset,
                                                  parser,
                                                  file_scan_param,
                                                  iterative_residual_buffer,
                                                  is_first_file_scan_call,
                                                  dummy);
 }

Here is the call graph for this function:

template<typename T >

ArrayDatum foreign_storage::encode_as_array_datum ( const std::vector< T > & data )

inline

Definition at line 33 of file GeospatialEncoder.h.

References heavydb.dtypes::T.

Referenced by foreign_storage::GeospatialEncoder::processGeoElement(), and foreign_storage::GeospatialEncoder::processNullGeoElement().

                                                                   {
   const size_t num_bytes = data.size() * sizeof(T);
   std::shared_ptr<int8_t> buffer(new int8_t[num_bytes], std::default_delete<int8_t[]>());
   memcpy(buffer.get(), data.data(), num_bytes);
   return ArrayDatum(num_bytes, buffer, false);
 }

Here is the caller graph for this function:

bool foreign_storage::fragment_maps_to_leaf ( const ChunkKey & key )

Definition at line 128 of file FsiChunkUtils.cpp.

References CHECK, g_distributed_leaf_idx, g_distributed_num_leaves, get_fragment(), dist::is_aggregator(), and dist::is_distributed().

Referenced by anonymous_namespace{ForeignStorageMgr.cpp}::filter_metadata_by_leaf(), foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), key_does_not_shard_to_leaf(), and foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

                                                 {
   CHECK(dist::is_distributed());
   CHECK(g_distributed_num_leaves > 0);
   CHECK(!dist::is_aggregator());
   return (get_fragment(key) % g_distributed_num_leaves == g_distributed_leaf_idx);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

const parquet::ColumnDescriptor * foreign_storage::get_column_descriptor	(	const parquet::arrow::FileReader *	reader,
		const int	logical_column_index
	)

Definition at line 46 of file ParquetShared.cpp.

Referenced by foreign_storage::LazyParquetChunkLoader::appendRowGroups(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_equal_schema().

                                     {
   return reader->parquet_reader()->metadata()->schema()->Column(logical_column_index);
 }

Here is the caller graph for this function:

std::set<ChunkKey> foreign_storage::get_column_key_set ( const ChunkKey & destination_chunk_key )

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), foreign_storage::CachingForeignStorageMgr::getOptionalKeysWithinSizeLimit(), and foreign_storage::CachingForeignStorageMgr::getRequiredBuffersSize().

Here is the caller graph for this function:

std::vector<ChunkKey> foreign_storage::get_column_key_vec ( const ChunkKey & destination_chunk_key )

std::string foreign_storage::get_db_name ( int32_t db_id )

Definition at line 31 of file InternalSystemDataWrapper.cpp.

References Catalog_Namespace::DBMetadata::dbName, Catalog_Namespace::SysCatalog::instance(), and kDeletedValueIndicator.

Referenced by foreign_storage::anonymous_namespace{InternalCatalogDataWrapper.cpp}::populate_import_buffers_for_catalog_dashboards(), foreign_storage::anonymous_namespace{InternalCatalogDataWrapper.cpp}::populate_import_buffers_for_catalog_permissions(), foreign_storage::anonymous_namespace{InternalCatalogDataWrapper.cpp}::populate_import_buffers_for_catalog_tables(), foreign_storage::anonymous_namespace{InternalCatalogDataWrapper.cpp}::populate_import_buffers_for_catalog_users(), foreign_storage::anonymous_namespace{InternalMemoryStatsDataWrapper.cpp}::populate_import_buffers_for_memory_details(), and foreign_storage::anonymous_namespace{InternalStorageStatsDataWrapper.cpp}::populate_import_buffers_for_storage_details().

                                      {
   Catalog_Namespace::DBMetadata db_metadata;
   auto& sys_catalog = Catalog_Namespace::SysCatalog::instance();
   if (sys_catalog.getMetadataForDBById(db_id, db_metadata)) {
     return db_metadata.dbName;
   } else {
     // Database has been deleted.
     return kDeletedValueIndicator;
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

const foreign_storage::ForeignTable & foreign_storage::get_foreign_table_for_key ( const ChunkKey & key )

Definition at line 101 of file FsiChunkUtils.cpp.

References CHECK, get_table_prefix(), Catalog_Namespace::SysCatalog::getCatalog(), and Catalog_Namespace::SysCatalog::instance().

Referenced by is_append_table_chunk_key(), foreign_storage::anonymous_namespace{CachingForeignStorageMgr.cpp}::is_in_memory_system_table_chunk_key(), is_replicated_table_chunk_key(), and is_system_table_chunk_key().

                                                                                   {
   auto [db_id, tb_id] = get_table_prefix(key);
   auto catalog = Catalog_Namespace::SysCatalog::instance().getCatalog(db_id);
   CHECK(catalog);
   auto table = catalog->getForeignTable(tb_id);
   CHECK(table);
   return *table;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

size_t foreign_storage::get_max_chunk_size ( const ChunkKey & key )

Referenced by foreign_storage::CachingForeignStorageMgr::getBufferSize().

Here is the caller graph for this function:

template<typename D , std::enable_if_t< std::is_integral< D >::value, int > = 0>

std::pair< D, D > foreign_storage::get_min_max_bounds ( )

inline

Definition at line 31 of file SharedMetadataValidator.h.

                                           {
   static_assert(std::is_signed<D>::value,
                 "'get_min_max_bounds' is only valid for signed types");
   return {get_null_value<D>() + 1, std::numeric_limits<D>::max()};
 }

std::optional<ParseBufferRequest> foreign_storage::get_next_scan_request ( MetadataScanMultiThreadingParams & multi_threading_params )

Gets the next metadata scan request object from the pending requests queue. A null optional is returned if there are no further requests to be processed.

Definition at line 529 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::continue_processing, foreign_storage::MetadataScanMultiThreadingParams::pending_requests, foreign_storage::MetadataScanMultiThreadingParams::pending_requests_condition, and foreign_storage::MetadataScanMultiThreadingParams::pending_requests_mutex.

Referenced by populate_chunks(), and scan_metadata().

                                                               {
   std::unique_lock<std::mutex> pending_requests_lock(
       multi_threading_params.pending_requests_mutex);
   multi_threading_params.pending_requests_condition.wait(
       pending_requests_lock, [&multi_threading_params] {
         return !multi_threading_params.pending_requests.empty() ||
                !multi_threading_params.continue_processing;
       });
   if (multi_threading_params.pending_requests.empty()) {
     return {};
   }
   auto request = std::move(multi_threading_params.pending_requests.front());
   multi_threading_params.pending_requests.pop();
   pending_requests_lock.unlock();
   multi_threading_params.pending_requests_condition.notify_all();
   return std::move(request);
 }

Here is the caller graph for this function:

template<typename V , std::enable_if_t< std::is_integral< V >::value, int > = 0>

V foreign_storage::get_null_value ( )

inline

Definition at line 21 of file SharedMetadataValidator.h.

                           {
   return inline_int_null_value<V>();
 }

size_t foreign_storage::get_num_threads ( const ForeignTable & table )

Definition at line 17 of file AbstractFileStorageDataWrapper.cpp.

References foreign_storage::OptionsContainer::getOption(), import_export::num_import_threads(), and foreign_storage::AbstractFileStorageDataWrapper::THREADS_KEY.

Referenced by foreign_storage::LazyParquetChunkLoader::metadataScan(), foreign_storage::ParquetDataWrapper::populateChunkBuffers(), and foreign_storage::LazyParquetChunkLoader::previewFiles().

                                                   {
   auto num_threads = 0;
   if (auto opt = table.getOption(AbstractFileStorageDataWrapper::THREADS_KEY);
       opt.has_value()) {
     num_threads = std::stoi(opt.value());
   }
   return import_export::num_import_threads(num_threads);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::pair< int, int > foreign_storage::get_parquet_table_size ( const ReaderPtr & reader )

Definition at line 39 of file ParquetShared.cpp.

Referenced by foreign_storage::LazyParquetChunkLoader::appendRowGroups(), foreign_storage::LazyParquetChunkLoader::loadRowGroups(), and foreign_storage::LazyParquetChunkLoader::metadataScan().

                                                                   {
   auto file_metadata = reader->parquet_reader()->metadata();
   const auto num_row_groups = file_metadata->num_row_groups();
   const auto num_columns = file_metadata->num_columns();
   return std::make_pair(num_row_groups, num_columns);
 }

Here is the caller graph for this function:

parquet::Type::type foreign_storage::get_physical_type	(	ReaderPtr &	reader,
		const int	logical_column_index
	)

Definition at line 52 of file ParquetShared.cpp.

Referenced by anonymous_namespace{ArrowResultSetConverter.cpp}::get_arrow_type(), and ArrowResultSetConverter::initializeColumnBuilder().

                                                                                      {
   return reader->parquet_reader()
       ->metadata()
       ->schema()
       ->Column(logical_column_index)
       ->physical_type();
 }

Here is the caller graph for this function:

std::shared_ptr< ChunkMetadata > foreign_storage::get_placeholder_metadata	(	const SQLTypeInfo &	type,
		size_t	num_elements
	)

Definition at line 77 of file FsiChunkUtils.cpp.

References SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_size(), Data_Namespace::AbstractBuffer::getEncoder(), Encoder::getMetadata(), Data_Namespace::AbstractBuffer::initEncoder(), SQLTypeInfo::is_array(), and SQLTypeInfo::is_varlen_indeed().

Referenced by foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::add_placeholder_metadata(), foreign_storage::InternalSystemDataWrapper::populateChunkMetadata(), and foreign_storage::AbstractTextFileDataWrapper::updateRolledOffChunks().

                                                                              {
   ForeignStorageBuffer empty_buffer;
   // Use default encoder metadata as in parquet wrapper
   empty_buffer.initEncoder(type);
 
   auto chunk_metadata = empty_buffer.getEncoder()->getMetadata(type);
   chunk_metadata->numElements = num_elements;
 
   if (!type.is_varlen_indeed()) {
     chunk_metadata->numBytes = type.get_size() * num_elements;
   }
   // min/max not set by default for arrays, so get from elem type encoder
   if (type.is_array()) {
     ForeignStorageBuffer scalar_buffer;
     scalar_buffer.initEncoder(type.get_elem_type());
     auto scalar_metadata = scalar_buffer.getEncoder()->getMetadata(type.get_elem_type());
     chunk_metadata->chunkStats.min = scalar_metadata->chunkStats.min;
     chunk_metadata->chunkStats.max = scalar_metadata->chunkStats.max;
   }
 
   return chunk_metadata;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

ParseBufferRequest foreign_storage::get_request_from_pool ( MetadataScanMultiThreadingParams & multi_threading_params )

Gets a request from the metadata scan request pool.

Definition at line 908 of file AbstractTextFileDataWrapper.cpp.

References CHECK, foreign_storage::MetadataScanMultiThreadingParams::request_pool, foreign_storage::MetadataScanMultiThreadingParams::request_pool_condition, and foreign_storage::MetadataScanMultiThreadingParams::request_pool_mutex.

Referenced by dispatch_scan_requests().

                                                               {
   std::unique_lock<std::mutex> request_pool_lock(
       multi_threading_params.request_pool_mutex);
   multi_threading_params.request_pool_condition.wait(
       request_pool_lock,
       [&multi_threading_params] { return !multi_threading_params.request_pool.empty(); });
   auto request = std::move(multi_threading_params.request_pool.front());
   multi_threading_params.request_pool.pop();
   request_pool_lock.unlock();
   CHECK(request.buffer);
   return request;
 }

Here is the caller graph for this function:

std::unique_ptr< ColumnDescriptor > foreign_storage::get_sub_type_column_descriptor ( const ColumnDescriptor * column )

Definition at line 76 of file ParquetShared.cpp.

References ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, ColumnDescriptor::db_id, SQLTypeInfo::get_elem_type(), SQLTypeInfo::set_size(), and ColumnDescriptor::tableId.

Referenced by foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_array_encoder(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_array_mapping().

                                     {
   auto column_type = column->columnType.get_elem_type();
   if (column_type.get_size() == -1 && column_type.is_dict_encoded_string()) {
     column_type.set_size(4);  // override default size of -1
   }
   return std::make_unique<ColumnDescriptor>(
       column->tableId, column->columnId, column->columnName, column_type, column->db_id);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::string foreign_storage::get_table_name	(	int32_t	db_id,
		int32_t	table_id
	)

Definition at line 42 of file InternalSystemDataWrapper.cpp.

References CHECK, Catalog_Namespace::SysCatalog::getCatalog(), Catalog_Namespace::SysCatalog::instance(), and kDeletedValueIndicator.

Referenced by anonymous_namespace{Execute.cpp}::checkWorkUnitWatchdog(), foreign_storage::anonymous_namespace{InternalMemoryStatsDataWrapper.cpp}::populate_import_buffers_for_memory_details(), and foreign_storage::anonymous_namespace{InternalStorageStatsDataWrapper.cpp}::populate_import_buffers_for_storage_details().

                                                           {
   auto catalog = Catalog_Namespace::SysCatalog::instance().getCatalog(db_id);
   CHECK(catalog);
   auto table_name = catalog->getTableName(table_id);
   if (table_name.has_value()) {
     return table_name.value();
   } else {
     // It is possible for the table to be concurrently deleted while querying the system
     // table.
     return kDeletedValueIndicator;
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::get_value	(	const rapidjson::Value &	json_val,
		FileRegion &	file_region
	)

Definition at line 44 of file CsvShared.cpp.

References CHECK, foreign_storage::FileRegion::first_row_file_offset, foreign_storage::FileRegion::first_row_index, foreign_storage::json_utils::get_value_from_object(), foreign_storage::FileRegion::region_size, and foreign_storage::FileRegion::row_count.

                                                                         {
   CHECK(json_val.IsObject());
   json_utils::get_value_from_object(
       json_val, file_region.first_row_file_offset, "first_row_file_offset");
   json_utils::get_value_from_object(
       json_val, file_region.first_row_index, "first_row_index");
   json_utils::get_value_from_object(json_val, file_region.region_size, "region_size");
   json_utils::get_value_from_object(json_val, file_region.row_count, "row_count");
   if (json_val.HasMember("filename")) {
     json_utils::get_value_from_object(json_val, file_region.filename, "filename");
   }
 }

Here is the call graph for this function:

void foreign_storage::get_value	(	const rapidjson::Value &	json_val,
		RowGroupInterval &	value
	)

Definition at line 671 of file ParquetDataWrapper.cpp.

References CHECK, foreign_storage::RowGroupInterval::end_index, foreign_storage::RowGroupInterval::file_path, foreign_storage::json_utils::get_value_from_object(), and foreign_storage::RowGroupInterval::start_index.

                                                                         {
   CHECK(json_val.IsObject());
   json_utils::get_value_from_object(json_val, value.file_path, "file_path");
   json_utils::get_value_from_object(json_val, value.start_index, "start_index");
   json_utils::get_value_from_object(json_val, value.end_index, "end_index");
 }

Here is the call graph for this function:

void foreign_storage::init_chunk_for_column	(	const ChunkKey &	chunk_key,
		const std::map< ChunkKey, std::shared_ptr< ChunkMetadata >> &	chunk_metadata_map,
		const std::map< ChunkKey, AbstractBuffer * > &	buffers,
		Chunk_NS::Chunk &	chunk
	)

Definition at line 21 of file FsiChunkUtils.cpp.

References CHECK, CHECK_EQ, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, Catalog_Namespace::SysCatalog::getCatalog(), Chunk_NS::Chunk::initEncoder(), Catalog_Namespace::SysCatalog::instance(), Data_Namespace::AbstractBuffer::reserve(), Chunk_NS::Chunk::setBuffer(), Chunk_NS::Chunk::setColumnDesc(), Chunk_NS::Chunk::setIndexBuffer(), Chunk_NS::Chunk::setPinnable(), Data_Namespace::AbstractBuffer::size(), and UNREACHABLE.

Referenced by foreign_storage::AbstractTextFileDataWrapper::populateChunkMapForColumns().

                           {
   auto catalog =
       Catalog_Namespace::SysCatalog::instance().getCatalog(chunk_key[CHUNK_KEY_DB_IDX]);
   CHECK(catalog);
 
   ChunkKey data_chunk_key = chunk_key;
   AbstractBuffer* data_buffer = nullptr;
   AbstractBuffer* index_buffer = nullptr;
   const auto column = catalog->getMetadataForColumn(chunk_key[CHUNK_KEY_TABLE_IDX],
                                                     chunk_key[CHUNK_KEY_COLUMN_IDX]);
 
   if (column->columnType.is_varlen_indeed()) {
     data_chunk_key.push_back(1);
     ChunkKey index_chunk_key = chunk_key;
     index_chunk_key.push_back(2);
 
     CHECK(buffers.find(data_chunk_key) != buffers.end());
     CHECK(buffers.find(index_chunk_key) != buffers.end());
 
     data_buffer = buffers.find(data_chunk_key)->second;
     index_buffer = buffers.find(index_chunk_key)->second;
     CHECK_EQ(data_buffer->size(), static_cast<size_t>(0));
     CHECK_EQ(index_buffer->size(), static_cast<size_t>(0));
 
     size_t index_offset_size{0};
     if (column->columnType.is_string() || column->columnType.is_geometry()) {
       index_offset_size = sizeof(StringOffsetT);
     } else if (column->columnType.is_array()) {
       index_offset_size = sizeof(ArrayOffsetT);
     } else {
       UNREACHABLE();
     }
     if (chunk_metadata_map.find(data_chunk_key) != chunk_metadata_map.end()) {
       index_buffer->reserve(index_offset_size *
                             (chunk_metadata_map.at(data_chunk_key)->numElements + 1));
     }
   } else {
     data_chunk_key = chunk_key;
     CHECK(buffers.find(data_chunk_key) != buffers.end());
     data_buffer = buffers.find(data_chunk_key)->second;
   }
   if (chunk_metadata_map.find(data_chunk_key) != chunk_metadata_map.end()) {
     data_buffer->reserve(chunk_metadata_map.at(data_chunk_key)->numBytes);
   }
 
   chunk.setPinnable(false);
   chunk.setColumnDesc(column);
   chunk.setBuffer(data_buffer);
   chunk.setIndexBuffer(index_buffer);
   chunk.initEncoder();
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::is_append_table_chunk_key ( const ChunkKey & chunk_key )

Definition at line 118 of file FsiChunkUtils.cpp.

References get_foreign_table_for_key(), and foreign_storage::ForeignTable::isAppendMode().

Referenced by foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

                                                           {
   return get_foreign_table_for_key(chunk_key).isAppendMode();
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::is_replicated_table_chunk_key ( const ChunkKey & chunk_key )

Definition at line 114 of file FsiChunkUtils.cpp.

References get_foreign_table_for_key(), and table_is_replicated().

Referenced by is_shardable_key().

                                                               {
   return table_is_replicated(&get_foreign_table_for_key(chunk_key));
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::is_shardable_key ( const ChunkKey & key )

Definition at line 122 of file FsiChunkUtils.cpp.

References dist::is_aggregator(), dist::is_distributed(), is_replicated_table_chunk_key(), and is_system_table_chunk_key().

Referenced by anonymous_namespace{ForeignStorageMgr.cpp}::filter_metadata_by_leaf(), foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), key_does_not_shard_to_leaf(), and foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

                                            {
   return (dist::is_distributed() && !dist::is_aggregator() &&
           !is_replicated_table_chunk_key(key) && !is_system_table_chunk_key(key));
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::is_system_table_chunk_key ( const ChunkKey & chunk_key )

Definition at line 110 of file FsiChunkUtils.cpp.

References get_foreign_table_for_key(), and TableDescriptor::is_system_table.

Referenced by is_shardable_key().

                                                           {
   return get_foreign_table_for_key(chunk_key).is_system_table;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::is_table_enabled_on_node ( const ChunkKey & key )

Referenced by foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecFromDataWrapper().

Here is the caller graph for this function:

bool foreign_storage::is_valid_source_type ( const import_export::CopyParams & copy_params )

Verify if source_type is valid.

Definition at line 120 of file ForeignDataWrapperFactory.cpp.

References import_export::kDelimitedFile, import_export::kParquetFile, import_export::kRegexParsedFile, and import_export::CopyParams::source_type.

Referenced by foreign_storage::ForeignDataWrapperFactory::createForeignServerProxy(), foreign_storage::ForeignDataWrapperFactory::createForeignTableProxy(), and import_export::ForeignDataImporter::importGeneral().

                                                                       {
   return
 #ifdef ENABLE_IMPORT_PARQUET
       copy_params.source_type == import_export::SourceType::kParquetFile ||
 #endif
       copy_params.source_type == import_export::SourceType::kDelimitedFile ||
       copy_params.source_type == import_export::SourceType::kRegexParsedFile;
 }

Here is the caller graph for this function:

bool foreign_storage::key_does_not_shard_to_leaf ( const ChunkKey & key )

Definition at line 135 of file FsiChunkUtils.cpp.

References fragment_maps_to_leaf(), and is_shardable_key().

Referenced by cache_blocks(), populate_string_dictionary(), and anonymous_namespace{RelAlgExecutor.cpp}::set_parallelism_hints().

                                                      {
   return (is_shardable_key(key) && !fragment_maps_to_leaf(key));
 }

Here is the call graph for this function:

Here is the caller graph for this function:

size_t foreign_storage::num_rows_to_process	(	const size_t	start_row_index,
		const size_t	max_fragment_size,
		const size_t	rows_remaining
	)

Number of rows to process given the current position defined by start_row_index, the max fragment size and the number of rows left to process.

Definition at line 483 of file AbstractTextFileDataWrapper.cpp.

Referenced by QueryExecutionContext::launchCpuCode(), and populate_chunks().

                                                         {
   size_t start_position_in_fragment = start_row_index % max_fragment_size;
   return std::min<size_t>(rows_remaining, max_fragment_size - start_position_in_fragment);
 }

Here is the caller graph for this function:

UniqueReaderPtr foreign_storage::open_parquet_table	(	const std::string &	file_path,
		std::shared_ptr< arrow::fs::FileSystem > &	file_system
	)

Definition at line 26 of file ParquetShared.cpp.

Referenced by foreign_storage::FileReaderMap::getOrInsert(), foreign_storage::FileReaderMap::insert(), and foreign_storage::LazyParquetChunkLoader::loadRowGroups().

                                                                                   {
   UniqueReaderPtr reader;
   auto file_result = file_system->OpenInputFile(file_path);
   if (!file_result.ok()) {
     throw std::runtime_error{"Unable to access " + file_system->type_name() + " file: " +
                              file_path + ". " + file_result.status().message()};
   }
   auto infile = file_result.ValueOrDie();
   PARQUET_THROW_NOT_OK(OpenFile(infile, arrow::default_memory_pool(), &reader));
   return reader;
 }

Here is the caller graph for this function:

ParseFileRegionResult foreign_storage::parse_file_regions	(	const FileRegions &	file_regions,
		const size_t	start_index,
		const size_t	end_index,
		FileReader &	file_reader,
		ParseBufferRequest &	parse_file_request,
		const std::map< int, Chunk_NS::Chunk > &	column_id_to_chunk_map,
		const TextFileBufferParser &	parser
	)

Parses a set of file regions given a handle to the file and range of indexes for the file regions to be parsed.

Definition at line 216 of file AbstractTextFileDataWrapper.cpp.

Referenced by foreign_storage::AbstractTextFileDataWrapper::populateChunks().

                                         {
   auto timer = DEBUG_TIMER(__func__);
   ParseFileRegionResult load_file_region_result{};
   load_file_region_result.file_offset = file_regions[start_index].first_row_file_offset;
   load_file_region_result.row_count = 0;
 
   ParseBufferResult result;
   for (size_t i = start_index; i <= end_index; i++) {
     CHECK(file_regions[i].region_size <= parse_file_request.buffer_size);
     auto read_size = file_reader.readRegion(parse_file_request.buffer.get(),
                                             file_regions[i].first_row_file_offset,
                                             file_regions[i].region_size);
     if (file_regions[i].region_size != read_size) {
       throw_unexpected_number_of_items(file_regions[i].region_size,
                                        read_size,
                                        "bytes",
                                        parse_file_request.getTableName());
     }
     parse_file_request.begin_pos = 0;
     parse_file_request.end_pos = file_regions[i].region_size;
     parse_file_request.first_row_index = file_regions[i].first_row_index;
     parse_file_request.file_offset = file_regions[i].first_row_file_offset;
     parse_file_request.process_row_count = file_regions[i].row_count;
 
     result = parser.parseBuffer(parse_file_request, i == end_index);
     CHECK_EQ(file_regions[i].row_count, result.row_count);
     for (const auto& rejected_row_index : result.rejected_rows) {
       load_file_region_result.rejected_row_indices.insert(
           load_file_region_result.row_count + rejected_row_index);
     }
     load_file_region_result.row_count += result.row_count;
   }
   load_file_region_result.column_id_to_data_blocks_map =
       result.column_id_to_data_blocks_map;
   return load_file_region_result;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::vector<size_t> foreign_storage::partition_by_fragment	(	const size_t	start_row_index,
		const size_t	max_fragment_size,
		const size_t	buffer_row_count
	)

Given a start row index, maximum fragment size, and number of rows in a buffer, this function returns a vector indicating how the rows in the buffer should be partitioned in order to fill up available fragment slots while staying within the capacity of fragments.

Definition at line 496 of file AbstractTextFileDataWrapper.cpp.

References CHECK.

Referenced by scan_metadata().

                                                                          {
   CHECK(buffer_row_count > 0);
   std::vector<size_t> partitions{};
   size_t remaining_rows_in_last_fragment;
   if (start_row_index % max_fragment_size == 0) {
     remaining_rows_in_last_fragment = 0;
   } else {
     remaining_rows_in_last_fragment =
         max_fragment_size - (start_row_index % max_fragment_size);
   }
   if (buffer_row_count <= remaining_rows_in_last_fragment) {
     partitions.emplace_back(buffer_row_count);
   } else {
     if (remaining_rows_in_last_fragment > 0) {
       partitions.emplace_back(remaining_rows_in_last_fragment);
     }
     size_t remaining_buffer_row_count =
         buffer_row_count - remaining_rows_in_last_fragment;
     while (remaining_buffer_row_count > 0) {
       partitions.emplace_back(
           std::min<size_t>(remaining_buffer_row_count, max_fragment_size));
       remaining_buffer_row_count -= partitions.back();
     }
   }
   return partitions;
 }

Here is the caller graph for this function:

std::pair<std::map<int, DataBlockPtr>, std::map<int, DataBlockPtr> > foreign_storage::partition_data_blocks	(	const std::map< int, const ColumnDescriptor * > &	column_by_id,
		const std::map< int, DataBlockPtr > &	data_blocks
	)

Partition data blocks such that dictionary encoded columns are disjoint of other columns

Returns: a pair of DataBlockPtr maps with the first being columns that are string dictionary encoded and the second being the complement

Definition at line 669 of file AbstractTextFileDataWrapper.cpp.

References shared::get_from_map().

Referenced by populate_chunks_using_data_blocks().

                                                   {
   std::map<int, DataBlockPtr> dict_encoded_data_blocks;
   std::map<int, DataBlockPtr> none_dict_encoded_data_blocks;
   for (auto& [column_id, data_block] : data_blocks) {
     const auto column = shared::get_from_map(column_by_id, column_id);
     if (column->columnType.is_dict_encoded_string()) {
       dict_encoded_data_blocks[column_id] = data_block;
     } else {
       none_dict_encoded_data_blocks[column_id] = data_block;
     }
   }
   return {dict_encoded_data_blocks, none_dict_encoded_data_blocks};
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >

auto foreign_storage::partition_for_threads	(	const std::set< T > &	items,
		size_t	max_threads
	)

Definition at line 41 of file FsiChunkUtils.h.

Referenced by create_futures_for_workers(), and foreign_storage::LazyParquetChunkLoader::metadataScan().

                                                                        {
   const size_t items_per_thread = (items.size() + (max_threads - 1)) / max_threads;
   std::list<std::set<T>> items_by_thread;
   auto i = 0U;
   for (auto item : items) {
     if (i++ % items_per_thread == 0) {
       items_by_thread.emplace_back(std::set<T>{});
     }
     items_by_thread.back().emplace(item);
   }
   return items_by_thread;
 }

Here is the caller graph for this function:

template<typename T >

auto foreign_storage::partition_for_threads	(	const std::vector< T > &	items,
		size_t	max_threads
	)

Definition at line 60 of file FsiChunkUtils.h.

                                                                           {
   const size_t items_per_thread = (items.size() + (max_threads - 1)) / max_threads;
   std::list<std::vector<T>> items_by_thread;
   auto i = 0U;
   for (auto item : items) {
     if (i++ % items_per_thread == 0) {
       items_by_thread.emplace_back(std::vector<T>{});
     }
     items_by_thread.back().emplace_back(item);
   }
   return items_by_thread;
 }

void foreign_storage::populate_chunks	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		std::map< int, FileRegions > &	fragment_id_to_file_regions_map,
		const TextFileBufferParser &	parser,
		foreign_storage::IterativeFileScanParameters &	file_scan_param
	)

Consumes and processes scan requests from a pending requests queue and populates chunks during an iterative file scan

Definition at line 978 of file AbstractTextFileDataWrapper.cpp.

Referenced by foreign_storage::AbstractTextFileDataWrapper::iterativeFileScan().

                                                                                   {
   std::map<int, const ColumnDescriptor*> column_by_id{};
   while (true) {
     auto request_opt = get_next_scan_request(multi_threading_params);
     if (!request_opt.has_value()) {
       break;
     }
     ParseBufferRequest& request = request_opt.value();
     try {
       if (column_by_id.empty()) {
         for (const auto column : request.getColumns()) {
           column_by_id[column->columnId] = column;
         }
       }
       CHECK_LE(request.processed_row_count, request.buffer_row_count);
       for (size_t num_rows_left_to_process =
                request.buffer_row_count - request.processed_row_count;
            num_rows_left_to_process > 0;
            num_rows_left_to_process =
                request.buffer_row_count - request.processed_row_count) {
         // NOTE: `request.begin_pos` state is required to be set correctly by this point
         // in execution
         size_t row_index = request.first_row_index + request.processed_row_count;
         int fragment_id = row_index / request.getMaxFragRows();
         if (fragment_id >
             file_scan_param.fragment_id) {  // processing must continue next iteration
           defer_scan_request(multi_threading_params, request);
           return;
         }
         request.process_row_count = num_rows_to_process(
             row_index, request.getMaxFragRows(), num_rows_left_to_process);
         for (const auto& import_buffer : request.import_buffers) {
           if (import_buffer != nullptr) {
             import_buffer->clear();
           }
         }
         auto result = parser.parseBuffer(request, true, true, true);
         size_t start_position_in_fragment = row_index % request.getMaxFragRows();
         populate_chunks_using_data_blocks(multi_threading_params,
                                           fragment_id,
                                           request,
                                           result,
                                           column_by_id,
                                           fragment_id_to_file_regions_map,
                                           file_scan_param,
                                           start_position_in_fragment);
 
         request.processed_row_count += result.row_count;
         request.begin_pos = result.row_offsets.back() - request.file_offset;
 
         update_delete_buffer(
             request, result, file_scan_param, start_position_in_fragment);
       }
 
     } catch (...) {
       // Re-add request to pool so we dont block any other threads
       {
         std::lock_guard<std::mutex> pending_requests_lock(
             multi_threading_params.pending_requests_mutex);
         multi_threading_params.continue_processing = false;
       }
       add_request_to_pool(multi_threading_params, request);
       throw;
     }
     add_request_to_pool(multi_threading_params, request);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::populate_chunks_using_data_blocks	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		int	fragment_id,
		const ParseBufferRequest &	request,
		ParseBufferResult &	result,
		std::map< int, const ColumnDescriptor * > &	column_by_id,
		std::map< int, FileRegions > &	fragment_id_to_file_regions_map,
		const foreign_storage::IterativeFileScanParameters &	file_scan_param,
		const size_t	expected_current_element_count
	)

Definition at line 707 of file AbstractTextFileDataWrapper.cpp.

References add_file_region(), append_data_block_to_chunk(), threading_serial::async(), CHECK_EQ, CHECK_GT, foreign_storage::MetadataScanMultiThreadingParams::chunk_encoder_buffers_mutex, foreign_storage::ParseBufferResult::column_id_to_data_blocks_map, foreign_storage::ParseBufferRequest::first_row_index, foreign_storage::IterativeFileScanParameters::fragment_id, foreign_storage::ParseBufferRequest::getFilePath(), foreign_storage::ParseBufferRequest::import_buffers, kENCODING_DICT, partition_data_blocks(), and foreign_storage::ParseBufferResult::row_count.

Referenced by populate_chunks().

                                                  {
   std::unique_lock<std::mutex> lock(multi_threading_params.chunk_encoder_buffers_mutex);
   // File regions should be added in same order as appendData
   add_file_region(fragment_id_to_file_regions_map,
                   fragment_id,
                   request.first_row_index,
                   result,
                   request.getFilePath());
   CHECK_EQ(fragment_id, file_scan_param.fragment_id);
 
   // start string encoding asynchronously
   std::vector<std::pair<const size_t, std::future<int8_t*>>>
       encoded_data_block_ptrs_futures;
 
   for (const auto& import_buffer : request.import_buffers) {
     if (import_buffer == nullptr) {
       continue;
     }
 
     if (import_buffer->getTypeInfo().is_dict_encoded_string()) {
       auto string_payload_ptr = import_buffer->getStringBuffer();
       CHECK_EQ(kENCODING_DICT, import_buffer->getTypeInfo().get_compression());
 
       auto column_id = import_buffer->getColumnDesc()->columnId;
       encoded_data_block_ptrs_futures.emplace_back(std::make_pair(
           column_id, std::async(std::launch::async, [&import_buffer, string_payload_ptr] {
             import_buffer->addDictEncodedString(*string_payload_ptr);
             return import_buffer->getStringDictBuffer();
           })));
     }
   }
 
   auto process_subset_of_data_blocks =
       [&](const std::map<int, DataBlockPtr>& data_blocks) {
         for (auto& [column_id, data_block] : data_blocks) {
           const auto column = column_by_id[column_id];
           lock.unlock();  // unlock the fragment based lock in order to achieve better
           // performance
           append_data_block_to_chunk(file_scan_param,
                                      data_block,
                                      result.row_count,
                                      column_id,
                                      column,
                                      expected_current_element_count);
           lock.lock();
         }
       };
 
   auto [dict_encoded_data_blocks, none_dict_encoded_data_blocks] =
       partition_data_blocks(column_by_id, result.column_id_to_data_blocks_map);
 
   process_subset_of_data_blocks(
       none_dict_encoded_data_blocks);  // skip dict string columns
 
   // wait for the async requests we made for string dictionary
   for (auto& encoded_ptr_future : encoded_data_block_ptrs_futures) {
     encoded_ptr_future.second.wait();
   }
   for (auto& encoded_ptr_future : encoded_data_block_ptrs_futures) {
     CHECK_GT(dict_encoded_data_blocks.count(encoded_ptr_future.first), 0UL);
     dict_encoded_data_blocks[encoded_ptr_future.first].numbersPtr =
         encoded_ptr_future.second.get();
   }
 
   process_subset_of_data_blocks(
       dict_encoded_data_blocks);  // process only dict string columns
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::populate_string_dictionary	(	int32_t	table_id,
		int32_t	col_id,
		int32_t	db_id
	)

Definition at line 205 of file Execute.cpp.

References CHECK, Data_Namespace::CPU_LEVEL, Catalog_Namespace::SysCatalog::getCatalog(), Chunk_NS::Chunk::getChunk(), Catalog_Namespace::SysCatalog::instance(), anonymous_namespace{Execute.cpp}::is_empty_table(), and key_does_not_shard_to_leaf().

Referenced by anonymous_namespace{RelAlgExecutor.cpp}::prepare_string_dictionaries().

                                                                                  {
   const auto catalog = Catalog_Namespace::SysCatalog::instance().getCatalog(db_id);
   CHECK(catalog);
   if (const auto foreign_table = dynamic_cast<const ForeignTable*>(
           catalog->getMetadataForTable(table_id, false))) {
     const auto col_desc = catalog->getMetadataForColumn(table_id, col_id);
     if (col_desc->columnType.is_dict_encoded_type()) {
       auto& fragmenter = foreign_table->fragmenter;
       CHECK(fragmenter != nullptr);
       if (is_empty_table(fragmenter.get())) {
         return;
       }
       for (const auto& frag : fragmenter->getFragmentsForQuery().fragments) {
         ChunkKey chunk_key = {db_id, table_id, col_id, frag.fragmentId};
         // If the key is sharded across leaves, only populate fragments that are sharded
         // to this leaf.
         if (key_does_not_shard_to_leaf(chunk_key)) {
           continue;
         }
 
         const ChunkMetadataMap& metadata_map = frag.getChunkMetadataMap();
         CHECK(metadata_map.find(col_id) != metadata_map.end());
         if (auto& meta = metadata_map.at(col_id); meta->isPlaceholder()) {
           // When this goes out of scope it will stay in CPU cache but become
           // evictable
           auto chunk = Chunk_NS::Chunk::getChunk(col_desc,
                                                  &(catalog->getDataMgr()),
                                                  chunk_key,
                                                  Data_Namespace::CPU_LEVEL,
                                                  0,
                                                  0,
                                                  0);
         }
       }
     }
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::process_data_blocks	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		int	fragment_id,
		const ParseBufferRequest &	request,
		ParseBufferResult &	result,
		std::map< int, const ColumnDescriptor * > &	column_by_id,
		std::map< int, FileRegions > &	fragment_id_to_file_regions_map
	)

Updates metadata encapsulated in encoders for all table columns given new data blocks gotten from parsing a new set of rows in a file buffer. If cache is available, also append the data_blocks to chunks in the cache

Definition at line 788 of file AbstractTextFileDataWrapper.cpp.

References add_file_region(), cache_blocks(), foreign_storage::MetadataScanMultiThreadingParams::cached_chunks, foreign_storage::MetadataScanMultiThreadingParams::chunk_encoder_buffers, foreign_storage::MetadataScanMultiThreadingParams::chunk_encoder_buffers_mutex, foreign_storage::ParseBufferResult::column_id_to_data_blocks_map, foreign_storage::ParseBufferRequest::db_id, foreign_storage::MetadataScanMultiThreadingParams::disable_cache, foreign_storage::ParseBufferRequest::first_row_index, foreign_storage::ParseBufferRequest::getFilePath(), foreign_storage::ParseBufferRequest::getTableId(), foreign_storage::ParseBufferResult::row_count, and update_stats().

Referenced by scan_metadata().

                                                                                     {
   std::lock_guard<std::mutex> lock(multi_threading_params.chunk_encoder_buffers_mutex);
   // File regions should be added in same order as appendData
   add_file_region(fragment_id_to_file_regions_map,
                   fragment_id,
                   request.first_row_index,
                   result,
                   request.getFilePath());
 
   for (auto& [column_id, data_block] : result.column_id_to_data_blocks_map) {
     ChunkKey chunk_key{request.db_id, request.getTableId(), column_id, fragment_id};
     const auto column = column_by_id[column_id];
     if (column->columnType.is_varlen_indeed()) {
       chunk_key.emplace_back(1);
     }
     if (multi_threading_params.chunk_encoder_buffers.find(chunk_key) ==
         multi_threading_params.chunk_encoder_buffers.end()) {
       multi_threading_params.chunk_encoder_buffers[chunk_key] =
           std::make_unique<ForeignStorageBuffer>();
       multi_threading_params.chunk_encoder_buffers[chunk_key]->initEncoder(
           column->columnType);
     }
     update_stats(multi_threading_params.chunk_encoder_buffers[chunk_key]->getEncoder(),
                  column->columnType,
                  data_block,
                  result.row_count);
     size_t num_elements = multi_threading_params.chunk_encoder_buffers[chunk_key]
                               ->getEncoder()
                               ->getNumElems() +
                           result.row_count;
     multi_threading_params.chunk_encoder_buffers[chunk_key]->getEncoder()->setNumElems(
         num_elements);
     cache_blocks(
         multi_threading_params.cached_chunks,
         data_block,
         result.row_count,
         chunk_key,
         column,
         (num_elements - result.row_count) == 0,  // Is the first block added to this chunk
         multi_threading_params.disable_cache);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::refresh_foreign_table	(	Catalog_Namespace::Catalog &	catalog,
		const std::string &	table_name,
		const bool	evict_cached_entries
	)

Definition at line 103 of file ForeignTableRefresh.cpp.

References CHECK, shared::contains(), StorageType::FOREIGN_TABLE, dist::is_leaf_node(), Catalog_Namespace::kAggregatorOnlySystemTables, and refresh_foreign_table_unlocked().

Referenced by RefreshForeignTablesCommand::execute(), and foreign_storage::ForeignTableRefreshScheduler::start().

                                                             {
   if (dist::is_leaf_node() &&
       shared::contains(Catalog_Namespace::kAggregatorOnlySystemTables, table_name)) {
     // Skip aggregator only system tables on leaf nodes.
     return;
   }
   auto table_lock =
       std::make_unique<lockmgr::TableSchemaLockContainer<lockmgr::WriteLock>>(
           lockmgr::TableSchemaLockContainer<lockmgr::WriteLock>::acquireTableDescriptor(
               catalog, table_name, false));
 
   const TableDescriptor* td = (*table_lock)();
   if (td->storageType != StorageType::FOREIGN_TABLE) {
     throw std::runtime_error{
         table_name +
         " is not a foreign table. Refreshes are applicable to only foreign tables."};
   }
 
   auto foreign_table = dynamic_cast<const ForeignTable*>(td);
   CHECK(foreign_table);
   refresh_foreign_table_unlocked(catalog, *foreign_table, evict_cached_entries);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::refresh_foreign_table_unlocked	(	Catalog_Namespace::Catalog &	catalog,
		const ForeignTable &	td,
		const bool	evict_cached_entries
	)

Definition at line 35 of file ForeignTableRefresh.cpp.

References CHECK, CHUNK_KEY_DB_IDX, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, foreign_storage::anonymous_namespace{ForeignTableRefresh.cpp}::clear_cpu_and_gpu_cache(), Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::Catalog::getCurrentDB(), Catalog_Namespace::Catalog::getDataMgr(), PostEvictionRefreshException::getOriginalException(), logger::INFO, CacheInvalidator< CACHE_HOLDING_TYPES >::invalidateCachesByTable(), foreign_storage::ForeignTable::isAppendMode(), LOG, Catalog_Namespace::Catalog::removeFragmenterForTable(), TableDescriptor::tableId, TableDescriptor::tableName, and Catalog_Namespace::Catalog::updateForeignTableRefreshTimes().

Referenced by refresh_foreign_table().

                                                                      {
   LOG(INFO) << "Starting refresh for table: " << td.tableName;
   auto& data_mgr = catalog.getDataMgr();
   ChunkKey table_key{catalog.getCurrentDB().dbId, td.tableId};
   ResultSetCacheInvalidator::invalidateCachesByTable(boost::hash_value(table_key));
   catalog.removeFragmenterForTable(td.tableId);
 
   auto fsm = data_mgr.getPersistentStorageMgr()->getForeignStorageMgr();
   CHECK(fsm);
   if (auto cfm = dynamic_cast<CachingForeignStorageMgr*>(fsm)) {
     if (!cfm->hasStoredDataWrapper(table_key[CHUNK_KEY_DB_IDX],
                                    table_key[CHUNK_KEY_TABLE_IDX])) {
       // If there is no wrapper stored on disk, then we have not populated the metadata
       // for this table and we are free to skip the refresh.
       catalog.updateForeignTableRefreshTimes(td.tableId);
       return;
     }
   }
 
   std::map<ChunkKey, std::shared_ptr<ChunkMetadata>> old_chunk_metadata_by_chunk_key;
   if (td.isAppendMode() && !evict_cached_entries) {
     ChunkMetadataVector metadata_vec;
     data_mgr.getChunkMetadataVecForKeyPrefix(metadata_vec, table_key);
     int last_fragment_id = 0;
     for (const auto& [key, metadata] : metadata_vec) {
       if (key[CHUNK_KEY_FRAGMENT_IDX] > last_fragment_id) {
         last_fragment_id = key[CHUNK_KEY_FRAGMENT_IDX];
       }
       old_chunk_metadata_by_chunk_key[key] = metadata;
     }
     for (const auto& [key, metadata] : metadata_vec) {
       if (key[CHUNK_KEY_FRAGMENT_IDX] == last_fragment_id) {
         clear_cpu_and_gpu_cache(data_mgr, key);
       }
     }
   } else {
     clear_cpu_and_gpu_cache(data_mgr, table_key);
   }
 
   try {
     fsm->refreshTable(table_key, evict_cached_entries);
     catalog.updateForeignTableRefreshTimes(td.tableId);
   } catch (PostEvictionRefreshException& e) {
     catalog.updateForeignTableRefreshTimes(td.tableId);
     clear_cpu_and_gpu_cache(data_mgr, table_key);
     throw e.getOriginalException();
   } catch (...) {
     clear_cpu_and_gpu_cache(data_mgr, table_key);
     throw;
   }
 
   // Delete cached rolled off/updated chunks.
   if (!old_chunk_metadata_by_chunk_key.empty()) {
     ChunkMetadataVector new_metadata_vec;
     data_mgr.getChunkMetadataVecForKeyPrefix(new_metadata_vec, table_key);
     for (const auto& [key, metadata] : new_metadata_vec) {
       auto it = old_chunk_metadata_by_chunk_key.find(key);
       if (it != old_chunk_metadata_by_chunk_key.end() &&
           it->second->numElements != metadata->numElements) {
         clear_cpu_and_gpu_cache(data_mgr, key);
       }
     }
   }
   LOG(INFO) << "Completed refresh for table: " << td.tableName;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::request_pool_non_empty ( MetadataScanMultiThreadingParams & multi_threading_params )

Definition at line 922 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::request_pool, and foreign_storage::MetadataScanMultiThreadingParams::request_pool_mutex.

Referenced by dispatch_scan_requests().

                                                                                       {
   std::unique_lock<std::mutex> request_pool_lock(
       multi_threading_params.request_pool_mutex);
   return !multi_threading_params.request_pool.empty();
 }

Here is the caller graph for this function:

void foreign_storage::reset_multithreading_params ( foreign_storage::MetadataScanMultiThreadingParams & multi_threading_params )

Definition at line 1063 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::cached_chunks, foreign_storage::MetadataScanMultiThreadingParams::chunk_encoder_buffers, foreign_storage::MetadataScanMultiThreadingParams::deferred_requests, foreign_storage::MetadataScanMultiThreadingParams::pending_requests, and foreign_storage::MetadataScanMultiThreadingParams::request_pool.

Referenced by foreign_storage::AbstractTextFileDataWrapper::iterativeFileScan().

                                                                              {
   multi_threading_params.request_pool = {};
   multi_threading_params.cached_chunks = {};
   multi_threading_params.pending_requests = {};
   multi_threading_params.deferred_requests = {};
   multi_threading_params.chunk_encoder_buffers.clear();
 }

Here is the caller graph for this function:

void foreign_storage::resize_buffer_if_needed	(	std::unique_ptr< char[]> &	buffer,
		size_t &	buffer_size,
		const size_t	alloc_size
	)

Optionally resizes the given buffer if the buffer size is less than the current buffer allocation size.

Definition at line 1053 of file AbstractTextFileDataWrapper.cpp.

References CHECK_LE.

Referenced by dispatch_scan_requests().

                                                       {
   CHECK_LE(buffer_size, alloc_size);
   if (buffer_size < alloc_size) {
     buffer = std::make_unique<char[]>(alloc_size);
     buffer_size = alloc_size;
   }
 }

Here is the caller graph for this function:

std::vector< Aws::S3::Model::Object > foreign_storage::s3_objects_filter_sort_files	(	const std::vector< Aws::S3::Model::Object > &	file_paths,
		const shared::FilePathOptions &	options
	)

Definition at line 22 of file S3FilePathUtil.cpp.

References shared::FilePathOptions::filter_regex, shared::PATHNAME_ORDER_TYPE, foreign_storage::anonymous_namespace{S3FilePathUtil.cpp}::s3_objects_regex_file_filter(), and shared::FilePathOptions::sort_by.

                                           {
   auto result_files =
       options.filter_regex.has_value()
           ? s3_objects_regex_file_filter(options.filter_regex.value(), file_paths)
           : file_paths;
   // initial lexicographical order ensures a determinisitc ordering for files not matching
   // sort_regex
   shared::FilePathOptions temp_options;
   temp_options.sort_by = shared::PATHNAME_ORDER_TYPE;
   auto initial_file_order = FileOrderS3(temp_options);
   auto lexi_comp = initial_file_order.getFileComparator();
   std::stable_sort(result_files.begin(), result_files.end(), lexi_comp);
 
   auto file_order = FileOrderS3(options);
   auto comp = file_order.getFileComparator();
   std::stable_sort(result_files.begin(), result_files.end(), comp);
   return result_files;
 }

Here is the call graph for this function:

void foreign_storage::scan_metadata	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		std::map< int, FileRegions > &	fragment_id_to_file_regions_map,
		const TextFileBufferParser &	parser
	)

Consumes and processes metadata scan requests from a pending requests queue and updates existing metadata objects based on newly scanned metadata.

Definition at line 853 of file AbstractTextFileDataWrapper.cpp.

References add_request_to_pool(), foreign_storage::MetadataScanMultiThreadingParams::continue_processing, get_next_scan_request(), foreign_storage::TextFileBufferParser::parseBuffer(), partition_by_fragment(), foreign_storage::MetadataScanMultiThreadingParams::pending_requests_mutex, process_data_blocks(), and run_benchmark_import::result.

Referenced by foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata().

                                                        {
   std::map<int, const ColumnDescriptor*> column_by_id{};
   while (true) {
     auto request_opt = get_next_scan_request(multi_threading_params);
     if (!request_opt.has_value()) {
       break;
     }
     auto& request = request_opt.value();
     try {
       if (column_by_id.empty()) {
         for (const auto column : request.getColumns()) {
           column_by_id[column->columnId] = column;
         }
       }
       auto partitions = partition_by_fragment(
           request.first_row_index, request.getMaxFragRows(), request.buffer_row_count);
       request.begin_pos = 0;
       size_t row_index = request.first_row_index;
       for (const auto partition : partitions) {
         request.process_row_count = partition;
         for (const auto& import_buffer : request.import_buffers) {
           if (import_buffer != nullptr) {
             import_buffer->clear();
           }
         }
         auto result = parser.parseBuffer(request, true);
         int fragment_id = row_index / request.getMaxFragRows();
         process_data_blocks(multi_threading_params,
                             fragment_id,
                             request,
                             result,
                             column_by_id,
                             fragment_id_to_file_regions_map);
         row_index += result.row_count;
         request.begin_pos = result.row_offsets.back() - request.file_offset;
       }
     } catch (...) {
       // Re-add request to pool so we dont block any other threads
       {
         std::lock_guard<std::mutex> pending_requests_lock(
             multi_threading_params.pending_requests_mutex);
         multi_threading_params.continue_processing = false;
       }
       add_request_to_pool(multi_threading_params, request);
       throw;
     }
     add_request_to_pool(multi_threading_params, request);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::set_comp	(	const ChunkKey &	left,
		const ChunkKey &	right
	)

void foreign_storage::set_node_name ( std::map< std::string, import_export::TypedImportBuffer * > & import_buffers )

Definition at line 55 of file InternalSystemDataWrapper.cpp.

References g_distributed_leaf_idx, dist::is_leaf_node(), and to_string().

Referenced by foreign_storage::anonymous_namespace{InternalMemoryStatsDataWrapper.cpp}::populate_import_buffers_for_memory_details(), foreign_storage::anonymous_namespace{InternalMemoryStatsDataWrapper.cpp}::populate_import_buffers_for_memory_summary(), and foreign_storage::anonymous_namespace{InternalStorageStatsDataWrapper.cpp}::populate_import_buffers_for_storage_details().

                                                                         {
   if (import_buffers.find("node") != import_buffers.end()) {
     if (dist::is_leaf_node()) {
       std::string leaf_string{"Leaf " + to_string(g_distributed_leaf_idx)};
       import_buffers["node"]->addString(leaf_string);
     } else {
       import_buffers["node"]->addString("Server");
     }
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::set_value	(	rapidjson::Value &	json_val,
		const FileRegion &	file_region,
		rapidjson::Document::AllocatorType &	allocator
	)

Definition at line 26 of file CsvShared.cpp.

References foreign_storage::json_utils::add_value_to_object(), foreign_storage::FileRegion::first_row_file_offset, foreign_storage::FileRegion::first_row_index, foreign_storage::FileRegion::region_size, and foreign_storage::FileRegion::row_count.

                                                           {
   json_val.SetObject();
   json_utils::add_value_to_object(
       json_val, file_region.first_row_file_offset, "first_row_file_offset", allocator);
   json_utils::add_value_to_object(
       json_val, file_region.first_row_index, "first_row_index", allocator);
   json_utils::add_value_to_object(
       json_val, file_region.region_size, "region_size", allocator);
   json_utils::add_value_to_object(
       json_val, file_region.row_count, "row_count", allocator);
   if (file_region.filename.size()) {
     json_utils::add_value_to_object(
         json_val, file_region.filename, "filename", allocator);
   }
 }

Here is the call graph for this function:

void foreign_storage::set_value	(	rapidjson::Value &	json_val,
		const RowGroupInterval &	value,
		rapidjson::Document::AllocatorType &	allocator
	)

Definition at line 662 of file ParquetDataWrapper.cpp.

References foreign_storage::json_utils::add_value_to_object(), foreign_storage::RowGroupInterval::end_index, foreign_storage::RowGroupInterval::file_path, and foreign_storage::RowGroupInterval::start_index.

                                                           {
   json_val.SetObject();
   json_utils::add_value_to_object(json_val, value.file_path, "file_path", allocator);
   json_utils::add_value_to_object(json_val, value.start_index, "start_index", allocator);
   json_utils::add_value_to_object(json_val, value.end_index, "end_index", allocator);
 }

Here is the call graph for this function:

void foreign_storage::throw_file_access_error	(	const std::string &	file_path,
		const std::string &	message
	)

inline

Definition at line 89 of file ForeignStorageException.h.

Referenced by foreign_storage::ParquetImporter::getAllFilePaths().

                                                               {
   std::string error_message{"Unable to access file \"" + file_path + "\". " + message};
   throw ForeignStorageException{error_message};
 }

Here is the caller graph for this function:

void foreign_storage::throw_file_not_found_error ( const std::string & file_path )

inline

Definition at line 95 of file ForeignStorageException.h.

Referenced by foreign_storage::ParquetImporter::getAllFilePaths().

                                                                    {
   throw ForeignStorageException{"File or directory \"" + file_path +
                                 "\" does not exist."};
 }

Here is the caller graph for this function:

void foreign_storage::throw_number_of_columns_mismatch_error	(	size_t	num_table_cols,
		size_t	num_file_cols,
		const std::string &	file_path
	)

inline

Definition at line 80 of file ForeignStorageException.h.

References to_string().

Referenced by foreign_storage::RegexFileBufferParser::regexMatchColumns(), foreign_storage::anonymous_namespace{CsvFileBufferParser.cpp}::validate_expected_column_count(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_number_of_columns().

                                                                                {
   throw ForeignStorageException{"Mismatched number of logical columns: (expected " +
                                 std::to_string(num_table_cols) + " columns, has " +
                                 std::to_string(num_file_cols) + "): in file '" +
                                 file_path + "'"};
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::throw_parquet_metadata_out_of_bounds_error	(	const std::string &	min_value,
		const std::string &	max_value,
		const std::string &	encountered_value
	)

inline

Definition at line 46 of file ParquetMetadataValidator.h.

Referenced by foreign_storage::TimestampBoundsValidator< T >::validateValue(), foreign_storage::IntegralFixedLengthBoundsValidator< T >::validateValue(), foreign_storage::BaseDateBoundsValidator< T, is_in_seconds >::validateValue(), and foreign_storage::FloatPointValidator< T >::validateValue().

                                         {
   std::stringstream error_message;
   error_message << "Parquet column contains values that are outside the range of the "
                    "HeavyDB column "
                    "type. Consider using a wider column type. Min allowed value: "
                 << min_value << ". Max allowed value: " << max_value
                 << ". Encountered value: " << encountered_value << ".";
   throw std::runtime_error(error_message.str());
 }

Here is the caller graph for this function:

void foreign_storage::throw_removed_file_error ( const std::string & file_path )

inline

Definition at line 73 of file ForeignStorageException.h.

Referenced by foreign_storage::LocalMultiFileReader::checkForMoreRows(), and foreign_storage::ParquetDataWrapper::fetchChunkMetadata().

                                                                  {
   throw ForeignStorageException{
       "Refresh of foreign table created with \"APPEND\" update type failed as "
       "file \"" +
       file_path + "\" was removed."};
 }

Here is the caller graph for this function:

void foreign_storage::throw_removed_row_in_file_error ( const std::string & file_path )

inline

Definition at line 66 of file ForeignStorageException.h.

Referenced by foreign_storage::SingleTextFileReader::checkForMoreRows(), and foreign_storage::ParquetDataWrapper::fetchChunkMetadata().

                                                                         {
   throw ForeignStorageException{
       "Refresh of foreign table created with \"APPEND\" update type failed as file "
       "reduced in size: \"" +
       file_path + "\""};
 }

Here is the caller graph for this function:

void foreign_storage::throw_removed_row_in_result_set_error ( const std::string & select_statement )

inline

Definition at line 58 of file ForeignStorageException.h.

                                                                                      {
   throw ForeignStorageException{
       "Refresh of foreign table created with \"APPEND\" update type failed as result set "
       "of select statement "
       "reduced in size: \"" +
       select_statement + "\""};
 }

void foreign_storage::throw_s3_compressed_extension	(	const std::string &	file_path,
		const std::string &	ext_type
	)

inline

Definition at line 107 of file ForeignStorageException.h.

                                                                      {
   throw ForeignStorageException{
       "File \"" + file_path + "\" has extension type \"" + ext_type +
       "\", compressed file formats are not supported by S3 Foreign Tables."};
 }

void foreign_storage::throw_s3_compressed_mime_type	(	const std::string &	file_path,
		const std::string &	mime_type
	)

inline

Definition at line 100 of file ForeignStorageException.h.

                                                                       {
   throw ForeignStorageException{
       "File \"" + file_path + "\" has mime type \"" + mime_type +
       "\", compressed file formats are not supported by S3 Foreign Tables."};
 }

void foreign_storage::throw_unexpected_number_of_items	(	const size_t &	num_expected,
		const size_t &	num_loaded,
		const std::string &	item_type
	)

inline

Definition at line 40 of file ForeignStorageException.h.

References to_string().

Referenced by parse_file_regions(), and foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::throw_unexpected_number_of_items().

                                                                          {
   throw ForeignStorageException(
       "Unexpected number of " + item_type +
       " while loading from foreign data source: expected " +
       std::to_string(num_expected) + " , obtained " + std::to_string(num_loaded) + " " +
       item_type +
       ". Please use the \"REFRESH FOREIGN TABLES\" command on the foreign table "
       "if data source has been updated.");
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::update_delete_buffer	(	const ParseBufferRequest &	request,
		const ParseBufferResult &	result,
		const foreign_storage::IterativeFileScanParameters &	file_scan_param,
		const size_t	start_position_in_fragment
	)

Definition at line 685 of file AbstractTextFileDataWrapper.cpp.

References CHECK, foreign_storage::IterativeFileScanParameters::delete_buffer, foreign_storage::IterativeFileScanParameters::delete_buffer_mutex, foreign_storage::ParseBufferRequest::processed_row_count, foreign_storage::ParseBufferResult::rejected_rows, and foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::resize_delete_buffer().

Referenced by populate_chunks().

                                              {
   if (file_scan_param.delete_buffer) {
     std::unique_lock delete_buffer_lock(file_scan_param.delete_buffer_mutex);
     auto& delete_buffer = file_scan_param.delete_buffer;
     auto chunk_offset = start_position_in_fragment;
     auto chunk_element_count = chunk_offset + request.processed_row_count;
 
     // ensure delete buffer is sized appropriately
     resize_delete_buffer(delete_buffer, chunk_element_count);
 
     auto delete_buffer_data = delete_buffer->getMemoryPtr();
     for (const auto rejected_row_index : result.rejected_rows) {
       CHECK(rejected_row_index + chunk_offset < delete_buffer->size());
       delete_buffer_data[rejected_row_index + chunk_offset] = true;
     }
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::update_stats	(	Encoder *	encoder,
		const SQLTypeInfo &	column_type,
		DataBlockPtr	data_block,
		const size_t	row_count
	)

Updates the statistics metadata encapsulated in the encoder given new data in a data block.

Definition at line 570 of file AbstractTextFileDataWrapper.cpp.

References DataBlockPtr::arraysPtr, SQLTypeInfo::is_array(), SQLTypeInfo::is_varlen(), DataBlockPtr::numbersPtr, DataBlockPtr::stringsPtr, and Encoder::updateStats().

Referenced by process_data_blocks(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumnMetadata(), and StorageIOFacility::yieldUpdateCallback().

                                           {
   if (column_type.is_array()) {
     encoder->updateStats(data_block.arraysPtr, 0, row_count);
   } else if (!column_type.is_varlen()) {
     encoder->updateStats(data_block.numbersPtr, row_count);
   } else {
     encoder->updateStats(data_block.stringsPtr, 0, row_count);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::shared_ptr< parquet::Statistics > foreign_storage::validate_and_get_column_metadata_statistics ( const parquet::ColumnChunkMetaData * column_metadata )

Definition at line 86 of file ParquetShared.cpp.

References CHECK.

Referenced by foreign_storage::ParquetEncoder::getRowGroupMetadata(), foreign_storage::TypedParquetInPlaceEncoder< V, V >::getRowGroupMetadata(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_definition_levels().

                                                        {
   CHECK(column_metadata->is_stats_set());
   std::shared_ptr<parquet::Statistics> stats = column_metadata->statistics();
   return stats;
 }

Here is the caller graph for this function:

void foreign_storage::validate_equal_column_descriptor	(	const parquet::ColumnDescriptor *	reference_descriptor,
		const parquet::ColumnDescriptor *	new_descriptor,
		const std::string &	reference_file_path,
		const std::string &	new_file_path
	)

Definition at line 60 of file ParquetShared.cpp.

Referenced by foreign_storage::LazyParquetChunkLoader::appendRowGroups(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_equal_schema().

                                     {
   if (!reference_descriptor->Equals(*new_descriptor)) {
     throw std::runtime_error{"Parquet file \"" + new_file_path +
                              "\" has a different schema. Please ensure that all Parquet "
                              "files use the same schema. Reference Parquet file: " +
                              reference_file_path +
                              ", column name: " + reference_descriptor->name() +
                              ". New Parquet file: " + new_file_path +
                              ", column name: " + new_descriptor->name() + "."};
   }
 }

Here is the caller graph for this function:

void foreign_storage::validate_non_foreign_table_write ( const TableDescriptor * table_descriptor )

inline

Definition at line 22 of file FsiUtils.h.

References StorageType::FOREIGN_TABLE, and TableDescriptor::storageType.

Referenced by Parser::InsertStmt::analyze(), Parser::TruncateTableStmt::execute(), Parser::InsertValuesStmt::execute(), Parser::InsertIntoTableAsSelectStmt::populateData(), and RelModify::RelModify().

                                                                                       {
   if (table_descriptor && table_descriptor->storageType == StorageType::FOREIGN_TABLE) {
     throw std::runtime_error{
         "DELETE, INSERT, TRUNCATE, OR UPDATE commands are not supported for foreign "
         "tables."};
   }
 }

Here is the caller graph for this function:

void foreign_storage::validate_regex_parser_options ( const import_export::CopyParams & copy_params )

Definition at line 114 of file ForeignDataWrapperFactory.cpp.

References import_export::CopyParams::line_regex.

Referenced by anonymous_namespace{ForeignDataImporter.cpp}::validate_copy_params().

                                                                                {
   if (copy_params.line_regex.empty()) {
     throw std::runtime_error{"Regex parser options must contain a line regex."};
   }
 }

Here is the caller graph for this function:

Variable Documentation

constexpr const char* foreign_storage::kDeletedValueIndicator {"<DELETED>"}

Definition at line 26 of file InternalSystemDataWrapper.h.

Referenced by foreign_storage::anonymous_namespace{InternalMemoryStatsDataWrapper.cpp}::get_column_name(), get_db_name(), foreign_storage::anonymous_namespace{InternalCatalogDataWrapper.cpp}::get_table_ddl(), get_table_name(), and foreign_storage::anonymous_namespace{InternalCatalogDataWrapper.cpp}::get_user_name().

Namespaces

Classes

Typedefs

Functions

Variables

Typedef Documentation

Function Documentation

Variable Documentation