Namespaces
	anonymous_namespace{AbstractFileStorageDataWrapper.cpp}

	anonymous_namespace{AbstractTextFileDataWrapper.cpp}

	anonymous_namespace{CachingForeignStorageMgr.cpp}

	anonymous_namespace{CsvFileBufferParser.cpp}

	anonymous_namespace{FileReader.cpp}

	anonymous_namespace{ForeignDataWrapperFactory.cpp}

	anonymous_namespace{ForeignStorageCache.cpp}

	anonymous_namespace{ForeignTableRefresh.cpp}

	anonymous_namespace{InternalCatalogDataWrapper.cpp}

	anonymous_namespace{InternalMemoryStatsDataWrapper.cpp}

	anonymous_namespace{InternalStorageStatsDataWrapper.cpp}

	anonymous_namespace{InternalSystemDataWrapper.cpp}

	anonymous_namespace{LazyParquetChunkLoader.cpp}

	anonymous_namespace{LogFileBufferParser.cpp}

	anonymous_namespace{ParquetDataWrapper.cpp}

	anonymous_namespace{RefreshTimeCalculator.cpp}

	anonymous_namespace{RegexFileBufferParser.cpp}

	anonymous_namespace{RegexParserDataWrapper.cpp}

	anonymous_namespace{S3FilePathUtil.cpp}

	anonymous_namespace{TextFileBufferParser.cpp}

	Csv

	json_utils

Classes
struct	ForeignServer

struct	ForeignTable

struct	OptionsContainer

struct	UserMappingType

struct	UserMapping

class	RefreshTimeCalculator

class	AbstractFileStorageDataWrapper

struct	ParseFileRegionResult

struct	MetadataScanMultiThreadingParams

struct	IterativeFileScanParameters

class	AbstractTextFileDataWrapper

class	CachingForeignStorageMgr

class	CsvDataWrapper

class	CsvFileBufferParser

struct	DataPreview

class	FileReader

class	SingleFileReader

class	SingleTextFileReader

class	ArchiveWrapper

class	CompressedFileReader

class	MultiFileReader

class	LocalMultiFileReader

struct	FileRegion

class	ForeignDataWrapper

struct	DataWrapperType
	Encapsulates an enumeration of foreign data wrapper type strings. More...

class	ForeignDataWrapperFactory

class	ForeignStorageBuffer

class	ForeignStorageCache

class	ForeignStorageException

class	MetadataScanInfeasibleFragmentSizeException

class	RequestedFragmentIdOutOfBoundsException

class	ChunkSizeValidator

class	MockForeignDataWrapper

class	ForeignStorageMgr

class	ForeignTableSchema

class	GeospatialEncoder

class	InternalCatalogDataWrapper

class	InternalLogsDataWrapper

class	InternalMemoryStatsDataWrapper

struct	StorageDetails

class	InternalStorageStatsDataWrapper

class	InternalSystemDataWrapper

struct	ColumnType

struct	FragmentType

struct	Interval

struct	ParquetBatchData

class	ParquetRowGroupReader

struct	PreviewContext

class	LazyParquetChunkLoader

class	LogFileBufferParser

class	OdbcGeospatialEncoder

class	ParquetArrayDetectEncoder

class	ParquetArrayEncoder

class	ParquetArrayImportEncoder

class	ParquetDataWrapper

class	ParquetDateInDaysFromTimestampEncoder

class	ParquetDateInSecondsEncoder

class	ParquetDecimalEncoder

class	ParquetDetectStringEncoder

class	ParquetEncoder

class	ParquetImportEncoder

class	ParquetScalarEncoder

class	ParquetFixedLengthArrayEncoder

class	ParquetFixedLengthEncoder

class	ParquetUnsignedFixedLengthEncoder

class	ParquetGeospatialEncoder

class	ParquetGeospatialImportEncoder

class	RowGroupIntervalTracker

class	ParquetImportBatchResult

class	AbstractRowGroupIntervalTracker

class	ParquetImporter

class	ParquetInPlaceEncoder

class	TypedParquetInPlaceEncoder

class	ParquetMetadataValidator

class	TimestampBoundsValidator

class	IntegralFixedLengthBoundsValidator

class	BaseDateBoundsValidator

class	FloatPointValidator

struct	RowGroupInterval

struct	RowGroupMetadata

class	FileReaderMap

class	ParquetStringEncoder

class	ParquetStringImportEncoder

class	ParquetStringNoneEncoder

class	ParquetTimeEncoder

class	ParquetTimestampEncoder

class	ParquetVariableLengthArrayEncoder

class	PassThroughBuffer

class	RegexFileBufferParser

class	RegexParserDataWrapper

class	FileOrderS3

struct	ParseBufferRequest

struct	ParseBufferResult

class	TextFileBufferParser

class	TypedParquetDetectBuffer

class	TypedParquetStorageBuffer

class	ForeignTableRefreshScheduler

Typedefs
using	OptionsMap = std::map< std::string, std::string, std::less<>>

using	SampleRows = std::vector< std::vector< std::string >>

using	FirstLineByFilePath = std::map< std::string, std::string >

using	FileRegions = std::vector< FileRegion >

using	ChunkToBufferMap = std::map< ChunkKey, AbstractBuffer * >

using	RenderGroupAnalyzerMap = std::map< int, std::unique_ptr< import_export::RenderGroupAnalyzer >>

using	read_lock = heavyai::shared_lock< heavyai::shared_mutex >

using	write_lock = heavyai::unique_lock< heavyai::shared_mutex >

using	FilePathAndRowGroup = std::pair< std::string, int32_t >

using	RejectedRowIndices = std::set< int64_t >

using	InvalidRowGroupIndices = std::set< int64_t >

template<typename T >
using	DateInSecondsBoundsValidator = BaseDateBoundsValidator< T, true >

template<typename T >
using	DateInDaysBoundsValidator = BaseDateBoundsValidator< T, false >

using	UniqueReaderPtr = std::unique_ptr< parquet::arrow::FileReader >

using	ReaderPtr = parquet::arrow::FileReader *

template<typename V , typename T , T conversion_denominator, typename NullType = V>
using	ParquetDateInSecondsFromTimestampEncoder = ParquetTimestampEncoder< V, T, conversion_denominator, NullType >

using	S3ObjectComparator = std::function< bool(const Aws::S3::Model::Object &, const Aws::S3::Model::Object &)>

Functions
size_t	get_num_threads (const ForeignTable &table)

ParseFileRegionResult	parse_file_regions (const FileRegions &file_regions, const size_t start_index, const size_t end_index, FileReader &file_reader, ParseBufferRequest &parse_file_request, const std::map< int, Chunk_NS::Chunk > &column_id_to_chunk_map, const TextFileBufferParser &parser)

size_t	num_rows_to_process (const size_t start_row_index, const size_t max_fragment_size, const size_t rows_remaining)

std::vector< size_t >	partition_by_fragment (const size_t start_row_index, const size_t max_fragment_size, const size_t buffer_row_count)

std::optional< ParseBufferRequest >	get_next_scan_request (MetadataScanMultiThreadingParams &multi_threading_params)

void	add_file_region (std::map< int, FileRegions > &fragment_id_to_file_regions_map, int fragment_id, size_t first_row_index, const ParseBufferResult &result, const std::string &file_path)

void	update_stats (Encoder *encoder, const SQLTypeInfo &column_type, DataBlockPtr data_block, const size_t row_count)

void	cache_blocks (std::map< ChunkKey, Chunk_NS::Chunk > &cached_chunks, DataBlockPtr data_block, size_t row_count, ChunkKey &chunk_key, const ColumnDescriptor *column, bool is_first_block, bool disable_cache)

void	append_data_block_to_chunk (const foreign_storage::IterativeFileScanParameters &file_scan_param, DataBlockPtr data_block, size_t row_count, const ChunkKey &chunk_key, const ColumnDescriptor *column, const std::set< size_t > &rejected_row_indices, const size_t element_count_required)

void	populate_chunks_using_data_blocks (MetadataScanMultiThreadingParams &multi_threading_params, int fragment_id, const ParseBufferRequest &request, ParseBufferResult &result, std::map< int, const ColumnDescriptor * > &column_by_id, std::map< int, FileRegions > &fragment_id_to_file_regions_map, const foreign_storage::IterativeFileScanParameters &file_scan_param)

void	process_data_blocks (MetadataScanMultiThreadingParams &multi_threading_params, int fragment_id, const ParseBufferRequest &request, ParseBufferResult &result, std::map< int, const ColumnDescriptor * > &column_by_id, std::map< int, FileRegions > &fragment_id_to_file_regions_map)

void	add_request_to_pool (MetadataScanMultiThreadingParams &multi_threading_params, ParseBufferRequest &request)

void	scan_metadata (MetadataScanMultiThreadingParams &multi_threading_params, std::map< int, FileRegions > &fragment_id_to_file_regions_map, const TextFileBufferParser &parser)

ParseBufferRequest	get_request_from_pool (MetadataScanMultiThreadingParams &multi_threading_params)

void	defer_scan_request (MetadataScanMultiThreadingParams &multi_threading_params, ParseBufferRequest &request)

void	dispatch_all_deferred_requests (MetadataScanMultiThreadingParams &multi_threading_params)

void	dispatch_scan_request (MetadataScanMultiThreadingParams &multi_threading_params, ParseBufferRequest &request)

void	populate_chunks (MetadataScanMultiThreadingParams &multi_threading_params, std::map< int, FileRegions > &fragment_id_to_file_regions_map, const TextFileBufferParser &parser, foreign_storage::IterativeFileScanParameters &file_scan_param)

void	resize_buffer_if_needed (std::unique_ptr< char[]> &buffer, size_t &buffer_size, const size_t alloc_size)

void	reset_multithreading_params (foreign_storage::MetadataScanMultiThreadingParams &multi_threading_params)

void	dispatch_scan_requests (const foreign_storage::ForeignTable table, const size_t &buffer_size, const std::string &file_path, FileReader &file_reader, const import_export::CopyParams &copy_params, MetadataScanMultiThreadingParams &multi_threading_params, size_t &first_row_index_in_buffer, size_t &current_file_offset, const TextFileBufferParser &parser, const foreign_storage::IterativeFileScanParameters file_scan_param, foreign_storage::AbstractTextFileDataWrapper::ResidualBuffer &iterative_residual_buffer, const bool is_first_file_scan_call)

void	dispatch_scan_requests_with_exception_handling (const foreign_storage::ForeignTable table, const size_t &buffer_size, const std::string &file_path, FileReader &file_reader, const import_export::CopyParams &copy_params, MetadataScanMultiThreadingParams &multi_threading_params, size_t &first_row_index_in_buffer, size_t &current_file_offset, const TextFileBufferParser &parser, const foreign_storage::IterativeFileScanParameters file_scan_param, foreign_storage::AbstractTextFileDataWrapper::ResidualBuffer &iterative_residual_buffer, const bool is_first_file_scan_call)

void	set_value (rapidjson::Value &json_val, const FileRegion &file_region, rapidjson::Document::AllocatorType &allocator)

void	get_value (const rapidjson::Value &json_val, FileRegion &file_region)

std::optional< SQLTypes >	detect_geo_type (const SampleRows &sample_rows, size_t column_index)

bool	is_s3_uri (const std::string &file_path)

std::tuple< std::unique_ptr < foreign_storage::ForeignServer > , std::unique_ptr < foreign_storage::UserMapping > , std::unique_ptr < foreign_storage::ForeignTable > >	create_proxy_fsi_objects (const std::string &copy_from_source, const import_export::CopyParams &copy_params, const int db_id, const TableDescriptor *table, const int32_t user_id)
	Create proxy fsi objects for use outside FSI. More...

std::tuple< std::unique_ptr < foreign_storage::ForeignServer > , std::unique_ptr < foreign_storage::UserMapping > , std::unique_ptr < foreign_storage::ForeignTable > >	create_proxy_fsi_objects (const std::string &copy_from_source, const import_export::CopyParams &copy_params, const TableDescriptor *table)
	Create proxy fsi objects for use outside FSI NOTE: parameters mirror function above. More...

void	validate_regex_parser_options (const import_export::CopyParams &copy_params)

bool	is_valid_source_type (const import_export::CopyParams &copy_params)

std::string	bool_to_option_value (const bool value)

void	throw_unexpected_number_of_items (const size_t &num_expected, const size_t &num_loaded, const std::string &item_type)

void	throw_removed_row_in_result_set_error (const std::string &select_statement)

void	throw_removed_row_in_file_error (const std::string &file_path)

void	throw_removed_file_error (const std::string &file_path)

void	throw_number_of_columns_mismatch_error (size_t num_table_cols, size_t num_file_cols, const std::string &file_path)

void	throw_file_access_error (const std::string &file_path, const std::string &message)

void	throw_file_not_found_error (const std::string &file_path)

void	throw_s3_compressed_mime_type (const std::string &file_path, const std::string &mime_type)

void	throw_s3_compressed_extension (const std::string &file_path, const std::string &ext_type)

bool	is_append_table_chunk_key (const ChunkKey &chunk_key)

bool	set_comp (const ChunkKey &left, const ChunkKey &right)

std::vector< ChunkKey >	get_column_key_vec (const ChunkKey &destination_chunk_key)

std::set< ChunkKey >	get_column_key_set (const ChunkKey &destination_chunk_key)

size_t	get_max_chunk_size (const ChunkKey &key)

bool	contains_fragment_key (const std::set< ChunkKey > &key_set, const ChunkKey &target_key)

bool	is_table_enabled_on_node (const ChunkKey &key)

void	refresh_foreign_table_unlocked (Catalog_Namespace::Catalog &catalog, const ForeignTable &td, const bool evict_cached_entries)

void	refresh_foreign_table (Catalog_Namespace::Catalog &catalog, const std::string &table_name, const bool evict_cached_entries)

void	init_chunk_for_column (const ChunkKey &chunk_key, const std::map< ChunkKey, std::shared_ptr< ChunkMetadata >> &chunk_metadata_map, const std::map< ChunkKey, AbstractBuffer * > &buffers, Chunk_NS::Chunk &chunk)

std::shared_ptr< ChunkMetadata >	get_placeholder_metadata (const SQLTypeInfo &type, size_t num_elements)

const foreign_storage::ForeignTable &	get_foreign_table_for_key (const ChunkKey &key)

bool	is_system_table_chunk_key (const ChunkKey &chunk_key)

bool	is_replicated_table_chunk_key (const ChunkKey &chunk_key)

bool	is_shardable_key (const ChunkKey &key)

bool	fragment_maps_to_leaf (const ChunkKey &key)

bool	key_does_not_shard_to_leaf (const ChunkKey &key)

template<typename T >
auto	partition_for_threads (const std::set< T > &items, size_t max_threads)

template<typename T >
auto	partition_for_threads (const std::vector< T > &items, size_t max_threads)

template<typename Container >
std::vector< std::future< void > >	create_futures_for_workers (const Container &items, size_t max_threads, std::function< void(const Container &)> lambda)

template<typename T >
ArrayDatum	encode_as_array_datum (const std::vector< T > &data)

std::string	get_db_name (int32_t db_id)

std::string	get_table_name (int32_t db_id, int32_t table_id)

void	set_node_name (std::map< std::string, import_export::TypedImportBuffer * > &import_buffers)

void	set_value (rapidjson::Value &json_val, const RowGroupInterval &value, rapidjson::Document::AllocatorType &allocator)

void	get_value (const rapidjson::Value &json_val, RowGroupInterval &value)

template<typename D , typename T >
bool	check_bounds (const T &value)

template<typename D >
std::string	datetime_to_string (const D &timestamp, const SQLTypeInfo &column_type)

void	throw_parquet_metadata_out_of_bounds_error (const std::string &min_value, const std::string &max_value, const std::string &encountered_value)

UniqueReaderPtr	open_parquet_table (const std::string &file_path, std::shared_ptr< arrow::fs::FileSystem > &file_system)

std::pair< int, int >	get_parquet_table_size (const ReaderPtr &reader)

const parquet::ColumnDescriptor *	get_column_descriptor (const parquet::arrow::FileReader *reader, const int logical_column_index)

parquet::Type::type	get_physical_type (ReaderPtr &reader, const int logical_column_index)

void	validate_equal_column_descriptor (const parquet::ColumnDescriptor reference_descriptor, const parquet::ColumnDescriptor new_descriptor, const std::string &reference_file_path, const std::string &new_file_path)

std::unique_ptr< ColumnDescriptor >	get_sub_type_column_descriptor (const ColumnDescriptor *column)

std::shared_ptr < parquet::Statistics >	validate_and_get_column_metadata_statistics (const parquet::ColumnChunkMetaData *column_metadata)

std::vector < Aws::S3::Model::Object >	s3_objects_filter_sort_files (const std::vector< Aws::S3::Model::Object > &file_paths, const shared::FilePathOptions &options)

template<typename V , std::enable_if_t< std::is_integral< V >::value, int > = 0>
V	get_null_value ()

template<typename D , std::enable_if_t< std::is_integral< D >::value, int > = 0>
std::pair< D, D >	get_min_max_bounds ()

void	populate_string_dictionary (const int32_t table_id, const int32_t col_id, const Catalog_Namespace::Catalog &catalog)

void	validate_non_foreign_table_write (const TableDescriptor *table_descriptor)

Variables
constexpr const char *	kDeletedValueIndicator {"<DELETED>"}

Typedef Documentation

using foreign_storage::ChunkToBufferMap = typedef std::map<ChunkKey, AbstractBuffer*>

Definition at line 34 of file ForeignDataWrapper.h.

template<typename T >

using foreign_storage::DateInDaysBoundsValidator = typedef BaseDateBoundsValidator<T, false>

Definition at line 277 of file ParquetMetadataValidator.h.

template<typename T >

using foreign_storage::DateInSecondsBoundsValidator = typedef BaseDateBoundsValidator<T, true>

Definition at line 274 of file ParquetMetadataValidator.h.

using foreign_storage::FilePathAndRowGroup = typedef std::pair<std::string, int32_t>

Definition at line 40 of file ParquetDataWrapper.h.

using foreign_storage::FileRegions = typedef std::vector<FileRegion>

Definition at line 60 of file FileRegions.h.

using foreign_storage::FirstLineByFilePath = typedef std::map<std::string, std::string>

Definition at line 37 of file FileReader.h.

using foreign_storage::InvalidRowGroupIndices = typedef std::set<int64_t>

Definition at line 123 of file ParquetEncoder.h.

using foreign_storage::OptionsMap = typedef std::map<std::string, std::string, std::less<>>

Definition at line 30 of file OptionsContainer.h.

template<typename V , typename T , T conversion_denominator, typename NullType = V>

using foreign_storage::ParquetDateInSecondsFromTimestampEncoder = typedef ParquetTimestampEncoder<V, T, conversion_denominator, NullType>

Definition at line 89 of file ParquetTimestampEncoder.h.

using foreign_storage::read_lock = typedef heavyai::shared_lock<heavyai::shared_mutex>

Definition at line 29 of file ForeignStorageCache.cpp.

using foreign_storage::ReaderPtr = typedef parquet::arrow::FileReader*

Definition at line 33 of file ParquetShared.h.

using foreign_storage::RejectedRowIndices = typedef std::set<int64_t>

Definition at line 28 of file ParquetEncoder.h.

using foreign_storage::RenderGroupAnalyzerMap = typedef std::map<int, std::unique_ptr<import_export::RenderGroupAnalyzer>>

Definition at line 37 of file ForeignDataWrapper.h.

using foreign_storage::S3ObjectComparator = typedef std::function<bool(const Aws::S3::Model::Object&, const Aws::S3::Model::Object&)>

Definition at line 17 of file S3FilePathUtil.h.

using foreign_storage::SampleRows = typedef std::vector<std::vector<std::string>>

Definition at line 26 of file DataPreview.h.

using foreign_storage::UniqueReaderPtr = typedef std::unique_ptr<parquet::arrow::FileReader>

Definition at line 32 of file ParquetShared.h.

using foreign_storage::write_lock = typedef heavyai::unique_lock<heavyai::shared_mutex>

Definition at line 30 of file ForeignStorageCache.cpp.

Function Documentation

void foreign_storage::add_file_region	(	std::map< int, FileRegions > &	fragment_id_to_file_regions_map,
		int	fragment_id,
		size_t	first_row_index,
		const ParseBufferResult &	result,
		const std::string &	file_path
	)

Creates a new file region based on metadata from parsed file buffers and adds the new region to the fragment id to file regions map.

Definition at line 549 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::ParseBufferResult::row_count, and foreign_storage::ParseBufferResult::row_offsets.

Referenced by populate_chunks_using_data_blocks(), and process_data_blocks().

                                                  {
   fragment_id_to_file_regions_map[fragment_id].emplace_back(
       // file naming is handled by FileReader
       FileRegion(file_path,
                  result.row_offsets.front(),
                  first_row_index,
                  result.row_count,
                  result.row_offsets.back() - result.row_offsets.front()));
 }

Here is the caller graph for this function:

void foreign_storage::add_request_to_pool	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		ParseBufferRequest &	request
	)

Adds the request object for a processed request back to the request pool for reuse in subsequent requests.

Definition at line 786 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::request_pool, foreign_storage::MetadataScanMultiThreadingParams::request_pool_condition, and foreign_storage::MetadataScanMultiThreadingParams::request_pool_mutex.

Referenced by dispatch_scan_requests(), populate_chunks(), and scan_metadata().

                                                       {
   std::unique_lock<std::mutex> completed_requests_queue_lock(
       multi_threading_params.request_pool_mutex);
   multi_threading_params.request_pool.emplace(std::move(request));
   completed_requests_queue_lock.unlock();
   multi_threading_params.request_pool_condition.notify_all();
 }

Here is the caller graph for this function:

void foreign_storage::append_data_block_to_chunk	(	const foreign_storage::IterativeFileScanParameters &	file_scan_param,
		DataBlockPtr	data_block,
		size_t	row_count,
		const ChunkKey &	chunk_key,
		const ColumnDescriptor *	column,
		const std::set< size_t > &	rejected_row_indices,
		const size_t	element_count_required
	)

Definition at line 636 of file AbstractTextFileDataWrapper.cpp.

References CHECK, CHUNK_KEY_COLUMN_IDX, foreign_storage::IterativeFileScanParameters::column_id_to_chunk_map, foreign_storage::IterativeFileScanParameters::delete_buffer, foreign_storage::IterativeFileScanParameters::delete_buffer_mutex, shared::get_from_map(), foreign_storage::IterativeFileScanParameters::getChunkConditionalVariable(), foreign_storage::IterativeFileScanParameters::getChunkMutex(), and foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::resize_delete_buffer().

Referenced by populate_chunks_using_data_blocks().

                                          {
   auto chunk = shared::get_from_map(file_scan_param.column_id_to_chunk_map,
                                     chunk_key[CHUNK_KEY_COLUMN_IDX]);
 
   auto& conditional_variable =
       file_scan_param.getChunkConditionalVariable(chunk_key[CHUNK_KEY_COLUMN_IDX]);
   {
     std::unique_lock<std::mutex> chunk_lock(
         file_scan_param.getChunkMutex(chunk_key[CHUNK_KEY_COLUMN_IDX]));
     conditional_variable.wait(chunk_lock, [element_count_required, &chunk]() {
       return chunk.getBuffer()->getEncoder()->getNumElems() == element_count_required;
     });
 
     chunk.appendData(data_block, row_count, 0);
   }
 
   conditional_variable
       .notify_all();  // notify any threads waiting on the correct element count
 
   if (file_scan_param.delete_buffer) {
     std::unique_lock delete_buffer_lock(file_scan_param.delete_buffer_mutex);
     auto& delete_buffer = file_scan_param.delete_buffer;
     auto chunk_offset = element_count_required;
     auto chunk_element_count = chunk_offset + row_count;
 
     // ensure delete buffer is sized appropriately
     resize_delete_buffer(delete_buffer, chunk_element_count);
 
     auto delete_buffer_data = delete_buffer->getMemoryPtr();
     for (const auto rejected_row_index : rejected_row_indices) {
       CHECK(rejected_row_index + chunk_offset < delete_buffer->size());
       delete_buffer_data[rejected_row_index + chunk_offset] = true;
     }
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::string foreign_storage::bool_to_option_value ( const bool value )

Definition at line 133 of file ForeignDataWrapperFactory.cpp.

Referenced by foreign_storage::ForeignDataWrapperFactory::createForeignTableProxy().

                                                  {
   return value ? "TRUE" : "FALSE";
 }

Here is the caller graph for this function:

void foreign_storage::cache_blocks	(	std::map< ChunkKey, Chunk_NS::Chunk > &	cached_chunks,
		DataBlockPtr	data_block,
		size_t	row_count,
		ChunkKey &	chunk_key,
		const ColumnDescriptor *	column,
		bool	is_first_block,
		bool	disable_cache
	)

Definition at line 595 of file AbstractTextFileDataWrapper.cpp.

References CHECK, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, ColumnDescriptor::columnType, foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::get_cache_if_enabled(), Catalog_Namespace::SysCatalog::getCatalog(), Catalog_Namespace::SysCatalog::instance(), SQLTypeInfo::is_varlen_indeed(), and key_does_not_shard_to_leaf().

Referenced by process_data_blocks().

                                       {
   auto catalog =
       Catalog_Namespace::SysCatalog::instance().getCatalog(chunk_key[CHUNK_KEY_DB_IDX]);
   CHECK(catalog);
   auto cache = get_cache_if_enabled(catalog, disable_cache);
   if (cache) {
     // This extra filter needs to be here because this wrapper is the only one that
     // accesses the cache directly and it should not be inserting chunks which are not
     // mapped to the current leaf (in distributed mode).
     if (key_does_not_shard_to_leaf(chunk_key)) {
       return;
     }
 
     ChunkKey index_key = {chunk_key[CHUNK_KEY_DB_IDX],
                           chunk_key[CHUNK_KEY_TABLE_IDX],
                           chunk_key[CHUNK_KEY_COLUMN_IDX],
                           chunk_key[CHUNK_KEY_FRAGMENT_IDX],
                           2};
     // Create actual data chunks to prepopulate cache
     if (cached_chunks.find(chunk_key) == cached_chunks.end()) {
       cached_chunks[chunk_key] = Chunk_NS::Chunk{column, false};
       cached_chunks[chunk_key].setBuffer(
           cache->getChunkBufferForPrecaching(chunk_key, is_first_block));
       if (column->columnType.is_varlen_indeed()) {
         cached_chunks[chunk_key].setIndexBuffer(
             cache->getChunkBufferForPrecaching(index_key, is_first_block));
       }
       if (is_first_block) {
         cached_chunks[chunk_key].initEncoder();
       }
     }
     cached_chunks[chunk_key].appendData(data_block, row_count, 0);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename D , typename T >

bool foreign_storage::check_bounds ( const T & value )

inline

Definition at line 32 of file ParquetMetadataValidator.h.

                                          {
   auto [min_value, max_value] = get_min_max_bounds<D>();
   return value >= min_value && value <= max_value;
 }

bool foreign_storage::contains_fragment_key	(	const std::set< ChunkKey > &	key_set,
		const ChunkKey &	target_key
	)

template<typename Container >

std::vector<std::future<void> > foreign_storage::create_futures_for_workers	(	const Container &	items,
		size_t	max_threads,
		std::function< void(const Container &)>	lambda
	)

Definition at line 74 of file FsiChunkUtils.h.

References threading_serial::async(), and partition_for_threads().

Referenced by import_export::ForeignDataImporter::importGeneralS3(), foreign_storage::ParquetDataWrapper::populateChunkBuffers(), and foreign_storage::LazyParquetChunkLoader::previewFiles().

                                                 {
   auto items_per_thread = partition_for_threads(items, max_threads);
   std::vector<std::future<void>> futures;
   for (const auto& items : items_per_thread) {
     futures.emplace_back(std::async(std::launch::async, lambda, items));
   }
 
   return futures;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::tuple< std::unique_ptr< foreign_storage::ForeignServer >, std::unique_ptr< foreign_storage::UserMapping >, std::unique_ptr< foreign_storage::ForeignTable > > foreign_storage::create_proxy_fsi_objects	(	const std::string &	copy_from_source,
		const import_export::CopyParams &	copy_params,
		const int	db_id,
		const TableDescriptor *	table,
		const int32_t	user_id
	)

Create proxy fsi objects for use outside FSI.

Parameters

copy_from_source	- the source that will be copied
copy_params	- CopyParams that specify parameters around use case
db_id	- db id of database in use case
table	- the table descriptor of the table in use case
user_id	- the user id of user in use case

Returns: tuple of FSI objects that can be used for particular use case (for example import or detect)

Definition at line 63 of file ForeignDataWrapperFactory.cpp.

References CHECK, foreign_storage::ForeignDataWrapperFactory::createForeignServerProxy(), foreign_storage::ForeignDataWrapperFactory::createForeignTableProxy(), and foreign_storage::ForeignDataWrapperFactory::createUserMappingProxyIfApplicable().

Referenced by create_proxy_fsi_objects(), and import_export::ForeignDataImporter::importGeneral().

                                                 {
   auto server = foreign_storage::ForeignDataWrapperFactory::createForeignServerProxy(
       db_id, user_id, copy_from_source, copy_params);
 
   CHECK(server);
   server->validate();
 
   auto user_mapping =
       foreign_storage::ForeignDataWrapperFactory::createUserMappingProxyIfApplicable(
           db_id, user_id, copy_from_source, copy_params, server.get());
 
   if (user_mapping) {
     user_mapping->validate(server.get());
   }
 
   auto foreign_table =
       foreign_storage::ForeignDataWrapperFactory::createForeignTableProxy(
           db_id, table, copy_from_source, copy_params, server.get());
 
   CHECK(foreign_table);
   foreign_table->validateOptionValues();
 
   return {std::move(server), std::move(user_mapping), std::move(foreign_table)};
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::tuple< std::unique_ptr< foreign_storage::ForeignServer >, std::unique_ptr< foreign_storage::UserMapping >, std::unique_ptr< foreign_storage::ForeignTable > > foreign_storage::create_proxy_fsi_objects	(	const std::string &	copy_from_source,
		const import_export::CopyParams &	copy_params,
		const TableDescriptor *	table
	)

Create proxy fsi objects for use outside FSI NOTE: parameters mirror function above.

Definition at line 95 of file ForeignDataWrapperFactory.cpp.

References create_proxy_fsi_objects().

                                                        {
   return create_proxy_fsi_objects(copy_from_source, copy_params, -1, table, -1);
 }

Here is the call graph for this function:

template<typename D >

std::string foreign_storage::datetime_to_string	(	const D &	timestamp,
		const SQLTypeInfo &	column_type
	)

inline

Definition at line 38 of file ParquetMetadataValidator.h.

References Datum::bigintval, CHECK, DatumToString(), SQLTypeInfo::is_date(), and SQLTypeInfo::is_timestamp().

Referenced by foreign_storage::TimestampBoundsValidator< T >::getMinMaxBoundsAsStrings(), foreign_storage::BaseDateBoundsValidator< T, is_in_seconds >::getMinMaxBoundsAsStrings(), foreign_storage::TimestampBoundsValidator< T >::validateValue(), and foreign_storage::BaseDateBoundsValidator< T, is_in_seconds >::validateValue().

                                                                       {
   CHECK(column_type.is_timestamp() || column_type.is_date());
   Datum d;
   d.bigintval = timestamp;
   return DatumToString(d, column_type);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::defer_scan_request	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		ParseBufferRequest &	request
	)

Definition at line 873 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::deferred_requests, and foreign_storage::MetadataScanMultiThreadingParams::deferred_requests_mutex.

Referenced by populate_chunks().

                                                      {
   std::unique_lock<std::mutex> deferred_requests_lock(
       multi_threading_params.deferred_requests_mutex);
   multi_threading_params.deferred_requests.emplace(std::move(request));
 }

Here is the caller graph for this function:

std::optional< SQLTypes > foreign_storage::detect_geo_type	(	const SampleRows &	sample_rows,
		size_t	column_index
	)

Definition at line 22 of file DataPreview.cpp.

References CHECK_EQ, CHECK_LT, kLINESTRING, kMULTIPOLYGON, kNULLT, kPOINT, kPOLYGON, and UNREACHABLE.

Referenced by foreign_storage::LazyParquetChunkLoader::previewFiles().

                                                              {
   std::optional<SQLTypes> tentative_geo_type{};
   for (const auto& row : sample_rows) {
     static std::regex geo_regex{
         "\\s*(POINT|LINESTRING|POLYGON|MULTIPOLYGON)\\s*\\(.+\\)\\s*"};
     std::smatch match;
     CHECK_LT(column_index, row.size());
     if (std::regex_match(row[column_index], match, geo_regex)) {
       CHECK_EQ(match.size(), static_cast<size_t>(2));
       SQLTypes geo_type{kNULLT};
       const auto& geo_type_str = match[1];
       if (geo_type_str == "POINT") {
         geo_type = kPOINT;
       } else if (geo_type_str == "LINESTRING") {
         geo_type = kLINESTRING;
       } else if (geo_type_str == "POLYGON") {
         geo_type = kPOLYGON;
       } else if (geo_type_str == "MULTIPOLYGON") {
         geo_type = kMULTIPOLYGON;
       } else {
         UNREACHABLE() << "Unexpected geo type match: " << geo_type_str;
       }
       if (tentative_geo_type.has_value()) {
         if (tentative_geo_type.value() != geo_type) {
           return {};  // geo type does not match between rows, can not be imported
         }
       } else {
         tentative_geo_type = geo_type;
       }
     }
   }
   return tentative_geo_type;
 }

Here is the caller graph for this function:

void foreign_storage::dispatch_all_deferred_requests ( MetadataScanMultiThreadingParams & multi_threading_params )

Definition at line 883 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::deferred_requests, foreign_storage::MetadataScanMultiThreadingParams::deferred_requests_mutex, foreign_storage::MetadataScanMultiThreadingParams::pending_requests, foreign_storage::MetadataScanMultiThreadingParams::pending_requests_condition, and foreign_storage::MetadataScanMultiThreadingParams::pending_requests_mutex.

Referenced by dispatch_scan_requests().

                                                               {
   std::unique_lock<std::mutex> deferred_requests_lock(
       multi_threading_params.deferred_requests_mutex);
   {
     std::unique_lock<std::mutex> pending_requests_lock(
         multi_threading_params.pending_requests_mutex);
 
     while (!multi_threading_params.deferred_requests.empty()) {
       auto& request = multi_threading_params.deferred_requests.front();
       multi_threading_params.pending_requests.emplace(std::move(request));
       multi_threading_params.deferred_requests.pop();
     }
     multi_threading_params.pending_requests_condition.notify_all();
   }
 }

Here is the caller graph for this function:

void foreign_storage::dispatch_scan_request	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		ParseBufferRequest &	request
	)

Dispatches a new metadata scan request by adding the request to the pending requests queue to be consumed by a worker thread.

Definition at line 904 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::pending_requests, foreign_storage::MetadataScanMultiThreadingParams::pending_requests_condition, and foreign_storage::MetadataScanMultiThreadingParams::pending_requests_mutex.

Referenced by dispatch_scan_requests().

                                                         {
   {
     std::unique_lock<std::mutex> pending_requests_lock(
         multi_threading_params.pending_requests_mutex);
     multi_threading_params.pending_requests.emplace(std::move(request));
   }
   multi_threading_params.pending_requests_condition.notify_all();
 }

Here is the caller graph for this function:

void foreign_storage::dispatch_scan_requests	(	const foreign_storage::ForeignTable *	table,
		const size_t &	buffer_size,
		const std::string &	file_path,
		FileReader &	file_reader,
		const import_export::CopyParams &	copy_params,
		MetadataScanMultiThreadingParams &	multi_threading_params,
		size_t &	first_row_index_in_buffer,
		size_t &	current_file_offset,
		const TextFileBufferParser &	parser,
		const foreign_storage::IterativeFileScanParameters *	file_scan_param,
		foreign_storage::AbstractTextFileDataWrapper::ResidualBuffer &	iterative_residual_buffer,
		const bool	is_first_file_scan_call
	)

Reads from a text file iteratively and dispatches metadata scan requests that are processed by worker threads.

Definition at line 1010 of file AbstractTextFileDataWrapper.cpp.

References add_request_to_pool(), foreign_storage::AbstractTextFileDataWrapper::ResidualBuffer::alloc_size, foreign_storage::MetadataScanMultiThreadingParams::continue_processing, dispatch_all_deferred_requests(), dispatch_scan_request(), foreign_storage::TextFileBufferParser::findRowEndPosition(), foreign_storage::IterativeFileScanParameters::fragment_id, get_request_from_pool(), foreign_storage::FileReader::getCurrentFilePath(), foreign_storage::FileReader::isScanFinished(), import_export::CopyParams::line_delim, TableDescriptor::maxFragRows, foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::no_deferred_requests(), foreign_storage::MetadataScanMultiThreadingParams::pending_requests, foreign_storage::MetadataScanMultiThreadingParams::pending_requests_condition, foreign_storage::MetadataScanMultiThreadingParams::pending_requests_mutex, foreign_storage::FileReader::read(), foreign_storage::AbstractTextFileDataWrapper::ResidualBuffer::residual_buffer_alloc_size, foreign_storage::AbstractTextFileDataWrapper::ResidualBuffer::residual_buffer_size, foreign_storage::AbstractTextFileDataWrapper::ResidualBuffer::residual_data, and resize_buffer_if_needed().

Referenced by dispatch_scan_requests_with_exception_handling().

                                         {
   auto& alloc_size = iterative_residual_buffer.alloc_size;
   auto& residual_buffer = iterative_residual_buffer.residual_data;
   auto& residual_buffer_size = iterative_residual_buffer.residual_buffer_size;
   auto& residual_buffer_alloc_size = iterative_residual_buffer.residual_buffer_alloc_size;
 
   if (is_first_file_scan_call) {
     alloc_size = buffer_size;
     residual_buffer = std::make_unique<char[]>(alloc_size);
     residual_buffer_size = 0;
     residual_buffer_alloc_size = alloc_size;
   } else if (!no_deferred_requests(multi_threading_params)) {
     dispatch_all_deferred_requests(multi_threading_params);
   }
 
   while (!file_reader.isScanFinished()) {
     {
       std::lock_guard<std::mutex> pending_requests_lock(
           multi_threading_params.pending_requests_mutex);
       if (!multi_threading_params.continue_processing) {
         break;
       }
     }
     auto request = get_request_from_pool(multi_threading_params);
     request.full_path = file_reader.getCurrentFilePath();
     resize_buffer_if_needed(request.buffer, request.buffer_alloc_size, alloc_size);
 
     if (residual_buffer_size > 0) {
       memcpy(request.buffer.get(), residual_buffer.get(), residual_buffer_size);
     }
     size_t size = residual_buffer_size;
     size += file_reader.read(request.buffer.get() + residual_buffer_size,
                              alloc_size - residual_buffer_size);
 
     if (size == 0) {
       // In some cases at the end of a file we will read 0 bytes even when
       // file_reader.isScanFinished() is false. Also add request back to the pool to be
       // picked up again in the next iteration.
       add_request_to_pool(multi_threading_params, request);
       continue;
     } else if (size == 1 && request.buffer[0] == copy_params.line_delim) {
       // In some cases files with newlines at the end will be encoded with a second
       // newline that can end up being the only thing in the buffer. Also add request
       // back to the pool to be picked up again in the next iteration.
       current_file_offset++;
       add_request_to_pool(multi_threading_params, request);
       continue;
     }
     unsigned int num_rows_in_buffer = 0;
     request.end_pos = parser.findRowEndPosition(alloc_size,
                                                 request.buffer,
                                                 size,
                                                 copy_params,
                                                 first_row_index_in_buffer,
                                                 num_rows_in_buffer,
                                                 &file_reader);
     request.buffer_size = size;
     request.buffer_alloc_size = alloc_size;
     request.first_row_index = first_row_index_in_buffer;
     request.file_offset = current_file_offset;
     request.buffer_row_count = num_rows_in_buffer;
     request.processed_row_count = 0;
     request.begin_pos = 0;
 
     residual_buffer_size = size - request.end_pos;
     if (residual_buffer_size > 0) {
       resize_buffer_if_needed(residual_buffer, residual_buffer_alloc_size, alloc_size);
       memcpy(residual_buffer.get(),
              request.buffer.get() + request.end_pos,
              residual_buffer_size);
     }
 
     current_file_offset += request.end_pos;
     first_row_index_in_buffer += num_rows_in_buffer;
 
     if (num_rows_in_buffer > 0) {
       dispatch_scan_request(multi_threading_params, request);
     } else {
       add_request_to_pool(multi_threading_params, request);
     }
 
     if (file_scan_param) {
       const int32_t last_fragment_index =
           (first_row_index_in_buffer) / table->maxFragRows;
       if (last_fragment_index > file_scan_param->fragment_id) {
         break;
       }
     }
   }
 
   std::unique_lock<std::mutex> pending_requests_queue_lock(
       multi_threading_params.pending_requests_mutex);
   multi_threading_params.pending_requests_condition.wait(
       pending_requests_queue_lock, [&multi_threading_params] {
         return multi_threading_params.pending_requests.empty() ||
                (multi_threading_params.continue_processing == false);
       });
   multi_threading_params.continue_processing = false;
   pending_requests_queue_lock.unlock();
   multi_threading_params.pending_requests_condition.notify_all();
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::dispatch_scan_requests_with_exception_handling	(	const foreign_storage::ForeignTable *	table,
		const size_t &	buffer_size,
		const std::string &	file_path,
		FileReader &	file_reader,
		const import_export::CopyParams &	copy_params,
		MetadataScanMultiThreadingParams &	multi_threading_params,
		size_t &	first_row_index_in_buffer,
		size_t &	current_file_offset,
		const TextFileBufferParser &	parser,
		const foreign_storage::IterativeFileScanParameters *	file_scan_param,
		foreign_storage::AbstractTextFileDataWrapper::ResidualBuffer &	iterative_residual_buffer,
		const bool	is_first_file_scan_call
	)

Definition at line 1125 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::continue_processing, dispatch_scan_requests(), foreign_storage::MetadataScanMultiThreadingParams::pending_requests_condition, and foreign_storage::MetadataScanMultiThreadingParams::pending_requests_mutex.

Referenced by foreign_storage::AbstractTextFileDataWrapper::iterativeFileScan(), and foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata().

                                         {
   try {
     dispatch_scan_requests(table,
                            buffer_size,
                            file_path,
                            file_reader,
                            copy_params,
                            multi_threading_params,
                            first_row_index_in_buffer,
                            current_file_offset,
                            parser,
                            file_scan_param,
                            iterative_residual_buffer,
                            is_first_file_scan_call);
   } catch (...) {
     {
       std::unique_lock<std::mutex> pending_requests_lock(
           multi_threading_params.pending_requests_mutex);
       multi_threading_params.continue_processing = false;
     }
     multi_threading_params.pending_requests_condition.notify_all();
     throw;
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >

ArrayDatum foreign_storage::encode_as_array_datum ( const std::vector< T > & data )

inline

Definition at line 33 of file GeospatialEncoder.h.

References heavydb.dtypes::T.

Referenced by foreign_storage::GeospatialEncoder::processGeoElement(), and foreign_storage::GeospatialEncoder::processNullGeoElement().

                                                                   {
   const size_t num_bytes = data.size() * sizeof(T);
   std::shared_ptr<int8_t> buffer(new int8_t[num_bytes], std::default_delete<int8_t[]>());
   memcpy(buffer.get(), data.data(), num_bytes);
   return ArrayDatum(num_bytes, buffer, false);
 }

Here is the caller graph for this function:

bool foreign_storage::fragment_maps_to_leaf ( const ChunkKey & key )

Definition at line 128 of file FsiChunkUtils.cpp.

References CHECK, g_distributed_leaf_idx, g_distributed_num_leaves, get_fragment(), dist::is_aggregator(), and dist::is_distributed().

Referenced by anonymous_namespace{ForeignStorageMgr.cpp}::filter_metadata_by_leaf(), foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), key_does_not_shard_to_leaf(), and foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

                                                 {
   CHECK(dist::is_distributed());
   CHECK(g_distributed_num_leaves > 0);
   CHECK(!dist::is_aggregator());
   return (get_fragment(key) % g_distributed_num_leaves == g_distributed_leaf_idx);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

const parquet::ColumnDescriptor * foreign_storage::get_column_descriptor	(	const parquet::arrow::FileReader *	reader,
		const int	logical_column_index
	)

Definition at line 46 of file ParquetShared.cpp.

Referenced by foreign_storage::LazyParquetChunkLoader::appendRowGroups(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_equal_schema().

                                     {
   return reader->parquet_reader()->metadata()->schema()->Column(logical_column_index);
 }

Here is the caller graph for this function:

std::set<ChunkKey> foreign_storage::get_column_key_set ( const ChunkKey & destination_chunk_key )

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), foreign_storage::CachingForeignStorageMgr::getOptionalKeysWithinSizeLimit(), and foreign_storage::CachingForeignStorageMgr::getRequiredBuffersSize().

Here is the caller graph for this function:

std::vector<ChunkKey> foreign_storage::get_column_key_vec ( const ChunkKey & destination_chunk_key )

std::string foreign_storage::get_db_name ( int32_t db_id )

Definition at line 31 of file InternalSystemDataWrapper.cpp.

References Catalog_Namespace::DBMetadata::dbName, Catalog_Namespace::SysCatalog::instance(), and kDeletedValueIndicator.

Referenced by foreign_storage::anonymous_namespace{InternalCatalogDataWrapper.cpp}::populate_import_buffers_for_catalog_dashboards(), foreign_storage::anonymous_namespace{InternalCatalogDataWrapper.cpp}::populate_import_buffers_for_catalog_permissions(), foreign_storage::anonymous_namespace{InternalCatalogDataWrapper.cpp}::populate_import_buffers_for_catalog_tables(), foreign_storage::anonymous_namespace{InternalCatalogDataWrapper.cpp}::populate_import_buffers_for_catalog_users(), foreign_storage::anonymous_namespace{InternalMemoryStatsDataWrapper.cpp}::populate_import_buffers_for_memory_details(), and foreign_storage::anonymous_namespace{InternalStorageStatsDataWrapper.cpp}::populate_import_buffers_for_storage_details().

                                      {
   Catalog_Namespace::DBMetadata db_metadata;
   auto& sys_catalog = Catalog_Namespace::SysCatalog::instance();
   if (sys_catalog.getMetadataForDBById(db_id, db_metadata)) {
     return db_metadata.dbName;
   } else {
     // Database has been deleted.
     return kDeletedValueIndicator;
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

const foreign_storage::ForeignTable & foreign_storage::get_foreign_table_for_key ( const ChunkKey & key )

Definition at line 101 of file FsiChunkUtils.cpp.

References CHECK, get_table_prefix(), Catalog_Namespace::SysCatalog::getCatalog(), and Catalog_Namespace::SysCatalog::instance().

Referenced by is_append_table_chunk_key(), foreign_storage::anonymous_namespace{CachingForeignStorageMgr.cpp}::is_in_memory_system_table_chunk_key(), is_replicated_table_chunk_key(), and is_system_table_chunk_key().

                                                                                   {
   auto [db_id, tb_id] = get_table_prefix(key);
   auto catalog = Catalog_Namespace::SysCatalog::instance().getCatalog(db_id);
   CHECK(catalog);
   auto table = catalog->getForeignTable(tb_id);
   CHECK(table);
   return *table;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

size_t foreign_storage::get_max_chunk_size ( const ChunkKey & key )

Referenced by foreign_storage::CachingForeignStorageMgr::getBufferSize().

Here is the caller graph for this function:

template<typename D , std::enable_if_t< std::is_integral< D >::value, int > = 0>

std::pair< D, D > foreign_storage::get_min_max_bounds ( )

inline

Definition at line 31 of file SharedMetadataValidator.h.

                                           {
   static_assert(std::is_signed<D>::value,
                 "'get_min_max_bounds' is only valid for signed types");
   return {get_null_value<D>() + 1, std::numeric_limits<D>::max()};
 }

std::optional<ParseBufferRequest> foreign_storage::get_next_scan_request ( MetadataScanMultiThreadingParams & multi_threading_params )

Gets the next metadata scan request object from the pending requests queue. A null optional is returned if there are no further requests to be processed.

Definition at line 526 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::continue_processing, foreign_storage::MetadataScanMultiThreadingParams::pending_requests, foreign_storage::MetadataScanMultiThreadingParams::pending_requests_condition, and foreign_storage::MetadataScanMultiThreadingParams::pending_requests_mutex.

Referenced by populate_chunks(), and scan_metadata().

                                                               {
   std::unique_lock<std::mutex> pending_requests_lock(
       multi_threading_params.pending_requests_mutex);
   multi_threading_params.pending_requests_condition.wait(
       pending_requests_lock, [&multi_threading_params] {
         return !multi_threading_params.pending_requests.empty() ||
                !multi_threading_params.continue_processing;
       });
   if (multi_threading_params.pending_requests.empty()) {
     return {};
   }
   auto request = std::move(multi_threading_params.pending_requests.front());
   multi_threading_params.pending_requests.pop();
   pending_requests_lock.unlock();
   multi_threading_params.pending_requests_condition.notify_all();
   return std::move(request);
 }

Here is the caller graph for this function:

template<typename V , std::enable_if_t< std::is_integral< V >::value, int > = 0>

V foreign_storage::get_null_value ( )

inline

Definition at line 21 of file SharedMetadataValidator.h.

                           {
   return inline_int_null_value<V>();
 }

size_t foreign_storage::get_num_threads ( const ForeignTable & table )

Definition at line 17 of file AbstractFileStorageDataWrapper.cpp.

References foreign_storage::OptionsContainer::getOption(), import_export::num_import_threads(), and foreign_storage::AbstractFileStorageDataWrapper::THREADS_KEY.

Referenced by foreign_storage::LazyParquetChunkLoader::metadataScan(), foreign_storage::ParquetDataWrapper::populateChunkBuffers(), and foreign_storage::LazyParquetChunkLoader::previewFiles().

                                                   {
   auto num_threads = 0;
   if (auto opt = table.getOption(AbstractFileStorageDataWrapper::THREADS_KEY);
       opt.has_value()) {
     num_threads = std::stoi(opt.value());
   }
   return import_export::num_import_threads(num_threads);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::pair< int, int > foreign_storage::get_parquet_table_size ( const ReaderPtr & reader )

Definition at line 39 of file ParquetShared.cpp.

Referenced by foreign_storage::LazyParquetChunkLoader::appendRowGroups(), foreign_storage::LazyParquetChunkLoader::loadRowGroups(), and foreign_storage::LazyParquetChunkLoader::metadataScan().

                                                                   {
   auto file_metadata = reader->parquet_reader()->metadata();
   const auto num_row_groups = file_metadata->num_row_groups();
   const auto num_columns = file_metadata->num_columns();
   return std::make_pair(num_row_groups, num_columns);
 }

Here is the caller graph for this function:

parquet::Type::type foreign_storage::get_physical_type	(	ReaderPtr &	reader,
		const int	logical_column_index
	)

Definition at line 52 of file ParquetShared.cpp.

Referenced by anonymous_namespace{ArrowResultSetConverter.cpp}::get_arrow_type(), and ArrowResultSetConverter::initializeColumnBuilder().

                                                                                      {
   return reader->parquet_reader()
       ->metadata()
       ->schema()
       ->Column(logical_column_index)
       ->physical_type();
 }

Here is the caller graph for this function:

std::shared_ptr< ChunkMetadata > foreign_storage::get_placeholder_metadata	(	const SQLTypeInfo &	type,
		size_t	num_elements
	)

Definition at line 77 of file FsiChunkUtils.cpp.

References SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_size(), Data_Namespace::AbstractBuffer::getEncoder(), Encoder::getMetadata(), Data_Namespace::AbstractBuffer::initEncoder(), SQLTypeInfo::is_array(), and SQLTypeInfo::is_varlen_indeed().

Referenced by foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::add_placeholder_metadata(), foreign_storage::InternalSystemDataWrapper::populateChunkMetadata(), and foreign_storage::AbstractTextFileDataWrapper::updateRolledOffChunks().

                                                                              {
   ForeignStorageBuffer empty_buffer;
   // Use default encoder metadata as in parquet wrapper
   empty_buffer.initEncoder(type);
 
   auto chunk_metadata = empty_buffer.getEncoder()->getMetadata(type);
   chunk_metadata->numElements = num_elements;
 
   if (!type.is_varlen_indeed()) {
     chunk_metadata->numBytes = type.get_size() * num_elements;
   }
   // min/max not set by default for arrays, so get from elem type encoder
   if (type.is_array()) {
     ForeignStorageBuffer scalar_buffer;
     scalar_buffer.initEncoder(type.get_elem_type());
     auto scalar_metadata = scalar_buffer.getEncoder()->getMetadata(type.get_elem_type());
     chunk_metadata->chunkStats.min = scalar_metadata->chunkStats.min;
     chunk_metadata->chunkStats.max = scalar_metadata->chunkStats.max;
   }
 
   return chunk_metadata;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

ParseBufferRequest foreign_storage::get_request_from_pool ( MetadataScanMultiThreadingParams & multi_threading_params )

Gets a request from the metadata scan request pool.

Definition at line 854 of file AbstractTextFileDataWrapper.cpp.

References CHECK, foreign_storage::MetadataScanMultiThreadingParams::request_pool, foreign_storage::MetadataScanMultiThreadingParams::request_pool_condition, and foreign_storage::MetadataScanMultiThreadingParams::request_pool_mutex.

Referenced by dispatch_scan_requests().

                                                               {
   std::unique_lock<std::mutex> request_pool_lock(
       multi_threading_params.request_pool_mutex);
   multi_threading_params.request_pool_condition.wait(
       request_pool_lock,
       [&multi_threading_params] { return !multi_threading_params.request_pool.empty(); });
   auto request = std::move(multi_threading_params.request_pool.front());
   multi_threading_params.request_pool.pop();
   request_pool_lock.unlock();
   CHECK(request.buffer);
   return request;
 }

Here is the caller graph for this function:

std::unique_ptr< ColumnDescriptor > foreign_storage::get_sub_type_column_descriptor ( const ColumnDescriptor * column )

Definition at line 76 of file ParquetShared.cpp.

References ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::get_elem_type(), SQLTypeInfo::set_size(), and ColumnDescriptor::tableId.

Referenced by foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_array_encoder(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_array_mapping().

                                     {
   auto column_type = column->columnType.get_elem_type();
   if (column_type.get_size() == -1 && column_type.is_dict_encoded_string()) {
     column_type.set_size(4);  // override default size of -1
   }
   return std::make_unique<ColumnDescriptor>(
       column->tableId, column->columnId, column->columnName, column_type);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::string foreign_storage::get_table_name	(	int32_t	db_id,
		int32_t	table_id
	)

Definition at line 42 of file InternalSystemDataWrapper.cpp.

References CHECK, Catalog_Namespace::SysCatalog::getCatalog(), Catalog_Namespace::SysCatalog::instance(), and kDeletedValueIndicator.

Referenced by anonymous_namespace{Execute.cpp}::checkWorkUnitWatchdog(), foreign_storage::anonymous_namespace{InternalMemoryStatsDataWrapper.cpp}::populate_import_buffers_for_memory_details(), and foreign_storage::anonymous_namespace{InternalStorageStatsDataWrapper.cpp}::populate_import_buffers_for_storage_details().

                                                           {
   auto catalog = Catalog_Namespace::SysCatalog::instance().getCatalog(db_id);
   CHECK(catalog);
   auto table_name = catalog->getTableName(table_id);
   if (table_name.has_value()) {
     return table_name.value();
   } else {
     // It is possible for the table to be concurrently deleted while querying the system
     // table.
     return kDeletedValueIndicator;
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::get_value	(	const rapidjson::Value &	json_val,
		FileRegion &	file_region
	)

Definition at line 44 of file CsvShared.cpp.

References CHECK, foreign_storage::FileRegion::first_row_file_offset, foreign_storage::FileRegion::first_row_index, foreign_storage::json_utils::get_value_from_object(), foreign_storage::FileRegion::region_size, and foreign_storage::FileRegion::row_count.

                                                                         {
   CHECK(json_val.IsObject());
   json_utils::get_value_from_object(
       json_val, file_region.first_row_file_offset, "first_row_file_offset");
   json_utils::get_value_from_object(
       json_val, file_region.first_row_index, "first_row_index");
   json_utils::get_value_from_object(json_val, file_region.region_size, "region_size");
   json_utils::get_value_from_object(json_val, file_region.row_count, "row_count");
   if (json_val.HasMember("filename")) {
     json_utils::get_value_from_object(json_val, file_region.filename, "filename");
   }
 }

Here is the call graph for this function:

void foreign_storage::get_value	(	const rapidjson::Value &	json_val,
		RowGroupInterval &	value
	)

Definition at line 671 of file ParquetDataWrapper.cpp.

References CHECK, foreign_storage::RowGroupInterval::end_index, foreign_storage::RowGroupInterval::file_path, foreign_storage::json_utils::get_value_from_object(), and foreign_storage::RowGroupInterval::start_index.

                                                                         {
   CHECK(json_val.IsObject());
   json_utils::get_value_from_object(json_val, value.file_path, "file_path");
   json_utils::get_value_from_object(json_val, value.start_index, "start_index");
   json_utils::get_value_from_object(json_val, value.end_index, "end_index");
 }

Here is the call graph for this function:

void foreign_storage::init_chunk_for_column	(	const ChunkKey &	chunk_key,
		const std::map< ChunkKey, std::shared_ptr< ChunkMetadata >> &	chunk_metadata_map,
		const std::map< ChunkKey, AbstractBuffer * > &	buffers,
		Chunk_NS::Chunk &	chunk
	)

Definition at line 21 of file FsiChunkUtils.cpp.

References CHECK, CHECK_EQ, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, Catalog_Namespace::SysCatalog::getCatalog(), Chunk_NS::Chunk::initEncoder(), Catalog_Namespace::SysCatalog::instance(), Data_Namespace::AbstractBuffer::reserve(), Chunk_NS::Chunk::setBuffer(), Chunk_NS::Chunk::setColumnDesc(), Chunk_NS::Chunk::setIndexBuffer(), Chunk_NS::Chunk::setPinnable(), Data_Namespace::AbstractBuffer::size(), and UNREACHABLE.

Referenced by foreign_storage::AbstractTextFileDataWrapper::populateChunkMapForColumns().

                           {
   auto catalog =
       Catalog_Namespace::SysCatalog::instance().getCatalog(chunk_key[CHUNK_KEY_DB_IDX]);
   CHECK(catalog);
 
   ChunkKey data_chunk_key = chunk_key;
   AbstractBuffer* data_buffer = nullptr;
   AbstractBuffer* index_buffer = nullptr;
   const auto column = catalog->getMetadataForColumn(chunk_key[CHUNK_KEY_TABLE_IDX],
                                                     chunk_key[CHUNK_KEY_COLUMN_IDX]);
 
   if (column->columnType.is_varlen_indeed()) {
     data_chunk_key.push_back(1);
     ChunkKey index_chunk_key = chunk_key;
     index_chunk_key.push_back(2);
 
     CHECK(buffers.find(data_chunk_key) != buffers.end());
     CHECK(buffers.find(index_chunk_key) != buffers.end());
 
     data_buffer = buffers.find(data_chunk_key)->second;
     index_buffer = buffers.find(index_chunk_key)->second;
     CHECK_EQ(data_buffer->size(), static_cast<size_t>(0));
     CHECK_EQ(index_buffer->size(), static_cast<size_t>(0));
 
     size_t index_offset_size{0};
     if (column->columnType.is_string() || column->columnType.is_geometry()) {
       index_offset_size = sizeof(StringOffsetT);
     } else if (column->columnType.is_array()) {
       index_offset_size = sizeof(ArrayOffsetT);
     } else {
       UNREACHABLE();
     }
     if (chunk_metadata_map.find(data_chunk_key) != chunk_metadata_map.end()) {
       index_buffer->reserve(index_offset_size *
                             (chunk_metadata_map.at(data_chunk_key)->numElements + 1));
     }
   } else {
     data_chunk_key = chunk_key;
     CHECK(buffers.find(data_chunk_key) != buffers.end());
     data_buffer = buffers.find(data_chunk_key)->second;
   }
   if (chunk_metadata_map.find(data_chunk_key) != chunk_metadata_map.end()) {
     data_buffer->reserve(chunk_metadata_map.at(data_chunk_key)->numBytes);
   }
 
   chunk.setPinnable(false);
   chunk.setColumnDesc(column);
   chunk.setBuffer(data_buffer);
   chunk.setIndexBuffer(index_buffer);
   chunk.initEncoder();
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::is_append_table_chunk_key ( const ChunkKey & chunk_key )

Definition at line 118 of file FsiChunkUtils.cpp.

References get_foreign_table_for_key(), and foreign_storage::ForeignTable::isAppendMode().

Referenced by foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

                                                           {
   return get_foreign_table_for_key(chunk_key).isAppendMode();
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::is_replicated_table_chunk_key ( const ChunkKey & chunk_key )

Definition at line 114 of file FsiChunkUtils.cpp.

References get_foreign_table_for_key(), and table_is_replicated().

Referenced by is_shardable_key().

                                                               {
   return table_is_replicated(&get_foreign_table_for_key(chunk_key));
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::is_s3_uri ( const std::string & file_path )

Definition at line 55 of file ForeignDataWrapperFactory.cpp.

Referenced by foreign_storage::ForeignDataWrapperFactory::createForeignServerProxy(), foreign_storage::ForeignDataWrapperFactory::createForeignTableProxy(), import_export::ForeignDataImporter::import(), and import_export::ForeignDataImporter::importGeneralS3().

                                            {
   const std::string s3_prefix = "s3://";
   return file_path.find(s3_prefix) != std::string::npos;
 }

Here is the caller graph for this function:

bool foreign_storage::is_shardable_key ( const ChunkKey & key )

Definition at line 122 of file FsiChunkUtils.cpp.

References dist::is_aggregator(), dist::is_distributed(), is_replicated_table_chunk_key(), and is_system_table_chunk_key().

Referenced by anonymous_namespace{ForeignStorageMgr.cpp}::filter_metadata_by_leaf(), foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), key_does_not_shard_to_leaf(), and foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

                                            {
   return (dist::is_distributed() && !dist::is_aggregator() &&
           !is_replicated_table_chunk_key(key) && !is_system_table_chunk_key(key));
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::is_system_table_chunk_key ( const ChunkKey & chunk_key )

Definition at line 110 of file FsiChunkUtils.cpp.

References get_foreign_table_for_key(), and TableDescriptor::is_system_table.

Referenced by is_shardable_key().

                                                           {
   return get_foreign_table_for_key(chunk_key).is_system_table;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::is_table_enabled_on_node ( const ChunkKey & key )

Referenced by foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecFromDataWrapper().

Here is the caller graph for this function:

bool foreign_storage::is_valid_source_type ( const import_export::CopyParams & copy_params )

Verify if source_type is valid.

Definition at line 124 of file ForeignDataWrapperFactory.cpp.

References import_export::kDelimitedFile, import_export::kParquetFile, import_export::kRegexParsedFile, and import_export::CopyParams::source_type.

Referenced by foreign_storage::ForeignDataWrapperFactory::createForeignServerProxy(), foreign_storage::ForeignDataWrapperFactory::createForeignTableProxy(), and import_export::ForeignDataImporter::importGeneral().

                                                                       {
   return
 #ifdef ENABLE_IMPORT_PARQUET
       copy_params.source_type == import_export::SourceType::kParquetFile ||
 #endif
       copy_params.source_type == import_export::SourceType::kDelimitedFile ||
       copy_params.source_type == import_export::SourceType::kRegexParsedFile;
 }

Here is the caller graph for this function:

bool foreign_storage::key_does_not_shard_to_leaf ( const ChunkKey & key )

Definition at line 135 of file FsiChunkUtils.cpp.

References fragment_maps_to_leaf(), and is_shardable_key().

Referenced by cache_blocks(), populate_string_dictionary(), and anonymous_namespace{RelAlgExecutor.cpp}::set_parallelism_hints().

                                                      {
   return (is_shardable_key(key) && !fragment_maps_to_leaf(key));
 }

Here is the call graph for this function:

Here is the caller graph for this function:

size_t foreign_storage::num_rows_to_process	(	const size_t	start_row_index,
		const size_t	max_fragment_size,
		const size_t	rows_remaining
	)

Number of rows to process given the current position defined by start_row_index, the max fragment size and the number of rows left to process.

Definition at line 480 of file AbstractTextFileDataWrapper.cpp.

Referenced by QueryExecutionContext::launchCpuCode(), and populate_chunks().

                                                         {
   size_t start_position_in_fragment = start_row_index % max_fragment_size;
   return max_fragment_size - start_position_in_fragment;
 }

Here is the caller graph for this function:

UniqueReaderPtr foreign_storage::open_parquet_table	(	const std::string &	file_path,
		std::shared_ptr< arrow::fs::FileSystem > &	file_system
	)

Definition at line 26 of file ParquetShared.cpp.

Referenced by foreign_storage::FileReaderMap::getOrInsert(), foreign_storage::FileReaderMap::insert(), and foreign_storage::LazyParquetChunkLoader::loadRowGroups().

                                                                                   {
   UniqueReaderPtr reader;
   auto file_result = file_system->OpenInputFile(file_path);
   if (!file_result.ok()) {
     throw std::runtime_error{"Unable to access " + file_system->type_name() + " file: " +
                              file_path + ". " + file_result.status().message()};
   }
   auto infile = file_result.ValueOrDie();
   PARQUET_THROW_NOT_OK(OpenFile(infile, arrow::default_memory_pool(), &reader));
   return reader;
 }

Here is the caller graph for this function:

ParseFileRegionResult foreign_storage::parse_file_regions	(	const FileRegions &	file_regions,
		const size_t	start_index,
		const size_t	end_index,
		FileReader &	file_reader,
		ParseBufferRequest &	parse_file_request,
		const std::map< int, Chunk_NS::Chunk > &	column_id_to_chunk_map,
		const TextFileBufferParser &	parser
	)

Parses a set of file regions given a handle to the file and range of indexes for the file regions to be parsed.

Definition at line 213 of file AbstractTextFileDataWrapper.cpp.

Referenced by foreign_storage::AbstractTextFileDataWrapper::populateChunks().

                                         {
   auto timer = DEBUG_TIMER(__func__);
   ParseFileRegionResult load_file_region_result{};
   load_file_region_result.file_offset = file_regions[start_index].first_row_file_offset;
   load_file_region_result.row_count = 0;
 
   ParseBufferResult result;
   for (size_t i = start_index; i <= end_index; i++) {
     CHECK(file_regions[i].region_size <= parse_file_request.buffer_size);
     auto read_size = file_reader.readRegion(parse_file_request.buffer.get(),
                                             file_regions[i].first_row_file_offset,
                                             file_regions[i].region_size);
     if (file_regions[i].region_size != read_size) {
       throw_unexpected_number_of_items(file_regions[i].region_size,
                                        read_size,
                                        "bytes",
                                        parse_file_request.getTableName());
     }
     parse_file_request.begin_pos = 0;
     parse_file_request.end_pos = file_regions[i].region_size;
     parse_file_request.first_row_index = file_regions[i].first_row_index;
     parse_file_request.file_offset = file_regions[i].first_row_file_offset;
     parse_file_request.process_row_count = file_regions[i].row_count;
 
     result = parser.parseBuffer(parse_file_request, i == end_index);
     CHECK_EQ(file_regions[i].row_count, result.row_count);
     for (const auto& rejected_row_index : result.rejected_rows) {
       load_file_region_result.rejected_row_indices.insert(
           load_file_region_result.row_count + rejected_row_index);
     }
     load_file_region_result.row_count += result.row_count;
   }
   load_file_region_result.column_id_to_data_blocks_map =
       result.column_id_to_data_blocks_map;
   return load_file_region_result;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::vector<size_t> foreign_storage::partition_by_fragment	(	const size_t	start_row_index,
		const size_t	max_fragment_size,
		const size_t	buffer_row_count
	)

Given a start row index, maximum fragment size, and number of rows in a buffer, this function returns a vector indicating how the rows in the buffer should be partitioned in order to fill up available fragment slots while staying within the capacity of fragments.

Definition at line 493 of file AbstractTextFileDataWrapper.cpp.

References CHECK.

Referenced by scan_metadata().

                                                                          {
   CHECK(buffer_row_count > 0);
   std::vector<size_t> partitions{};
   size_t remaining_rows_in_last_fragment;
   if (start_row_index % max_fragment_size == 0) {
     remaining_rows_in_last_fragment = 0;
   } else {
     remaining_rows_in_last_fragment =
         max_fragment_size - (start_row_index % max_fragment_size);
   }
   if (buffer_row_count <= remaining_rows_in_last_fragment) {
     partitions.emplace_back(buffer_row_count);
   } else {
     if (remaining_rows_in_last_fragment > 0) {
       partitions.emplace_back(remaining_rows_in_last_fragment);
     }
     size_t remaining_buffer_row_count =
         buffer_row_count - remaining_rows_in_last_fragment;
     while (remaining_buffer_row_count > 0) {
       partitions.emplace_back(
           std::min<size_t>(remaining_buffer_row_count, max_fragment_size));
       remaining_buffer_row_count -= partitions.back();
     }
   }
   return partitions;
 }

Here is the caller graph for this function:

template<typename T >

auto foreign_storage::partition_for_threads	(	const std::set< T > &	items,
		size_t	max_threads
	)

Definition at line 41 of file FsiChunkUtils.h.

Referenced by create_futures_for_workers(), and foreign_storage::LazyParquetChunkLoader::metadataScan().

                                                                        {
   const size_t items_per_thread = (items.size() + (max_threads - 1)) / max_threads;
   std::list<std::set<T>> items_by_thread;
   auto i = 0U;
   for (auto item : items) {
     if (i++ % items_per_thread == 0) {
       items_by_thread.emplace_back(std::set<T>{});
     }
     items_by_thread.back().emplace(item);
   }
   return items_by_thread;
 }

Here is the caller graph for this function:

template<typename T >

auto foreign_storage::partition_for_threads	(	const std::vector< T > &	items,
		size_t	max_threads
	)

Definition at line 60 of file FsiChunkUtils.h.

                                                                           {
   const size_t items_per_thread = (items.size() + (max_threads - 1)) / max_threads;
   std::list<std::vector<T>> items_by_thread;
   auto i = 0U;
   for (auto item : items) {
     if (i++ % items_per_thread == 0) {
       items_by_thread.emplace_back(std::vector<T>{});
     }
     items_by_thread.back().emplace_back(item);
   }
   return items_by_thread;
 }

void foreign_storage::populate_chunks	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		std::map< int, FileRegions > &	fragment_id_to_file_regions_map,
		const TextFileBufferParser &	parser,
		foreign_storage::IterativeFileScanParameters &	file_scan_param
	)

Consumes and processes scan requests from a pending requests queue and populates chunks during an iterative file scan

Definition at line 918 of file AbstractTextFileDataWrapper.cpp.

Referenced by foreign_storage::AbstractTextFileDataWrapper::iterativeFileScan().

                                                                                   {
   std::map<int, const ColumnDescriptor*> column_by_id{};
   while (true) {
     auto request_opt = get_next_scan_request(multi_threading_params);
     if (!request_opt.has_value()) {
       break;
     }
     ParseBufferRequest& request = request_opt.value();
     try {
       if (column_by_id.empty()) {
         for (const auto column : request.getColumns()) {
           column_by_id[column->columnId] = column;
         }
       }
       CHECK_LE(request.processed_row_count, request.buffer_row_count);
       for (size_t num_rows_left_to_process =
                request.buffer_row_count - request.processed_row_count;
            num_rows_left_to_process > 0;
            num_rows_left_to_process =
                request.buffer_row_count - request.processed_row_count) {
         // NOTE: `request.begin_pos` state is required to be set correctly by this point
         // in execution
         size_t row_index = request.first_row_index + request.processed_row_count;
         int fragment_id = row_index / request.getMaxFragRows();
         if (fragment_id >
             file_scan_param.fragment_id) {  // processing must continue next iteration
           defer_scan_request(multi_threading_params, request);
           return;
         }
         request.process_row_count = num_rows_to_process(
             row_index, request.getMaxFragRows(), num_rows_left_to_process);
         for (const auto& import_buffer : request.import_buffers) {
           if (import_buffer != nullptr) {
             import_buffer->clear();
           }
         }
         auto result = parser.parseBuffer(request, true);
         populate_chunks_using_data_blocks(multi_threading_params,
                                           fragment_id,
                                           request,
                                           result,
                                           column_by_id,
                                           fragment_id_to_file_regions_map,
                                           file_scan_param);
         request.processed_row_count += result.row_count;
         request.begin_pos = result.row_offsets.back() - request.file_offset;
       }
 
     } catch (...) {
       // Re-add request to pool so we dont block any other threads
       {
         std::lock_guard<std::mutex> pending_requests_lock(
             multi_threading_params.pending_requests_mutex);
         multi_threading_params.continue_processing = false;
       }
       add_request_to_pool(multi_threading_params, request);
       throw;
     }
     add_request_to_pool(multi_threading_params, request);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::populate_chunks_using_data_blocks	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		int	fragment_id,
		const ParseBufferRequest &	request,
		ParseBufferResult &	result,
		std::map< int, const ColumnDescriptor * > &	column_by_id,
		std::map< int, FileRegions > &	fragment_id_to_file_regions_map,
		const foreign_storage::IterativeFileScanParameters &	file_scan_param
	)

Definition at line 679 of file AbstractTextFileDataWrapper.cpp.

References add_file_region(), append_data_block_to_chunk(), CHECK_EQ, foreign_storage::MetadataScanMultiThreadingParams::chunk_encoder_buffers, foreign_storage::MetadataScanMultiThreadingParams::chunk_encoder_buffers_mutex, foreign_storage::ParseBufferResult::column_id_to_data_blocks_map, foreign_storage::ParseBufferRequest::db_id, foreign_storage::ParseBufferRequest::first_row_index, foreign_storage::IterativeFileScanParameters::fragment_id, foreign_storage::ParseBufferRequest::getFilePath(), foreign_storage::ParseBufferRequest::getTableId(), foreign_storage::ParseBufferResult::rejected_rows, and foreign_storage::ParseBufferResult::row_count.

Referenced by populate_chunks().

                                                                        {
   std::unique_lock<std::mutex> lock(multi_threading_params.chunk_encoder_buffers_mutex);
   // File regions should be added in same order as appendData
   add_file_region(fragment_id_to_file_regions_map,
                   fragment_id,
                   request.first_row_index,
                   result,
                   request.getFilePath());
   CHECK_EQ(fragment_id, file_scan_param.fragment_id);
 
   for (auto& [column_id, data_block] : result.column_id_to_data_blocks_map) {
     ChunkKey chunk_key{request.db_id, request.getTableId(), column_id, fragment_id};
     const auto column = column_by_id[column_id];
     if (column->columnType.is_varlen_indeed()) {
       chunk_key.emplace_back(1);
     }
     if (multi_threading_params.chunk_encoder_buffers.find(chunk_key) ==
         multi_threading_params.chunk_encoder_buffers.end()) {
       multi_threading_params.chunk_encoder_buffers[chunk_key] =
           std::make_unique<ForeignStorageBuffer>();
       multi_threading_params.chunk_encoder_buffers[chunk_key]->initEncoder(
           column->columnType);
     }
     size_t current_element_count =
         multi_threading_params.chunk_encoder_buffers[chunk_key]
             ->getEncoder()
             ->getNumElems();
     size_t num_elements = current_element_count + result.row_count;
     multi_threading_params.chunk_encoder_buffers[chunk_key]->getEncoder()->setNumElems(
         num_elements);
     lock.unlock();  // unlock the fragment based lock in order to achieve better
                     // performance
     append_data_block_to_chunk(file_scan_param,
                                data_block,
                                result.row_count,
                                chunk_key,
                                column,
                                result.rejected_rows,
                                current_element_count);
     lock.lock();
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::populate_string_dictionary	(	const int32_t	table_id,
		const int32_t	col_id,
		const Catalog_Namespace::Catalog &	catalog
	)

Definition at line 205 of file Execute.cpp.

References CHECK, Data_Namespace::CPU_LEVEL, Chunk_NS::Chunk::getChunk(), Catalog_Namespace::Catalog::getDatabaseId(), Catalog_Namespace::Catalog::getDataMgr(), Catalog_Namespace::Catalog::getMetadataForColumn(), Catalog_Namespace::Catalog::getMetadataForTable(), anonymous_namespace{Execute.cpp}::is_empty_table(), and key_does_not_shard_to_leaf().

Referenced by anonymous_namespace{RelAlgExecutor.cpp}::prepare_string_dictionaries().

                                                                          {
   if (const auto foreign_table = dynamic_cast<const ForeignTable*>(
           catalog.getMetadataForTable(table_id, false))) {
     const auto col_desc = catalog.getMetadataForColumn(table_id, col_id);
     if (col_desc->columnType.is_dict_encoded_type()) {
       auto& fragmenter = foreign_table->fragmenter;
       CHECK(fragmenter != nullptr);
       if (is_empty_table(fragmenter.get())) {
         return;
       }
       for (const auto& frag : fragmenter->getFragmentsForQuery().fragments) {
         ChunkKey chunk_key = {catalog.getDatabaseId(), table_id, col_id, frag.fragmentId};
         // If the key is sharded across leaves, only populate fragments that are sharded
         // to this leaf.
         if (key_does_not_shard_to_leaf(chunk_key)) {
           continue;
         }
 
         const ChunkMetadataMap& metadata_map = frag.getChunkMetadataMap();
         CHECK(metadata_map.find(col_id) != metadata_map.end());
         if (auto& meta = metadata_map.at(col_id); meta->isPlaceholder()) {
           // When this goes out of scope it will stay in CPU cache but become
           // evictable
           auto chunk = Chunk_NS::Chunk::getChunk(col_desc,
                                                  &(catalog.getDataMgr()),
                                                  chunk_key,
                                                  Data_Namespace::CPU_LEVEL,
                                                  0,
                                                  0,
                                                  0);
         }
       }
     }
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::process_data_blocks	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		int	fragment_id,
		const ParseBufferRequest &	request,
		ParseBufferResult &	result,
		std::map< int, const ColumnDescriptor * > &	column_by_id,
		std::map< int, FileRegions > &	fragment_id_to_file_regions_map
	)

Updates metadata encapsulated in encoders for all table columns given new data blocks gotten from parsing a new set of rows in a file buffer. If cache is available, also append the data_blocks to chunks in the cache

Definition at line 734 of file AbstractTextFileDataWrapper.cpp.

References add_file_region(), cache_blocks(), foreign_storage::MetadataScanMultiThreadingParams::cached_chunks, foreign_storage::MetadataScanMultiThreadingParams::chunk_encoder_buffers, foreign_storage::MetadataScanMultiThreadingParams::chunk_encoder_buffers_mutex, foreign_storage::ParseBufferResult::column_id_to_data_blocks_map, foreign_storage::ParseBufferRequest::db_id, foreign_storage::MetadataScanMultiThreadingParams::disable_cache, foreign_storage::ParseBufferRequest::first_row_index, foreign_storage::ParseBufferRequest::getFilePath(), foreign_storage::ParseBufferRequest::getTableId(), foreign_storage::ParseBufferResult::row_count, and update_stats().

Referenced by scan_metadata().

                                                                                     {
   std::lock_guard<std::mutex> lock(multi_threading_params.chunk_encoder_buffers_mutex);
   // File regions should be added in same order as appendData
   add_file_region(fragment_id_to_file_regions_map,
                   fragment_id,
                   request.first_row_index,
                   result,
                   request.getFilePath());
 
   for (auto& [column_id, data_block] : result.column_id_to_data_blocks_map) {
     ChunkKey chunk_key{request.db_id, request.getTableId(), column_id, fragment_id};
     const auto column = column_by_id[column_id];
     if (column->columnType.is_varlen_indeed()) {
       chunk_key.emplace_back(1);
     }
     if (multi_threading_params.chunk_encoder_buffers.find(chunk_key) ==
         multi_threading_params.chunk_encoder_buffers.end()) {
       multi_threading_params.chunk_encoder_buffers[chunk_key] =
           std::make_unique<ForeignStorageBuffer>();
       multi_threading_params.chunk_encoder_buffers[chunk_key]->initEncoder(
           column->columnType);
     }
     update_stats(multi_threading_params.chunk_encoder_buffers[chunk_key]->getEncoder(),
                  column->columnType,
                  data_block,
                  result.row_count);
     size_t num_elements = multi_threading_params.chunk_encoder_buffers[chunk_key]
                               ->getEncoder()
                               ->getNumElems() +
                           result.row_count;
     multi_threading_params.chunk_encoder_buffers[chunk_key]->getEncoder()->setNumElems(
         num_elements);
     cache_blocks(
         multi_threading_params.cached_chunks,
         data_block,
         result.row_count,
         chunk_key,
         column,
         (num_elements - result.row_count) == 0,  // Is the first block added to this chunk
         multi_threading_params.disable_cache);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::refresh_foreign_table	(	Catalog_Namespace::Catalog &	catalog,
		const std::string &	table_name,
		const bool	evict_cached_entries
	)

Definition at line 103 of file ForeignTableRefresh.cpp.

References CHECK, shared::contains(), StorageType::FOREIGN_TABLE, dist::is_leaf_node(), Catalog_Namespace::kAggregatorOnlySystemTables, and refresh_foreign_table_unlocked().

Referenced by RefreshForeignTablesCommand::execute(), and foreign_storage::ForeignTableRefreshScheduler::start().

                                                             {
   if (dist::is_leaf_node() &&
       shared::contains(Catalog_Namespace::kAggregatorOnlySystemTables, table_name)) {
     // Skip aggregator only system tables on leaf nodes.
     return;
   }
   auto table_lock =
       std::make_unique<lockmgr::TableSchemaLockContainer<lockmgr::WriteLock>>(
           lockmgr::TableSchemaLockContainer<lockmgr::WriteLock>::acquireTableDescriptor(
               catalog, table_name, false));
 
   const TableDescriptor* td = (*table_lock)();
   if (td->storageType != StorageType::FOREIGN_TABLE) {
     throw std::runtime_error{
         table_name +
         " is not a foreign table. Refreshes are applicable to only foreign tables."};
   }
 
   auto foreign_table = dynamic_cast<const ForeignTable*>(td);
   CHECK(foreign_table);
   refresh_foreign_table_unlocked(catalog, *foreign_table, evict_cached_entries);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::refresh_foreign_table_unlocked	(	Catalog_Namespace::Catalog &	catalog,
		const ForeignTable &	td,
		const bool	evict_cached_entries
	)

Definition at line 35 of file ForeignTableRefresh.cpp.

References CHECK, CHUNK_KEY_DB_IDX, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, foreign_storage::anonymous_namespace{ForeignTableRefresh.cpp}::clear_cpu_and_gpu_cache(), Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::Catalog::getCurrentDB(), Catalog_Namespace::Catalog::getDataMgr(), PostEvictionRefreshException::getOriginalException(), logger::INFO, CacheInvalidator< CACHE_HOLDING_TYPES >::invalidateCachesByTable(), foreign_storage::ForeignTable::isAppendMode(), LOG, Catalog_Namespace::Catalog::removeFragmenterForTable(), TableDescriptor::tableId, TableDescriptor::tableName, and Catalog_Namespace::Catalog::updateForeignTableRefreshTimes().

Referenced by refresh_foreign_table().

                                                                      {
   LOG(INFO) << "Starting refresh for table: " << td.tableName;
   auto& data_mgr = catalog.getDataMgr();
   ChunkKey table_key{catalog.getCurrentDB().dbId, td.tableId};
   ResultSetCacheInvalidator::invalidateCachesByTable(boost::hash_value(table_key));
   catalog.removeFragmenterForTable(td.tableId);
 
   auto fsm = data_mgr.getPersistentStorageMgr()->getForeignStorageMgr();
   CHECK(fsm);
   if (auto cfm = dynamic_cast<CachingForeignStorageMgr*>(fsm)) {
     if (!cfm->hasStoredDataWrapper(table_key[CHUNK_KEY_DB_IDX],
                                    table_key[CHUNK_KEY_TABLE_IDX])) {
       // If there is no wrapper stored on disk, then we have not populated the metadata
       // for this table and we are free to skip the refresh.
       catalog.updateForeignTableRefreshTimes(td.tableId);
       return;
     }
   }
 
   std::map<ChunkKey, std::shared_ptr<ChunkMetadata>> old_chunk_metadata_by_chunk_key;
   if (td.isAppendMode() && !evict_cached_entries) {
     ChunkMetadataVector metadata_vec;
     data_mgr.getChunkMetadataVecForKeyPrefix(metadata_vec, table_key);
     int last_fragment_id = 0;
     for (const auto& [key, metadata] : metadata_vec) {
       if (key[CHUNK_KEY_FRAGMENT_IDX] > last_fragment_id) {
         last_fragment_id = key[CHUNK_KEY_FRAGMENT_IDX];
       }
       old_chunk_metadata_by_chunk_key[key] = metadata;
     }
     for (const auto& [key, metadata] : metadata_vec) {
       if (key[CHUNK_KEY_FRAGMENT_IDX] == last_fragment_id) {
         clear_cpu_and_gpu_cache(data_mgr, key);
       }
     }
   } else {
     clear_cpu_and_gpu_cache(data_mgr, table_key);
   }
 
   try {
     fsm->refreshTable(table_key, evict_cached_entries);
     catalog.updateForeignTableRefreshTimes(td.tableId);
   } catch (PostEvictionRefreshException& e) {
     catalog.updateForeignTableRefreshTimes(td.tableId);
     clear_cpu_and_gpu_cache(data_mgr, table_key);
     throw e.getOriginalException();
   } catch (...) {
     clear_cpu_and_gpu_cache(data_mgr, table_key);
     throw;
   }
 
   // Delete cached rolled off/updated chunks.
   if (!old_chunk_metadata_by_chunk_key.empty()) {
     ChunkMetadataVector new_metadata_vec;
     data_mgr.getChunkMetadataVecForKeyPrefix(new_metadata_vec, table_key);
     for (const auto& [key, metadata] : new_metadata_vec) {
       auto it = old_chunk_metadata_by_chunk_key.find(key);
       if (it != old_chunk_metadata_by_chunk_key.end() &&
           it->second->numElements != metadata->numElements) {
         clear_cpu_and_gpu_cache(data_mgr, key);
       }
     }
   }
   LOG(INFO) << "Completed refresh for table: " << td.tableName;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::reset_multithreading_params ( foreign_storage::MetadataScanMultiThreadingParams & multi_threading_params )

Definition at line 997 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::cached_chunks, foreign_storage::MetadataScanMultiThreadingParams::chunk_encoder_buffers, foreign_storage::MetadataScanMultiThreadingParams::deferred_requests, foreign_storage::MetadataScanMultiThreadingParams::pending_requests, and foreign_storage::MetadataScanMultiThreadingParams::request_pool.

Referenced by foreign_storage::AbstractTextFileDataWrapper::iterativeFileScan().

                                                                              {
   multi_threading_params.request_pool = {};
   multi_threading_params.cached_chunks = {};
   multi_threading_params.pending_requests = {};
   multi_threading_params.deferred_requests = {};
   multi_threading_params.chunk_encoder_buffers.clear();
 }

Here is the caller graph for this function:

void foreign_storage::resize_buffer_if_needed	(	std::unique_ptr< char[]> &	buffer,
		size_t &	buffer_size,
		const size_t	alloc_size
	)

Optionally resizes the given buffer if the buffer size is less than the current buffer allocation size.

Definition at line 987 of file AbstractTextFileDataWrapper.cpp.

References CHECK_LE.

Referenced by dispatch_scan_requests().

                                                       {
   CHECK_LE(buffer_size, alloc_size);
   if (buffer_size < alloc_size) {
     buffer = std::make_unique<char[]>(alloc_size);
     buffer_size = alloc_size;
   }
 }

Here is the caller graph for this function:

std::vector< Aws::S3::Model::Object > foreign_storage::s3_objects_filter_sort_files	(	const std::vector< Aws::S3::Model::Object > &	file_paths,
		const shared::FilePathOptions &	options
	)

Definition at line 22 of file S3FilePathUtil.cpp.

References shared::FilePathOptions::filter_regex, shared::PATHNAME_ORDER_TYPE, foreign_storage::anonymous_namespace{S3FilePathUtil.cpp}::s3_objects_regex_file_filter(), and shared::FilePathOptions::sort_by.

                                           {
   auto result_files =
       options.filter_regex.has_value()
           ? s3_objects_regex_file_filter(options.filter_regex.value(), file_paths)
           : file_paths;
   // initial lexicographical order ensures a determinisitc ordering for files not matching
   // sort_regex
   shared::FilePathOptions temp_options;
   temp_options.sort_by = shared::PATHNAME_ORDER_TYPE;
   auto initial_file_order = FileOrderS3(temp_options);
   auto lexi_comp = initial_file_order.getFileComparator();
   std::stable_sort(result_files.begin(), result_files.end(), lexi_comp);
 
   auto file_order = FileOrderS3(options);
   auto comp = file_order.getFileComparator();
   std::stable_sort(result_files.begin(), result_files.end(), comp);
   return result_files;
 }

Here is the call graph for this function:

void foreign_storage::scan_metadata	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		std::map< int, FileRegions > &	fragment_id_to_file_regions_map,
		const TextFileBufferParser &	parser
	)

Consumes and processes metadata scan requests from a pending requests queue and updates existing metadata objects based on newly scanned metadata.

Definition at line 799 of file AbstractTextFileDataWrapper.cpp.

References add_request_to_pool(), foreign_storage::MetadataScanMultiThreadingParams::continue_processing, get_next_scan_request(), foreign_storage::TextFileBufferParser::parseBuffer(), partition_by_fragment(), foreign_storage::MetadataScanMultiThreadingParams::pending_requests_mutex, process_data_blocks(), and run_benchmark_import::result.

Referenced by foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata().

                                                        {
   std::map<int, const ColumnDescriptor*> column_by_id{};
   while (true) {
     auto request_opt = get_next_scan_request(multi_threading_params);
     if (!request_opt.has_value()) {
       break;
     }
     auto& request = request_opt.value();
     try {
       if (column_by_id.empty()) {
         for (const auto column : request.getColumns()) {
           column_by_id[column->columnId] = column;
         }
       }
       auto partitions = partition_by_fragment(
           request.first_row_index, request.getMaxFragRows(), request.buffer_row_count);
       request.begin_pos = 0;
       size_t row_index = request.first_row_index;
       for (const auto partition : partitions) {
         request.process_row_count = partition;
         for (const auto& import_buffer : request.import_buffers) {
           if (import_buffer != nullptr) {
             import_buffer->clear();
           }
         }
         auto result = parser.parseBuffer(request, true);
         int fragment_id = row_index / request.getMaxFragRows();
         process_data_blocks(multi_threading_params,
                             fragment_id,
                             request,
                             result,
                             column_by_id,
                             fragment_id_to_file_regions_map);
         row_index += result.row_count;
         request.begin_pos = result.row_offsets.back() - request.file_offset;
       }
     } catch (...) {
       // Re-add request to pool so we dont block any other threads
       {
         std::lock_guard<std::mutex> pending_requests_lock(
             multi_threading_params.pending_requests_mutex);
         multi_threading_params.continue_processing = false;
       }
       add_request_to_pool(multi_threading_params, request);
       throw;
     }
     add_request_to_pool(multi_threading_params, request);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::set_comp	(	const ChunkKey &	left,
		const ChunkKey &	right
	)

void foreign_storage::set_node_name ( std::map< std::string, import_export::TypedImportBuffer * > & import_buffers )

Definition at line 55 of file InternalSystemDataWrapper.cpp.

References g_distributed_leaf_idx, dist::is_leaf_node(), and to_string().

Referenced by foreign_storage::anonymous_namespace{InternalMemoryStatsDataWrapper.cpp}::populate_import_buffers_for_memory_details(), foreign_storage::anonymous_namespace{InternalMemoryStatsDataWrapper.cpp}::populate_import_buffers_for_memory_summary(), and foreign_storage::anonymous_namespace{InternalStorageStatsDataWrapper.cpp}::populate_import_buffers_for_storage_details().

                                                                         {
   if (import_buffers.find("node") != import_buffers.end()) {
     if (dist::is_leaf_node()) {
       std::string leaf_string{"Leaf " + to_string(g_distributed_leaf_idx)};
       import_buffers["node"]->addString(leaf_string);
     } else {
       import_buffers["node"]->addString("Server");
     }
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::set_value	(	rapidjson::Value &	json_val,
		const FileRegion &	file_region,
		rapidjson::Document::AllocatorType &	allocator
	)

Definition at line 26 of file CsvShared.cpp.

References foreign_storage::json_utils::add_value_to_object(), foreign_storage::FileRegion::first_row_file_offset, foreign_storage::FileRegion::first_row_index, foreign_storage::FileRegion::region_size, and foreign_storage::FileRegion::row_count.

                                                           {
   json_val.SetObject();
   json_utils::add_value_to_object(
       json_val, file_region.first_row_file_offset, "first_row_file_offset", allocator);
   json_utils::add_value_to_object(
       json_val, file_region.first_row_index, "first_row_index", allocator);
   json_utils::add_value_to_object(
       json_val, file_region.region_size, "region_size", allocator);
   json_utils::add_value_to_object(
       json_val, file_region.row_count, "row_count", allocator);
   if (file_region.filename.size()) {
     json_utils::add_value_to_object(
         json_val, file_region.filename, "filename", allocator);
   }
 }

Here is the call graph for this function:

void foreign_storage::set_value	(	rapidjson::Value &	json_val,
		const RowGroupInterval &	value,
		rapidjson::Document::AllocatorType &	allocator
	)

Definition at line 662 of file ParquetDataWrapper.cpp.

References foreign_storage::json_utils::add_value_to_object(), foreign_storage::RowGroupInterval::end_index, foreign_storage::RowGroupInterval::file_path, and foreign_storage::RowGroupInterval::start_index.

                                                           {
   json_val.SetObject();
   json_utils::add_value_to_object(json_val, value.file_path, "file_path", allocator);
   json_utils::add_value_to_object(json_val, value.start_index, "start_index", allocator);
   json_utils::add_value_to_object(json_val, value.end_index, "end_index", allocator);
 }

Here is the call graph for this function:

void foreign_storage::throw_file_access_error	(	const std::string &	file_path,
		const std::string &	message
	)

inline

Definition at line 89 of file ForeignStorageException.h.

Referenced by foreign_storage::ParquetImporter::getAllFilePaths().

                                                               {
   std::string error_message{"Unable to access file \"" + file_path + "\". " + message};
   throw ForeignStorageException{error_message};
 }

Here is the caller graph for this function:

void foreign_storage::throw_file_not_found_error ( const std::string & file_path )

inline

Definition at line 95 of file ForeignStorageException.h.

Referenced by foreign_storage::ParquetImporter::getAllFilePaths().

                                                                    {
   throw ForeignStorageException{"File or directory \"" + file_path +
                                 "\" does not exist."};
 }

Here is the caller graph for this function:

void foreign_storage::throw_number_of_columns_mismatch_error	(	size_t	num_table_cols,
		size_t	num_file_cols,
		const std::string &	file_path
	)

inline

Definition at line 80 of file ForeignStorageException.h.

References to_string().

Referenced by foreign_storage::RegexFileBufferParser::regexMatchColumns(), foreign_storage::anonymous_namespace{CsvFileBufferParser.cpp}::validate_expected_column_count(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_number_of_columns().

                                                                                {
   throw ForeignStorageException{"Mismatched number of logical columns: (expected " +
                                 std::to_string(num_table_cols) + " columns, has " +
                                 std::to_string(num_file_cols) + "): in file '" +
                                 file_path + "'"};
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::throw_parquet_metadata_out_of_bounds_error	(	const std::string &	min_value,
		const std::string &	max_value,
		const std::string &	encountered_value
	)

inline

Definition at line 46 of file ParquetMetadataValidator.h.

Referenced by foreign_storage::TimestampBoundsValidator< T >::validateValue(), foreign_storage::IntegralFixedLengthBoundsValidator< T >::validateValue(), foreign_storage::BaseDateBoundsValidator< T, is_in_seconds >::validateValue(), and foreign_storage::FloatPointValidator< T >::validateValue().

                                         {
   std::stringstream error_message;
   error_message << "Parquet column contains values that are outside the range of the "
                    "HeavyDB column "
                    "type. Consider using a wider column type. Min allowed value: "
                 << min_value << ". Max allowed value: " << max_value
                 << ". Encountered value: " << encountered_value << ".";
   throw std::runtime_error(error_message.str());
 }

Here is the caller graph for this function:

void foreign_storage::throw_removed_file_error ( const std::string & file_path )

inline

Definition at line 73 of file ForeignStorageException.h.

Referenced by foreign_storage::LocalMultiFileReader::checkForMoreRows(), and foreign_storage::ParquetDataWrapper::fetchChunkMetadata().

                                                                  {
   throw ForeignStorageException{
       "Refresh of foreign table created with \"APPEND\" update type failed as "
       "file \"" +
       file_path + "\" was removed."};
 }

Here is the caller graph for this function:

void foreign_storage::throw_removed_row_in_file_error ( const std::string & file_path )

inline

Definition at line 66 of file ForeignStorageException.h.

Referenced by foreign_storage::SingleTextFileReader::checkForMoreRows(), and foreign_storage::ParquetDataWrapper::fetchChunkMetadata().

                                                                         {
   throw ForeignStorageException{
       "Refresh of foreign table created with \"APPEND\" update type failed as file "
       "reduced in size: \"" +
       file_path + "\""};
 }

Here is the caller graph for this function:

void foreign_storage::throw_removed_row_in_result_set_error ( const std::string & select_statement )

inline

Definition at line 58 of file ForeignStorageException.h.

                                                                                      {
   throw ForeignStorageException{
       "Refresh of foreign table created with \"APPEND\" update type failed as result set "
       "of select statement "
       "reduced in size: \"" +
       select_statement + "\""};
 }

void foreign_storage::throw_s3_compressed_extension	(	const std::string &	file_path,
		const std::string &	ext_type
	)

inline

Definition at line 107 of file ForeignStorageException.h.

                                                                      {
   throw ForeignStorageException{
       "File \"" + file_path + "\" has extension type \"" + ext_type +
       "\", compressed file formats are not supported by S3 Foreign Tables."};
 }

void foreign_storage::throw_s3_compressed_mime_type	(	const std::string &	file_path,
		const std::string &	mime_type
	)

inline

Definition at line 100 of file ForeignStorageException.h.

                                                                       {
   throw ForeignStorageException{
       "File \"" + file_path + "\" has mime type \"" + mime_type +
       "\", compressed file formats are not supported by S3 Foreign Tables."};
 }

void foreign_storage::throw_unexpected_number_of_items	(	const size_t &	num_expected,
		const size_t &	num_loaded,
		const std::string &	item_type
	)

inline

Definition at line 40 of file ForeignStorageException.h.

References to_string().

Referenced by parse_file_regions(), and foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::throw_unexpected_number_of_items().

                                                                          {
   throw ForeignStorageException(
       "Unexpected number of " + item_type +
       " while loading from foreign data source: expected " +
       std::to_string(num_expected) + " , obtained " + std::to_string(num_loaded) + " " +
       item_type +
       ". Please use the \"REFRESH FOREIGN TABLES\" command on the foreign table "
       "if data source has been updated.");
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::update_stats	(	Encoder *	encoder,
		const SQLTypeInfo &	column_type,
		DataBlockPtr	data_block,
		const size_t	row_count
	)

Updates the statistics metadata encapsulated in the encoder given new data in a data block.

Definition at line 567 of file AbstractTextFileDataWrapper.cpp.

References DataBlockPtr::arraysPtr, SQLTypeInfo::is_array(), SQLTypeInfo::is_varlen(), DataBlockPtr::numbersPtr, DataBlockPtr::stringsPtr, and Encoder::updateStats().

Referenced by process_data_blocks(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumnMetadata(), and StorageIOFacility::yieldUpdateCallback().

                                           {
   if (column_type.is_array()) {
     encoder->updateStats(data_block.arraysPtr, 0, row_count);
   } else if (!column_type.is_varlen()) {
     encoder->updateStats(data_block.numbersPtr, row_count);
   } else {
     encoder->updateStats(data_block.stringsPtr, 0, row_count);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::shared_ptr< parquet::Statistics > foreign_storage::validate_and_get_column_metadata_statistics ( const parquet::ColumnChunkMetaData * column_metadata )

Definition at line 86 of file ParquetShared.cpp.

References CHECK.

Referenced by foreign_storage::ParquetEncoder::getRowGroupMetadata(), foreign_storage::TypedParquetInPlaceEncoder< V, V >::getRowGroupMetadata(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_definition_levels().

                                                        {
   CHECK(column_metadata->is_stats_set());
   std::shared_ptr<parquet::Statistics> stats = column_metadata->statistics();
   return stats;
 }

Here is the caller graph for this function:

void foreign_storage::validate_equal_column_descriptor	(	const parquet::ColumnDescriptor *	reference_descriptor,
		const parquet::ColumnDescriptor *	new_descriptor,
		const std::string &	reference_file_path,
		const std::string &	new_file_path
	)

Definition at line 60 of file ParquetShared.cpp.

Referenced by foreign_storage::LazyParquetChunkLoader::appendRowGroups(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_equal_schema().

                                     {
   if (!reference_descriptor->Equals(*new_descriptor)) {
     throw std::runtime_error{"Parquet file \"" + new_file_path +
                              "\" has a different schema. Please ensure that all Parquet "
                              "files use the same schema. Reference Parquet file: " +
                              reference_file_path +
                              ", column name: " + reference_descriptor->name() +
                              ". New Parquet file: " + new_file_path +
                              ", column name: " + new_descriptor->name() + "."};
   }
 }

Here is the caller graph for this function:

void foreign_storage::validate_non_foreign_table_write ( const TableDescriptor * table_descriptor )

inline

Definition at line 22 of file FsiUtils.h.

References StorageType::FOREIGN_TABLE, and TableDescriptor::storageType.

Referenced by Parser::InsertStmt::analyze(), Parser::TruncateTableStmt::execute(), Parser::InsertValuesStmt::execute(), Parser::InsertIntoTableAsSelectStmt::populateData(), and RelModify::RelModify().

                                                                                       {
   if (table_descriptor && table_descriptor->storageType == StorageType::FOREIGN_TABLE) {
     throw std::runtime_error{
         "DELETE, INSERT, TRUNCATE, OR UPDATE commands are not supported for foreign "
         "tables."};
   }
 }

Here is the caller graph for this function:

void foreign_storage::validate_regex_parser_options ( const import_export::CopyParams & copy_params )

Definition at line 118 of file ForeignDataWrapperFactory.cpp.

References import_export::CopyParams::line_regex.

Referenced by anonymous_namespace{ForeignDataImporter.cpp}::validate_copy_params().

                                                                                {
   if (copy_params.line_regex.empty()) {
     throw std::runtime_error{"Regex parser options must contain a line regex."};
   }
 }

Here is the caller graph for this function:

Variable Documentation

constexpr const char* foreign_storage::kDeletedValueIndicator {"<DELETED>"}

Definition at line 26 of file InternalSystemDataWrapper.h.

Referenced by foreign_storage::anonymous_namespace{InternalMemoryStatsDataWrapper.cpp}::get_column_name(), get_db_name(), foreign_storage::anonymous_namespace{InternalCatalogDataWrapper.cpp}::get_table_ddl(), get_table_name(), and foreign_storage::anonymous_namespace{InternalCatalogDataWrapper.cpp}::get_user_name().

Namespaces

Classes

Typedefs

Functions

Variables

Typedef Documentation

Function Documentation

Variable Documentation