Namespaces
	anonymous_namespace{AbstractFileStorageDataWrapper.cpp}

	anonymous_namespace{AbstractTextFileDataWrapper.cpp}

	anonymous_namespace{CachingForeignStorageMgr.cpp}

	anonymous_namespace{CsvFileBufferParser.cpp}

	anonymous_namespace{FileReader.cpp}

	anonymous_namespace{ForeignDataWrapperFactory.cpp}

	anonymous_namespace{ForeignStorageCache.cpp}

	anonymous_namespace{ForeignTableRefresh.cpp}

	anonymous_namespace{InternalCatalogDataWrapper.cpp}

	anonymous_namespace{InternalMemoryStatsDataWrapper.cpp}

	anonymous_namespace{InternalStorageStatsDataWrapper.cpp}

	anonymous_namespace{InternalSystemDataWrapper.cpp}

	anonymous_namespace{LazyParquetChunkLoader.cpp}

	anonymous_namespace{LogFileBufferParser.cpp}

	anonymous_namespace{ParquetDataWrapper.cpp}

	anonymous_namespace{RefreshTimeCalculator.cpp}

	anonymous_namespace{RegexFileBufferParser.cpp}

	anonymous_namespace{RegexParserDataWrapper.cpp}

	anonymous_namespace{S3FilePathUtil.cpp}

	anonymous_namespace{TextFileBufferParser.cpp}

	Csv

	json_utils

Classes
struct	ForeignServer

struct	ForeignTable

struct	OptionsContainer

struct	UserMappingType

struct	UserMapping

class	RefreshTimeCalculator

class	AbstractFileStorageDataWrapper

struct	ParseFileRegionResult

struct	MetadataScanMultiThreadingParams

class	AbstractTextFileDataWrapper

class	CachingForeignStorageMgr

class	CsvDataWrapper

class	CsvFileBufferParser

struct	DataPreview

class	FileReader

class	SingleFileReader

class	SingleTextFileReader

class	ArchiveWrapper

class	CompressedFileReader

class	MultiFileReader

class	LocalMultiFileReader

struct	FileRegion

class	ForeignDataWrapper

struct	DataWrapperType
	Encapsulates an enumeration of foreign data wrapper type strings. More...

class	ForeignDataWrapperFactory

class	ForeignStorageBuffer

class	ForeignStorageCache

class	ForeignStorageException

class	MetadataScanInfeasibleFragmentSizeException

class	ChunkSizeValidator

class	MockForeignDataWrapper

class	ForeignStorageMgr

class	ForeignTableSchema

class	GeospatialEncoder

class	InternalCatalogDataWrapper

class	InternalLogsDataWrapper

class	InternalMemoryStatsDataWrapper

struct	StorageDetails

class	InternalStorageStatsDataWrapper

class	InternalSystemDataWrapper

struct	ColumnType

struct	FragmentType

struct	Interval

struct	ParquetBatchData

class	ParquetRowGroupReader

struct	PreviewContext

class	LazyParquetChunkLoader

class	LogFileBufferParser

class	OdbcGeospatialEncoder

class	ParquetArrayDetectEncoder

class	ParquetArrayEncoder

class	ParquetArrayImportEncoder

class	ParquetDataWrapper

class	ParquetDateInDaysFromTimestampEncoder

class	ParquetDateInSecondsEncoder

class	ParquetDecimalEncoder

class	ParquetDetectStringEncoder

class	ParquetEncoder

class	ParquetImportEncoder

class	ParquetScalarEncoder

class	ParquetFixedLengthArrayEncoder

class	ParquetFixedLengthEncoder

class	ParquetUnsignedFixedLengthEncoder

class	ParquetGeospatialEncoder

class	ParquetGeospatialImportEncoder

class	RowGroupIntervalTracker

class	ParquetImportBatchResult

class	AbstractRowGroupIntervalTracker

class	ParquetImporter

class	ParquetInPlaceEncoder

class	TypedParquetInPlaceEncoder

class	ParquetMetadataValidator

class	TimestampBoundsValidator

class	IntegralFixedLengthBoundsValidator

class	BaseDateBoundsValidator

class	FloatPointValidator

struct	RowGroupInterval

struct	RowGroupMetadata

class	FileReaderMap

class	ParquetStringEncoder

class	ParquetStringImportEncoder

class	ParquetStringNoneEncoder

class	ParquetTimeEncoder

class	ParquetTimestampEncoder

class	ParquetVariableLengthArrayEncoder

class	PassThroughBuffer

class	RegexFileBufferParser

class	RegexParserDataWrapper

class	FileOrderS3

struct	ParseBufferRequest

struct	ParseBufferResult

class	TextFileBufferParser

class	TypedParquetDetectBuffer

class	TypedParquetStorageBuffer

class	ForeignTableRefreshScheduler

Typedefs
using	OptionsMap = std::map< std::string, std::string, std::less<>>

using	SampleRows = std::vector< std::vector< std::string >>

using	FirstLineByFilePath = std::map< std::string, std::string >

using	FileRegions = std::vector< FileRegion >

using	ChunkToBufferMap = std::map< ChunkKey, AbstractBuffer * >

using	RenderGroupAnalyzerMap = std::map< int, std::unique_ptr< import_export::RenderGroupAnalyzer >>

using	read_lock = heavyai::shared_lock< heavyai::shared_mutex >

using	write_lock = heavyai::unique_lock< heavyai::shared_mutex >

using	FilePathAndRowGroup = std::pair< std::string, int32_t >

using	RejectedRowIndices = std::set< int64_t >

using	InvalidRowGroupIndices = std::set< int64_t >

template<typename T >
using	DateInSecondsBoundsValidator = BaseDateBoundsValidator< T, true >

template<typename T >
using	DateInDaysBoundsValidator = BaseDateBoundsValidator< T, false >

using	UniqueReaderPtr = std::unique_ptr< parquet::arrow::FileReader >

using	ReaderPtr = parquet::arrow::FileReader *

template<typename V , typename T , T conversion_denominator, typename NullType = V>
using	ParquetDateInSecondsFromTimestampEncoder = ParquetTimestampEncoder< V, T, conversion_denominator, NullType >

using	S3ObjectComparator = std::function< bool(const Aws::S3::Model::Object &, const Aws::S3::Model::Object &)>

Functions
ParseFileRegionResult	parse_file_regions (const FileRegions &file_regions, const size_t start_index, const size_t end_index, FileReader &file_reader, ParseBufferRequest &parse_file_request, const std::map< int, Chunk_NS::Chunk > &column_id_to_chunk_map, const TextFileBufferParser &parser)

size_t	get_buffer_size (const import_export::CopyParams &copy_params, const bool size_known, const size_t file_size)

size_t	get_buffer_size (const FileRegions &file_regions)

size_t	get_thread_count (const import_export::CopyParams &copy_params, const bool size_known, const size_t file_size, const size_t buffer_size)

size_t	get_thread_count (const import_export::CopyParams &copy_params, const FileRegions &file_regions)

std::vector< size_t >	partition_by_fragment (const size_t start_row_index, const size_t max_fragment_size, const size_t buffer_row_count)

std::optional< ParseBufferRequest >	get_next_metadata_scan_request (MetadataScanMultiThreadingParams &multi_threading_params)

void	add_file_region (std::map< int, FileRegions > &fragment_id_to_file_regions_map, int fragment_id, size_t first_row_index, const ParseBufferResult &result, const std::string &file_path)

void	update_stats (Encoder *encoder, const SQLTypeInfo &column_type, DataBlockPtr data_block, const size_t row_count)

void	cache_blocks (std::map< ChunkKey, Chunk_NS::Chunk > &cached_chunks, DataBlockPtr data_block, size_t row_count, ChunkKey &chunk_key, const ColumnDescriptor *column, bool is_first_block, bool is_last_block, bool disable_cache)

void	process_data_blocks (MetadataScanMultiThreadingParams &multi_threading_params, int fragment_id, const ParseBufferRequest &request, ParseBufferResult &result, std::map< int, const ColumnDescriptor * > &column_by_id, std::map< int, FileRegions > &fragment_id_to_file_regions_map)

void	add_request_to_pool (MetadataScanMultiThreadingParams &multi_threading_params, ParseBufferRequest &request)

void	scan_metadata (MetadataScanMultiThreadingParams &multi_threading_params, std::map< int, FileRegions > &fragment_id_to_file_regions_map, const TextFileBufferParser &parser)

ParseBufferRequest	get_request_from_pool (MetadataScanMultiThreadingParams &multi_threading_params)

void	dispatch_metadata_scan_request (MetadataScanMultiThreadingParams &multi_threading_params, ParseBufferRequest &request)

void	resize_buffer_if_needed (std::unique_ptr< char[]> &buffer, size_t &buffer_size, const size_t alloc_size)

void	dispatch_metadata_scan_requests (const size_t &buffer_size, const std::string &file_path, FileReader &file_reader, const import_export::CopyParams &copy_params, MetadataScanMultiThreadingParams &multi_threading_params, size_t &first_row_index_in_buffer, size_t &current_file_offset, const TextFileBufferParser &parser)

void	set_value (rapidjson::Value &json_val, const FileRegion &file_region, rapidjson::Document::AllocatorType &allocator)

void	get_value (const rapidjson::Value &json_val, FileRegion &file_region)

std::optional< SQLTypes >	detect_geo_type (const SampleRows &sample_rows, size_t column_index)

bool	is_s3_uri (const std::string &file_path)

std::tuple< std::unique_ptr < foreign_storage::ForeignServer > , std::unique_ptr < foreign_storage::UserMapping > , std::unique_ptr < foreign_storage::ForeignTable > >	create_proxy_fsi_objects (const std::string &copy_from_source, const import_export::CopyParams &copy_params, const int db_id, const TableDescriptor *table, const int32_t user_id)
	Create proxy fsi objects for use outside FSI. More...

std::tuple< std::unique_ptr < foreign_storage::ForeignServer > , std::unique_ptr < foreign_storage::UserMapping > , std::unique_ptr < foreign_storage::ForeignTable > >	create_proxy_fsi_objects (const std::string &copy_from_source, const import_export::CopyParams &copy_params, const TableDescriptor *table)
	Create proxy fsi objects for use outside FSI NOTE: parameters mirror function above. More...

void	validate_regex_parser_options (const import_export::CopyParams &copy_params)

bool	is_valid_source_type (const import_export::CopyParams &copy_params)

std::string	bool_to_option_value (const bool value)

void	throw_unexpected_number_of_items (const size_t &num_expected, const size_t &num_loaded, const std::string &item_type)

void	throw_removed_row_in_result_set_error (const std::string &select_statement)

void	throw_removed_row_in_file_error (const std::string &file_path)

void	throw_removed_file_error (const std::string &file_path)

void	throw_number_of_columns_mismatch_error (size_t num_table_cols, size_t num_file_cols, const std::string &file_path)

void	throw_file_access_error (const std::string &file_path, const std::string &message)

void	throw_file_not_found_error (const std::string &file_path)

void	throw_s3_compressed_mime_type (const std::string &file_path, const std::string &mime_type)

void	throw_s3_compressed_extension (const std::string &file_path, const std::string &ext_type)

bool	is_append_table_chunk_key (const ChunkKey &chunk_key)

bool	set_comp (const ChunkKey &left, const ChunkKey &right)

std::vector< ChunkKey >	get_column_key_vec (const ChunkKey &destination_chunk_key)

std::set< ChunkKey >	get_column_key_set (const ChunkKey &destination_chunk_key)

size_t	get_max_chunk_size (const ChunkKey &key)

bool	contains_fragment_key (const std::set< ChunkKey > &key_set, const ChunkKey &target_key)

bool	is_table_enabled_on_node (const ChunkKey &key)

void	refresh_foreign_table_unlocked (Catalog_Namespace::Catalog &catalog, const ForeignTable &td, const bool evict_cached_entries)

void	refresh_foreign_table (Catalog_Namespace::Catalog &catalog, const std::string &table_name, const bool evict_cached_entries)

void	init_chunk_for_column (const ChunkKey &chunk_key, const std::map< ChunkKey, std::shared_ptr< ChunkMetadata >> &chunk_metadata_map, const std::map< ChunkKey, AbstractBuffer * > &buffers, Chunk_NS::Chunk &chunk)

std::shared_ptr< ChunkMetadata >	get_placeholder_metadata (const SQLTypeInfo &type, size_t num_elements)

const foreign_storage::ForeignTable &	get_foreign_table_for_key (const ChunkKey &key)

bool	is_system_table_chunk_key (const ChunkKey &chunk_key)

bool	is_replicated_table_chunk_key (const ChunkKey &chunk_key)

bool	is_shardable_key (const ChunkKey &key)

bool	fragment_maps_to_leaf (const ChunkKey &key)

bool	key_does_not_shard_to_leaf (const ChunkKey &key)

template<typename T >
auto	partition_for_threads (const std::set< T > &items, size_t max_threads)

template<typename T >
auto	partition_for_threads (const std::vector< T > &items, size_t max_threads)

template<typename Container >
std::vector< std::future< void > >	create_futures_for_workers (const Container &items, size_t max_threads, std::function< void(const Container &)> lambda)

template<typename T >
ArrayDatum	encode_as_array_datum (const std::vector< T > &data)

std::string	get_db_name (int32_t db_id)

std::string	get_table_name (int32_t db_id, int32_t table_id)

void	set_node_name (std::map< std::string, import_export::TypedImportBuffer * > &import_buffers)

void	set_value (rapidjson::Value &json_val, const RowGroupInterval &value, rapidjson::Document::AllocatorType &allocator)

void	get_value (const rapidjson::Value &json_val, RowGroupInterval &value)

template<typename D , typename T >
bool	check_bounds (const T &value)

template<typename D >
std::string	datetime_to_string (const D &timestamp, const SQLTypeInfo &column_type)

void	throw_parquet_metadata_out_of_bounds_error (const std::string &min_value, const std::string &max_value, const std::string &encountered_value)

UniqueReaderPtr	open_parquet_table (const std::string &file_path, std::shared_ptr< arrow::fs::FileSystem > &file_system)

std::pair< int, int >	get_parquet_table_size (const ReaderPtr &reader)

const parquet::ColumnDescriptor *	get_column_descriptor (const parquet::arrow::FileReader *reader, const int logical_column_index)

parquet::Type::type	get_physical_type (ReaderPtr &reader, const int logical_column_index)

void	validate_equal_column_descriptor (const parquet::ColumnDescriptor reference_descriptor, const parquet::ColumnDescriptor new_descriptor, const std::string &reference_file_path, const std::string &new_file_path)

std::unique_ptr< ColumnDescriptor >	get_sub_type_column_descriptor (const ColumnDescriptor *column)

std::shared_ptr < parquet::Statistics >	validate_and_get_column_metadata_statistics (const parquet::ColumnChunkMetaData *column_metadata)

std::vector < Aws::S3::Model::Object >	s3_objects_filter_sort_files (const std::vector< Aws::S3::Model::Object > &file_paths, const shared::FilePathOptions &options)

template<typename V , std::enable_if_t< std::is_integral< V >::value, int > = 0>
V	get_null_value ()

template<typename D , std::enable_if_t< std::is_integral< D >::value, int > = 0>
std::pair< D, D >	get_min_max_bounds ()

void	populate_string_dictionary (const int32_t table_id, const int32_t col_id, const Catalog_Namespace::Catalog &catalog)

void	validate_non_foreign_table_write (const TableDescriptor *table_descriptor)

Variables
constexpr const char *	kDeletedValueIndicator {"<DELETED>"}

Typedef Documentation

using foreign_storage::ChunkToBufferMap = typedef std::map<ChunkKey, AbstractBuffer*>

Definition at line 34 of file ForeignDataWrapper.h.

template<typename T >

using foreign_storage::DateInDaysBoundsValidator = typedef BaseDateBoundsValidator<T, false>

Definition at line 277 of file ParquetMetadataValidator.h.

template<typename T >

using foreign_storage::DateInSecondsBoundsValidator = typedef BaseDateBoundsValidator<T, true>

Definition at line 274 of file ParquetMetadataValidator.h.

using foreign_storage::FilePathAndRowGroup = typedef std::pair<std::string, int32_t>

Definition at line 40 of file ParquetDataWrapper.h.

using foreign_storage::FileRegions = typedef std::vector<FileRegion>

Definition at line 60 of file FileRegions.h.

using foreign_storage::FirstLineByFilePath = typedef std::map<std::string, std::string>

Definition at line 37 of file FileReader.h.

using foreign_storage::InvalidRowGroupIndices = typedef std::set<int64_t>

Definition at line 123 of file ParquetEncoder.h.

using foreign_storage::OptionsMap = typedef std::map<std::string, std::string, std::less<>>

Definition at line 30 of file OptionsContainer.h.

template<typename V , typename T , T conversion_denominator, typename NullType = V>

using foreign_storage::ParquetDateInSecondsFromTimestampEncoder = typedef ParquetTimestampEncoder<V, T, conversion_denominator, NullType>

Definition at line 89 of file ParquetTimestampEncoder.h.

using foreign_storage::read_lock = typedef heavyai::shared_lock<heavyai::shared_mutex>

Definition at line 29 of file ForeignStorageCache.cpp.

using foreign_storage::ReaderPtr = typedef parquet::arrow::FileReader*

Definition at line 33 of file ParquetShared.h.

using foreign_storage::RejectedRowIndices = typedef std::set<int64_t>

Definition at line 28 of file ParquetEncoder.h.

using foreign_storage::RenderGroupAnalyzerMap = typedef std::map<int, std::unique_ptr<import_export::RenderGroupAnalyzer>>

Definition at line 37 of file ForeignDataWrapper.h.

using foreign_storage::S3ObjectComparator = typedef std::function<bool(const Aws::S3::Model::Object&, const Aws::S3::Model::Object&)>

Definition at line 17 of file S3FilePathUtil.h.

using foreign_storage::SampleRows = typedef std::vector<std::vector<std::string>>

Definition at line 26 of file DataPreview.h.

using foreign_storage::UniqueReaderPtr = typedef std::unique_ptr<parquet::arrow::FileReader>

Definition at line 32 of file ParquetShared.h.

using foreign_storage::write_lock = typedef heavyai::unique_lock<heavyai::shared_mutex>

Definition at line 30 of file ForeignStorageCache.cpp.

Function Documentation

void foreign_storage::add_file_region	(	std::map< int, FileRegions > &	fragment_id_to_file_regions_map,
		int	fragment_id,
		size_t	first_row_index,
		const ParseBufferResult &	result,
		const std::string &	file_path
	)

Creates a new file region based on metadata from parsed file buffers and adds the new region to the fragment id to file regions map.

Definition at line 478 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::ParseBufferResult::row_count, and foreign_storage::ParseBufferResult::row_offsets.

Referenced by process_data_blocks().

                                                  {
   fragment_id_to_file_regions_map[fragment_id].emplace_back(
       // file naming is handled by FileReader
       FileRegion(file_path,
                  result.row_offsets.front(),
                  first_row_index,
                  result.row_count,
                  result.row_offsets.back() - result.row_offsets.front()));
 }

Here is the caller graph for this function:

void foreign_storage::add_request_to_pool	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		ParseBufferRequest &	request
	)

Adds the request object for a processed request back to the request pool for reuse in subsequent requests.

Definition at line 624 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::request_pool, foreign_storage::MetadataScanMultiThreadingParams::request_pool_condition, and foreign_storage::MetadataScanMultiThreadingParams::request_pool_mutex.

Referenced by dispatch_metadata_scan_requests(), and scan_metadata().

                                                       {
   std::unique_lock<std::mutex> completed_requests_queue_lock(
       multi_threading_params.request_pool_mutex);
   multi_threading_params.request_pool.emplace(std::move(request));
   completed_requests_queue_lock.unlock();
   multi_threading_params.request_pool_condition.notify_all();
 }

Here is the caller graph for this function:

std::string foreign_storage::bool_to_option_value ( const bool value )

Definition at line 115 of file ForeignDataWrapperFactory.cpp.

Referenced by foreign_storage::ForeignDataWrapperFactory::createForeignTableProxy().

                                                  {
   return value ? "TRUE" : "FALSE";
 }

Here is the caller graph for this function:

void foreign_storage::cache_blocks	(	std::map< ChunkKey, Chunk_NS::Chunk > &	cached_chunks,
		DataBlockPtr	data_block,
		size_t	row_count,
		ChunkKey &	chunk_key,
		const ColumnDescriptor *	column,
		bool	is_first_block,
		bool	is_last_block,
		bool	disable_cache
	)

Definition at line 524 of file AbstractTextFileDataWrapper.cpp.

References CHECK, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_FRAGMENT_IDX, CHUNK_KEY_TABLE_IDX, ColumnDescriptor::columnType, foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::get_cache_if_enabled(), Catalog_Namespace::SysCatalog::getCatalog(), Catalog_Namespace::SysCatalog::instance(), SQLTypeInfo::is_varlen_indeed(), and key_does_not_shard_to_leaf().

Referenced by process_data_blocks().

                                       {
   auto catalog =
       Catalog_Namespace::SysCatalog::instance().getCatalog(chunk_key[CHUNK_KEY_DB_IDX]);
   CHECK(catalog);
   auto cache = get_cache_if_enabled(catalog, disable_cache);
   if (cache) {
     // This extra filter needs to be here because this wrapper is the only one that
     // accesses the cache directly and it should not be inserting chunks which are not
     // mapped to the current leaf (in distributed mode).
     if (key_does_not_shard_to_leaf(chunk_key)) {
       return;
     }
 
     ChunkKey index_key = {chunk_key[CHUNK_KEY_DB_IDX],
                           chunk_key[CHUNK_KEY_TABLE_IDX],
                           chunk_key[CHUNK_KEY_COLUMN_IDX],
                           chunk_key[CHUNK_KEY_FRAGMENT_IDX],
                           2};
     // Create actual data chunks to prepopulate cache
     if (cached_chunks.find(chunk_key) == cached_chunks.end()) {
       cached_chunks[chunk_key] = Chunk_NS::Chunk{column, false};
       cached_chunks[chunk_key].setBuffer(
           cache->getChunkBufferForPrecaching(chunk_key, is_first_block));
       if (column->columnType.is_varlen_indeed()) {
         cached_chunks[chunk_key].setIndexBuffer(
             cache->getChunkBufferForPrecaching(index_key, is_first_block));
       }
       if (is_first_block) {
         cached_chunks[chunk_key].initEncoder();
       }
     }
     cached_chunks[chunk_key].appendData(data_block, row_count, 0);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename D , typename T >

bool foreign_storage::check_bounds ( const T & value )

inline

Definition at line 32 of file ParquetMetadataValidator.h.

                                          {
   auto [min_value, max_value] = get_min_max_bounds<D>();
   return value >= min_value && value <= max_value;
 }

bool foreign_storage::contains_fragment_key	(	const std::set< ChunkKey > &	key_set,
		const ChunkKey &	target_key
	)

template<typename Container >

std::vector<std::future<void> > foreign_storage::create_futures_for_workers	(	const Container &	items,
		size_t	max_threads,
		std::function< void(const Container &)>	lambda
	)

Definition at line 74 of file FsiChunkUtils.h.

References threading_serial::async(), and partition_for_threads().

Referenced by import_export::ForeignDataImporter::importGeneralS3(), foreign_storage::ParquetDataWrapper::populateChunkBuffers(), and foreign_storage::LazyParquetChunkLoader::previewFiles().

                                                 {
   auto items_per_thread = partition_for_threads(items, max_threads);
   std::vector<std::future<void>> futures;
   for (const auto& items : items_per_thread) {
     futures.emplace_back(std::async(std::launch::async, lambda, items));
   }
 
   return futures;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::tuple< std::unique_ptr< foreign_storage::ForeignServer >, std::unique_ptr< foreign_storage::UserMapping >, std::unique_ptr< foreign_storage::ForeignTable > > foreign_storage::create_proxy_fsi_objects	(	const std::string &	copy_from_source,
		const import_export::CopyParams &	copy_params,
		const int	db_id,
		const TableDescriptor *	table,
		const int32_t	user_id
	)

Create proxy fsi objects for use outside FSI.

Parameters

copy_from_source	- the source that will be copied
copy_params	- CopyParams that specify parameters around use case
db_id	- db id of database in use case
table	- the table descriptor of the table in use case
user_id	- the user id of user in use case

Returns: tuple of FSI objects that can be used for particular use case (for example import or detect)

Definition at line 44 of file ForeignDataWrapperFactory.cpp.

References CHECK, foreign_storage::ForeignDataWrapperFactory::createForeignServerProxy(), foreign_storage::ForeignDataWrapperFactory::createForeignTableProxy(), and foreign_storage::ForeignDataWrapperFactory::createUserMappingProxyIfApplicable().

Referenced by create_proxy_fsi_objects(), and import_export::ForeignDataImporter::importGeneral().

                                                 {
   auto server = foreign_storage::ForeignDataWrapperFactory::createForeignServerProxy(
       db_id, user_id, copy_from_source, copy_params);
 
   CHECK(server);
   server->validate();
 
   auto user_mapping =
       foreign_storage::ForeignDataWrapperFactory::createUserMappingProxyIfApplicable(
           db_id, user_id, copy_from_source, copy_params, server.get());
 
   if (user_mapping) {
     user_mapping->validate(server.get());
   }
 
   auto foreign_table =
       foreign_storage::ForeignDataWrapperFactory::createForeignTableProxy(
           db_id, table, copy_from_source, copy_params, server.get());
 
   CHECK(foreign_table);
   foreign_table->validateOptionValues();
 
   return {std::move(server), std::move(user_mapping), std::move(foreign_table)};
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::tuple< std::unique_ptr< foreign_storage::ForeignServer >, std::unique_ptr< foreign_storage::UserMapping >, std::unique_ptr< foreign_storage::ForeignTable > > foreign_storage::create_proxy_fsi_objects	(	const std::string &	copy_from_source,
		const import_export::CopyParams &	copy_params,
		const TableDescriptor *	table
	)

Create proxy fsi objects for use outside FSI NOTE: parameters mirror function above.

Definition at line 76 of file ForeignDataWrapperFactory.cpp.

References create_proxy_fsi_objects().

                                                        {
   return create_proxy_fsi_objects(copy_from_source, copy_params, -1, table, -1);
 }

Here is the call graph for this function:

template<typename D >

std::string foreign_storage::datetime_to_string	(	const D &	timestamp,
		const SQLTypeInfo &	column_type
	)

inline

Definition at line 38 of file ParquetMetadataValidator.h.

References Datum::bigintval, CHECK, DatumToString(), SQLTypeInfo::is_date(), and SQLTypeInfo::is_timestamp().

Referenced by foreign_storage::TimestampBoundsValidator< T >::getMinMaxBoundsAsStrings(), foreign_storage::BaseDateBoundsValidator< T, is_in_seconds >::getMinMaxBoundsAsStrings(), foreign_storage::TimestampBoundsValidator< T >::validateValue(), and foreign_storage::BaseDateBoundsValidator< T, is_in_seconds >::validateValue().

                                                                       {
   CHECK(column_type.is_timestamp() || column_type.is_date());
   Datum d;
   d.bigintval = timestamp;
   return DatumToString(d, column_type);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::optional< SQLTypes > foreign_storage::detect_geo_type	(	const SampleRows &	sample_rows,
		size_t	column_index
	)

Definition at line 22 of file DataPreview.cpp.

References CHECK_EQ, CHECK_LT, kLINESTRING, kMULTIPOLYGON, kNULLT, kPOINT, kPOLYGON, and UNREACHABLE.

Referenced by foreign_storage::LazyParquetChunkLoader::previewFiles().

                                                              {
   std::optional<SQLTypes> tentative_geo_type{};
   for (const auto& row : sample_rows) {
     static std::regex geo_regex{
         "\\s*(POINT|LINESTRING|POLYGON|MULTIPOLYGON)\\s*\\(.+\\)\\s*"};
     std::smatch match;
     CHECK_LT(column_index, row.size());
     if (std::regex_match(row[column_index], match, geo_regex)) {
       CHECK_EQ(match.size(), static_cast<size_t>(2));
       SQLTypes geo_type{kNULLT};
       const auto& geo_type_str = match[1];
       if (geo_type_str == "POINT") {
         geo_type = kPOINT;
       } else if (geo_type_str == "LINESTRING") {
         geo_type = kLINESTRING;
       } else if (geo_type_str == "POLYGON") {
         geo_type = kPOLYGON;
       } else if (geo_type_str == "MULTIPOLYGON") {
         geo_type = kMULTIPOLYGON;
       } else {
         UNREACHABLE() << "Unexpected geo type match: " << geo_type_str;
       }
       if (tentative_geo_type.has_value()) {
         if (tentative_geo_type.value() != geo_type) {
           return {};  // geo type does not match between rows, can not be imported
         }
       } else {
         tentative_geo_type = geo_type;
       }
     }
   }
   return tentative_geo_type;
 }

Here is the caller graph for this function:

void foreign_storage::dispatch_metadata_scan_request	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		ParseBufferRequest &	request
	)

Dispatches a new metadata scan request by adding the request to the pending requests queue to be consumed by a worker thread.

Definition at line 710 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::pending_requests, foreign_storage::MetadataScanMultiThreadingParams::pending_requests_condition, and foreign_storage::MetadataScanMultiThreadingParams::pending_requests_mutex.

Referenced by dispatch_metadata_scan_requests().

                                  {
   {
     std::unique_lock<std::mutex> pending_requests_lock(
         multi_threading_params.pending_requests_mutex);
     multi_threading_params.pending_requests.emplace(std::move(request));
   }
   multi_threading_params.pending_requests_condition.notify_all();
 }

Here is the caller graph for this function:

void foreign_storage::dispatch_metadata_scan_requests	(	const size_t &	buffer_size,
		const std::string &	file_path,
		FileReader &	file_reader,
		const import_export::CopyParams &	copy_params,
		MetadataScanMultiThreadingParams &	multi_threading_params,
		size_t &	first_row_index_in_buffer,
		size_t &	current_file_offset,
		const TextFileBufferParser &	parser
	)

Reads from a text file iteratively and dispatches metadata scan requests that are processed by worker threads.

Definition at line 739 of file AbstractTextFileDataWrapper.cpp.

References add_request_to_pool(), foreign_storage::MetadataScanMultiThreadingParams::continue_processing, dispatch_metadata_scan_request(), foreign_storage::TextFileBufferParser::findRowEndPosition(), get_request_from_pool(), foreign_storage::FileReader::getCurrentFilePath(), foreign_storage::FileReader::isScanFinished(), import_export::CopyParams::line_delim, foreign_storage::MetadataScanMultiThreadingParams::pending_requests, foreign_storage::MetadataScanMultiThreadingParams::pending_requests_condition, foreign_storage::MetadataScanMultiThreadingParams::pending_requests_mutex, foreign_storage::FileReader::read(), and resize_buffer_if_needed().

Referenced by foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata().

                                         {
   auto alloc_size = buffer_size;
   auto residual_buffer = std::make_unique<char[]>(alloc_size);
   size_t residual_buffer_size = 0;
   size_t residual_buffer_alloc_size = alloc_size;
 
   while (!file_reader.isScanFinished()) {
     {
       std::lock_guard<std::mutex> pending_requests_lock(
           multi_threading_params.pending_requests_mutex);
       if (!multi_threading_params.continue_processing) {
         break;
       }
     }
     auto request = get_request_from_pool(multi_threading_params);
     request.full_path = file_reader.getCurrentFilePath();
     resize_buffer_if_needed(request.buffer, request.buffer_alloc_size, alloc_size);
 
     if (residual_buffer_size > 0) {
       memcpy(request.buffer.get(), residual_buffer.get(), residual_buffer_size);
     }
     size_t size = residual_buffer_size;
     size += file_reader.read(request.buffer.get() + residual_buffer_size,
                              alloc_size - residual_buffer_size);
 
     if (size == 0) {
       // In some cases at the end of a file we will read 0 bytes even when
       // file_reader.isScanFinished() is false. Also add request back to the pool to be
       // picked up again in the next iteration.
       add_request_to_pool(multi_threading_params, request);
       continue;
     } else if (size == 1 && request.buffer[0] == copy_params.line_delim) {
       // In some cases files with newlines at the end will be encoded with a second
       // newline that can end up being the only thing in the buffer. Also add request
       // back to the pool to be picked up again in the next iteration.
       current_file_offset++;
       add_request_to_pool(multi_threading_params, request);
       continue;
     }
     unsigned int num_rows_in_buffer = 0;
     request.end_pos = parser.findRowEndPosition(alloc_size,
                                                 request.buffer,
                                                 size,
                                                 copy_params,
                                                 first_row_index_in_buffer,
                                                 num_rows_in_buffer,
                                                 &file_reader);
     request.buffer_size = size;
     request.buffer_alloc_size = alloc_size;
     request.first_row_index = first_row_index_in_buffer;
     request.file_offset = current_file_offset;
     request.buffer_row_count = num_rows_in_buffer;
 
     residual_buffer_size = size - request.end_pos;
     if (residual_buffer_size > 0) {
       resize_buffer_if_needed(residual_buffer, residual_buffer_alloc_size, alloc_size);
       memcpy(residual_buffer.get(),
              request.buffer.get() + request.end_pos,
              residual_buffer_size);
     }
 
     current_file_offset += request.end_pos;
     first_row_index_in_buffer += num_rows_in_buffer;
 
     if (num_rows_in_buffer > 0) {
       dispatch_metadata_scan_request(multi_threading_params, request);
     } else {
       add_request_to_pool(multi_threading_params, request);
     }
   }
 
   std::unique_lock<std::mutex> pending_requests_queue_lock(
       multi_threading_params.pending_requests_mutex);
   multi_threading_params.pending_requests_condition.wait(
       pending_requests_queue_lock, [&multi_threading_params] {
         return multi_threading_params.pending_requests.empty() ||
                (multi_threading_params.continue_processing == false);
       });
   multi_threading_params.continue_processing = false;
   pending_requests_queue_lock.unlock();
   multi_threading_params.pending_requests_condition.notify_all();
 }

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >

ArrayDatum foreign_storage::encode_as_array_datum ( const std::vector< T > & data )

inline

Definition at line 33 of file GeospatialEncoder.h.

References heavydb.dtypes::T.

Referenced by foreign_storage::GeospatialEncoder::processGeoElement(), and foreign_storage::GeospatialEncoder::processNullGeoElement().

                                                                   {
   const size_t num_bytes = data.size() * sizeof(T);
   std::shared_ptr<int8_t> buffer(new int8_t[num_bytes], std::default_delete<int8_t[]>());
   memcpy(buffer.get(), data.data(), num_bytes);
   return ArrayDatum(num_bytes, buffer, false);
 }

Here is the caller graph for this function:

bool foreign_storage::fragment_maps_to_leaf ( const ChunkKey & key )

Definition at line 126 of file FsiChunkUtils.cpp.

References CHECK, g_distributed_leaf_idx, g_distributed_num_leaves, get_fragment(), dist::is_aggregator(), and dist::is_distributed().

Referenced by anonymous_namespace{ForeignStorageMgr.cpp}::filter_metadata_by_leaf(), foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), key_does_not_shard_to_leaf(), and foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

                                                 {
   CHECK(dist::is_distributed());
   CHECK(g_distributed_num_leaves > 0);
   CHECK(!dist::is_aggregator());
   return (get_fragment(key) % g_distributed_num_leaves == g_distributed_leaf_idx);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

size_t foreign_storage::get_buffer_size	(	const import_export::CopyParams &	copy_params,
		const bool	size_known,
		const size_t	file_size
	)

Gets the appropriate buffer size to be used when processing file(s).

Definition at line 241 of file AbstractTextFileDataWrapper.cpp.

References import_export::CopyParams::buffer_size.

Referenced by foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata(), and foreign_storage::AbstractTextFileDataWrapper::populateChunks().

                                                {
   size_t buffer_size = copy_params.buffer_size;
   if (size_known && file_size < buffer_size) {
     buffer_size = file_size + 1;  // +1 for end of line character, if missing
   }
   return buffer_size;
 }

Here is the caller graph for this function:

size_t foreign_storage::get_buffer_size ( const FileRegions & file_regions )

Definition at line 251 of file AbstractTextFileDataWrapper.cpp.

References CHECK.

                                                         {
   size_t buffer_size = 0;
   for (const auto& file_region : file_regions) {
     buffer_size = std::max(buffer_size, file_region.region_size);
   }
   CHECK(buffer_size);
   return buffer_size;
 }

const parquet::ColumnDescriptor * foreign_storage::get_column_descriptor	(	const parquet::arrow::FileReader *	reader,
		const int	logical_column_index
	)

Definition at line 46 of file ParquetShared.cpp.

Referenced by foreign_storage::LazyParquetChunkLoader::appendRowGroups(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_equal_schema().

                                     {
   return reader->parquet_reader()->metadata()->schema()->Column(logical_column_index);
 }

Here is the caller graph for this function:

std::set<ChunkKey> foreign_storage::get_column_key_set ( const ChunkKey & destination_chunk_key )

Referenced by foreign_storage::CachingForeignStorageMgr::fetchBuffer(), foreign_storage::CachingForeignStorageMgr::getOptionalKeysWithinSizeLimit(), and foreign_storage::CachingForeignStorageMgr::getRequiredBuffersSize().

Here is the caller graph for this function:

std::vector<ChunkKey> foreign_storage::get_column_key_vec ( const ChunkKey & destination_chunk_key )

std::string foreign_storage::get_db_name ( int32_t db_id )

Definition at line 31 of file InternalSystemDataWrapper.cpp.

References Catalog_Namespace::DBMetadata::dbName, Catalog_Namespace::SysCatalog::instance(), and kDeletedValueIndicator.

Referenced by foreign_storage::anonymous_namespace{InternalCatalogDataWrapper.cpp}::populate_import_buffers_for_catalog_dashboards(), foreign_storage::anonymous_namespace{InternalCatalogDataWrapper.cpp}::populate_import_buffers_for_catalog_permissions(), foreign_storage::anonymous_namespace{InternalCatalogDataWrapper.cpp}::populate_import_buffers_for_catalog_tables(), foreign_storage::anonymous_namespace{InternalCatalogDataWrapper.cpp}::populate_import_buffers_for_catalog_users(), foreign_storage::anonymous_namespace{InternalMemoryStatsDataWrapper.cpp}::populate_import_buffers_for_memory_details(), and foreign_storage::anonymous_namespace{InternalStorageStatsDataWrapper.cpp}::populate_import_buffers_for_storage_details().

                                      {
   Catalog_Namespace::DBMetadata db_metadata;
   auto& sys_catalog = Catalog_Namespace::SysCatalog::instance();
   if (sys_catalog.getMetadataForDBById(db_id, db_metadata)) {
     return db_metadata.dbName;
   } else {
     // Database has been deleted.
     return kDeletedValueIndicator;
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

const foreign_storage::ForeignTable & foreign_storage::get_foreign_table_for_key ( const ChunkKey & key )

Definition at line 99 of file FsiChunkUtils.cpp.

References CHECK, get_table_prefix(), Catalog_Namespace::SysCatalog::getCatalog(), and Catalog_Namespace::SysCatalog::instance().

Referenced by is_append_table_chunk_key(), foreign_storage::anonymous_namespace{CachingForeignStorageMgr.cpp}::is_in_memory_system_table_chunk_key(), is_replicated_table_chunk_key(), and is_system_table_chunk_key().

                                                                                   {
   auto [db_id, tb_id] = get_table_prefix(key);
   auto catalog = Catalog_Namespace::SysCatalog::instance().getCatalog(db_id);
   CHECK(catalog);
   auto table = catalog->getForeignTable(tb_id);
   CHECK(table);
   return *table;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

size_t foreign_storage::get_max_chunk_size ( const ChunkKey & key )

Referenced by foreign_storage::CachingForeignStorageMgr::getBufferSize().

Here is the caller graph for this function:

template<typename D , std::enable_if_t< std::is_integral< D >::value, int > = 0>

std::pair< D, D > foreign_storage::get_min_max_bounds ( )

inline

Definition at line 31 of file SharedMetadataValidator.h.

                                           {
   static_assert(std::is_signed<D>::value,
                 "'get_min_max_bounds' is only valid for signed types");
   return {get_null_value<D>() + 1, std::numeric_limits<D>::max()};
 }

std::optional<ParseBufferRequest> foreign_storage::get_next_metadata_scan_request ( MetadataScanMultiThreadingParams & multi_threading_params )

Gets the next metadata scan request object from the pending requests queue. A null optional is returned if there are no further requests to be processed.

Definition at line 455 of file AbstractTextFileDataWrapper.cpp.

References foreign_storage::MetadataScanMultiThreadingParams::continue_processing, foreign_storage::MetadataScanMultiThreadingParams::pending_requests, foreign_storage::MetadataScanMultiThreadingParams::pending_requests_condition, and foreign_storage::MetadataScanMultiThreadingParams::pending_requests_mutex.

Referenced by scan_metadata().

                                                               {
   std::unique_lock<std::mutex> pending_requests_lock(
       multi_threading_params.pending_requests_mutex);
   multi_threading_params.pending_requests_condition.wait(
       pending_requests_lock, [&multi_threading_params] {
         return !multi_threading_params.pending_requests.empty() ||
                !multi_threading_params.continue_processing;
       });
   if (multi_threading_params.pending_requests.empty()) {
     return {};
   }
   auto request = std::move(multi_threading_params.pending_requests.front());
   multi_threading_params.pending_requests.pop();
   pending_requests_lock.unlock();
   multi_threading_params.pending_requests_condition.notify_all();
   return std::move(request);
 }

Here is the caller graph for this function:

template<typename V , std::enable_if_t< std::is_integral< V >::value, int > = 0>

V foreign_storage::get_null_value ( )

inline

Definition at line 21 of file SharedMetadataValidator.h.

                           {
   return inline_int_null_value<V>();
 }

std::pair< int, int > foreign_storage::get_parquet_table_size ( const ReaderPtr & reader )

Definition at line 39 of file ParquetShared.cpp.

Referenced by foreign_storage::LazyParquetChunkLoader::appendRowGroups(), foreign_storage::LazyParquetChunkLoader::loadRowGroups(), and foreign_storage::LazyParquetChunkLoader::metadataScan().

                                                                   {
   auto file_metadata = reader->parquet_reader()->metadata();
   const auto num_row_groups = file_metadata->num_row_groups();
   const auto num_columns = file_metadata->num_columns();
   return std::make_pair(num_row_groups, num_columns);
 }

Here is the caller graph for this function:

parquet::Type::type foreign_storage::get_physical_type	(	ReaderPtr &	reader,
		const int	logical_column_index
	)

Definition at line 52 of file ParquetShared.cpp.

Referenced by anonymous_namespace{ArrowResultSetConverter.cpp}::get_arrow_type(), and ArrowResultSetConverter::initializeColumnBuilder().

                                                                                      {
   return reader->parquet_reader()
       ->metadata()
       ->schema()
       ->Column(logical_column_index)
       ->physical_type();
 }

Here is the caller graph for this function:

std::shared_ptr< ChunkMetadata > foreign_storage::get_placeholder_metadata	(	const SQLTypeInfo &	type,
		size_t	num_elements
	)

Definition at line 75 of file FsiChunkUtils.cpp.

References SQLTypeInfo::get_elem_type(), SQLTypeInfo::get_size(), Data_Namespace::AbstractBuffer::getEncoder(), Encoder::getMetadata(), Data_Namespace::AbstractBuffer::initEncoder(), SQLTypeInfo::is_array(), and SQLTypeInfo::is_varlen_indeed().

Referenced by foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::add_placeholder_metadata(), foreign_storage::InternalSystemDataWrapper::populateChunkMetadata(), and foreign_storage::AbstractTextFileDataWrapper::updateRolledOffChunks().

                                                                              {
   ForeignStorageBuffer empty_buffer;
   // Use default encoder metadata as in parquet wrapper
   empty_buffer.initEncoder(type);
 
   auto chunk_metadata = empty_buffer.getEncoder()->getMetadata(type);
   chunk_metadata->numElements = num_elements;
 
   if (!type.is_varlen_indeed()) {
     chunk_metadata->numBytes = type.get_size() * num_elements;
   }
   // min/max not set by default for arrays, so get from elem type encoder
   if (type.is_array()) {
     ForeignStorageBuffer scalar_buffer;
     scalar_buffer.initEncoder(type.get_elem_type());
     auto scalar_metadata = scalar_buffer.getEncoder()->getMetadata(type.get_elem_type());
     chunk_metadata->chunkStats.min = scalar_metadata->chunkStats.min;
     chunk_metadata->chunkStats.max = scalar_metadata->chunkStats.max;
   }
 
   return chunk_metadata;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

ParseBufferRequest foreign_storage::get_request_from_pool ( MetadataScanMultiThreadingParams & multi_threading_params )

Gets a request from the metadata scan request pool.

Definition at line 692 of file AbstractTextFileDataWrapper.cpp.

References CHECK, foreign_storage::MetadataScanMultiThreadingParams::request_pool, foreign_storage::MetadataScanMultiThreadingParams::request_pool_condition, and foreign_storage::MetadataScanMultiThreadingParams::request_pool_mutex.

Referenced by dispatch_metadata_scan_requests().

                                                               {
   std::unique_lock<std::mutex> request_pool_lock(
       multi_threading_params.request_pool_mutex);
   multi_threading_params.request_pool_condition.wait(
       request_pool_lock,
       [&multi_threading_params] { return !multi_threading_params.request_pool.empty(); });
   auto request = std::move(multi_threading_params.request_pool.front());
   multi_threading_params.request_pool.pop();
   request_pool_lock.unlock();
   CHECK(request.buffer);
   return request;
 }

Here is the caller graph for this function:

std::unique_ptr< ColumnDescriptor > foreign_storage::get_sub_type_column_descriptor ( const ColumnDescriptor * column )

Definition at line 76 of file ParquetShared.cpp.

References ColumnDescriptor::columnId, ColumnDescriptor::columnName, ColumnDescriptor::columnType, SQLTypeInfo::get_elem_type(), SQLTypeInfo::set_size(), and ColumnDescriptor::tableId.

Referenced by foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::create_parquet_array_encoder(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_array_mapping().

                                     {
   auto column_type = column->columnType.get_elem_type();
   if (column_type.get_size() == -1 && column_type.is_dict_encoded_string()) {
     column_type.set_size(4);  // override default size of -1
   }
   return std::make_unique<ColumnDescriptor>(
       column->tableId, column->columnId, column->columnName, column_type);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::string foreign_storage::get_table_name	(	int32_t	db_id,
		int32_t	table_id
	)

Definition at line 42 of file InternalSystemDataWrapper.cpp.

References CHECK, Catalog_Namespace::SysCatalog::getCatalog(), Catalog_Namespace::SysCatalog::instance(), and kDeletedValueIndicator.

Referenced by anonymous_namespace{Execute.cpp}::checkWorkUnitWatchdog(), foreign_storage::anonymous_namespace{InternalMemoryStatsDataWrapper.cpp}::populate_import_buffers_for_memory_details(), and foreign_storage::anonymous_namespace{InternalStorageStatsDataWrapper.cpp}::populate_import_buffers_for_storage_details().

                                                           {
   auto catalog = Catalog_Namespace::SysCatalog::instance().getCatalog(db_id);
   CHECK(catalog);
   auto table_name = catalog->getTableName(table_id);
   if (table_name.has_value()) {
     return table_name.value();
   } else {
     // It is possible for the table to be concurrently deleted while querying the system
     // table.
     return kDeletedValueIndicator;
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

size_t foreign_storage::get_thread_count	(	const import_export::CopyParams &	copy_params,
		const bool	size_known,
		const size_t	file_size,
		const size_t	buffer_size
	)

Gets the appropriate number of threads to be used for concurrent processing within the data wrapper.

Definition at line 264 of file AbstractTextFileDataWrapper.cpp.

References CHECK_GT, and import_export::CopyParams::threads.

Referenced by foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata(), and foreign_storage::AbstractTextFileDataWrapper::populateChunks().

                                                   {
   size_t thread_count = copy_params.threads;
   if (thread_count == 0) {
     thread_count = std::thread::hardware_concurrency();
   }
   if (size_known && file_size > 0) {
     size_t num_buffers_in_file = (file_size + buffer_size - 1) / buffer_size;
     if (num_buffers_in_file < thread_count) {
       thread_count = num_buffers_in_file;
     }
   }
   CHECK_GT(thread_count, static_cast<size_t>(0));
   return thread_count;
 }

Here is the caller graph for this function:

size_t foreign_storage::get_thread_count	(	const import_export::CopyParams &	copy_params,
		const FileRegions &	file_regions
	)

Definition at line 282 of file AbstractTextFileDataWrapper.cpp.

References CHECK_GT, and import_export::CopyParams::threads.

                                                          {
   size_t thread_count = copy_params.threads;
   if (thread_count == 0) {
     thread_count =
         std::min<size_t>(std::thread::hardware_concurrency(), file_regions.size());
   }
   CHECK_GT(thread_count, static_cast<size_t>(0));
   return thread_count;
 }

void foreign_storage::get_value	(	const rapidjson::Value &	json_val,
		FileRegion &	file_region
	)

Definition at line 44 of file CsvShared.cpp.

References CHECK, foreign_storage::FileRegion::first_row_file_offset, foreign_storage::FileRegion::first_row_index, foreign_storage::json_utils::get_value_from_object(), foreign_storage::FileRegion::region_size, and foreign_storage::FileRegion::row_count.

                                                                         {
   CHECK(json_val.IsObject());
   json_utils::get_value_from_object(
       json_val, file_region.first_row_file_offset, "first_row_file_offset");
   json_utils::get_value_from_object(
       json_val, file_region.first_row_index, "first_row_index");
   json_utils::get_value_from_object(json_val, file_region.region_size, "region_size");
   json_utils::get_value_from_object(json_val, file_region.row_count, "row_count");
   if (json_val.HasMember("filename")) {
     json_utils::get_value_from_object(json_val, file_region.filename, "filename");
   }
 }

Here is the call graph for this function:

void foreign_storage::get_value	(	const rapidjson::Value &	json_val,
		RowGroupInterval &	value
	)

Definition at line 669 of file ParquetDataWrapper.cpp.

References CHECK, foreign_storage::RowGroupInterval::end_index, foreign_storage::RowGroupInterval::file_path, foreign_storage::json_utils::get_value_from_object(), and foreign_storage::RowGroupInterval::start_index.

                                                                         {
   CHECK(json_val.IsObject());
   json_utils::get_value_from_object(json_val, value.file_path, "file_path");
   json_utils::get_value_from_object(json_val, value.start_index, "start_index");
   json_utils::get_value_from_object(json_val, value.end_index, "end_index");
 }

Here is the call graph for this function:

void foreign_storage::init_chunk_for_column	(	const ChunkKey &	chunk_key,
		const std::map< ChunkKey, std::shared_ptr< ChunkMetadata >> &	chunk_metadata_map,
		const std::map< ChunkKey, AbstractBuffer * > &	buffers,
		Chunk_NS::Chunk &	chunk
	)

Definition at line 21 of file FsiChunkUtils.cpp.

References CHECK, CHECK_EQ, CHUNK_KEY_COLUMN_IDX, CHUNK_KEY_DB_IDX, CHUNK_KEY_TABLE_IDX, Catalog_Namespace::SysCatalog::getCatalog(), Chunk_NS::Chunk::initEncoder(), Catalog_Namespace::SysCatalog::instance(), Data_Namespace::AbstractBuffer::reserve(), Chunk_NS::Chunk::setBuffer(), Chunk_NS::Chunk::setColumnDesc(), Chunk_NS::Chunk::setIndexBuffer(), Chunk_NS::Chunk::setPinnable(), Data_Namespace::AbstractBuffer::size(), and UNREACHABLE.

Referenced by foreign_storage::AbstractTextFileDataWrapper::populateChunkMapForColumns().

                           {
   auto catalog =
       Catalog_Namespace::SysCatalog::instance().getCatalog(chunk_key[CHUNK_KEY_DB_IDX]);
   CHECK(catalog);
 
   ChunkKey data_chunk_key = chunk_key;
   AbstractBuffer* data_buffer = nullptr;
   AbstractBuffer* index_buffer = nullptr;
   const auto column = catalog->getMetadataForColumn(chunk_key[CHUNK_KEY_TABLE_IDX],
                                                     chunk_key[CHUNK_KEY_COLUMN_IDX]);
 
   if (column->columnType.is_varlen_indeed()) {
     data_chunk_key.push_back(1);
     ChunkKey index_chunk_key = chunk_key;
     index_chunk_key.push_back(2);
 
     CHECK(buffers.find(data_chunk_key) != buffers.end());
     CHECK(buffers.find(index_chunk_key) != buffers.end());
 
     data_buffer = buffers.find(data_chunk_key)->second;
     index_buffer = buffers.find(index_chunk_key)->second;
     CHECK_EQ(data_buffer->size(), static_cast<size_t>(0));
     CHECK_EQ(index_buffer->size(), static_cast<size_t>(0));
 
     size_t index_offset_size{0};
     if (column->columnType.is_string() || column->columnType.is_geometry()) {
       index_offset_size = sizeof(StringOffsetT);
     } else if (column->columnType.is_array()) {
       index_offset_size = sizeof(ArrayOffsetT);
     } else {
       UNREACHABLE();
     }
     CHECK(chunk_metadata_map.find(data_chunk_key) != chunk_metadata_map.end());
     index_buffer->reserve(index_offset_size *
                           (chunk_metadata_map.at(data_chunk_key)->numElements + 1));
   } else {
     data_chunk_key = chunk_key;
     CHECK(buffers.find(data_chunk_key) != buffers.end());
     data_buffer = buffers.find(data_chunk_key)->second;
   }
   CHECK(chunk_metadata_map.find(data_chunk_key) != chunk_metadata_map.end());
   data_buffer->reserve(chunk_metadata_map.at(data_chunk_key)->numBytes);
 
   chunk.setPinnable(false);
   chunk.setColumnDesc(column);
   chunk.setBuffer(data_buffer);
   chunk.setIndexBuffer(index_buffer);
   chunk.initEncoder();
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::is_append_table_chunk_key ( const ChunkKey & chunk_key )

Definition at line 116 of file FsiChunkUtils.cpp.

References get_foreign_table_for_key(), and foreign_storage::ForeignTable::isAppendMode().

Referenced by foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

                                                           {
   return get_foreign_table_for_key(chunk_key).isAppendMode();
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::is_replicated_table_chunk_key ( const ChunkKey & chunk_key )

Definition at line 112 of file FsiChunkUtils.cpp.

References get_foreign_table_for_key(), and table_is_replicated().

Referenced by is_shardable_key().

                                                               {
   return table_is_replicated(&get_foreign_table_for_key(chunk_key));
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::is_s3_uri ( const std::string & file_path )

Definition at line 36 of file ForeignDataWrapperFactory.cpp.

Referenced by foreign_storage::ForeignDataWrapperFactory::createForeignServerProxy(), foreign_storage::ForeignDataWrapperFactory::createForeignTableProxy(), import_export::ForeignDataImporter::import(), and import_export::ForeignDataImporter::importGeneralS3().

                                            {
   const std::string s3_prefix = "s3://";
   return file_path.find(s3_prefix) != std::string::npos;
 }

Here is the caller graph for this function:

bool foreign_storage::is_shardable_key ( const ChunkKey & key )

Definition at line 120 of file FsiChunkUtils.cpp.

References dist::is_aggregator(), dist::is_distributed(), is_replicated_table_chunk_key(), and is_system_table_chunk_key().

Referenced by anonymous_namespace{ForeignStorageMgr.cpp}::filter_metadata_by_leaf(), foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecForKeyPrefix(), key_does_not_shard_to_leaf(), and foreign_storage::CachingForeignStorageMgr::refreshTableInCache().

                                            {
   return (dist::is_distributed() && !dist::is_aggregator() &&
           !is_replicated_table_chunk_key(key) && !is_system_table_chunk_key(key));
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::is_system_table_chunk_key ( const ChunkKey & chunk_key )

Definition at line 108 of file FsiChunkUtils.cpp.

References get_foreign_table_for_key(), and TableDescriptor::is_system_table.

Referenced by is_shardable_key().

                                                           {
   return get_foreign_table_for_key(chunk_key).is_system_table;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::is_table_enabled_on_node ( const ChunkKey & key )

Referenced by foreign_storage::CachingForeignStorageMgr::getChunkMetadataVecFromDataWrapper().

Here is the caller graph for this function:

bool foreign_storage::is_valid_source_type ( const import_export::CopyParams & copy_params )

Verify if source_type is valid.

Definition at line 106 of file ForeignDataWrapperFactory.cpp.

References import_export::kDelimitedFile, import_export::kParquetFile, import_export::kRegexParsedFile, and import_export::CopyParams::source_type.

Referenced by foreign_storage::ForeignDataWrapperFactory::createForeignServerProxy(), foreign_storage::ForeignDataWrapperFactory::createForeignTableProxy(), and import_export::ForeignDataImporter::importGeneral().

                                                                       {
   return
 #ifdef ENABLE_IMPORT_PARQUET
       copy_params.source_type == import_export::SourceType::kParquetFile ||
 #endif
       copy_params.source_type == import_export::SourceType::kDelimitedFile ||
       copy_params.source_type == import_export::SourceType::kRegexParsedFile;
 }

Here is the caller graph for this function:

bool foreign_storage::key_does_not_shard_to_leaf ( const ChunkKey & key )

Definition at line 133 of file FsiChunkUtils.cpp.

References fragment_maps_to_leaf(), and is_shardable_key().

Referenced by cache_blocks(), populate_string_dictionary(), and anonymous_namespace{RelAlgExecutor.cpp}::set_parallelism_hints().

                                                      {
   return (is_shardable_key(key) && !fragment_maps_to_leaf(key));
 }

Here is the call graph for this function:

Here is the caller graph for this function:

UniqueReaderPtr foreign_storage::open_parquet_table	(	const std::string &	file_path,
		std::shared_ptr< arrow::fs::FileSystem > &	file_system
	)

Definition at line 26 of file ParquetShared.cpp.

Referenced by foreign_storage::FileReaderMap::getOrInsert(), foreign_storage::FileReaderMap::insert(), and foreign_storage::LazyParquetChunkLoader::loadRowGroups().

                                                                                   {
   UniqueReaderPtr reader;
   auto file_result = file_system->OpenInputFile(file_path);
   if (!file_result.ok()) {
     throw std::runtime_error{"Unable to access " + file_system->type_name() + " file: " +
                              file_path + ". " + file_result.status().message()};
   }
   auto infile = file_result.ValueOrDie();
   PARQUET_THROW_NOT_OK(OpenFile(infile, arrow::default_memory_pool(), &reader));
   return reader;
 }

Here is the caller graph for this function:

ParseFileRegionResult foreign_storage::parse_file_regions	(	const FileRegions &	file_regions,
		const size_t	start_index,
		const size_t	end_index,
		FileReader &	file_reader,
		ParseBufferRequest &	parse_file_request,
		const std::map< int, Chunk_NS::Chunk > &	column_id_to_chunk_map,
		const TextFileBufferParser &	parser
	)

Parses a set of file regions given a handle to the file and range of indexes for the file regions to be parsed.

Definition at line 194 of file AbstractTextFileDataWrapper.cpp.

Referenced by foreign_storage::AbstractTextFileDataWrapper::populateChunks().

                                         {
   auto timer = DEBUG_TIMER(__func__);
   ParseFileRegionResult load_file_region_result{};
   load_file_region_result.file_offset = file_regions[start_index].first_row_file_offset;
   load_file_region_result.row_count = 0;
 
   ParseBufferResult result;
   for (size_t i = start_index; i <= end_index; i++) {
     CHECK(file_regions[i].region_size <= parse_file_request.buffer_size);
     auto read_size = file_reader.readRegion(parse_file_request.buffer.get(),
                                             file_regions[i].first_row_file_offset,
                                             file_regions[i].region_size);
     if (file_regions[i].region_size != read_size) {
       throw_unexpected_number_of_items(file_regions[i].region_size,
                                        read_size,
                                        "bytes",
                                        parse_file_request.getTableName());
     }
     parse_file_request.begin_pos = 0;
     parse_file_request.end_pos = file_regions[i].region_size;
     parse_file_request.first_row_index = file_regions[i].first_row_index;
     parse_file_request.file_offset = file_regions[i].first_row_file_offset;
     parse_file_request.process_row_count = file_regions[i].row_count;
 
     result = parser.parseBuffer(parse_file_request, i == end_index);
     CHECK_EQ(file_regions[i].row_count, result.row_count);
     for (const auto& rejected_row_index : result.rejected_rows) {
       load_file_region_result.rejected_row_indices.insert(
           load_file_region_result.row_count + rejected_row_index);
     }
     load_file_region_result.row_count += result.row_count;
   }
   load_file_region_result.column_id_to_data_blocks_map =
       result.column_id_to_data_blocks_map;
   return load_file_region_result;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::vector<size_t> foreign_storage::partition_by_fragment	(	const size_t	start_row_index,
		const size_t	max_fragment_size,
		const size_t	buffer_row_count
	)

Given a start row index, maximum fragment size, and number of rows in a buffer, this function returns a vector indicating how the rows in the buffer should be partitioned in order to fill up available fragment slots while staying within the capacity of fragments.

Definition at line 403 of file AbstractTextFileDataWrapper.cpp.

References CHECK.

Referenced by scan_metadata().

                                                                          {
   CHECK(buffer_row_count > 0);
   std::vector<size_t> partitions{};
   size_t remaining_rows_in_last_fragment;
   if (start_row_index % max_fragment_size == 0) {
     remaining_rows_in_last_fragment = 0;
   } else {
     remaining_rows_in_last_fragment =
         max_fragment_size - (start_row_index % max_fragment_size);
   }
   if (buffer_row_count <= remaining_rows_in_last_fragment) {
     partitions.emplace_back(buffer_row_count);
   } else {
     if (remaining_rows_in_last_fragment > 0) {
       partitions.emplace_back(remaining_rows_in_last_fragment);
     }
     size_t remaining_buffer_row_count =
         buffer_row_count - remaining_rows_in_last_fragment;
     while (remaining_buffer_row_count > 0) {
       partitions.emplace_back(
           std::min<size_t>(remaining_buffer_row_count, max_fragment_size));
       remaining_buffer_row_count -= partitions.back();
     }
   }
   return partitions;
 }

Here is the caller graph for this function:

template<typename T >

auto foreign_storage::partition_for_threads	(	const std::set< T > &	items,
		size_t	max_threads
	)

Definition at line 41 of file FsiChunkUtils.h.

Referenced by create_futures_for_workers(), and foreign_storage::LazyParquetChunkLoader::metadataScan().

                                                                        {
   const size_t items_per_thread = (items.size() + (max_threads - 1)) / max_threads;
   std::list<std::set<T>> items_by_thread;
   auto i = 0U;
   for (auto item : items) {
     if (i++ % items_per_thread == 0) {
       items_by_thread.emplace_back(std::set<T>{});
     }
     items_by_thread.back().emplace(item);
   }
   return items_by_thread;
 }

Here is the caller graph for this function:

template<typename T >

auto foreign_storage::partition_for_threads	(	const std::vector< T > &	items,
		size_t	max_threads
	)

Definition at line 60 of file FsiChunkUtils.h.

                                                                           {
   const size_t items_per_thread = (items.size() + (max_threads - 1)) / max_threads;
   std::list<std::vector<T>> items_by_thread;
   auto i = 0U;
   for (auto item : items) {
     if (i++ % items_per_thread == 0) {
       items_by_thread.emplace_back(std::vector<T>{});
     }
     items_by_thread.back().emplace_back(item);
   }
   return items_by_thread;
 }

void foreign_storage::populate_string_dictionary	(	const int32_t	table_id,
		const int32_t	col_id,
		const Catalog_Namespace::Catalog &	catalog
	)

Definition at line 205 of file Execute.cpp.

References CHECK, Data_Namespace::CPU_LEVEL, Chunk_NS::Chunk::getChunk(), Catalog_Namespace::Catalog::getDatabaseId(), Catalog_Namespace::Catalog::getDataMgr(), Catalog_Namespace::Catalog::getMetadataForColumn(), Catalog_Namespace::Catalog::getMetadataForTable(), anonymous_namespace{Execute.cpp}::is_empty_table(), and key_does_not_shard_to_leaf().

Referenced by anonymous_namespace{RelAlgExecutor.cpp}::prepare_string_dictionaries().

                                                                          {
   if (const auto foreign_table = dynamic_cast<const ForeignTable*>(
           catalog.getMetadataForTable(table_id, false))) {
     const auto col_desc = catalog.getMetadataForColumn(table_id, col_id);
     if (col_desc->columnType.is_dict_encoded_type()) {
       auto& fragmenter = foreign_table->fragmenter;
       CHECK(fragmenter != nullptr);
       if (is_empty_table(fragmenter.get())) {
         return;
       }
       for (const auto& frag : fragmenter->getFragmentsForQuery().fragments) {
         ChunkKey chunk_key = {catalog.getDatabaseId(), table_id, col_id, frag.fragmentId};
         // If the key is sharded across leaves, only populate fragments that are sharded
         // to this leaf.
         if (key_does_not_shard_to_leaf(chunk_key)) {
           continue;
         }
 
         const ChunkMetadataMap& metadata_map = frag.getChunkMetadataMap();
         CHECK(metadata_map.find(col_id) != metadata_map.end());
         if (auto& meta = metadata_map.at(col_id); meta->isPlaceholder()) {
           // When this goes out of scope it will stay in CPU cache but become
           // evictable
           auto chunk = Chunk_NS::Chunk::getChunk(col_desc,
                                                  &(catalog.getDataMgr()),
                                                  chunk_key,
                                                  Data_Namespace::CPU_LEVEL,
                                                  0,
                                                  0,
                                                  0);
         }
       }
     }
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::process_data_blocks	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		int	fragment_id,
		const ParseBufferRequest &	request,
		ParseBufferResult &	result,
		std::map< int, const ColumnDescriptor * > &	column_by_id,
		std::map< int, FileRegions > &	fragment_id_to_file_regions_map
	)

Updates metadata encapsulated in encoders for all table columns given new data blocks gotten from parsing a new set of rows in a file buffer. If cache is available, also append the data_blocks to chunks in the cache

Definition at line 571 of file AbstractTextFileDataWrapper.cpp.

References add_file_region(), cache_blocks(), foreign_storage::MetadataScanMultiThreadingParams::cached_chunks, foreign_storage::MetadataScanMultiThreadingParams::chunk_encoder_buffers, foreign_storage::MetadataScanMultiThreadingParams::chunk_encoder_buffers_mutex, foreign_storage::ParseBufferResult::column_id_to_data_blocks_map, foreign_storage::ParseBufferRequest::db_id, foreign_storage::MetadataScanMultiThreadingParams::disable_cache, foreign_storage::ParseBufferRequest::first_row_index, foreign_storage::ParseBufferRequest::getFilePath(), foreign_storage::ParseBufferRequest::getMaxFragRows(), foreign_storage::ParseBufferRequest::getTableId(), foreign_storage::ParseBufferResult::row_count, and update_stats().

Referenced by scan_metadata().

                                                                                     {
   std::lock_guard<std::mutex> lock(multi_threading_params.chunk_encoder_buffers_mutex);
   // File regions should be added in same order as appendData
   add_file_region(fragment_id_to_file_regions_map,
                   fragment_id,
                   request.first_row_index,
                   result,
                   request.getFilePath());
 
   for (auto& [column_id, data_block] : result.column_id_to_data_blocks_map) {
     ChunkKey chunk_key{request.db_id, request.getTableId(), column_id, fragment_id};
     const auto column = column_by_id[column_id];
     if (column->columnType.is_varlen_indeed()) {
       chunk_key.emplace_back(1);
     }
     if (multi_threading_params.chunk_encoder_buffers.find(chunk_key) ==
         multi_threading_params.chunk_encoder_buffers.end()) {
       multi_threading_params.chunk_encoder_buffers[chunk_key] =
           std::make_unique<ForeignStorageBuffer>();
       multi_threading_params.chunk_encoder_buffers[chunk_key]->initEncoder(
           column->columnType);
     }
     update_stats(multi_threading_params.chunk_encoder_buffers[chunk_key]->getEncoder(),
                  column->columnType,
                  data_block,
                  result.row_count);
     size_t num_elements = multi_threading_params.chunk_encoder_buffers[chunk_key]
                               ->getEncoder()
                               ->getNumElems() +
                           result.row_count;
     multi_threading_params.chunk_encoder_buffers[chunk_key]->getEncoder()->setNumElems(
         num_elements);
     cache_blocks(
         multi_threading_params.cached_chunks,
         data_block,
         result.row_count,
         chunk_key,
         column,
         (num_elements - result.row_count) == 0,  // Is the first block added to this chunk
         num_elements == request.getMaxFragRows(),  // Is the last block for this chunk
         multi_threading_params.disable_cache);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::refresh_foreign_table	(	Catalog_Namespace::Catalog &	catalog,
		const std::string &	table_name,
		const bool	evict_cached_entries
	)

Definition at line 90 of file ForeignTableRefresh.cpp.

References CHECK, shared::contains(), StorageType::FOREIGN_TABLE, dist::is_leaf_node(), Catalog_Namespace::kAggregatorOnlySystemTables, and refresh_foreign_table_unlocked().

Referenced by RefreshForeignTablesCommand::execute(), and foreign_storage::ForeignTableRefreshScheduler::start().

                                                             {
   if (dist::is_leaf_node() &&
       shared::contains(Catalog_Namespace::kAggregatorOnlySystemTables, table_name)) {
     // Skip aggregator only system tables on leaf nodes.
     return;
   }
   auto table_lock =
       std::make_unique<lockmgr::TableSchemaLockContainer<lockmgr::WriteLock>>(
           lockmgr::TableSchemaLockContainer<lockmgr::WriteLock>::acquireTableDescriptor(
               catalog, table_name, false));
 
   const TableDescriptor* td = (*table_lock)();
   if (td->storageType != StorageType::FOREIGN_TABLE) {
     throw std::runtime_error{
         table_name +
         " is not a foreign table. Refreshes are applicable to only foreign tables."};
   }
 
   auto foreign_table = dynamic_cast<const ForeignTable*>(td);
   CHECK(foreign_table);
   refresh_foreign_table_unlocked(catalog, *foreign_table, evict_cached_entries);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::refresh_foreign_table_unlocked	(	Catalog_Namespace::Catalog &	catalog,
		const ForeignTable &	td,
		const bool	evict_cached_entries
	)

Definition at line 34 of file ForeignTableRefresh.cpp.

References CHUNK_KEY_FRAGMENT_IDX, foreign_storage::anonymous_namespace{ForeignTableRefresh.cpp}::clear_cpu_and_gpu_cache(), Catalog_Namespace::DBMetadata::dbId, Catalog_Namespace::Catalog::getCurrentDB(), Catalog_Namespace::Catalog::getDataMgr(), Catalog_Namespace::Catalog::getForeignTable(), PostEvictionRefreshException::getOriginalException(), CacheInvalidator< CACHE_HOLDING_TYPES >::invalidateCachesByTable(), foreign_storage::ForeignTable::isAppendMode(), Catalog_Namespace::Catalog::removeFragmenterForTable(), TableDescriptor::tableId, and Catalog_Namespace::Catalog::updateForeignTableRefreshTimes().

Referenced by refresh_foreign_table().

                                                                      {
   auto& data_mgr = catalog.getDataMgr();
   ChunkKey table_key{catalog.getCurrentDB().dbId, td.tableId};
   ResultSetCacheInvalidator::invalidateCachesByTable(boost::hash_value(table_key));
 
   catalog.removeFragmenterForTable(td.tableId);
 
   std::map<ChunkKey, std::shared_ptr<ChunkMetadata>> old_chunk_metadata_by_chunk_key;
   if (catalog.getForeignTable(td.tableId)->isAppendMode() && !evict_cached_entries) {
     ChunkMetadataVector metadata_vec;
     data_mgr.getChunkMetadataVecForKeyPrefix(metadata_vec, table_key);
     int last_fragment_id = 0;
     for (const auto& [key, metadata] : metadata_vec) {
       if (key[CHUNK_KEY_FRAGMENT_IDX] > last_fragment_id) {
         last_fragment_id = key[CHUNK_KEY_FRAGMENT_IDX];
       }
       old_chunk_metadata_by_chunk_key[key] = metadata;
     }
     for (const auto& [key, metadata] : metadata_vec) {
       if (key[CHUNK_KEY_FRAGMENT_IDX] == last_fragment_id) {
         clear_cpu_and_gpu_cache(data_mgr, key);
       }
     }
   } else {
     clear_cpu_and_gpu_cache(data_mgr, table_key);
   }
 
   try {
     data_mgr.getPersistentStorageMgr()->getForeignStorageMgr()->refreshTable(
         table_key, evict_cached_entries);
     catalog.updateForeignTableRefreshTimes(td.tableId);
   } catch (PostEvictionRefreshException& e) {
     catalog.updateForeignTableRefreshTimes(td.tableId);
     clear_cpu_and_gpu_cache(data_mgr, table_key);
     throw e.getOriginalException();
   } catch (...) {
     clear_cpu_and_gpu_cache(data_mgr, table_key);
     throw;
   }
 
   // Delete cached rolled off/updated chunks.
   if (!old_chunk_metadata_by_chunk_key.empty()) {
     ChunkMetadataVector new_metadata_vec;
     data_mgr.getChunkMetadataVecForKeyPrefix(new_metadata_vec, table_key);
     for (const auto& [key, metadata] : new_metadata_vec) {
       auto it = old_chunk_metadata_by_chunk_key.find(key);
       if (it != old_chunk_metadata_by_chunk_key.end() &&
           it->second->numElements != metadata->numElements) {
         clear_cpu_and_gpu_cache(data_mgr, key);
       }
     }
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::resize_buffer_if_needed	(	std::unique_ptr< char[]> &	buffer,
		size_t &	buffer_size,
		const size_t	alloc_size
	)

Optionally resizes the given buffer if the buffer size is less than the current buffer allocation size.

Definition at line 725 of file AbstractTextFileDataWrapper.cpp.

References CHECK_LE.

Referenced by dispatch_metadata_scan_requests().

                                                       {
   CHECK_LE(buffer_size, alloc_size);
   if (buffer_size < alloc_size) {
     buffer = std::make_unique<char[]>(alloc_size);
     buffer_size = alloc_size;
   }
 }

Here is the caller graph for this function:

std::vector< Aws::S3::Model::Object > foreign_storage::s3_objects_filter_sort_files	(	const std::vector< Aws::S3::Model::Object > &	file_paths,
		const shared::FilePathOptions &	options
	)

Definition at line 22 of file S3FilePathUtil.cpp.

References shared::FilePathOptions::filter_regex, shared::PATHNAME_ORDER_TYPE, foreign_storage::anonymous_namespace{S3FilePathUtil.cpp}::s3_objects_regex_file_filter(), and shared::FilePathOptions::sort_by.

                                           {
   auto result_files =
       options.filter_regex.has_value()
           ? s3_objects_regex_file_filter(options.filter_regex.value(), file_paths)
           : file_paths;
   // initial lexicographical order ensures a determinisitc ordering for files not matching
   // sort_regex
   shared::FilePathOptions temp_options;
   temp_options.sort_by = shared::PATHNAME_ORDER_TYPE;
   auto initial_file_order = FileOrderS3(temp_options);
   auto lexi_comp = initial_file_order.getFileComparator();
   std::stable_sort(result_files.begin(), result_files.end(), lexi_comp);
 
   auto file_order = FileOrderS3(options);
   auto comp = file_order.getFileComparator();
   std::stable_sort(result_files.begin(), result_files.end(), comp);
   return result_files;
 }

Here is the call graph for this function:

void foreign_storage::scan_metadata	(	MetadataScanMultiThreadingParams &	multi_threading_params,
		std::map< int, FileRegions > &	fragment_id_to_file_regions_map,
		const TextFileBufferParser &	parser
	)

Consumes and processes metadata scan requests from a pending requests queue and updates existing metadata objects based on newly scanned metadata.

Definition at line 637 of file AbstractTextFileDataWrapper.cpp.

References add_request_to_pool(), foreign_storage::MetadataScanMultiThreadingParams::continue_processing, get_next_metadata_scan_request(), foreign_storage::TextFileBufferParser::parseBuffer(), partition_by_fragment(), foreign_storage::MetadataScanMultiThreadingParams::pending_requests_mutex, process_data_blocks(), and run_benchmark_import::result.

Referenced by foreign_storage::AbstractTextFileDataWrapper::populateChunkMetadata().

                                                        {
   std::map<int, const ColumnDescriptor*> column_by_id{};
   while (true) {
     auto request_opt = get_next_metadata_scan_request(multi_threading_params);
     if (!request_opt.has_value()) {
       break;
     }
     auto& request = request_opt.value();
     try {
       if (column_by_id.empty()) {
         for (const auto column : request.getColumns()) {
           column_by_id[column->columnId] = column;
         }
       }
       auto partitions = partition_by_fragment(
           request.first_row_index, request.getMaxFragRows(), request.buffer_row_count);
       request.begin_pos = 0;
       size_t row_index = request.first_row_index;
       for (const auto partition : partitions) {
         request.process_row_count = partition;
         for (const auto& import_buffer : request.import_buffers) {
           if (import_buffer != nullptr) {
             import_buffer->clear();
           }
         }
         auto result = parser.parseBuffer(request, true);
         int fragment_id = row_index / request.getMaxFragRows();
         process_data_blocks(multi_threading_params,
                             fragment_id,
                             request,
                             result,
                             column_by_id,
                             fragment_id_to_file_regions_map);
         row_index += result.row_count;
         request.begin_pos = result.row_offsets.back() - request.file_offset;
       }
     } catch (...) {
       // Re-add request to pool so we dont block any other threads
       {
         std::lock_guard<std::mutex> pending_requests_lock(
             multi_threading_params.pending_requests_mutex);
         multi_threading_params.continue_processing = false;
       }
       add_request_to_pool(multi_threading_params, request);
       throw;
     }
     add_request_to_pool(multi_threading_params, request);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

bool foreign_storage::set_comp	(	const ChunkKey &	left,
		const ChunkKey &	right
	)

void foreign_storage::set_node_name ( std::map< std::string, import_export::TypedImportBuffer * > & import_buffers )

Definition at line 55 of file InternalSystemDataWrapper.cpp.

References g_distributed_leaf_idx, dist::is_leaf_node(), and to_string().

Referenced by foreign_storage::anonymous_namespace{InternalMemoryStatsDataWrapper.cpp}::populate_import_buffers_for_memory_details(), foreign_storage::anonymous_namespace{InternalMemoryStatsDataWrapper.cpp}::populate_import_buffers_for_memory_summary(), and foreign_storage::anonymous_namespace{InternalStorageStatsDataWrapper.cpp}::populate_import_buffers_for_storage_details().

                                                                         {
   if (import_buffers.find("node") != import_buffers.end()) {
     if (dist::is_leaf_node()) {
       std::string leaf_string{"Leaf " + to_string(g_distributed_leaf_idx)};
       import_buffers["node"]->addString(leaf_string);
     } else {
       import_buffers["node"]->addString("Server");
     }
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::set_value	(	rapidjson::Value &	json_val,
		const FileRegion &	file_region,
		rapidjson::Document::AllocatorType &	allocator
	)

Definition at line 26 of file CsvShared.cpp.

References foreign_storage::json_utils::add_value_to_object(), foreign_storage::FileRegion::first_row_file_offset, foreign_storage::FileRegion::first_row_index, foreign_storage::FileRegion::region_size, and foreign_storage::FileRegion::row_count.

                                                           {
   json_val.SetObject();
   json_utils::add_value_to_object(
       json_val, file_region.first_row_file_offset, "first_row_file_offset", allocator);
   json_utils::add_value_to_object(
       json_val, file_region.first_row_index, "first_row_index", allocator);
   json_utils::add_value_to_object(
       json_val, file_region.region_size, "region_size", allocator);
   json_utils::add_value_to_object(
       json_val, file_region.row_count, "row_count", allocator);
   if (file_region.filename.size()) {
     json_utils::add_value_to_object(
         json_val, file_region.filename, "filename", allocator);
   }
 }

Here is the call graph for this function:

void foreign_storage::set_value	(	rapidjson::Value &	json_val,
		const RowGroupInterval &	value,
		rapidjson::Document::AllocatorType &	allocator
	)

Definition at line 660 of file ParquetDataWrapper.cpp.

References foreign_storage::json_utils::add_value_to_object(), foreign_storage::RowGroupInterval::end_index, foreign_storage::RowGroupInterval::file_path, and foreign_storage::RowGroupInterval::start_index.

                                                           {
   json_val.SetObject();
   json_utils::add_value_to_object(json_val, value.file_path, "file_path", allocator);
   json_utils::add_value_to_object(json_val, value.start_index, "start_index", allocator);
   json_utils::add_value_to_object(json_val, value.end_index, "end_index", allocator);
 }

Here is the call graph for this function:

void foreign_storage::throw_file_access_error	(	const std::string &	file_path,
		const std::string &	message
	)

inline

Definition at line 83 of file ForeignStorageException.h.

Referenced by foreign_storage::ParquetImporter::getAllFilePaths().

                                                               {
   std::string error_message{"Unable to access file \"" + file_path + "\". " + message};
   throw ForeignStorageException{error_message};
 }

Here is the caller graph for this function:

void foreign_storage::throw_file_not_found_error ( const std::string & file_path )

inline

Definition at line 89 of file ForeignStorageException.h.

Referenced by foreign_storage::ParquetImporter::getAllFilePaths().

                                                                    {
   throw ForeignStorageException{"File or directory \"" + file_path +
                                 "\" does not exist."};
 }

Here is the caller graph for this function:

void foreign_storage::throw_number_of_columns_mismatch_error	(	size_t	num_table_cols,
		size_t	num_file_cols,
		const std::string &	file_path
	)

inline

Definition at line 74 of file ForeignStorageException.h.

References to_string().

Referenced by foreign_storage::RegexFileBufferParser::regexMatchColumns(), foreign_storage::anonymous_namespace{CsvFileBufferParser.cpp}::validate_expected_column_count(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_number_of_columns().

                                                                                {
   throw ForeignStorageException{"Mismatched number of logical columns: (expected " +
                                 std::to_string(num_table_cols) + " columns, has " +
                                 std::to_string(num_file_cols) + "): in file '" +
                                 file_path + "'"};
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::throw_parquet_metadata_out_of_bounds_error	(	const std::string &	min_value,
		const std::string &	max_value,
		const std::string &	encountered_value
	)

inline

Definition at line 46 of file ParquetMetadataValidator.h.

Referenced by foreign_storage::TimestampBoundsValidator< T >::validateValue(), foreign_storage::IntegralFixedLengthBoundsValidator< T >::validateValue(), foreign_storage::BaseDateBoundsValidator< T, is_in_seconds >::validateValue(), and foreign_storage::FloatPointValidator< T >::validateValue().

                                         {
   std::stringstream error_message;
   error_message << "Parquet column contains values that are outside the range of the "
                    "HeavyDB column "
                    "type. Consider using a wider column type. Min allowed value: "
                 << min_value << ". Max allowed value: " << max_value
                 << ". Encountered value: " << encountered_value << ".";
   throw std::runtime_error(error_message.str());
 }

Here is the caller graph for this function:

void foreign_storage::throw_removed_file_error ( const std::string & file_path )

inline

Definition at line 67 of file ForeignStorageException.h.

Referenced by foreign_storage::LocalMultiFileReader::checkForMoreRows(), and foreign_storage::ParquetDataWrapper::fetchChunkMetadata().

                                                                  {
   throw ForeignStorageException{
       "Refresh of foreign table created with \"APPEND\" update type failed as "
       "file \"" +
       file_path + "\" was removed."};
 }

Here is the caller graph for this function:

void foreign_storage::throw_removed_row_in_file_error ( const std::string & file_path )

inline

Definition at line 60 of file ForeignStorageException.h.

Referenced by foreign_storage::SingleTextFileReader::checkForMoreRows(), and foreign_storage::ParquetDataWrapper::fetchChunkMetadata().

                                                                         {
   throw ForeignStorageException{
       "Refresh of foreign table created with \"APPEND\" update type failed as file "
       "reduced in size: \"" +
       file_path + "\""};
 }

Here is the caller graph for this function:

void foreign_storage::throw_removed_row_in_result_set_error ( const std::string & select_statement )

inline

Definition at line 52 of file ForeignStorageException.h.

                                                                                      {
   throw ForeignStorageException{
       "Refresh of foreign table created with \"APPEND\" update type failed as result set "
       "of select statement "
       "reduced in size: \"" +
       select_statement + "\""};
 }

void foreign_storage::throw_s3_compressed_extension	(	const std::string &	file_path,
		const std::string &	ext_type
	)

inline

Definition at line 101 of file ForeignStorageException.h.

                                                                      {
   throw ForeignStorageException{
       "File \"" + file_path + "\" has extension type \"" + ext_type +
       "\", compressed file formats are not supported by S3 Foreign Tables."};
 }

void foreign_storage::throw_s3_compressed_mime_type	(	const std::string &	file_path,
		const std::string &	mime_type
	)

inline

Definition at line 94 of file ForeignStorageException.h.

                                                                       {
   throw ForeignStorageException{
       "File \"" + file_path + "\" has mime type \"" + mime_type +
       "\", compressed file formats are not supported by S3 Foreign Tables."};
 }

void foreign_storage::throw_unexpected_number_of_items	(	const size_t &	num_expected,
		const size_t &	num_loaded,
		const std::string &	item_type
	)

inline

Definition at line 40 of file ForeignStorageException.h.

References to_string().

Referenced by parse_file_regions(), and foreign_storage::anonymous_namespace{AbstractTextFileDataWrapper.cpp}::throw_unexpected_number_of_items().

                                                                          {
   throw ForeignStorageException(
       "Unexpected number of " + item_type +
       " while loading from foreign data source: expected " +
       std::to_string(num_expected) + " , obtained " + std::to_string(num_loaded) + " " +
       item_type +
       ". Please use the \"REFRESH FOREIGN TABLES\" command on the foreign table "
       "if data source has been updated.");
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void foreign_storage::update_stats	(	Encoder *	encoder,
		const SQLTypeInfo &	column_type,
		DataBlockPtr	data_block,
		const size_t	row_count
	)

Updates the statistics metadata encapsulated in the encoder given new data in a data block.

Definition at line 496 of file AbstractTextFileDataWrapper.cpp.

References DataBlockPtr::arraysPtr, SQLTypeInfo::is_array(), SQLTypeInfo::is_varlen(), DataBlockPtr::numbersPtr, DataBlockPtr::stringsPtr, and Encoder::updateStats().

Referenced by process_data_blocks(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumn(), Fragmenter_Namespace::InsertOrderFragmenter::updateColumnMetadata(), and StorageIOFacility::yieldUpdateCallback().

                                           {
   if (column_type.is_array()) {
     encoder->updateStats(data_block.arraysPtr, 0, row_count);
   } else if (!column_type.is_varlen()) {
     encoder->updateStats(data_block.numbersPtr, row_count);
   } else {
     encoder->updateStats(data_block.stringsPtr, 0, row_count);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

std::shared_ptr< parquet::Statistics > foreign_storage::validate_and_get_column_metadata_statistics ( const parquet::ColumnChunkMetaData * column_metadata )

Definition at line 86 of file ParquetShared.cpp.

References CHECK.

Referenced by foreign_storage::ParquetEncoder::getRowGroupMetadata(), foreign_storage::TypedParquetInPlaceEncoder< V, V >::getRowGroupMetadata(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_definition_levels().

                                                        {
   CHECK(column_metadata->is_stats_set());
   std::shared_ptr<parquet::Statistics> stats = column_metadata->statistics();
   return stats;
 }

Here is the caller graph for this function:

void foreign_storage::validate_equal_column_descriptor	(	const parquet::ColumnDescriptor *	reference_descriptor,
		const parquet::ColumnDescriptor *	new_descriptor,
		const std::string &	reference_file_path,
		const std::string &	new_file_path
	)

Definition at line 60 of file ParquetShared.cpp.

Referenced by foreign_storage::LazyParquetChunkLoader::appendRowGroups(), and foreign_storage::anonymous_namespace{LazyParquetChunkLoader.cpp}::validate_equal_schema().

                                     {
   if (!reference_descriptor->Equals(*new_descriptor)) {
     throw std::runtime_error{"Parquet file \"" + new_file_path +
                              "\" has a different schema. Please ensure that all Parquet "
                              "files use the same schema. Reference Parquet file: " +
                              reference_file_path +
                              ", column name: " + reference_descriptor->name() +
                              ". New Parquet file: " + new_file_path +
                              ", column name: " + new_descriptor->name() + "."};
   }
 }

Here is the caller graph for this function:

void foreign_storage::validate_non_foreign_table_write ( const TableDescriptor * table_descriptor )

inline

Definition at line 22 of file FsiUtils.h.

References StorageType::FOREIGN_TABLE, and TableDescriptor::storageType.

Referenced by Parser::InsertStmt::analyze(), Parser::TruncateTableStmt::execute(), Parser::InsertValuesStmt::execute(), Parser::InsertIntoTableAsSelectStmt::populateData(), and RelModify::RelModify().

                                                                                       {
   if (table_descriptor && table_descriptor->storageType == StorageType::FOREIGN_TABLE) {
     throw std::runtime_error{
         "DELETE, INSERT, TRUNCATE, OR UPDATE commands are not supported for foreign "
         "tables."};
   }
 }

Here is the caller graph for this function:

void foreign_storage::validate_regex_parser_options ( const import_export::CopyParams & copy_params )

Definition at line 100 of file ForeignDataWrapperFactory.cpp.

References import_export::CopyParams::line_regex.

Referenced by anonymous_namespace{ForeignDataImporter.cpp}::validate_copy_params().

                                                                                {
   if (copy_params.line_regex.empty()) {
     throw std::runtime_error{"Regex parser options must contain a line regex."};
   }
 }

Here is the caller graph for this function:

Variable Documentation

constexpr const char* foreign_storage::kDeletedValueIndicator {"<DELETED>"}

Definition at line 26 of file InternalSystemDataWrapper.h.

Referenced by foreign_storage::anonymous_namespace{InternalMemoryStatsDataWrapper.cpp}::get_column_name(), get_db_name(), foreign_storage::anonymous_namespace{InternalCatalogDataWrapper.cpp}::get_table_ddl(), get_table_name(), and foreign_storage::anonymous_namespace{InternalCatalogDataWrapper.cpp}::get_user_name().

Namespaces

Classes

Typedefs

Functions

Variables

Typedef Documentation

Function Documentation

Variable Documentation