20 #include <unordered_set>
38 namespace foreign_storage {
46 const bool do_metadata_stats_validation =
true);
52 std::shared_ptr<arrow::fs::FileSystem> file_system);
63 const std::string& file_path,
84 const bool reserve_buffers_and_set_stats =
false);
87 const int fragment_id,
99 bool isNewFile(
const std::string& file_path)
const;
101 void addNewFile(
const std::string& file_path);
std::string getSerializedDataWrapper() const override
size_t last_fragment_row_count_
void finalizeFragmentMap()
std::set< std::string > getProcessedFilePaths()
std::unique_ptr< FileReaderMap > file_reader_cache_
void restoreDataWrapperInternals(const std::string &file_path, const ChunkMetadataVector &chunk_metadata_vector) override
std::vector< std::string > getAllFilePaths()
std::unique_ptr< ForeignTableSchema > schema_
void populateChunkBuffers(const ChunkToBufferMap &required_buffers, const ChunkToBufferMap &optional_buffers, AbstractBuffer *delete_buffer) override
std::map< ChunkKey, AbstractBuffer * > ChunkToBufferMap
std::mutex delete_buffer_mutex_
std::map< int, std::vector< RowGroupInterval > > fragment_to_row_group_interval_map_
void resetParquetMetadata()
std::map< ChunkKey, std::shared_ptr< ChunkMetadata > > chunk_metadata_map_
ParallelismLevel getCachedParallelismLevel() const override
const bool do_metadata_stats_validation_
void addNewFile(const std::string &file_path)
void fetchChunkMetadata()
DataPreview getDataPreview(const size_t num_rows)
const ForeignTable * foreign_table_
RenderGroupAnalyzerMap render_group_analyzer_map_
void loadBuffersUsingLazyParquetChunkLoader(const int logical_column_id, const int fragment_id, const ChunkToBufferMap &required_buffers, AbstractBuffer *delete_buffer)
void addNewFragment(int row_group, const std::string &file_path)
void initializeChunkBuffers(const int fragment_index, const Interval< ColumnType > &column_interval, const ChunkToBufferMap &required_buffers, const bool reserve_buffers_and_set_stats=false)
An AbstractBuffer is a unit of data management for a data manager.
bool isNewFile(const std::string &file_path) const
bool moveToNextFragment(size_t new_rows_count) const
std::shared_ptr< arrow::fs::FileSystem > file_system_
std::list< const ColumnDescriptor * > getColumnsToInitialize(const Interval< ColumnType > &column_interval)
ParallelismLevel getNonCachedParallelismLevel() const override
void metadataScanFiles(const std::vector< std::string > &file_paths)
void populateChunkMetadata(ChunkMetadataVector &chunk_metadata_vector) override
std::map< int, std::unique_ptr< import_export::RenderGroupAnalyzer >> RenderGroupAnalyzerMap
bool isRestored() const override
void createRenderGroupAnalyzers() override
Create RenderGroupAnalyzers for poly columns.