OmniSciDB  085a039ca4
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
AbstractTextFileDataWrapper.h
Go to the documentation of this file.
1 /*
2  * Copyright 2021 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #pragma once
18 
19 #include <map>
20 #include <set>
21 
23 #include "Catalog/CatalogFwd.h"
24 #include "Catalog/ForeignTable.h"
25 #include "DataMgr/Chunk/Chunk.h"
29 
30 namespace foreign_storage {
31 
33  public:
35 
36  AbstractTextFileDataWrapper(const int db_id, const ForeignTable* foreign_table);
37 
38  AbstractTextFileDataWrapper(const int db_id,
39  const ForeignTable* foreign_table,
40  const UserMapping* user_mapping,
41  const bool disable_cache);
42 
43  void populateChunkMetadata(ChunkMetadataVector& chunk_metadata_vector) override;
44 
45  void populateChunkBuffers(const ChunkToBufferMap& required_buffers,
46  const ChunkToBufferMap& optional_buffers,
47  AbstractBuffer* delete_buffer) override;
48 
49  std::string getSerializedDataWrapper() const override;
50 
51  void restoreDataWrapperInternals(const std::string& file_path,
52  const ChunkMetadataVector& chunk_metadata) override;
53  bool isRestored() const override;
54 
56 
58  return INTRA_FRAGMENT;
59  }
60 
61  void createRenderGroupAnalyzers() override;
62 
63  protected:
64  virtual const TextFileBufferParser& getFileBufferParser() const = 0;
65 
66  private:
67  AbstractTextFileDataWrapper(const ForeignTable* foreign_table);
68 
77  void populateChunks(std::map<int, Chunk_NS::Chunk>& column_id_to_chunk_map,
78  int fragment_id,
79  AbstractBuffer* delete_buffer);
80 
81  void populateChunkMapForColumns(const std::set<const ColumnDescriptor*>& columns,
82  const int fragment_id,
83  const ChunkToBufferMap& buffers,
84  std::map<int, Chunk_NS::Chunk>& column_id_to_chunk_map);
85 
86  void updateMetadata(std::map<int, Chunk_NS::Chunk>& column_id_to_chunk_map,
87  int fragment_id);
88 
89  std::map<ChunkKey, std::shared_ptr<ChunkMetadata>> chunk_metadata_map_;
90  std::map<int, FileRegions> fragment_id_to_file_regions_map_;
91 
92  std::unique_ptr<FileReader> file_reader_;
93 
94  const int db_id_;
96 
97  // Data needed for append workflow
98  std::map<ChunkKey, std::unique_ptr<ForeignStorageBuffer>> chunk_encoder_buffers_;
99  // How many rows have been read
100  size_t num_rows_;
101  // What byte offset we left off at in the file_reader
103  // Is this datawrapper restored from disk
105 
107 
108  // Force cache to be disabled
109  const bool disable_cache_;
110 
111  // declared in three derived classes to avoid
112  // polluting ForeignDataWrapper virtual base
113  // @TODO refactor to lower class if needed
115 };
116 } // namespace foreign_storage
std::map< ChunkKey, AbstractBuffer * > ChunkToBufferMap
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_encoder_buffers_
virtual const TextFileBufferParser & getFileBufferParser() const =0
void createRenderGroupAnalyzers() override
Create RenderGroupAnalyzers for poly columns.
void populateChunkMetadata(ChunkMetadataVector &chunk_metadata_vector) override
std::map< ChunkKey, std::shared_ptr< ChunkMetadata > > chunk_metadata_map_
void updateMetadata(std::map< int, Chunk_NS::Chunk > &column_id_to_chunk_map, int fragment_id)
ParallelismLevel getNonCachedParallelismLevel() const override
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
An AbstractBuffer is a unit of data management for a data manager.
void populateChunks(std::map< int, Chunk_NS::Chunk > &column_id_to_chunk_map, int fragment_id, AbstractBuffer *delete_buffer)
ParallelismLevel getCachedParallelismLevel() const override
void populateChunkBuffers(const ChunkToBufferMap &required_buffers, const ChunkToBufferMap &optional_buffers, AbstractBuffer *delete_buffer) override
void restoreDataWrapperInternals(const std::string &file_path, const ChunkMetadataVector &chunk_metadata) override
void populateChunkMapForColumns(const std::set< const ColumnDescriptor * > &columns, const int fragment_id, const ChunkToBufferMap &buffers, std::map< int, Chunk_NS::Chunk > &column_id_to_chunk_map)
std::map< int, std::unique_ptr< import_export::RenderGroupAnalyzer >> RenderGroupAnalyzerMap