OmniSciDB  a667adc9c8
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
CsvDataWrapper.h
Go to the documentation of this file.
1 /*
2  * Copyright 2020 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #pragma once
18 
19 #include <map>
20 #include <vector>
21 
23 #include "Catalog/Catalog.h"
24 #include "Catalog/ForeignTable.h"
25 #include "DataMgr/Chunk/Chunk.h"
28 #include "ForeignDataWrapper.h"
29 #include "ImportExport/Importer.h"
30 
31 namespace foreign_storage {
32 
34  public:
36 
37  CsvDataWrapper(const int db_id, const ForeignTable* foreign_table);
38 
39  void populateChunkMetadata(ChunkMetadataVector& chunk_metadata_vector) override;
40 
41  void populateChunkBuffers(const ChunkToBufferMap& required_buffers,
42  const ChunkToBufferMap& optional_buffers) override;
43 
44  void validateTableOptions(const ForeignTable* foreign_table) const override;
45 
46  const std::set<std::string_view>& getSupportedTableOptions() const override;
47 
48  void serializeDataWrapperInternals(const std::string& file_path) const override;
49 
50  void restoreDataWrapperInternals(const std::string& file_path,
51  const ChunkMetadataVector& chunk_metadata) override;
52  bool isRestored() const override;
53 
55 
57  return INTRA_FRAGMENT;
58  }
59 
60  private:
61  CsvDataWrapper(const ForeignTable* foreign_table);
62 
70  void populateChunks(std::map<int, Chunk_NS::Chunk>& column_id_to_chunk_map,
71  int fragment_id);
72 
73  void populateChunkMapForColumns(const std::set<const ColumnDescriptor*>& columns,
74  const int fragment_id,
75  const ChunkToBufferMap& buffers,
76  std::map<int, Chunk_NS::Chunk>& column_id_to_chunk_map);
77 
78  void updateMetadata(std::map<int, Chunk_NS::Chunk>& column_id_to_chunk_map,
79  int fragment_id);
80 
81  std::set<std::string_view> getAllCsvTableOptions() const;
82 
83  std::map<ChunkKey, std::shared_ptr<ChunkMetadata>> chunk_metadata_map_;
84  std::map<int, FileRegions> fragment_id_to_file_regions_map_;
85 
86  std::unique_ptr<CsvReader> csv_reader_;
87 
88  const int db_id_;
90 
91  // Data needed for append workflow
92  std::map<ChunkKey, std::unique_ptr<ForeignStorageBuffer>> chunk_encoder_buffers_;
93  std::map<ChunkKey, size_t> chunk_byte_count_;
94  // How many rows have been read
95  size_t num_rows_;
96  // What byte offset we left off at in the csv_reader
98  // Is this datawrapper restored from disk
100  static const std::set<std::string_view> csv_table_options_;
101 };
102 } // namespace foreign_storage
bool isRestored() const override
void populateChunkBuffers(const ChunkToBufferMap &required_buffers, const ChunkToBufferMap &optional_buffers) override
void serializeDataWrapperInternals(const std::string &file_path) const override
std::map< ChunkKey, size_t > chunk_byte_count_
static const std::set< std::string_view > csv_table_options_
void restoreDataWrapperInternals(const std::string &file_path, const ChunkMetadataVector &chunk_metadata) override
std::map< ChunkKey, std::unique_ptr< ForeignStorageBuffer > > chunk_encoder_buffers_
std::map< ChunkKey, AbstractBuffer * > ChunkToBufferMap
ParallelismLevel getCachedParallelismLevel() const override
std::unique_ptr< CsvReader > csv_reader_
std::set< std::string_view > getAllCsvTableOptions() const
This file contains the class specification and related data structures for Catalog.
void populateChunkMapForColumns(const std::set< const ColumnDescriptor * > &columns, const int fragment_id, const ChunkToBufferMap &buffers, std::map< int, Chunk_NS::Chunk > &column_id_to_chunk_map)
void populateChunkMetadata(ChunkMetadataVector &chunk_metadata_vector) override
std::map< int, FileRegions > fragment_id_to_file_regions_map_
std::vector< std::pair< ChunkKey, std::shared_ptr< ChunkMetadata >>> ChunkMetadataVector
void updateMetadata(std::map< int, Chunk_NS::Chunk > &column_id_to_chunk_map, int fragment_id)
void validateTableOptions(const ForeignTable *foreign_table) const override
std::map< ChunkKey, std::shared_ptr< ChunkMetadata > > chunk_metadata_map_
const std::set< std::string_view > & getSupportedTableOptions() const override
const ForeignTable * foreign_table_
void populateChunks(std::map< int, Chunk_NS::Chunk > &column_id_to_chunk_map, int fragment_id)
ParallelismLevel getNonCachedParallelismLevel() const override