OmniSciDB  2e3a973ef4
CsvReaderS3.h
Go to the documentation of this file.
1 /*
2  * Copyright 2020 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #pragma once
18 
19 #include <aws/core/Aws.h>
20 #include <aws/s3/S3Client.h>
21 
22 #include "Catalog/ForeignServer.h"
23 #include "Catalog/ee/UserMapping.h"
25 namespace foreign_storage {
26 
27 // Single S3 file or directory with multiple files
29  public:
30  MultiS3Reader(const std::string& file_path,
31  const import_export::CopyParams& copy_params,
32  const ForeignServer* server_options,
33  const UserMapping* user_mapping);
34 
35  MultiS3Reader(const std::string& file_path,
36  const import_export::CopyParams& copy_params,
37  const ForeignServer* server_options,
38  const UserMapping* user_mapping,
39  const rapidjson::Value& value);
40  void checkForMoreRows(size_t file_offset,
41  const ForeignServer* server_options,
42  const UserMapping* user_mapping) override;
43  void serialize(rapidjson::Value& value,
44  rapidjson::Document::AllocatorType& allocator) const override;
45 
46  private:
47  void skipHeader();
48  size_t file_size_;
49  // We've reached the end of the file
51  std::unique_ptr<Aws::S3::S3Client> s3_client_;
52  std::vector<size_t> file_sizes_;
53  std::string bucket_name_;
54 };
55 
56 class CsvReaderS3 : public CsvReader {
57  public:
58  CsvReaderS3(const std::string& obj_key,
59  size_t file_size,
60  const import_export::CopyParams& copy_params,
61  const ForeignServer* server_options,
62  const UserMapping* user_mapping);
63 
64  CsvReaderS3(const std::string& obj_key,
65  const import_export::CopyParams& copy_params,
66  const ForeignServer* server_options,
67  const UserMapping* user_mapping,
68  const rapidjson::Value& value);
69 
70  size_t read(void* buffer, size_t max_size) override;
71  size_t readRegion(void* buffer, size_t offset, size_t size) override {
73  current_offset_ = offset;
74  return read(buffer, size);
75  }
76 
77  bool isScanFinished() override { return scan_finished_; }
78 
79  size_t getRemainingSize() override { return file_size_ - current_offset_; }
80 
81  bool isRemainingSizeKnown() override { return true; };
82 
83  void serialize(rapidjson::Value& value,
84  rapidjson::Document::AllocatorType& allocator) const override;
85 
86  // Increase file size and continue metadata scan
87  void increaseFileSize(size_t new_size);
88 
89  private:
90  void skipHeader();
91  size_t file_size_;
92  // We've reached the end of the file
94  std::unique_ptr<Aws::S3::S3Client> s3_client_;
95 
96  std::string obj_key_;
97  std::string bucket_name_;
99 
102 };
103 
104 } // namespace foreign_storage
MultiS3Reader(const std::string &file_path, const import_export::CopyParams &copy_params, const ForeignServer *server_options, const UserMapping *user_mapping)
std::vector< size_t > file_sizes_
Definition: CsvReaderS3.h:52
size_t getRemainingSize() override
Definition: CsvReaderS3.h:79
void checkForMoreRows(size_t file_offset, const ForeignServer *server_options, const UserMapping *user_mapping) override
size_t readRegion(void *buffer, size_t offset, size_t size) override
Definition: CsvReaderS3.h:71
std::unique_ptr< Aws::S3::S3Client > s3_client_
Definition: CsvReaderS3.h:51
bool isRemainingSizeKnown() override
Definition: CsvReaderS3.h:81
bool isScanFinished() override
Definition: CsvReaderS3.h:77
std::unique_ptr< Aws::S3::S3Client > s3_client_
Definition: CsvReaderS3.h:94
size_t read(void *buffer, size_t max_size) override
Definition: CsvReader.cpp:644
#define CHECK(condition)
Definition: Logger.h:197
bool isScanFinished() override
Definition: CsvReader.h:307
void serialize(rapidjson::Value &value, rapidjson::Document::AllocatorType &allocator) const override
import_export::CopyParams copy_params_
Definition: CsvReaderS3.h:98
size_t file_size(const int fd)
Definition: omnisci_fs.cpp:31