OmniSciDB  a5dc49c757
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
ParquetS3DetectFileSystem.h
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #pragma once
18 
19 #if defined(HAVE_AWS_S3)
20 #include <map>
21 #include <optional>
22 
23 #include <arrow/filesystem/filesystem.h>
24 #include <arrow/filesystem/s3fs.h>
25 #include <arrow/result.h>
26 #include <arrow/status.h>
27 #include <aws/core/Globals.h>
28 #include <aws/core/auth/AWSCredentialsProviderChain.h>
29 
31 
32 namespace foreign_storage {
33 class ParquetS3DetectFileSystem {
34  public:
35  struct ParquetS3DetectFileSystemConfiguration {
36  std::optional<std::string> s3_access_key = std::nullopt;
37  std::optional<std::string> s3_secret_key = std::nullopt;
38  std::optional<std::string> s3_session_token = std::nullopt;
39  std::string s3_region;
40  };
41 
42  static std::shared_ptr<arrow::fs::FileSystem> create(
43  const ParquetS3DetectFileSystemConfiguration& config) {
44  auto s3_options = arrow::fs::S3Options::Anonymous();
45  if (config.s3_access_key.has_value() || config.s3_secret_key.has_value() ||
46  config.s3_session_token.has_value()) {
47  if (config.s3_access_key.has_value() && config.s3_secret_key.has_value()) {
48  if (config.s3_session_token.has_value()) {
49  s3_options =
50  arrow::fs::S3Options::FromAccessKey(config.s3_access_key.value(),
51  config.s3_secret_key.value(),
52  config.s3_session_token.value());
53  } else {
54  s3_options = arrow::fs::S3Options::FromAccessKey(config.s3_access_key.value(),
55  config.s3_secret_key.value());
56  }
57  }
58  } else if (g_allow_s3_server_privileges) {
59  Aws::Auth::DefaultAWSCredentialsProviderChain default_provider;
60  if ((default_provider.GetAWSCredentials().GetAWSAccessKeyId().size() > 0) &&
61  (default_provider.GetAWSCredentials().GetAWSSecretKey().size() > 0)) {
62  if (default_provider.GetAWSCredentials().GetSessionToken().size() > 0) {
63  s3_options = arrow::fs::S3Options::FromAccessKey(
64  default_provider.GetAWSCredentials().GetAWSAccessKeyId(),
65  default_provider.GetAWSCredentials().GetAWSSecretKey(),
66  default_provider.GetAWSCredentials().GetSessionToken());
67  } else {
68  s3_options = arrow::fs::S3Options::FromAccessKey(
69  default_provider.GetAWSCredentials().GetAWSAccessKeyId(),
70  default_provider.GetAWSCredentials().GetAWSSecretKey());
71  }
72  }
73  }
74  s3_options.region = config.s3_region;
75  auto fs_result = arrow::fs::S3FileSystem::Make(s3_options);
76  if (!fs_result.ok()) {
77  throw std::runtime_error{"An error occurred when setting up S3 connection. " +
78  fs_result.status().message()};
79  }
80  return fs_result.ValueOrDie();
81  }
82 };
83 } // namespace foreign_storage
84 #endif // defined(HAVE_AWS_S3)
std::pair< FILE *, std::string > create(const std::string &basePath, const int fileId, const size_t pageSize, const size_t numPages)
Definition: File.cpp:55
bool g_allow_s3_server_privileges
Definition: S3Archive.cpp:34