OmniSciDB  2e3a973ef4
ParquetS3FileSystem.h
Go to the documentation of this file.
1 /*
2  * Copyright 2020 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #pragma once
18 
19 #include <map>
20 
21 #include <arrow/filesystem/filesystem.h>
22 #include <arrow/filesystem/s3fs.h>
23 #include <arrow/result.h>
24 #include <arrow/status.h>
25 #include <aws/core/Globals.h>
26 
27 #include "Catalog/ForeignServer.h"
28 #include "Catalog/ee/UserMapping.h"
29 
30 namespace foreign_storage {
32  public:
33  static std::shared_ptr<arrow::fs::FileSystem> create(
34  const ForeignServer* foreign_server,
35  const UserMapping* user_mapping) {
36  static bool is_initialized = false;
37  if (!is_initialized) {
38  // Arrow requires its own S3 initialization (which does an AWS SDK initialization)
39  // before any S3 related call can be made. However, S3Archive also does an
40  // AWS SDK initialization. Both classes do not work if their own initialization
41  // is not done. When both initializations are done, AWS SDK does not correctly
42  // clean up before the second initialization. The following ensures that is done.
43  // TODO: Figure out a way to have a single AWS SDK initialization that works for
44  // both Arrow and S3Archive.
45  Aws::CleanupEnumOverflowContainer();
46  is_initialized = true;
47  }
48 
49  arrow::fs::EnsureS3Initialized();
50  auto s3_options = arrow::fs::S3Options::Anonymous();
51  if (user_mapping) {
52  const auto options = user_mapping->getUnencryptedOptions();
53  if (options.find(UserMapping::S3_ACCESS_KEY) != options.end() &&
54  options.find(UserMapping::S3_SECRET_KEY) != options.end()) {
55  s3_options = arrow::fs::S3Options::FromAccessKey(
56  options.find(UserMapping::S3_ACCESS_KEY)->second,
57  options.find(UserMapping::S3_SECRET_KEY)->second);
58  }
59  }
60  s3_options.region =
61  foreign_server->options.find(ForeignServer::AWS_REGION_KEY)->second;
62  auto fs_result = arrow::fs::S3FileSystem::Make(s3_options);
63  if (!fs_result.ok()) {
64  throw std::runtime_error{"An error occurred when setting up S3 connection. " +
65  fs_result.status().message()};
66  }
67  return fs_result.ValueOrDie();
68  }
69 };
70 } // namespace foreign_storage
std::map< std::string, std::string, std::less<> > options
static std::shared_ptr< arrow::fs::FileSystem > create(const ForeignServer *foreign_server, const UserMapping *user_mapping)