OmniSciDB  a5dc49c757
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
HeavyDbAwsSdk.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "HeavyDbAwsSdk.h"
18 
19 #include <arrow/filesystem/s3fs.h>
20 #include <arrow/status.h>
21 #include <boost/filesystem.hpp>
22 
23 #ifdef ARROW_HAS_PRIVATE_AWS_SDK
24 #include <aws/core/Aws.h>
25 #endif
26 
27 #include "Logger/Logger.h"
28 
29 #ifdef ARROW_HAS_PRIVATE_AWS_SDK
30 static Aws::SDKOptions awsapi_options;
31 #endif
32 
34  auto ssl_config = heavydb_aws_sdk::get_ssl_config();
35  arrow::fs::FileSystemGlobalOptions global_options;
36  global_options.tls_ca_dir_path = ssl_config.ca_path;
37  global_options.tls_ca_file_path = ssl_config.ca_file;
38  arrow::fs::Initialize(global_options);
39  arrow::fs::S3GlobalOptions s3_global_options;
40  s3_global_options.log_level = arrow::fs::S3LogLevel::Off;
41  auto status = arrow::fs::InitializeS3(s3_global_options);
42  CHECK(status.ok()) << "InitializeS3 resulted in an error: " << status.message();
43 #ifdef ARROW_HAS_PRIVATE_AWS_SDK
44  // Directly initialize the AWS SDK, if Arrow uses a private version of the SDK
45  Aws::InitAPI(awsapi_options);
46 #endif
47 }
48 
50  auto status = arrow::fs::FinalizeS3();
51  CHECK(status.ok()) << "FinalizeS3 resulted in an error: " << status.message();
52 #ifdef ARROW_HAS_PRIVATE_AWS_SDK
53  // Directly shutdown the AWS SDK, if Arrow uses a private version of the SDK
54  Aws::ShutdownAPI(awsapi_options);
55 #endif
56 }
57 
60  SslConfig ssl_config;
61  /*
62  Fix a wrong ca path established at building libcurl on Centos being carried to
63  Ubuntu. To fix the issue, this is this sequence of locating ca file: 1) if
64  `SSL_CERT_DIR` or `SSL_CERT_FILE` is set, set it to S3 ClientConfiguration. 2) if
65  none ^ is set, heavydb searches a list of known ca file paths. 3) if 2)
66  finds nothing, it is users' call to set correct SSL_CERT_DIR or SSL_CERT_FILE. S3
67  c++ sdk: "we only want to override the default path if someone has explicitly told
68  us to."
69  */
70  std::list<std::string> v_known_ca_paths({
71  "/etc/ssl/certs/ca-certificates.crt",
72  "/etc/pki/tls/certs/ca-bundle.crt",
73  "/usr/share/ssl/certs/ca-bundle.crt",
74  "/usr/local/share/certs/ca-root.crt",
75  "/etc/ssl/cert.pem",
76  "/etc/ssl/ca-bundle.pem",
77  });
78  char* env;
79  if (nullptr != (env = getenv("SSL_CERT_DIR"))) {
80  ssl_config.ca_path = env;
81  }
82  if (nullptr != (env = getenv("SSL_CERT_FILE"))) {
83  v_known_ca_paths.push_front(env);
84  }
85  for (const auto& known_ca_path : v_known_ca_paths) {
86  if (boost::filesystem::exists(known_ca_path)) {
87  ssl_config.ca_file = known_ca_path;
88  break;
89  }
90  }
91  return ssl_config;
92 }
SslConfig get_ssl_config()
#define CHECK(condition)
Definition: Logger.h:291