OmniSciDB  2e3a973ef4
foreign_storage::anonymous_namespace{CsvReaderS3.cpp} Namespace Reference

Typedefs

using S3FileInfo = std::pair< std::string, size_t >
 

Functions

Aws::Client::ClientConfiguration get_s3_config (const ForeignServer *server_options)
 
Aws::S3::Model::GetObjectRequest create_request (const std::string &bucket_name, const std::string &obj_name, size_t start=0, size_t end=0)
 
std::string get_access_error_message (const std::string &bucket, const std::string &object_name, const std::string &exception_name, const std::string &message)
 
std::shared_ptr< Aws::Auth::AWSCredentialsProvider > get_credentials (const UserMapping *user_mapping)
 
void list_files_s3 (std::unique_ptr< Aws::S3::S3Client > &s3_client, const std::string &prefix_name, const std::string &bucket_name, std::set< S3FileInfo > &file_info_set)
 

Typedef Documentation

◆ S3FileInfo

using foreign_storage::anonymous_namespace{CsvReaderS3.cpp}::S3FileInfo = typedef std::pair<std::string, size_t>

Definition at line 215 of file CsvReaderS3.cpp.

Function Documentation

◆ create_request()

Aws::S3::Model::GetObjectRequest foreign_storage::anonymous_namespace{CsvReaderS3.cpp}::create_request ( const std::string &  bucket_name,
const std::string &  obj_name,
size_t  start = 0,
size_t  end = 0 
)

Definition at line 58 of file CsvReaderS3.cpp.

References CHECK, parse_ast::end, and to_string().

Referenced by foreign_storage::CsvReaderS3::read(), and foreign_storage::CsvReaderS3::skipHeader().

61  {
62  CHECK(start <= end);
63  Aws::S3::Model::GetObjectRequest object_request;
64  object_request.WithBucket(bucket_name).WithKey(obj_name);
65  if (end > 0) {
66  object_request.SetRange(std::string("bytes=") + std::to_string(start) + "-" +
68  }
69  return object_request;
70 }
std::string to_string(char const *&&v)
#define CHECK(condition)
Definition: Logger.h:197
+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ get_access_error_message()

std::string foreign_storage::anonymous_namespace{CsvReaderS3.cpp}::get_access_error_message ( const std::string &  bucket,
const std::string &  object_name,
const std::string &  exception_name,
const std::string &  message 
)

Definition at line 72 of file CsvReaderS3.cpp.

Referenced by list_files_s3(), foreign_storage::CsvReaderS3::read(), and foreign_storage::CsvReaderS3::skipHeader().

75  {
76  return "Unable to access s3 file: " + bucket + "/" + object_name + ". " +
77  exception_name + ": " + message;
78 }
+ Here is the caller graph for this function:

◆ get_credentials()

std::shared_ptr<Aws::Auth::AWSCredentialsProvider> foreign_storage::anonymous_namespace{CsvReaderS3.cpp}::get_credentials ( const UserMapping *  user_mapping)

Definition at line 80 of file CsvReaderS3.cpp.

Referenced by foreign_storage::CsvReaderS3::CsvReaderS3(), and foreign_storage::MultiS3Reader::MultiS3Reader().

81  {
82  if (user_mapping) {
83  const auto options = user_mapping->getUnencryptedOptions();
84  if (options.find(UserMapping::S3_ACCESS_KEY) != options.end() &&
85  options.find(UserMapping::S3_SECRET_KEY) != options.end()) {
86  return std::make_shared<Aws::Auth::SimpleAWSCredentialsProvider>(
87  options.find(UserMapping::S3_ACCESS_KEY)->second,
88  options.find(UserMapping::S3_SECRET_KEY)->second);
89  }
90  }
91  return std::make_shared<Aws::Auth::AnonymousAWSCredentialsProvider>();
92 }
+ Here is the caller graph for this function:

◆ get_s3_config()

Aws::Client::ClientConfiguration foreign_storage::anonymous_namespace{CsvReaderS3.cpp}::get_s3_config ( const ForeignServer server_options)

Definition at line 29 of file CsvReaderS3.cpp.

References foreign_storage::OptionsContainer::options.

Referenced by foreign_storage::CsvReaderS3::CsvReaderS3(), and foreign_storage::MultiS3Reader::MultiS3Reader().

29  {
30  Aws::Client::ClientConfiguration s3_config;
31  s3_config.region = server_options->options.find(ForeignServer::AWS_REGION_KEY)->second;
32 
33  // Find SSL certificate trust store to connect to S3
34  std::list<std::string> v_known_ca_paths({
35  "/etc/ssl/certs/ca-certificates.crt",
36  "/etc/pki/tls/certs/ca-bundle.crt",
37  "/usr/share/ssl/certs/ca-bundle.crt",
38  "/usr/local/share/certs/ca-root.crt",
39  "/etc/ssl/cert.pem",
40  "/etc/ssl/ca-bundle.pem",
41  });
42  char* env;
43  if (nullptr != (env = getenv("SSL_CERT_DIR"))) {
44  s3_config.caPath = env;
45  }
46  if (nullptr != (env = getenv("SSL_CERT_FILE"))) {
47  v_known_ca_paths.push_front(env);
48  }
49  for (const auto& known_ca_path : v_known_ca_paths) {
50  if (boost::filesystem::exists(known_ca_path)) {
51  s3_config.caFile = known_ca_path;
52  break;
53  }
54  }
55  return s3_config;
56 }
+ Here is the caller graph for this function:

◆ list_files_s3()

void foreign_storage::anonymous_namespace{CsvReaderS3.cpp}::list_files_s3 ( std::unique_ptr< Aws::S3::S3Client > &  s3_client,
const std::string &  prefix_name,
const std::string &  bucket_name,
std::set< S3FileInfo > &  file_info_set 
)

Definition at line 216 of file CsvReaderS3.cpp.

References get_access_error_message().

Referenced by foreign_storage::MultiS3Reader::checkForMoreRows(), and foreign_storage::MultiS3Reader::MultiS3Reader().

219  {
220  Aws::S3::Model::ListObjectsV2Request objects_request;
221  objects_request.WithBucket(bucket_name);
222  objects_request.WithPrefix(prefix_name);
223  auto list_objects_outcome = s3_client->ListObjectsV2(objects_request);
224  if (list_objects_outcome.IsSuccess()) {
225  auto object_list = list_objects_outcome.GetResult().GetContents();
226  if (0 == object_list.size()) {
227  throw std::runtime_error{get_access_error_message(
228  bucket_name, prefix_name, "Error", "No object was found at the given path.")};
229  }
230  // Instantiate CsvReaderS3 for each valid object
231  for (auto const& obj : object_list) {
232  std::string objkey = obj.GetKey().c_str();
233 
234  // skip keys with trailing / or basename with heading '.'
235  boost::filesystem::path path{objkey};
236  if (0 == obj.GetSize()) {
237  continue;
238  }
239  if ('/' == objkey.back()) {
240  continue;
241  }
242  if ('.' == path.filename().string().front()) {
243  continue;
244  }
245  // TODO: remove filename restriction on txt when new S3 test datasests are added
246  if (boost::filesystem::extension(path) != ".csv" &&
247  boost::filesystem::extension(path) != ".tsv") {
248  continue;
249  }
250  file_info_set.insert(S3FileInfo(objkey, obj.GetSize()));
251  }
252  } else {
253  throw std::runtime_error{
254  get_access_error_message(bucket_name,
255  prefix_name,
256  list_objects_outcome.GetError().GetExceptionName(),
257  list_objects_outcome.GetError().GetMessage())};
258  }
259 }
std::string get_access_error_message(const std::string &bucket, const std::string &object_name, const std::string &exception_name, const std::string &message)
Definition: CsvReaderS3.cpp:72
+ Here is the call graph for this function:
+ Here is the caller graph for this function: