OmniSciDB  6686921089
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
S3FilePathUtil.h
Go to the documentation of this file.
1 #include <optional>
2 #include <string>
3 #include <vector>
4 
5 #include <aws/s3/model/Object.h>
6 
7 // #include "DataMgr/OmniSciAwsSdk.h"
10 
11 namespace foreign_storage {
12 std::vector<Aws::S3::Model::Object> s3_objects_filter_sort_files(
13  const std::vector<Aws::S3::Model::Object>& file_paths,
14  const std::optional<std::string>& filter_regex,
15  const std::optional<std::string>& sort_by,
16  const std::optional<std::string>& sort_regex);
17 
18 using S3ObjectComparator =
19  std::function<bool(const Aws::S3::Model::Object&, const Aws::S3::Model::Object&)>;
20 class FileOrderS3 : public shared::FileOrderBase<S3ObjectComparator> {
21  public:
22  FileOrderS3(const std::optional<std::string>& sort_regex,
23  const std::optional<std::string>& sort_by)
24  : FileOrderBase<S3ObjectComparator>(sort_regex, sort_by) {}
25 
27  auto comparator_pair = comparator_map_.find(getSortBy());
28  CHECK(comparator_pair != comparator_map_.end());
29  return comparator_pair->second;
30  }
31 
32  protected:
33  const std::map<std::string, S3ObjectComparator> comparator_map_{
35  [](const Aws::S3::Model::Object& lhs, const Aws::S3::Model::Object& rhs) -> bool {
36  return lhs.GetKey() < rhs.GetKey();
37  }},
39  [](const Aws::S3::Model::Object& lhs, const Aws::S3::Model::Object& rhs) -> bool {
40  return lhs.GetLastModified() < rhs.GetLastModified();
41  }},
43  [this](const Aws::S3::Model::Object& lhs,
44  const Aws::S3::Model::Object& rhs) -> bool {
45  auto lhs_name = lhs.GetKey();
46  auto rhs_name = rhs.GetKey();
47  return this->concatCaptureGroups(lhs_name) < this->concatCaptureGroups(rhs_name);
48  }},
50  [this](const Aws::S3::Model::Object& lhs,
51  const Aws::S3::Model::Object& rhs) -> bool {
52  return shared::common_regex_date_comp_(this->concatCaptureGroups(lhs.GetKey()),
53  this->concatCaptureGroups(rhs.GetKey()));
54  }},
56  [this](const Aws::S3::Model::Object& lhs,
57  const Aws::S3::Model::Object& rhs) -> bool {
59  this->concatCaptureGroups(lhs.GetKey()),
60  this->concatCaptureGroups(rhs.GetKey()));
61  }}};
62 };
63 } // namespace foreign_storage
S3ObjectComparator getFileComparator() override
const std::string REGEX_NUMBER_ORDER_TYPE
const std::string REGEX_ORDER_TYPE
shared utility for globbing files, paths can be specified as either a single file, directory or wildcards
const std::map< std::string, S3ObjectComparator > comparator_map_
const std::string REGEX_DATE_ORDER_TYPE
std::function< bool(const Aws::S3::Model::Object &, const Aws::S3::Model::Object &)> S3ObjectComparator
const std::string PATHNAME_ORDER_TYPE
FileOrderBase(const std::optional< std::string > &sort_regex, const std::optional< std::string > &sort_by)
const std::string DATE_MODIFIED_ORDER_TYPE
FileOrderS3(const std::optional< std::string > &sort_regex, const std::optional< std::string > &sort_by)
virtual std::string concatCaptureGroups(const std::string &file_name) const
#define CHECK(condition)
Definition: Logger.h:209
const std::function< bool(const std::string &, const std::string &)> common_regex_number_comp_
const std::function< bool(const std::string &, const std::string &)> common_regex_date_comp_
std::vector< Aws::S3::Model::Object > s3_objects_filter_sort_files(const std::vector< Aws::S3::Model::Object > &file_paths, const std::optional< std::string > &filter_regex, const std::optional< std::string > &sort_by, const std::optional< std::string > &sort_regex)