OmniSciDB  a987f07e93
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
S3FilePathUtil.cpp
Go to the documentation of this file.
2 
3 namespace foreign_storage {
4 namespace {
5 std::vector<Aws::S3::Model::Object> s3_objects_regex_file_filter(
6  const std::string& pattern,
7  const std::vector<Aws::S3::Model::Object>& objects_list) {
8  boost::regex regex_pattern(pattern);
9  std::vector<Aws::S3::Model::Object> matched_objects_list;
10  for (const auto& object : objects_list) {
11  if (boost::regex_match(object.GetKey(), regex_pattern)) {
12  matched_objects_list.emplace_back(object);
13  }
14  }
15  if (matched_objects_list.empty()) {
17  }
18  return matched_objects_list;
19 }
20 } // namespace
21 
22 std::vector<Aws::S3::Model::Object> s3_objects_filter_sort_files(
23  const std::vector<Aws::S3::Model::Object>& file_paths,
24  const shared::FilePathOptions& options) {
25  auto result_files =
26  options.filter_regex.has_value()
27  ? s3_objects_regex_file_filter(options.filter_regex.value(), file_paths)
28  : file_paths;
29  // initial lexicographical order ensures a determinisitc ordering for files not matching
30  // sort_regex
31  shared::FilePathOptions temp_options;
32  temp_options.sort_by = shared::PATHNAME_ORDER_TYPE;
33  auto initial_file_order = FileOrderS3(temp_options);
34  auto lexi_comp = initial_file_order.getFileComparator();
35  std::stable_sort(result_files.begin(), result_files.end(), lexi_comp);
36 
37  auto file_order = FileOrderS3(options);
38  auto comp = file_order.getFileComparator();
39  std::stable_sort(result_files.begin(), result_files.end(), comp);
40  return result_files;
41 }
42 } // namespace foreign_storage
std::optional< std::string > filter_regex
void throw_no_filter_match(const std::string &pattern)
std::vector< Aws::S3::Model::Object > s3_objects_filter_sort_files(const std::vector< Aws::S3::Model::Object > &file_paths, const shared::FilePathOptions &options)
std::vector< Aws::S3::Model::Object > s3_objects_regex_file_filter(const std::string &pattern, const std::vector< Aws::S3::Model::Object > &objects_list)
const std::string PATHNAME_ORDER_TYPE
std::optional< std::string > sort_by