OmniSciDB  c1a53651b2
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
file_path_util.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
24 #include "Shared/file_path_util.h"
25 
26 #include "Logger/Logger.h"
28 #include "Shared/misc.h"
29 
30 namespace shared {
31 
32 void validate_sort_options(const FilePathOptions& options) {
33  const auto sort_by_str = to_upper(options.sort_by.value_or(PATHNAME_ORDER_TYPE));
34 
36  throw std::runtime_error{FILE_SORT_ORDER_BY_KEY +
37  " must be one of the following options: " +
39  }
40 
41  if (shared::contains(non_regex_sort_order_types, sort_by_str) &&
42  options.sort_regex.has_value()) {
43  throw std::runtime_error{"Option \"" + FILE_SORT_REGEX_KEY +
44  "\" must not be set for selected option \"" +
45  FILE_SORT_ORDER_BY_KEY + "='" + sort_by_str + "'\"."};
46  }
47 
48  if (shared::contains(regex_sort_order_types, sort_by_str) &&
49  !options.sort_regex.has_value()) {
50  throw std::runtime_error{"Option \"" + FILE_SORT_REGEX_KEY +
51  "\" must be set for selected option \"" +
52  FILE_SORT_ORDER_BY_KEY + "='" + sort_by_str + "'\"."};
53  }
54 }
55 
56 namespace {
57 
58 std::vector<std::string> glob_local_recursive_files(const std::string& file_path,
59  const bool recurse) {
60  std::vector<std::string> file_paths;
61 
62  if (boost::filesystem::is_regular_file(file_path)) {
63  file_paths.emplace_back(file_path);
64  } else if (recurse && boost::filesystem::is_directory(file_path)) {
65  for (boost::filesystem::recursive_directory_iterator
66  it(file_path, boost::filesystem::symlink_option::recurse),
67  eit;
68  it != eit;
69  ++it) {
70  if (!boost::filesystem::is_directory(it->path())) {
71  file_paths.emplace_back(it->path().string());
72  }
73  }
74  // empty directories will not throw an error
75  } else {
76  auto glob_results = heavyai::glob(file_path);
77  for (const auto& path : glob_results) {
78  if (recurse && boost::filesystem::is_directory(path)) {
79  auto expanded_paths = glob_local_recursive_files(path, true);
80  file_paths.insert(file_paths.end(), expanded_paths.begin(), expanded_paths.end());
81  } else {
82  file_paths.emplace_back(path);
83  }
84  }
85  if (file_paths.empty()) {
86  throw_file_not_found(file_path);
87  }
88  }
89  return file_paths;
90 }
91 
92 std::vector<std::string> regex_file_filter(const std::string& pattern,
93  const std::vector<std::string>& file_paths) {
94  boost::regex regex_pattern(pattern);
95  std::vector<std::string> matched_file_paths;
96  for (const auto& path : file_paths) {
97  if (boost::regex_match(path, regex_pattern)) {
98  matched_file_paths.emplace_back(path);
99  }
100  }
101  if (matched_file_paths.empty()) {
102  throw_no_filter_match(pattern);
103  }
104  return matched_file_paths;
105 }
106 
107 } // namespace
108 
109 std::vector<std::string> local_glob_filter_sort_files(const std::string& file_path,
110  const FilePathOptions& options,
111  const bool recurse) {
112  auto result_files = glob_local_recursive_files(file_path, recurse);
113  if (options.filter_regex.has_value()) {
114  result_files = regex_file_filter(options.filter_regex.value(), result_files);
115  }
116  // initial lexicographical order ensures a determinisitc ordering for files not matching
117  // sort_regex
118  FilePathOptions temp_options;
119  temp_options.sort_by = PATHNAME_ORDER_TYPE;
120  auto initial_file_order = FileOrderLocal(temp_options);
121  auto lexi_comp = initial_file_order.getFileComparator();
122  std::stable_sort(result_files.begin(), result_files.end(), lexi_comp);
123 
124  auto file_order = FileOrderLocal(options);
125  auto comp = file_order.getFileComparator();
126  std::stable_sort(result_files.begin(), result_files.end(), comp);
127  return result_files;
128 }
129 
130 #ifdef HAVE_AWS_S3
131 namespace {
132 
133 std::vector<arrow::fs::FileInfo> arrow_fs_regex_file_filter(
134  const std::string& pattern,
135  const std::vector<arrow::fs::FileInfo>& file_info_list) {
136  boost::regex regex_pattern(pattern);
137  std::vector<arrow::fs::FileInfo> matched_file_info_list;
138  for (const auto& file_info : file_info_list) {
139  if (boost::regex_match(file_info.path(), regex_pattern)) {
140  matched_file_info_list.emplace_back(file_info);
141  }
142  }
143  if (matched_file_info_list.empty()) {
144  throw_no_filter_match(pattern);
145  }
146  return matched_file_info_list;
147 }
148 
149 } // namespace
150 
151 std::vector<arrow::fs::FileInfo> arrow_fs_filter_sort_files(
152  const std::vector<arrow::fs::FileInfo>& file_paths,
153  const FilePathOptions& options) {
154  auto result_files =
155  options.filter_regex.has_value()
156  ? arrow_fs_regex_file_filter(options.filter_regex.value(), file_paths)
157  : file_paths;
158  // initial lexicographical order ensures a determinisitc ordering for files not matching
159  // sort_regex
160  FilePathOptions temp_options;
161  temp_options.sort_by = PATHNAME_ORDER_TYPE;
162  auto initial_file_order = FileOrderArrow(temp_options);
163  auto lexi_comp = initial_file_order.getFileComparator();
164  std::stable_sort(result_files.begin(), result_files.end(), lexi_comp);
165 
166  auto file_order = FileOrderArrow(options);
167  auto comp = file_order.getFileComparator();
168  std::stable_sort(result_files.begin(), result_files.end(), comp);
169  return result_files;
170 }
171 
172 #endif // HAVE_AWS_S3
173 
174 bool file_or_glob_path_exists(const std::string& path) {
175  return boost::filesystem::exists(path) || !heavyai::glob(path).empty();
176 }
177 
178 std::set<std::string> check_for_rolled_off_file_paths(
179  const std::vector<std::string>& all_file_paths,
180  std::vector<std::string>& processed_file_paths) {
181  std::set<std::string> rolled_off_file_paths;
182  if (all_file_paths.empty()) {
183  // An empty all_file_paths vector implies that all files have been rolled off
184  rolled_off_file_paths.insert(processed_file_paths.begin(),
185  processed_file_paths.end());
186  } else {
187  auto roll_off_end_it = std::find(
188  processed_file_paths.begin(), processed_file_paths.end(), all_file_paths[0]);
189  for (auto it = processed_file_paths.begin(); it != roll_off_end_it; it++) {
190  rolled_off_file_paths.emplace(*it);
191  }
192  }
193  if (!rolled_off_file_paths.empty()) {
194  processed_file_paths.erase(
195  processed_file_paths.begin(),
196  processed_file_paths.begin() + rolled_off_file_paths.size());
197  }
198  return rolled_off_file_paths;
199 }
200 
201 bool is_s3_uri(const std::string& file_path) {
202  const std::string s3_prefix = "s3://";
203  return file_path.find(s3_prefix) != std::string::npos;
204 }
205 } // namespace shared
bool contains(const T &container, const U &element)
Definition: misc.h:195
const std::array< std::string, 2 > non_regex_sort_order_types
std::optional< std::string > filter_regex
void throw_no_filter_match(const std::string &pattern)
const std::string FILE_SORT_REGEX_KEY
shared utility for globbing files, paths can be specified as either a single file, directory or wildcards
bool is_s3_uri(const std::string &file_path)
std::optional< std::string > sort_regex
void validate_sort_options(const FilePathOptions &options)
std::string join(T const &container, std::string const &delim)
void throw_file_not_found(const std::string &file_path)
std::set< std::string > check_for_rolled_off_file_paths(const std::vector< std::string > &all_file_paths, std::vector< std::string > &processed_file_paths)
const std::string PATHNAME_ORDER_TYPE
std::vector< std::string > glob_local_recursive_files(const std::string &file_path, const bool recurse)
const std::string FILE_SORT_ORDER_BY_KEY
std::string to_upper(const std::string &str)
bool file_or_glob_path_exists(const std::string &path)
const std::array< std::string, 5 > supported_file_sort_order_types
std::vector< std::string > local_glob_filter_sort_files(const std::string &file_path, const FilePathOptions &options, const bool recurse)
const std::array< std::string, 3 > regex_sort_order_types
std::vector< std::string > glob(const std::string &pattern)
std::vector< std::string > regex_file_filter(const std::string &pattern, const std::vector< std::string > &file_paths)
std::optional< std::string > sort_by