OmniSciDB  085a039ca4
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
file_path_util.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2021 OmniSci, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
24 #include "Shared/file_path_util.h"
25 
26 #include "Logger/Logger.h"
28 #include "Shared/misc.h"
29 
30 namespace shared {
31 
32 void validate_sort_options(const std::optional<std::string>& sort_by,
33  const std::optional<std::string>& sort_regex) {
34  const auto sort_by_str = to_upper(sort_by.value_or(PATHNAME_ORDER_TYPE));
35 
37  throw std::runtime_error{FILE_SORT_ORDER_BY_KEY +
38  " must be one of the following options: " +
40  }
41 
42  if (shared::contains(non_regex_sort_order_types, sort_by_str) &&
43  sort_regex.has_value()) {
44  throw std::runtime_error{"Option \"" + FILE_SORT_REGEX_KEY +
45  "\" must not be set for selected option \"" +
46  FILE_SORT_ORDER_BY_KEY + "='" + sort_by_str + "'\"."};
47  }
48 
49  if (shared::contains(regex_sort_order_types, sort_by_str) && !sort_regex.has_value()) {
50  throw std::runtime_error{"Option \"" + FILE_SORT_REGEX_KEY +
51  "\" must be set for selected option \"" +
52  FILE_SORT_ORDER_BY_KEY + "='" + sort_by_str + "'\"."};
53  }
54 }
55 
56 namespace {
57 
58 std::vector<std::string> glob_local_recursive_files(const std::string& file_path,
59  const bool recurse) {
60  std::vector<std::string> file_paths;
61 
62  if (boost::filesystem::is_regular_file(file_path)) {
63  file_paths.emplace_back(file_path);
64  } else if (recurse && boost::filesystem::is_directory(file_path)) {
65  for (boost::filesystem::recursive_directory_iterator
66  it(file_path, boost::filesystem::symlink_option::recurse),
67  eit;
68  it != eit;
69  ++it) {
70  if (!boost::filesystem::is_directory(it->path())) {
71  file_paths.emplace_back(it->path().string());
72  }
73  }
74  // empty directories will not throw an error
75  } else {
76  auto glob_results = heavyai::glob(file_path);
77  for (const auto& path : glob_results) {
78  if (recurse && boost::filesystem::is_directory(path)) {
79  auto expanded_paths = glob_local_recursive_files(path, true);
80  file_paths.insert(file_paths.end(), expanded_paths.begin(), expanded_paths.end());
81  } else {
82  file_paths.emplace_back(path);
83  }
84  }
85  if (file_paths.empty()) {
86  throw_file_not_found(file_path);
87  }
88  }
89  return file_paths;
90 }
91 
92 std::vector<std::string> regex_file_filter(const std::string& pattern,
93  const std::vector<std::string>& file_paths) {
94  boost::regex regex_pattern(pattern);
95  std::vector<std::string> matched_file_paths;
96  for (const auto& path : file_paths) {
97  if (boost::regex_match(path, regex_pattern)) {
98  matched_file_paths.emplace_back(path);
99  }
100  }
101  if (matched_file_paths.empty()) {
102  throw_no_filter_match(pattern);
103  }
104  return matched_file_paths;
105 }
106 
107 } // namespace
108 
109 std::vector<std::string> local_glob_filter_sort_files(
110  const std::string& file_path,
111  const std::optional<std::string>& filter_regex,
112  const std::optional<std::string>& sort_by,
113  const std::optional<std::string>& sort_regex,
114  const bool recurse) {
115  auto result_files = glob_local_recursive_files(file_path, recurse);
116  if (filter_regex.has_value()) {
117  result_files = regex_file_filter(filter_regex.value(), result_files);
118  }
119  // initial lexicographical order ensures a determinisitc ordering for files not matching
120  // sort_regex
121  auto initial_file_order = FileOrderLocal(std::nullopt, PATHNAME_ORDER_TYPE);
122  auto lexi_comp = initial_file_order.getFileComparator();
123  std::stable_sort(result_files.begin(), result_files.end(), lexi_comp);
124 
125  auto file_order = FileOrderLocal(sort_regex, sort_by);
126  auto comp = file_order.getFileComparator();
127  std::stable_sort(result_files.begin(), result_files.end(), comp);
128  return result_files;
129 }
130 
131 #ifdef HAVE_AWS_S3
132 namespace {
133 
134 std::vector<arrow::fs::FileInfo> arrow_fs_regex_file_filter(
135  const std::string& pattern,
136  const std::vector<arrow::fs::FileInfo>& file_info_list) {
137  boost::regex regex_pattern(pattern);
138  std::vector<arrow::fs::FileInfo> matched_file_info_list;
139  for (const auto& file_info : file_info_list) {
140  if (boost::regex_match(file_info.path(), regex_pattern)) {
141  matched_file_info_list.emplace_back(file_info);
142  }
143  }
144  if (matched_file_info_list.empty()) {
145  throw_no_filter_match(pattern);
146  }
147  return matched_file_info_list;
148 }
149 
150 } // namespace
151 
152 std::vector<arrow::fs::FileInfo> arrow_fs_filter_sort_files(
153  const std::vector<arrow::fs::FileInfo>& file_paths,
154  const std::optional<std::string>& filter_regex,
155  const std::optional<std::string>& sort_by,
156  const std::optional<std::string>& sort_regex) {
157  auto result_files = filter_regex.has_value()
158  ? arrow_fs_regex_file_filter(filter_regex.value(), file_paths)
159  : file_paths;
160  // initial lexicographical order ensures a determinisitc ordering for files not matching
161  // sort_regex
162  auto initial_file_order = FileOrderArrow(std::nullopt, PATHNAME_ORDER_TYPE);
163  auto lexi_comp = initial_file_order.getFileComparator();
164  std::stable_sort(result_files.begin(), result_files.end(), lexi_comp);
165 
166  auto file_order = FileOrderArrow(sort_regex, sort_by);
167  auto comp = file_order.getFileComparator();
168  std::stable_sort(result_files.begin(), result_files.end(), comp);
169  return result_files;
170 }
171 
172 #endif // HAVE_AWS_S3
173 
174 bool file_or_glob_path_exists(const std::string& path) {
175  return boost::filesystem::exists(path) || !heavyai::glob(path).empty();
176 }
177 
178 } // namespace shared
bool contains(const T &container, const U &element)
Definition: misc.h:196
const std::array< std::string, 2 > non_regex_sort_order_types
void throw_no_filter_match(const std::string &pattern)
const std::string FILE_SORT_REGEX_KEY
shared utility for globbing files, paths can be specified as either a single file, directory or wildcards
void validate_sort_options(const std::optional< std::string > &sort_by, const std::optional< std::string > &sort_regex)
std::string join(T const &container, std::string const &delim)
void throw_file_not_found(const std::string &file_path)
const std::string PATHNAME_ORDER_TYPE
std::vector< std::string > glob_local_recursive_files(const std::string &file_path, const bool recurse)
const std::string FILE_SORT_ORDER_BY_KEY
std::string to_upper(const std::string &str)
bool file_or_glob_path_exists(const std::string &path)
std::vector< std::string > local_glob_filter_sort_files(const std::string &file_path, const std::optional< std::string > &filter_regex, const std::optional< std::string > &sort_by, const std::optional< std::string > &sort_regex, const bool recurse)
const std::array< std::string, 5 > supported_file_sort_order_types
const std::array< std::string, 3 > regex_sort_order_types
std::vector< std::string > glob(const std::string &pattern)
std::vector< std::string > regex_file_filter(const std::string &pattern, const std::vector< std::string > &file_paths)