37 " must be one of the following options: " +
44 "\" must not be set for selected option \"" +
51 "\" must be set for selected option \"" +
60 std::vector<std::string> file_paths;
62 if (boost::filesystem::is_regular_file(file_path)) {
63 file_paths.emplace_back(file_path);
64 }
else if (recurse && boost::filesystem::is_directory(file_path)) {
65 for (boost::filesystem::recursive_directory_iterator
66 it(file_path, boost::filesystem::symlink_option::recurse),
70 if (!boost::filesystem::is_directory(it->path())) {
71 file_paths.emplace_back(it->path().string());
77 for (
const auto& path : glob_results) {
78 if (recurse && boost::filesystem::is_directory(path)) {
80 file_paths.insert(file_paths.end(), expanded_paths.begin(), expanded_paths.end());
82 file_paths.emplace_back(path);
85 if (file_paths.empty()) {
93 const std::vector<std::string>& file_paths) {
94 boost::regex regex_pattern(pattern);
95 std::vector<std::string> matched_file_paths;
96 for (
const auto& path : file_paths) {
97 if (boost::regex_match(path, regex_pattern)) {
98 matched_file_paths.emplace_back(path);
101 if (matched_file_paths.empty()) {
104 return matched_file_paths;
111 const bool recurse) {
121 auto lexi_comp = initial_file_order.getFileComparator();
122 std::stable_sort(result_files.begin(), result_files.end(), lexi_comp);
125 auto comp = file_order.getFileComparator();
126 std::stable_sort(result_files.begin(), result_files.end(), comp);
133 std::vector<arrow::fs::FileInfo> arrow_fs_regex_file_filter(
134 const std::string& pattern,
135 const std::vector<arrow::fs::FileInfo>& file_info_list) {
136 boost::regex regex_pattern(pattern);
137 std::vector<arrow::fs::FileInfo> matched_file_info_list;
138 for (
const auto& file_info : file_info_list) {
139 if (boost::regex_match(file_info.path(), regex_pattern)) {
140 matched_file_info_list.emplace_back(file_info);
143 if (matched_file_info_list.empty()) {
146 return matched_file_info_list;
151 std::vector<arrow::fs::FileInfo> arrow_fs_filter_sort_files(
152 const std::vector<arrow::fs::FileInfo>& file_paths,
153 const FilePathOptions& options) {
155 options.filter_regex.has_value()
156 ? arrow_fs_regex_file_filter(options.filter_regex.value(), file_paths)
160 FilePathOptions temp_options;
162 auto initial_file_order = FileOrderArrow(temp_options);
163 auto lexi_comp = initial_file_order.getFileComparator();
164 std::stable_sort(result_files.begin(), result_files.end(), lexi_comp);
166 auto file_order = FileOrderArrow(options);
167 auto comp = file_order.getFileComparator();
168 std::stable_sort(result_files.begin(), result_files.end(), comp);
172 #endif // HAVE_AWS_S3
175 return boost::filesystem::exists(path) || !
heavyai::glob(path).empty();
179 const std::vector<std::string>& all_file_paths,
180 std::vector<std::string>& processed_file_paths) {
181 std::set<std::string> rolled_off_file_paths;
182 if (all_file_paths.empty()) {
184 rolled_off_file_paths.insert(processed_file_paths.begin(),
185 processed_file_paths.end());
187 auto roll_off_end_it = std::find(
188 processed_file_paths.begin(), processed_file_paths.end(), all_file_paths[0]);
189 for (
auto it = processed_file_paths.begin(); it != roll_off_end_it; it++) {
190 rolled_off_file_paths.emplace(*it);
193 if (!rolled_off_file_paths.empty()) {
194 processed_file_paths.erase(
195 processed_file_paths.begin(),
196 processed_file_paths.begin() + rolled_off_file_paths.size());
198 return rolled_off_file_paths;
202 const std::string s3_prefix =
"s3://";
203 return file_path.find(s3_prefix) != std::string::npos;
bool contains(const T &container, const U &element)
const std::array< std::string, 2 > non_regex_sort_order_types
std::optional< std::string > filter_regex
void throw_no_filter_match(const std::string &pattern)
const std::string FILE_SORT_REGEX_KEY
shared utility for globbing files, paths can be specified as either a single file, directory or wildcards
bool is_s3_uri(const std::string &file_path)
std::optional< std::string > sort_regex
void validate_sort_options(const FilePathOptions &options)
void throw_file_not_found(const std::string &file_path)
std::set< std::string > check_for_rolled_off_file_paths(const std::vector< std::string > &all_file_paths, std::vector< std::string > &processed_file_paths)
const std::string PATHNAME_ORDER_TYPE
std::vector< std::string > glob_local_recursive_files(const std::string &file_path, const bool recurse)
const std::string FILE_SORT_ORDER_BY_KEY
bool file_or_glob_path_exists(const std::string &path)
const std::array< std::string, 5 > supported_file_sort_order_types
std::vector< std::string > local_glob_filter_sort_files(const std::string &file_path, const FilePathOptions &options, const bool recurse)
const std::array< std::string, 3 > regex_sort_order_types
std::vector< std::string > glob(const std::string &pattern)
std::vector< std::string > regex_file_filter(const std::string &pattern, const std::vector< std::string > &file_paths)
std::optional< std::string > sort_by