26 #include <tbb/parallel_for.h>
27 #include <tbb/task_arena.h>
29 #define NANOSECONDS_PER_SECOND 1000000000
33 T col_min = std::numeric_limits<T>::max();
34 T col_max = std::numeric_limits<T>::lowest();
35 const int64_t num_rows = col.
size();
36 const size_t max_thread_count = std::thread::hardware_concurrency();
38 const size_t num_threads = std::min(
39 max_thread_count, ((num_rows + max_inputs_per_thread - 1) / max_inputs_per_thread));
41 std::vector<T> local_col_mins(num_threads, std::numeric_limits<T>::max());
42 std::vector<T> local_col_maxes(num_threads, std::numeric_limits<T>::lowest());
43 tbb::task_arena limited_arena(num_threads);
45 limited_arena.execute([&] {
47 [&](
const tbb::blocked_range<int64_t>& r) {
48 const int64_t start_idx = r.begin();
49 const int64_t end_idx = r.end();
50 T local_col_min = std::numeric_limits<T>::max();
51 T local_col_max = std::numeric_limits<T>::lowest();
52 for (int64_t r = start_idx; r < end_idx; ++r) {
54 if (val == inline_null_value<T>()) {
57 if (val < local_col_min) {
60 if (val > local_col_max) {
64 size_t thread_idx = tbb::this_task_arena::current_thread_index();
65 if (local_col_min < local_col_mins[thread_idx]) {
66 local_col_mins[thread_idx] = local_col_min;
68 if (local_col_max > local_col_maxes[thread_idx]) {
69 local_col_maxes[thread_idx] = local_col_max;
74 for (
size_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
75 if (local_col_mins[thread_idx] < col_min) {
76 col_min = local_col_mins[thread_idx];
78 if (local_col_maxes[thread_idx] > col_max) {
79 col_max = local_col_maxes[thread_idx];
82 return std::make_pair(col_min, col_max);
109 template <
typename T>
112 const size_t max_thread_count = std::thread::hardware_concurrency();
114 const size_t num_threads = std::min(
115 max_thread_count, ((num_rows + max_inputs_per_thread - 1) / max_inputs_per_thread));
117 std::vector<double> local_col_sums(num_threads, 0.);
118 std::vector<int64_t> local_col_non_null_counts(num_threads, 0L);
119 tbb::task_arena limited_arena(num_threads);
120 limited_arena.execute([&] {
122 [&](
const tbb::blocked_range<int64_t>& r) {
123 const int64_t start_idx = r.begin();
124 const int64_t end_idx = r.end();
125 double local_col_sum = 0.;
126 int64_t local_col_non_null_count = 0;
127 for (int64_t r = start_idx; r < end_idx; ++r) {
128 const T val = data[r];
129 if (val == inline_null_value<T>()) {
132 local_col_sum += data[r];
133 local_col_non_null_count++;
135 size_t thread_idx = tbb::this_task_arena::current_thread_index();
136 local_col_sums[thread_idx] += local_col_sum;
137 local_col_non_null_counts[thread_idx] += local_col_non_null_count;
141 double col_sum = 0.0;
142 int64_t col_non_null_count = 0L;
144 for (
size_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
145 col_sum += local_col_sums[thread_idx];
146 col_non_null_count += local_col_non_null_counts[thread_idx];
149 return col_non_null_count == 0 ? 0 : col_sum / col_non_null_count;
153 const int64_t num_rows);
156 const int64_t num_rows);
159 const int64_t num_rows);
162 const int64_t num_rows);
165 const int64_t num_rows);
168 const int64_t num_rows);
170 template <
typename T>
182 template <
typename T>
196 template <
typename T>
198 const int64_t num_rows,
201 const size_t max_thread_count = std::thread::hardware_concurrency();
203 const size_t num_threads = std::min(
204 max_thread_count, ((num_rows + max_inputs_per_thread - 1) / max_inputs_per_thread));
206 std::vector<double> local_col_squared_residuals(num_threads, 0.);
207 std::vector<int64_t> local_col_non_null_counts(num_threads, 0L);
208 tbb::task_arena limited_arena(num_threads);
210 limited_arena.execute([&] {
212 [&](
const tbb::blocked_range<int64_t>& r) {
213 const int64_t start_idx = r.begin();
214 const int64_t end_idx = r.end();
215 double local_col_squared_residual = 0.;
216 int64_t local_col_non_null_count = 0;
217 for (int64_t r = start_idx; r < end_idx; ++r) {
218 const T val = data[r];
219 if (val == inline_null_value<T>()) {
222 const double residual = val - mean;
223 local_col_squared_residual += (residual * residual);
224 local_col_non_null_count++;
226 size_t thread_idx = tbb::this_task_arena::current_thread_index();
227 local_col_squared_residuals[thread_idx] +=
228 local_col_squared_residual;
229 local_col_non_null_counts[thread_idx] += local_col_non_null_count;
233 double col_sum_squared_residual = 0.0;
234 int64_t col_non_null_count = 0;
236 for (
size_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
237 col_sum_squared_residual += local_col_squared_residuals[thread_idx];
238 col_non_null_count += local_col_non_null_counts[thread_idx];
241 return col_non_null_count == 0 ? 0
242 : sqrt(col_sum_squared_residual / col_non_null_count);
246 const int64_t num_rows,
249 const int64_t num_rows,
252 const int64_t num_rows,
255 const int64_t num_rows,
258 template <
typename T>
260 T col_min = std::numeric_limits<T>::max();
261 T col_max = std::numeric_limits<T>::lowest();
262 bool has_nulls =
false;
263 const int64_t num_rows = col.
size();
264 const size_t max_thread_count = std::thread::hardware_concurrency();
266 const size_t num_threads = std::min(
267 max_thread_count, ((num_rows + max_inputs_per_thread - 1) / max_inputs_per_thread));
269 std::vector<T> local_col_mins(num_threads, std::numeric_limits<T>::max());
270 std::vector<T> local_col_maxes(num_threads, std::numeric_limits<T>::lowest());
271 std::vector<bool> local_col_has_nulls(num_threads,
false);
272 tbb::task_arena limited_arena(num_threads);
274 limited_arena.execute([&] {
276 [&](
const tbb::blocked_range<int64_t>& r) {
277 const int64_t start_idx = r.begin();
278 const int64_t end_idx = r.end();
279 T local_col_min = std::numeric_limits<T>::max();
280 T local_col_max = std::numeric_limits<T>::lowest();
281 bool local_has_nulls =
false;
282 for (int64_t r = start_idx; r < end_idx; ++r) {
284 local_has_nulls =
true;
287 if (col[r] < local_col_min) {
288 local_col_min = col[r];
290 if (col[r] > local_col_max) {
291 local_col_max = col[r];
294 const size_t thread_idx =
295 tbb::this_task_arena::current_thread_index();
296 if (local_has_nulls) {
297 local_col_has_nulls[thread_idx] =
true;
299 if (local_col_min < local_col_mins[thread_idx]) {
300 local_col_mins[thread_idx] = local_col_min;
302 if (local_col_max > local_col_maxes[thread_idx]) {
303 local_col_maxes[thread_idx] = local_col_max;
308 for (
size_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
309 if (local_col_has_nulls[thread_idx]) {
312 if (local_col_mins[thread_idx] < col_min) {
313 col_min = local_col_mins[thread_idx];
315 if (local_col_maxes[thread_idx] > col_max) {
316 col_max = local_col_maxes[thread_idx];
319 return {col_min, col_max, has_nulls};
341 template <
typename T>
344 const int64_t num_rows,
346 const double std_dev) {
347 if (std_dev <= 0.0) {
348 throw std::runtime_error(
"Standard deviation cannot be <= 0");
350 const double inv_std_dev = 1.0 / std_dev;
353 [&](
const tbb::blocked_range<int64_t>& r) {
354 const int64_t start_idx = r.begin();
355 const int64_t end_idx = r.end();
356 for (int64_t row_idx = start_idx; row_idx < end_idx; ++row_idx) {
357 output_data[row_idx] = (input_data[row_idx] - mean) * inv_std_dev;
364 const int64_t num_rows,
366 const double std_dev);
369 const int64_t num_rows,
371 const double std_dev);
373 template <
typename T>
375 const int64_t num_rows) {
376 const int64_t num_features = input_data.size();
377 std::vector<std::vector<T>> normalized_data(num_features);
378 for (int64_t feature_idx = 0; feature_idx < num_features; ++feature_idx) {
381 normalized_data[feature_idx].resize(num_rows);
383 normalized_data[feature_idx].data(),
388 return normalized_data;
392 const std::vector<float*>& input_data,
393 const int64_t num_rows);
395 const std::vector<double*>& input_data,
396 const int64_t num_rows);
398 template <
typename T1,
typename T2>
401 T1 latitudeArc = (fromlat - tolat) * 0.017453292519943295769236907684886;
402 T1 longitudeArc = (fromlon - tolon) * 0.017453292519943295769236907684886;
403 T1 latitudeH = sin(latitudeArc * 0.5);
404 latitudeH *= latitudeH;
405 T1 lontitudeH = sin(longitudeArc * 0.5);
406 lontitudeH *= lontitudeH;
407 T1 tmp = cos(fromlat * 0.017453292519943295769236907684886) *
408 cos(tolat * 0.017453292519943295769236907684886);
409 return 6372797.560856 * (2.0 * asin(sqrt(latitudeH + tmp * lontitudeH)));
423 const double fromlat,
428 const double fromlat,
432 namespace FileUtilities {
439 std::string regex_string{glob};
441 regex_string = std::regex_replace(regex_string, std::regex(
"\\\\"),
"\\\\");
442 regex_string = std::regex_replace(regex_string, std::regex(
"\\^"),
"\\^");
443 regex_string = std::regex_replace(regex_string, std::regex(
"\\."),
"\\.");
444 regex_string = std::regex_replace(regex_string, std::regex(
"\\$"),
"\\$");
445 regex_string = std::regex_replace(regex_string, std::regex(
"\\|"),
"\\|");
446 regex_string = std::regex_replace(regex_string, std::regex(
"\\("),
"\\(");
447 regex_string = std::regex_replace(regex_string, std::regex(
"\\)"),
"\\)");
448 regex_string = std::regex_replace(regex_string, std::regex(
"\\{"),
"\\{");
449 regex_string = std::regex_replace(regex_string, std::regex(
"\\{"),
"\\}");
450 regex_string = std::regex_replace(regex_string, std::regex(
"\\["),
"\\[");
451 regex_string = std::regex_replace(regex_string, std::regex(
"\\]"),
"\\]");
452 regex_string = std::regex_replace(regex_string, std::regex(
"\\+"),
"\\+");
453 regex_string = std::regex_replace(regex_string, std::regex(
"\\/"),
"\\/");
455 regex_string = std::regex_replace(regex_string, std::regex(
"\\?"),
".");
456 regex_string = std::regex_replace(regex_string, std::regex(
"\\*"),
".*");
460 case_sensitive ? std::regex_constants::ECMAScript : std::regex_constants::icase);
463 std::vector<std::filesystem::path>
get_fs_paths(
const std::string& file_or_directory) {
464 const std::filesystem::path file_or_directory_path(file_or_directory);
465 const auto file_status = std::filesystem::status(file_or_directory_path);
467 std::vector<std::filesystem::path> fs_paths;
468 if (std::filesystem::is_regular_file(file_status)) {
469 fs_paths.emplace_back(file_or_directory_path);
471 }
else if (std::filesystem::is_directory(file_status)) {
472 for (std::filesystem::directory_entry
const& entry :
473 std::filesystem::directory_iterator(file_or_directory_path)) {
474 if (std::filesystem::is_regular_file(std::filesystem::status(entry))) {
475 fs_paths.emplace_back(entry.path());
480 const auto parent_path = file_or_directory_path.parent_path();
481 const auto parent_status = std::filesystem::status(parent_path);
482 if (std::filesystem::is_directory(parent_status)) {
483 const auto file_glob = file_or_directory_path.filename();
484 const std::regex glob_regex{
glob_to_regex(file_glob.string(),
false)};
486 for (std::filesystem::directory_entry
const& entry :
487 std::filesystem::directory_iterator(parent_path)) {
488 if (std::filesystem::is_regular_file(std::filesystem::status(entry))) {
489 const auto entry_filename = entry.path().filename().string();
490 if (std::regex_match(entry_filename, glob_regex)) {
491 fs_paths.emplace_back(entry.path());
503 template <
typename T>
508 switch (bounds_type) {
510 switch (interval_type) {
512 return input >= bounds_val;
514 return input > bounds_val;
519 switch (interval_type) {
521 return input <= bounds_val;
523 return input < bounds_val;
536 const int32_t bounds_val,
541 const int64_t bounds_val,
546 const float bounds_val,
551 const double bounds_val,
555 #endif // #ifndef __CUDACC__
std::regex glob_to_regex(const std::string &glob, bool case_sensitive=false)
NEVER_INLINE HOST std::pair< T, T > get_column_min_max(const Column< T > &col)
DEVICE int64_t size() const
DEVICE T * getPtr() const
void z_std_normalize_col(const T *input_data, T *output_data, const int64_t num_rows, const double mean, const double std_dev)
std::vector< std::filesystem::path > get_fs_paths(const std::string &file_or_directory)
const size_t max_inputs_per_thread
DEVICE TextEncodingDict * getPtr() const
DEVICE bool isNull(int64_t index) const
void parallel_for(const blocked_range< Int > &range, const Body &body, const Partitioner &p=Partitioner())
EXTENSION_NOINLINE double distance_in_meters(const double fromlon, const double fromlat, const double tolon, const double tolat)
Computes the distance, in meters, between two WGS-84 positions.
NEVER_INLINE HOST std::tuple< T, T, bool > get_column_metadata(const Column< T > &col)
std::vector< std::vector< T > > z_std_normalize_data(const std::vector< T * > &input_data, const int64_t num_rows)
NEVER_INLINE HOST double get_column_std_dev(const Column< T > &col, const double mean)
DEVICE int64_t size() const
std::vector< std::string > glob(const std::string &pattern)
NEVER_INLINE HOST bool is_valid_tf_input(const T input, const T bounds_val, const BoundsType bounds_type, const IntervalType interval_type)
NEVER_INLINE HOST double get_column_mean(const T *data, const int64_t num_rows)