TableFunctionsCommon.hpp File Reference

#include <filesystem>
#include <mutex>
#include <shared_mutex>
#include <string>
#include <tuple>
#include <unordered_map>
#include <vector>
#include "QueryEngine/heavydbTypes.h"

Include dependency graph for TableFunctionsCommon.hpp:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Namespaces
	FileUtilities

Enumerations
enum	BoundsType { Min, Max }
enum	IntervalType { Inclusive, Exclusive }

Functions
template<typename T >
NEVER_INLINE HOST std::pair< T, T >	get_column_min_max (const Column< T > &col)
NEVER_INLINE HOST std::pair < int32_t, int32_t >	get_column_min_max (const Column< TextEncodingDict > &col)
template<typename T >
NEVER_INLINE HOST double	get_column_mean (const T *data, const int64_t num_rows)
template<typename T >
NEVER_INLINE HOST double	get_column_mean (const Column< T > &col)
template<typename T >
NEVER_INLINE HOST double	get_column_std_dev (const Column< T > &col, const double mean)
template<typename T >
NEVER_INLINE HOST double	get_column_std_dev (const T *data, const int64_t num_rows, const double mean)
template<typename T >
void	z_std_normalize_col (const T *input_data, T *output_data, const int64_t num_rows, const double mean, const double std_dev)
template<typename T >
std::vector< std::vector< T > >	z_std_normalize_data (const std::vector< T * > &input_data, const int64_t num_rows)
template<typename T >
NEVER_INLINE HOST std::tuple < T, T, bool >	get_column_metadata (const Column< T > &col)
NEVER_INLINE HOST std::tuple < int32_t, int32_t, bool >	get_column_metadata (const Column< TextEncodingDict > &col)
template<typename T1 , typename T2 >
NEVER_INLINE HOST T1	distance_in_meters (const T1 fromlon, const T1 fromlat, const T2 tolon, const T2 tolat)
int64_t	x_y_bin_to_bin_index (const int64_t x_bin, const int64_t y_bin, const int64_t num_x_bins)
std::pair< int64_t, int64_t >	bin_to_x_y_bin_indexes (const int64_t bin, const int64_t num_x_bins)
std::vector < std::filesystem::path >	FileUtilities::get_fs_paths (const std::string &file_or_directory)
template<typename T >
NEVER_INLINE HOST bool	is_valid_tf_input (const T input, const T bounds_val, const BoundsType bounds_type, const IntervalType interval_type)

Enumeration Type Documentation

enum BoundsType

Enumerator
Min
Max

Definition at line 89 of file TableFunctionsCommon.hpp.

{ Min, Max };

enum IntervalType

Enumerator
Inclusive
Exclusive

Definition at line 91 of file TableFunctionsCommon.hpp.

{ Inclusive, Exclusive };

Function Documentation

std::pair<int64_t, int64_t> bin_to_x_y_bin_indexes ( const int64_t  bin, const int64_t  num_x_bins  )

inline

Definition at line 80 of file TableFunctionsCommon.hpp.

Referenced by GeoRaster< T, Z >::get_xy_coords_for_bin_idx().

                                                                                    {
  return std::make_pair(bin % num_x_bins, bin / num_x_bins);
}

Here is the caller graph for this function:

template<typename T1 , typename T2 >

NEVER_INLINE HOST T1 distance_in_meters	(	const T1	fromlon,
		const T1	fromlat,
		const T2	tolon,
		const T2	tolat
	)

Definition at line 400 of file TableFunctionsCommon.cpp.

                                                                                       {
  T1 latitudeArc = (fromlat - tolat) * 0.017453292519943295769236907684886;
  T1 longitudeArc = (fromlon - tolon) * 0.017453292519943295769236907684886;
  T1 latitudeH = sin(latitudeArc * 0.5);
  latitudeH *= latitudeH;
  T1 lontitudeH = sin(longitudeArc * 0.5);
  lontitudeH *= lontitudeH;
  T1 tmp = cos(fromlat * 0.017453292519943295769236907684886) *
           cos(tolat * 0.017453292519943295769236907684886);
  return 6372797.560856 * (2.0 * asin(sqrt(latitudeH + tmp * lontitudeH)));
}

template<typename T >

NEVER_INLINE HOST double get_column_mean	(	const T *	data,
		const int64_t	num_rows
	)

Definition at line 110 of file TableFunctionsCommon.cpp.

References max_inputs_per_thread, threading_serial::parallel_for(), and heavydb.dtypes::T.

Referenced by get_column_mean(), and z_std_normalize_data().

                                                                                {
  // const int64_t num_rows = col.size();
  const size_t max_thread_count = std::thread::hardware_concurrency();
  const size_t max_inputs_per_thread = 20000;
  const size_t num_threads = std::min(
      max_thread_count, ((num_rows + max_inputs_per_thread - 1) / max_inputs_per_thread));

  std::vector<double> local_col_sums(num_threads, 0.);
  std::vector<int64_t> local_col_non_null_counts(num_threads, 0L);
  tbb::task_arena limited_arena(num_threads);
  limited_arena.execute([&] {
    tbb::parallel_for(tbb::blocked_range<int64_t>(0, num_rows),
                      [&](const tbb::blocked_range<int64_t>& r) {
                        const int64_t start_idx = r.begin();
                        const int64_t end_idx = r.end();
                        double local_col_sum = 0.;
                        int64_t local_col_non_null_count = 0;
                        for (int64_t r = start_idx; r < end_idx; ++r) {
                          const T val = data[r];
                          if (val == inline_null_value<T>()) {
                            continue;
                          }
                          local_col_sum += data[r];
                          local_col_non_null_count++;
                        }
                        size_t thread_idx = tbb::this_task_arena::current_thread_index();
                        local_col_sums[thread_idx] += local_col_sum;
                        local_col_non_null_counts[thread_idx] += local_col_non_null_count;
                      });
  });

  double col_sum = 0.0;
  int64_t col_non_null_count = 0L;

  for (size_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
    col_sum += local_col_sums[thread_idx];
    col_non_null_count += local_col_non_null_counts[thread_idx];
  }

  return col_non_null_count == 0 ? 0 : col_sum / col_non_null_count;
}

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >

NEVER_INLINE HOST double get_column_mean

(

const Column< T > &

col

)

Definition at line 171 of file TableFunctionsCommon.cpp.

References get_column_mean(), Column< T >::getPtr(), and Column< T >::size().

                                                               {
  return get_column_mean(col.getPtr(), col.size());
}

Here is the call graph for this function:

template<typename T >

NEVER_INLINE HOST std::tuple<T, T, bool> get_column_metadata

(

const Column< T > &

col

)

Definition at line 259 of file TableFunctionsCommon.cpp.

References Column< T >::isNull(), max_inputs_per_thread, threading_serial::parallel_for(), Column< T >::size(), and heavydb.dtypes::T.

Referenced by get_column_metadata().

                                                                                 {
  T col_min = std::numeric_limits<T>::max();
  T col_max = std::numeric_limits<T>::lowest();
  bool has_nulls = false;
  const int64_t num_rows = col.size();
  const size_t max_thread_count = std::thread::hardware_concurrency();
  const size_t max_inputs_per_thread = 200000;
  const size_t num_threads = std::min(
      max_thread_count, ((num_rows + max_inputs_per_thread - 1) / max_inputs_per_thread));

  std::vector<T> local_col_mins(num_threads, std::numeric_limits<T>::max());
  std::vector<T> local_col_maxes(num_threads, std::numeric_limits<T>::lowest());
  std::vector<bool> local_col_has_nulls(num_threads, false);
  tbb::task_arena limited_arena(num_threads);

  limited_arena.execute([&] {
    tbb::parallel_for(tbb::blocked_range<int64_t>(0, num_rows),
                      [&](const tbb::blocked_range<int64_t>& r) {
                        const int64_t start_idx = r.begin();
                        const int64_t end_idx = r.end();
                        T local_col_min = std::numeric_limits<T>::max();
                        T local_col_max = std::numeric_limits<T>::lowest();
                        bool local_has_nulls = false;
                        for (int64_t r = start_idx; r < end_idx; ++r) {
                          if (col.isNull(r)) {
                            local_has_nulls = true;
                            continue;
                          }
                          if (col[r] < local_col_min) {
                            local_col_min = col[r];
                          }
                          if (col[r] > local_col_max) {
                            local_col_max = col[r];
                          }
                        }
                        const size_t thread_idx =
                            tbb::this_task_arena::current_thread_index();
                        if (local_has_nulls) {
                          local_col_has_nulls[thread_idx] = true;
                        }
                        if (local_col_min < local_col_mins[thread_idx]) {
                          local_col_mins[thread_idx] = local_col_min;
                        }
                        if (local_col_max > local_col_maxes[thread_idx]) {
                          local_col_maxes[thread_idx] = local_col_max;
                        }
                      });
  });

  for (size_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
    if (local_col_has_nulls[thread_idx]) {
      has_nulls = true;
    }
    if (local_col_mins[thread_idx] < col_min) {
      col_min = local_col_mins[thread_idx];
    }
    if (local_col_maxes[thread_idx] > col_max) {
      col_max = local_col_maxes[thread_idx];
    }
  }
  return {col_min, col_max, has_nulls};
}

Here is the call graph for this function:

Here is the caller graph for this function:

NEVER_INLINE HOST std::tuple<int32_t, int32_t, bool> get_column_metadata

(

const Column< TextEncodingDict > &

col

)

Definition at line 335 of file TableFunctionsCommon.cpp.

References get_column_metadata(), Column< TextEncodingDict >::getPtr(), and Column< TextEncodingDict >::size().

                                         {
  Column<int32_t> int_alias_col(reinterpret_cast<int32_t*>(col.getPtr()), col.size());
  return get_column_metadata(int_alias_col);
}

Here is the call graph for this function:

template<typename T >

NEVER_INLINE HOST std::pair<T, T> get_column_min_max

(

const Column< T > &

col

)

Definition at line 32 of file TableFunctionsCommon.cpp.

References max_inputs_per_thread, threading_serial::parallel_for(), Column< T >::size(), and heavydb.dtypes::T.

Referenced by ct_union_pushdown_stats__cpu_template(), GeoRaster< T, Z >::GeoRaster(), get_column_min_max(), get_min_or_max(), and get_min_or_max_union().

                                                                         {
  T col_min = std::numeric_limits<T>::max();
  T col_max = std::numeric_limits<T>::lowest();
  const int64_t num_rows = col.size();
  const size_t max_thread_count = std::thread::hardware_concurrency();
  const size_t max_inputs_per_thread = 20000;
  const size_t num_threads = std::min(
      max_thread_count, ((num_rows + max_inputs_per_thread - 1) / max_inputs_per_thread));

  std::vector<T> local_col_mins(num_threads, std::numeric_limits<T>::max());
  std::vector<T> local_col_maxes(num_threads, std::numeric_limits<T>::lowest());
  tbb::task_arena limited_arena(num_threads);

  limited_arena.execute([&] {
    tbb::parallel_for(tbb::blocked_range<int64_t>(0, num_rows),
                      [&](const tbb::blocked_range<int64_t>& r) {
                        const int64_t start_idx = r.begin();
                        const int64_t end_idx = r.end();
                        T local_col_min = std::numeric_limits<T>::max();
                        T local_col_max = std::numeric_limits<T>::lowest();
                        for (int64_t r = start_idx; r < end_idx; ++r) {
                          const T val = col[r];
                          if (val == inline_null_value<T>()) {
                            continue;
                          }
                          if (val < local_col_min) {
                            local_col_min = val;
                          }
                          if (val > local_col_max) {
                            local_col_max = val;
                          }
                        }
                        size_t thread_idx = tbb::this_task_arena::current_thread_index();
                        if (local_col_min < local_col_mins[thread_idx]) {
                          local_col_mins[thread_idx] = local_col_min;
                        }
                        if (local_col_max > local_col_maxes[thread_idx]) {
                          local_col_maxes[thread_idx] = local_col_max;
                        }
                      });
  });

  for (size_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
    if (local_col_mins[thread_idx] < col_min) {
      col_min = local_col_mins[thread_idx];
    }
    if (local_col_maxes[thread_idx] > col_max) {
      col_max = local_col_maxes[thread_idx];
    }
  }
  return std::make_pair(col_min, col_max);
}

Here is the call graph for this function:

Here is the caller graph for this function:

NEVER_INLINE HOST std::pair<int32_t, int32_t> get_column_min_max

(

const Column< TextEncodingDict > &

col

)

Definition at line 98 of file TableFunctionsCommon.cpp.

References get_column_min_max(), Column< TextEncodingDict >::getPtr(), and Column< TextEncodingDict >::size().

                                                                                  {
  Column<int32_t> int_alias_col(reinterpret_cast<int32_t*>(col.getPtr()), col.size());
  return get_column_min_max(int_alias_col);
}

Here is the call graph for this function:

template<typename T >

NEVER_INLINE HOST double get_column_std_dev	(	const Column< T > &	col,
		const double	mean
	)

Definition at line 183 of file TableFunctionsCommon.cpp.

References get_column_std_dev(), Column< T >::getPtr(), and Column< T >::size().

Referenced by get_column_std_dev(), and z_std_normalize_data().

                                                                                     {
  return get_column_std_dev(col.getPtr(), col.size(), mean);
}

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >

NEVER_INLINE HOST double get_column_std_dev	(	const T *	data,
		const int64_t	num_rows,
		const double	mean
	)

Definition at line 197 of file TableFunctionsCommon.cpp.

References max_inputs_per_thread, threading_serial::parallel_for(), and heavydb.dtypes::T.

                                                               {
  // const int64_t num_rows = col.size();
  const size_t max_thread_count = std::thread::hardware_concurrency();
  const size_t max_inputs_per_thread = 200000;
  const size_t num_threads = std::min(
      max_thread_count, ((num_rows + max_inputs_per_thread - 1) / max_inputs_per_thread));

  std::vector<double> local_col_squared_residuals(num_threads, 0.);
  std::vector<int64_t> local_col_non_null_counts(num_threads, 0L);
  tbb::task_arena limited_arena(num_threads);

  limited_arena.execute([&] {
    tbb::parallel_for(tbb::blocked_range<int64_t>(0, num_rows),
                      [&](const tbb::blocked_range<int64_t>& r) {
                        const int64_t start_idx = r.begin();
                        const int64_t end_idx = r.end();
                        double local_col_squared_residual = 0.;
                        int64_t local_col_non_null_count = 0;
                        for (int64_t r = start_idx; r < end_idx; ++r) {
                          const T val = data[r];
                          if (val == inline_null_value<T>()) {
                            continue;
                          }
                          const double residual = val - mean;
                          local_col_squared_residual += (residual * residual);
                          local_col_non_null_count++;
                        }
                        size_t thread_idx = tbb::this_task_arena::current_thread_index();
                        local_col_squared_residuals[thread_idx] +=
                            local_col_squared_residual;
                        local_col_non_null_counts[thread_idx] += local_col_non_null_count;
                      });
  });

  double col_sum_squared_residual = 0.0;
  int64_t col_non_null_count = 0;

  for (size_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
    col_sum_squared_residual += local_col_squared_residuals[thread_idx];
    col_non_null_count += local_col_non_null_counts[thread_idx];
  }

  return col_non_null_count == 0 ? 0
                                 : sqrt(col_sum_squared_residual / col_non_null_count);
}

Here is the call graph for this function:

template<typename T >

NEVER_INLINE HOST bool is_valid_tf_input	(	const T	input,
		const T	bounds_val,
		const BoundsType	bounds_type,
		const IntervalType	interval_type
	)

Definition at line 504 of file TableFunctionsCommon.cpp.

References Exclusive, Inclusive, Max, Min, and UNREACHABLE.

                                                                           {
  switch (bounds_type) {
    case BoundsType::Min:
      switch (interval_type) {
        case IntervalType::Inclusive:
          return input >= bounds_val;
        case IntervalType::Exclusive:
          return input > bounds_val;
        default:
          UNREACHABLE();
      }
    case BoundsType::Max:
      switch (interval_type) {
        case IntervalType::Inclusive:
          return input <= bounds_val;
        case IntervalType::Exclusive:
          return input < bounds_val;
        default:
          UNREACHABLE();
      }
      break;
    default:
      UNREACHABLE();
  }
  UNREACHABLE();
  return false;  // To address compiler warning
}

int64_t x_y_bin_to_bin_index ( const int64_t  x_bin, const int64_t  y_bin, const int64_t  num_x_bins  )

inline

Definition at line 74 of file TableFunctionsCommon.hpp.

Referenced by GeoRaster< T, Z >::calculate_slope_and_aspect(), GeoRaster< T, Z >::computeParallelImpl(), GeoRaster< T, Z >::computeSerialImpl(), GeoRaster< T, Z >::fill_bin_from_avg_box_neighborhood(), GeoRaster< T, Z >::fill_bin_from_box_neighborhood(), GeoRaster< T, Z >::fill_bins_from_box_neighborhood(), GeoRaster< T, Z >::fill_bins_from_gaussian_neighborhood(), GeoRaster< T, Z >::get_bin_idx_and_z_val_for_xy_bin(), GeoRaster< T, Z >::get_bin_idx_for_xy_coords(), GeoRaster< T, Z >::get_nxn_neighbors_if_not_null(), GeoRaster< T, Z >::offset_source_z_from_raster_z(), and GeoRaster< T, Z >::outputDenseColumns().

                                                              {
  return y_bin * num_x_bins + x_bin;
}

Here is the caller graph for this function:

template<typename T >

void z_std_normalize_col	(	const T *	input_data,
		T *	output_data,
		const int64_t	num_rows,
		const double	mean,
		const double	std_dev
	)

Definition at line 342 of file TableFunctionsCommon.cpp.

References threading_serial::parallel_for().

Referenced by z_std_normalize_data().

                                               {
  if (std_dev <= 0.0) {
    throw std::runtime_error("Standard deviation cannot be <= 0");
  }
  const double inv_std_dev = 1.0 / std_dev;

  tbb::parallel_for(tbb::blocked_range<int64_t>(0, num_rows),
                    [&](const tbb::blocked_range<int64_t>& r) {
                      const int64_t start_idx = r.begin();
                      const int64_t end_idx = r.end();
                      for (int64_t row_idx = start_idx; row_idx < end_idx; ++row_idx) {
                        output_data[row_idx] = (input_data[row_idx] - mean) * inv_std_dev;
                      }
                    });
}

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >

std::vector<std::vector<T> > z_std_normalize_data	(	const std::vector< T * > &	input_data,
		const int64_t	num_rows
	)

Definition at line 374 of file TableFunctionsCommon.cpp.

References get_column_mean(), get_column_std_dev(), and z_std_normalize_col().

Referenced by dbscan__cpu_template(), and kmeans__cpu_template().

                                                                         {
  const int64_t num_features = input_data.size();
  std::vector<std::vector<T>> normalized_data(num_features);
  for (int64_t feature_idx = 0; feature_idx < num_features; ++feature_idx) {
    const auto mean = get_column_mean(input_data[feature_idx], num_rows);
    const auto std_dev = get_column_std_dev(input_data[feature_idx], num_rows, mean);
    normalized_data[feature_idx].resize(num_rows);
    z_std_normalize_col(input_data[feature_idx],
                        normalized_data[feature_idx].data(),
                        num_rows,
                        mean,
                        std_dev);
  }
  return normalized_data;
}

Here is the call graph for this function:

Here is the caller graph for this function: