UtilityTableFunctions.cpp File Reference

#include <string>
#include "Shared/ThreadInfo.h"
#include "UtilityTableFunctions.h"
#include <chrono>
#include <random>
#include <thread>

Include dependency graph for UtilityTableFunctions.cpp:

Go to the source code of this file.

Functions
int64_t	numStepsBetween (int64_t start, int64_t stop, int64_t step)
template<typename T >
int64_t	numStepsBetween (Timestamp start, Timestamp stop, T step)
template<typename T , typename K >
int32_t	generate_series_parallel (const T start, const T stop, const K step, Column< T > &series_output)
template<typename T , typename K >
NEVER_INLINE HOST int32_t	generate_series__cpu_template (TableFunctionManager &mgr, const T start, const T stop, const K step, Column< T > &series_output)
template<typename T >
NEVER_INLINE HOST int32_t	generate_series__cpu_template (TableFunctionManager &mgr, const T start, const T stop, Column< T > &series_output)
HOST std::string	gen_random_str (std::mt19937 &generator, const int64_t str_len)
EXTENSION_NOINLINE_HOST int32_t	generate_random_strings__cpu_ (TableFunctionManager &mgr, const int64_t num_strings, const int64_t string_length, Column< int64_t > &output_id, Column< TextEncodingDict > &output_strings)
template int64_t	numStepsBetween (Timestamp, Timestamp, DayTimeInterval)
template int64_t	numStepsBetween (Timestamp, Timestamp, YearMonthTimeInterval)
template int32_t	generate_series_parallel (int64_t, int64_t, int64_t, Column< int64_t > &)
template int32_t	generate_series_parallel (Timestamp, Timestamp, DayTimeInterval, Column< Timestamp > &)
template int32_t	generate_series_parallel (Timestamp, Timestamp, YearMonthTimeInterval, Column< Timestamp > &)
template int32_t	generate_series__cpu_template (TableFunctionManager &, int64_t, int64_t, int64_t, Column< int64_t > &)
template int32_t	generate_series__cpu_template (TableFunctionManager &, Timestamp, Timestamp, DayTimeInterval, Column< Timestamp > &)
template int32_t	generate_series__cpu_template (TableFunctionManager &, Timestamp, Timestamp, YearMonthTimeInterval, Column< Timestamp > &)
template int32_t	generate_series__cpu_template (TableFunctionManager &, int64_t, int64_t, Column< int64_t > &)

Function Documentation

HOST std::string gen_random_str	(	std::mt19937 &	generator,
		const int64_t	str_len
	)

Definition at line 99 of file UtilityTableFunctions.cpp.

Referenced by generate_random_strings__cpu_().

                                                                            {
  constexpr char alphanum_lookup_table[] =
      "0123456789"
      "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
      "abcdefghijklmnopqrstuvwxyz";
  constexpr size_t char_mod = sizeof(alphanum_lookup_table) - 1;
  std::uniform_int_distribution<int32_t> rand_distribution(0, char_mod);

  std::string tmp_s;
  tmp_s.reserve(str_len);
  for (int i = 0; i < str_len; ++i) {
    tmp_s += alphanum_lookup_table[rand_distribution(generator)];
  }
  return tmp_s;
}

Here is the caller graph for this function:

EXTENSION_NOINLINE_HOST int32_t generate_random_strings__cpu_	(	TableFunctionManager &	mgr,
		const int64_t	num_strings,
		const int64_t	string_length,
		Column< int64_t > &	output_id,
		Column< TextEncodingDict > &	output_strings
	)

Definition at line 121 of file UtilityTableFunctions.cpp.

References CHECK_LE, DEBUG_TIMER, gen_random_str(), ThreadInfo::num_elems_per_thread, ThreadInfo::num_threads, threading_serial::parallel_for(), and TableFunctionManager::set_output_row_size().

                                                                                {
  auto timer = DEBUG_TIMER(__func__);
  // Check for out-of-range errors for the input parameters
  // in the function instead of with require due to issue encountered
  // with require over multiple variables
  constexpr int64_t max_strings{10000000L};
  constexpr int64_t max_str_len{10000L};
  if (num_strings > max_strings) {
    return mgr.ERROR_MESSAGE(
        "generate_random_strings: num_strings must be between 0 and 10,000,000.");
  }
  if (string_length > max_str_len) {
    return mgr.ERROR_MESSAGE(
        "generate_random_strings: string_length must be between 1 and 10,000.");
  }
  if (num_strings == 0L) {
    // Bail early as there is no work to be done
    return 0;
  }

  mgr.set_output_row_size(num_strings);
  constexpr int64_t target_strings_per_thread{5000};
  const ThreadInfo thread_info(
      std::thread::hardware_concurrency(), num_strings, target_strings_per_thread);
  std::vector<std::mt19937> per_thread_rand_generators;
  per_thread_rand_generators.reserve(thread_info.num_threads);
  for (int64_t thread_idx = 0; thread_idx < thread_info.num_threads; ++thread_idx) {
    const uint64_t seed = std::chrono::duration_cast<std::chrono::nanoseconds>(
                              std::chrono::system_clock::now().time_since_epoch())
                              .count() +
                          thread_idx * 971;
    per_thread_rand_generators.emplace_back(seed);
  }
  std::vector<std::string> rand_strings(num_strings);
  tbb::task_arena limited_arena(thread_info.num_threads);
  limited_arena.execute([&] {
    CHECK_LE(tbb::this_task_arena::max_concurrency(), thread_info.num_threads);
    tbb::parallel_for(
        tbb::blocked_range<int64_t>(0, num_strings, thread_info.num_elems_per_thread),
        [&](const tbb::blocked_range<int64_t>& r) {
          const int64_t tbb_thread_idx = tbb::this_task_arena::current_thread_index();
          const int64_t start_out_idx = r.begin();
          const int64_t end_out_idx = r.end();
          for (int64_t out_idx = start_out_idx; out_idx != end_out_idx; ++out_idx) {
            rand_strings[out_idx] =
                gen_random_str(per_thread_rand_generators[tbb_thread_idx], string_length);
          }
        },
        tbb::simple_partitioner());
  });
  const std::vector<int32_t> rand_string_ids =
      output_strings.string_dict_proxy_->getOrAddTransientBulk(rand_strings);
  for (int64_t row_idx = 0; row_idx < num_strings; row_idx++) {
    output_id[row_idx] = row_idx;
    output_strings[row_idx] = rand_string_ids[row_idx];
  }
  return num_strings;
}

Here is the call graph for this function:

template<typename T , typename K >

NEVER_INLINE HOST int32_t generate_series__cpu_template	(	TableFunctionManager &	mgr,
		const T	start,
		const T	stop,
		const K	step,
		Column< T > &	series_output
	)

Definition at line 58 of file UtilityTableFunctions.cpp.

References generate_series_parallel(), numStepsBetween(), TableFunctionManager::set_output_row_size(), and SUCCESS.

Referenced by generate_series__cpu_template().

                                                                                  {
  const int64_t PARALLEL_THRESHOLD{10000L};
  const int64_t num_rows = numStepsBetween(start, stop, step) + 1;
  if (num_rows <= 0) {
    mgr.set_output_row_size(0);
    return 0;
  }
  // set_output_row_size ensures that the output buffer size will be
  // in a reasonable range (up to 16 TiB), and if it is not, an
  // OutOfHostMemory exception will be thrown.
  mgr.set_output_row_size(num_rows);

#ifdef HAVE_TBB
  if (num_rows > PARALLEL_THRESHOLD) {
    return generate_series_parallel(start, stop, step, series_output);
  }
#endif

  for (int64_t out_idx = 0; out_idx != num_rows; ++out_idx) {
    series_output[out_idx] = start + (step * out_idx);
  }
  return SUCCESS;
}

Here is the call graph for this function:

Here is the caller graph for this function:

template<typename T >

NEVER_INLINE HOST int32_t generate_series__cpu_template	(	TableFunctionManager &	mgr,
		const T	start,
		const T	stop,
		Column< T > &	series_output
	)

Definition at line 87 of file UtilityTableFunctions.cpp.

References generate_series__cpu_template().

                                                                                  {
  return generate_series__cpu_template(
      mgr, start, stop, static_cast<int64_t>(1), series_output);
}

Here is the call graph for this function:

template int32_t generate_series__cpu_template	(	TableFunctionManager &	,
		int64_t	,
		int64_t	,
		int64_t	,
		Column< int64_t > &
	)

template int32_t generate_series__cpu_template	(	TableFunctionManager &	,
		Timestamp	,
		Timestamp	,
		DayTimeInterval	,
		Column< Timestamp > &
	)

template int32_t generate_series__cpu_template	(	TableFunctionManager &	,
		Timestamp	,
		Timestamp	,
		YearMonthTimeInterval	,
		Column< Timestamp > &
	)

template int32_t generate_series__cpu_template	(	TableFunctionManager &	,
		int64_t	,
		int64_t	,
		Column< int64_t > &
	)

template<typename T , typename K >

int32_t generate_series_parallel	(	const T	start,
		const T	stop,
		const K	step,
		Column< T > &	series_output
	)

Definition at line 39 of file UtilityTableFunctions.cpp.

References numStepsBetween(), threading_serial::parallel_for(), and SUCCESS.

Referenced by generate_series__cpu_template().

                                                           {
  const int64_t num_rows = numStepsBetween(start, stop, step) + 1;

  tbb::parallel_for(tbb::blocked_range<int64_t>(0, num_rows),
                    [&](const tbb::blocked_range<int64_t>& r) {
                      const int64_t start_out_idx = r.begin();
                      const int64_t end_out_idx = r.end();
                      for (int64_t out_idx = start_out_idx; out_idx != end_out_idx;
                           ++out_idx) {
                        series_output[out_idx] = start + (step * out_idx);
                      }
                    });
  return SUCCESS;
}

Here is the call graph for this function:

Here is the caller graph for this function:

template int32_t generate_series_parallel	(	int64_t	,
		int64_t	,
		int64_t	,
		Column< int64_t > &
	)

template int32_t generate_series_parallel	(	Timestamp	,
		Timestamp	,
		DayTimeInterval	,
		Column< Timestamp > &
	)

template int32_t generate_series_parallel	(	Timestamp	,
		Timestamp	,
		YearMonthTimeInterval	,
		Column< Timestamp > &
	)

int64_t numStepsBetween	(	int64_t	start,
		int64_t	stop,
		int64_t	step
	)

Definition at line 29 of file UtilityTableFunctions.cpp.

Referenced by generate_series__cpu_template(), and generate_series_parallel().

                                                                   {
  return (stop - start) / step;
}

Here is the caller graph for this function:

template<typename T >

int64_t numStepsBetween	(	Timestamp	start,
		Timestamp	stop,
		T	step
	)

Definition at line 34 of file UtilityTableFunctions.cpp.

                                                                 {
  return step.numStepsBetween(start, stop);
}

template int64_t numStepsBetween	(	Timestamp	,
		Timestamp	,
		DayTimeInterval
	)

template int64_t numStepsBetween	(	Timestamp	,
		Timestamp	,
		YearMonthTimeInterval
	)