UtilityTableFunctions.h File Reference

#include "QueryEngine/heavydbTypes.h"

Include dependency graph for UtilityTableFunctions.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Functions
EXTENSION_NOINLINE_HOST int32_t	generate_series_parallel (const int64_t start, const int64_t stop, const int64_t step, Column< int64_t > &series_output)
EXTENSION_NOINLINE_HOST int32_t	generate_series__cpu_1 (TableFunctionManager &mgr, const int64_t start, const int64_t stop, const int64_t step, Column< int64_t > &series_output)
EXTENSION_NOINLINE_HOST int32_t	generate_series__cpu_2 (TableFunctionManager &mgr, const int64_t start, const int64_t stop, Column< int64_t > &series_output)
EXTENSION_NOINLINE_HOST int32_t	generate_random_strings__cpu_ (TableFunctionManager &mgr, const int64_t num_strings, const int64_t string_length, Column< int64_t > &output_id, Column< TextEncodingDict > &output_strings)

Function Documentation

EXTENSION_NOINLINE_HOST int32_t generate_random_strings__cpu_	(	TableFunctionManager &	mgr,
		const int64_t	num_strings,
		const int64_t	string_length,
		Column< int64_t > &	output_id,
		Column< TextEncodingDict > &	output_strings
	)

Definition at line 130 of file UtilityTableFunctions.cpp.

References CHECK_LE, DEBUG_TIMER, gen_random_str(), ThreadInfo::num_elems_per_thread, ThreadInfo::num_threads, threading_serial::parallel_for(), and TableFunctionManager::set_output_row_size().

                                                                                {
  auto timer = DEBUG_TIMER(__func__);
  // Check for out-of-range errors for the input parameters
  // in the function instead of with require due to issue encountered
  // with require over multiple variables
  constexpr int64_t max_strings{10000000L};
  constexpr int64_t max_str_len{10000L};
  if (num_strings > max_strings) {
    return mgr.ERROR_MESSAGE(
        "generate_random_strings: num_strings must be between 0 and 10,000,000.");
  }
  if (string_length > max_str_len) {
    return mgr.ERROR_MESSAGE(
        "generate_random_strings: string_length must be between 1 and 10,000.");
  }
  if (num_strings == 0L) {
    // Bail early as there is no work to be done
    return 0;
  }

  mgr.set_output_row_size(num_strings);
  constexpr int64_t target_strings_per_thread{5000};
  const ThreadInfo thread_info(
      std::thread::hardware_concurrency(), num_strings, target_strings_per_thread);
  std::vector<std::mt19937> per_thread_rand_generators;
  per_thread_rand_generators.reserve(thread_info.num_threads);
  for (int64_t thread_idx = 0; thread_idx < thread_info.num_threads; ++thread_idx) {
    const uint64_t seed = std::chrono::duration_cast<std::chrono::nanoseconds>(
                              std::chrono::system_clock::now().time_since_epoch())
                              .count() +
                          thread_idx * 971;
    per_thread_rand_generators.emplace_back(seed);
  }
  std::vector<std::string> rand_strings(num_strings);
  tbb::task_arena limited_arena(thread_info.num_threads);
  limited_arena.execute([&] {
    CHECK_LE(tbb::this_task_arena::max_concurrency(), thread_info.num_threads);
    tbb::parallel_for(
        tbb::blocked_range<int64_t>(0, num_strings, thread_info.num_elems_per_thread),
        [&](const tbb::blocked_range<int64_t>& r) {
          const int64_t tbb_thread_idx = tbb::this_task_arena::current_thread_index();
          const int64_t start_out_idx = r.begin();
          const int64_t end_out_idx = r.end();
          for (int64_t out_idx = start_out_idx; out_idx != end_out_idx; ++out_idx) {
            rand_strings[out_idx] =
                gen_random_str(per_thread_rand_generators[tbb_thread_idx], string_length);
          }
        },
        tbb::simple_partitioner());
  });
  const std::vector<int32_t> rand_string_ids =
      output_strings.string_dict_proxy_->getOrAddTransientBulk(rand_strings);
  for (int64_t row_idx = 0; row_idx < num_strings; row_idx++) {
    output_id[row_idx] = row_idx;
    output_strings[row_idx] = rand_string_ids[row_idx];
  }
  return num_strings;
}

Here is the call graph for this function:

EXTENSION_NOINLINE_HOST int32_t generate_series__cpu_1	(	TableFunctionManager &	mgr,
		const int64_t	start,
		const int64_t	stop,
		const int64_t	step,
		Column< int64_t > &	series_output
	)

Definition at line 59 of file UtilityTableFunctions.cpp.

References generate_series_parallel(), TableFunctionManager::set_output_row_size(), and to_string().

Referenced by generate_series__cpu_2().

                                                               {
  const int64_t MAX_ROWS{1L << 30};
  const int64_t PARALLEL_THRESHOLD{10000L};
  const int64_t num_rows = ((stop - start) / step) + 1;
  if (num_rows <= 0) {
    mgr.set_output_row_size(0);
    return 0;
  }
  mgr.set_output_row_size(num_rows);

  if (num_rows > MAX_ROWS) {
    return mgr.ERROR_MESSAGE(
        "Invocation of generate_series would result in " + std::to_string(num_rows) +
        " rows, which exceeds the max limit of " + std::to_string(MAX_ROWS) + " rows.");
  }

#ifdef HAVE_TBB
  if (num_rows > PARALLEL_THRESHOLD) {
    return generate_series_parallel(start, stop, step, series_output);
  }
#endif

  for (int64_t out_idx = 0; out_idx != num_rows; ++out_idx) {
    series_output[out_idx] = start + out_idx * step;
  }
  return num_rows;
}

Here is the call graph for this function:

Here is the caller graph for this function:

EXTENSION_NOINLINE_HOST int32_t generate_series__cpu_2	(	TableFunctionManager &	mgr,
		const int64_t	start,
		const int64_t	stop,
		Column< int64_t > &	series_output
	)

Definition at line 97 of file UtilityTableFunctions.cpp.

References generate_series__cpu_1().

                                                               {
  return generate_series__cpu_1(mgr, start, stop, 1, series_output);
}

Here is the call graph for this function:

EXTENSION_NOINLINE_HOST int32_t generate_series_parallel	(	const int64_t	start,
		const int64_t	stop,
		const int64_t	step,
		Column< int64_t > &	series_output
	)

Definition at line 35 of file UtilityTableFunctions.cpp.

References threading_serial::parallel_for().

Referenced by generate_series__cpu_1().

                                                                 {
  const int64_t num_rows = ((stop - start) / step) + 1;

  tbb::parallel_for(tbb::blocked_range<int64_t>(0, num_rows),
                    [&](const tbb::blocked_range<int64_t>& r) {
                      const int64_t start_out_idx = r.begin();
                      const int64_t end_out_idx = r.end();
                      for (int64_t out_idx = start_out_idx; out_idx != end_out_idx;
                           ++out_idx) {
                        series_output[out_idx] = start + out_idx * step;
                      }
                    });
  return num_rows;
}

Here is the call graph for this function:

Here is the caller graph for this function: