OmniSciDB  ba1bac9284
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE > Struct Template Reference
+ Collaboration diagram for ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >:

Public Types

using BufferIteratorType = BUFFER_ITERATOR_TYPE
 

Public Member Functions

 ResultSetComparator (const std::list< Analyzer::OrderEntry > &order_entries, const ResultSet *result_set, const PermutationView permutation, const Executor *executor, const bool single_threaded)
 
void materializeCountDistinctColumns ()
 
ApproxMedianBuffers materializeApproxMedianColumns () const
 
std::vector< int64_t > materializeCountDistinctColumn (const Analyzer::OrderEntry &order_entry) const
 
ApproxMedianBuffers::value_type materializeApproxMedianColumn (const Analyzer::OrderEntry &order_entry) const
 
bool operator() (const PermutationIdx lhs, const PermutationIdx rhs) const
 

Public Attributes

const std::list
< Analyzer::OrderEntry > & 
order_entries_
 
const ResultSetresult_set_
 
const PermutationView permutation_
 
const BufferIteratorType buffer_itr_
 
const Executorexecutor_
 
const bool single_threaded_
 
std::vector< std::vector
< int64_t > > 
count_distinct_materialized_buffers_
 
const ApproxMedianBuffers approx_median_materialized_buffers_
 

Detailed Description

template<typename BUFFER_ITERATOR_TYPE>
struct ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >

Definition at line 627 of file ResultSet.h.

Member Typedef Documentation

template<typename BUFFER_ITERATOR_TYPE>
using ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::BufferIteratorType = BUFFER_ITERATOR_TYPE

Definition at line 628 of file ResultSet.h.

Constructor & Destructor Documentation

template<typename BUFFER_ITERATOR_TYPE>
ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::ResultSetComparator ( const std::list< Analyzer::OrderEntry > &  order_entries,
const ResultSet result_set,
const PermutationView  permutation,
const Executor executor,
const bool  single_threaded 
)
inline

Definition at line 630 of file ResultSet.h.

References ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::materializeCountDistinctColumns().

635  : order_entries_(order_entries)
636  , result_set_(result_set)
637  , permutation_(permutation)
638  , buffer_itr_(result_set)
639  , executor_(executor)
640  , single_threaded_(single_threaded)
643  }
const BufferIteratorType buffer_itr_
Definition: ResultSet.h:658
const Executor * executor_
Definition: ResultSet.h:659
ApproxMedianBuffers materializeApproxMedianColumns() const
Definition: ResultSet.cpp:683
const ResultSet * result_set_
Definition: ResultSet.h:656
const std::list< Analyzer::OrderEntry > & order_entries_
Definition: ResultSet.h:655
const PermutationView permutation_
Definition: ResultSet.h:657
const ApproxMedianBuffers approx_median_materialized_buffers_
Definition: ResultSet.h:662

+ Here is the call graph for this function:

Member Function Documentation

template<typename BUFFER_ITERATOR_TYPE >
ResultSet::ApproxMedianBuffers::value_type ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::materializeApproxMedianColumn ( const Analyzer::OrderEntry order_entry) const

Definition at line 739 of file ResultSet.cpp.

References cpu_threads(), i, threadpool::FuturesThreadPool< T, ENABLE >::join(), NULL_DOUBLE, threadpool::FuturesThreadPoolBase< T >::spawn(), and Analyzer::OrderEntry::tle_no.

740  {
741  ResultSet::ApproxMedianBuffers::value_type materialized_buffer(
742  result_set_->query_mem_desc_.getEntryCount());
743  const size_t size = permutation_.size();
744  const auto work = [&](const size_t start, const size_t end) {
745  for (size_t i = start; i < end; ++i) {
746  const PermutationIdx permuted_idx = permutation_[i];
747  const auto storage_lookup_result = result_set_->findStorage(permuted_idx);
748  const auto storage = storage_lookup_result.storage_ptr;
749  const auto off = storage_lookup_result.fixedup_entry_idx;
750  const auto value = buffer_itr_.getColumnInternal(
751  storage->buff_, off, order_entry.tle_no - 1, storage_lookup_result);
752  materialized_buffer[permuted_idx] =
753  value.i1
754  ? calculateQuantile(reinterpret_cast<quantile::TDigest*>(value.i1), 0.5)
755  : NULL_DOUBLE;
756  }
757  };
758  if (single_threaded_) {
759  work(0, size);
760  } else {
762  for (auto interval : makeIntervals<size_t>(0, size, cpu_threads())) {
763  thread_pool.spawn(work, interval.begin, interval.end);
764  }
765  thread_pool.join();
766  }
767  return materialized_buffer;
768 }
#define NULL_DOUBLE
const BufferIteratorType buffer_itr_
Definition: ResultSet.h:658
int tle_no
Definition: Analyzer.h:1418
DEVICE size_type size() const
Definition: VectorView.h:84
const ResultSet * result_set_
Definition: ResultSet.h:656
const PermutationView permutation_
Definition: ResultSet.h:657
static double calculateQuantile(quantile::TDigest *const t_digest, double const q)
Definition: ResultSet.cpp:729
uint32_t PermutationIdx
Definition: ResultSet.h:152
void spawn(Function &&f, Args &&...args)
Definition: threadpool.h:33
int cpu_threads()
Definition: thread_count.h:24

+ Here is the call graph for this function:

template<typename BUFFER_ITERATOR_TYPE >
ResultSet::ApproxMedianBuffers ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::materializeApproxMedianColumns ( ) const

Definition at line 683 of file ResultSet.cpp.

References kAPPROX_MEDIAN.

683  {
684  ResultSet::ApproxMedianBuffers approx_median_materialized_buffers;
685  for (const auto& order_entry : order_entries_) {
686  if (result_set_->targets_[order_entry.tle_no - 1].agg_kind == kAPPROX_MEDIAN) {
687  approx_median_materialized_buffers.emplace_back(
688  materializeApproxMedianColumn(order_entry));
689  }
690  }
691  return approx_median_materialized_buffers;
692 }
const ResultSet * result_set_
Definition: ResultSet.h:656
const std::list< Analyzer::OrderEntry > & order_entries_
Definition: ResultSet.h:655
std::vector< std::vector< double >> ApproxMedianBuffers
Definition: ResultSet.h:624
ApproxMedianBuffers::value_type materializeApproxMedianColumn(const Analyzer::OrderEntry &order_entry) const
Definition: ResultSet.cpp:739
template<typename BUFFER_ITERATOR_TYPE >
std::vector< int64_t > ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::materializeCountDistinctColumn ( const Analyzer::OrderEntry order_entry) const

Definition at line 696 of file ResultSet.cpp.

References count_distinct_set_size(), cpu_threads(), i, threadpool::FuturesThreadPool< T, ENABLE >::join(), threadpool::FuturesThreadPoolBase< T >::spawn(), and Analyzer::OrderEntry::tle_no.

697  {
698  const size_t num_storage_entries = result_set_->query_mem_desc_.getEntryCount();
699  std::vector<int64_t> count_distinct_materialized_buffer(num_storage_entries);
700  const CountDistinctDescriptor count_distinct_descriptor =
701  result_set_->query_mem_desc_.getCountDistinctDescriptor(order_entry.tle_no - 1);
702  const size_t num_non_empty_entries = permutation_.size();
703  const auto work = [&](const size_t start, const size_t end) {
704  for (size_t i = start; i < end; ++i) {
705  const PermutationIdx permuted_idx = permutation_[i];
706  const auto storage_lookup_result = result_set_->findStorage(permuted_idx);
707  const auto storage = storage_lookup_result.storage_ptr;
708  const auto off = storage_lookup_result.fixedup_entry_idx;
709  const auto value = buffer_itr_.getColumnInternal(
710  storage->buff_, off, order_entry.tle_no - 1, storage_lookup_result);
711  count_distinct_materialized_buffer[permuted_idx] =
712  count_distinct_set_size(value.i1, count_distinct_descriptor);
713  }
714  };
715  // TODO(tlm): Allow use of tbb after we determine how to easily encapsulate the choice
716  // between thread pool types
717  if (single_threaded_) {
718  work(0, num_non_empty_entries);
719  } else {
721  for (auto interval : makeIntervals<size_t>(0, num_non_empty_entries, cpu_threads())) {
722  thread_pool.spawn(work, interval.begin, interval.end);
723  }
724  thread_pool.join();
725  }
726  return count_distinct_materialized_buffer;
727 }
const BufferIteratorType buffer_itr_
Definition: ResultSet.h:658
int tle_no
Definition: Analyzer.h:1418
DEVICE size_type size() const
Definition: VectorView.h:84
int64_t count_distinct_set_size(const int64_t set_handle, const CountDistinctDescriptor &count_distinct_desc)
Definition: CountDistinct.h:75
const ResultSet * result_set_
Definition: ResultSet.h:656
const PermutationView permutation_
Definition: ResultSet.h:657
uint32_t PermutationIdx
Definition: ResultSet.h:152
void spawn(Function &&f, Args &&...args)
Definition: threadpool.h:33
int cpu_threads()
Definition: thread_count.h:24

+ Here is the call graph for this function:

template<typename BUFFER_ITERATOR_TYPE >
void ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::materializeCountDistinctColumns ( )

Definition at line 672 of file ResultSet.cpp.

References is_distinct_target().

Referenced by ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::ResultSetComparator().

672  {
673  for (const auto& order_entry : order_entries_) {
674  if (is_distinct_target(result_set_->targets_[order_entry.tle_no - 1])) {
676  materializeCountDistinctColumn(order_entry));
677  }
678  }
679 }
std::vector< int64_t > materializeCountDistinctColumn(const Analyzer::OrderEntry &order_entry) const
Definition: ResultSet.cpp:696
const ResultSet * result_set_
Definition: ResultSet.h:656
const std::list< Analyzer::OrderEntry > & order_entries_
Definition: ResultSet.h:655
bool is_distinct_target(const TargetInfo &target_info)
Definition: TargetInfo.h:130
std::vector< std::vector< int64_t > > count_distinct_materialized_buffers_
Definition: ResultSet.h:661

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

template<typename BUFFER_ITERATOR_TYPE >
bool ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::operator() ( const PermutationIdx  lhs,
const PermutationIdx  rhs 
) const

Definition at line 771 of file ResultSet.cpp.

References CHECK, CHECK_EQ, CHECK_GE, CHECK_LT, get_compact_type(), is_distinct_target(), kAPPROX_MEDIAN, kENCODING_DICT, kFLOAT, LIKELY, NULL_DOUBLE, pair_to_double(), takes_float_argument(), and UNLIKELY.

773  {
774  // NB: The compare function must define a strict weak ordering, otherwise
775  // std::sort will trigger a segmentation fault (or corrupt memory).
776  const auto lhs_storage_lookup_result = result_set_->findStorage(lhs);
777  const auto rhs_storage_lookup_result = result_set_->findStorage(rhs);
778  const auto lhs_storage = lhs_storage_lookup_result.storage_ptr;
779  const auto rhs_storage = rhs_storage_lookup_result.storage_ptr;
780  const auto fixedup_lhs = lhs_storage_lookup_result.fixedup_entry_idx;
781  const auto fixedup_rhs = rhs_storage_lookup_result.fixedup_entry_idx;
782  size_t materialized_count_distinct_buffer_idx{0};
783  size_t materialized_approx_median_buffer_idx{0};
784 
785  for (const auto& order_entry : order_entries_) {
786  CHECK_GE(order_entry.tle_no, 1);
787  const auto& agg_info = result_set_->targets_[order_entry.tle_no - 1];
788  const auto entry_ti = get_compact_type(agg_info);
789  bool float_argument_input = takes_float_argument(agg_info);
790  // Need to determine if the float value has been stored as float
791  // or if it has been compacted to a different (often larger 8 bytes)
792  // in distributed case the floats are actually 4 bytes
793  // TODO the above takes_float_argument() is widely used wonder if this problem
794  // exists elsewhere
795  if (entry_ti.get_type() == kFLOAT) {
796  const auto is_col_lazy =
797  !result_set_->lazy_fetch_info_.empty() &&
798  result_set_->lazy_fetch_info_[order_entry.tle_no - 1].is_lazily_fetched;
799  if (result_set_->query_mem_desc_.getPaddedSlotWidthBytes(order_entry.tle_no - 1) ==
800  sizeof(float)) {
801  float_argument_input =
802  result_set_->query_mem_desc_.didOutputColumnar() ? !is_col_lazy : true;
803  }
804  }
805 
806  if (UNLIKELY(is_distinct_target(agg_info))) {
807  CHECK_LT(materialized_count_distinct_buffer_idx,
809 
810  const auto& count_distinct_materialized_buffer =
811  count_distinct_materialized_buffers_[materialized_count_distinct_buffer_idx];
812  const auto lhs_sz = count_distinct_materialized_buffer[lhs];
813  const auto rhs_sz = count_distinct_materialized_buffer[rhs];
814  ++materialized_count_distinct_buffer_idx;
815  if (lhs_sz == rhs_sz) {
816  continue;
817  }
818  return (lhs_sz < rhs_sz) != order_entry.is_desc;
819  } else if (UNLIKELY(agg_info.agg_kind == kAPPROX_MEDIAN)) {
820  CHECK_LT(materialized_approx_median_buffer_idx,
822  const auto& approx_median_materialized_buffer =
823  approx_median_materialized_buffers_[materialized_approx_median_buffer_idx];
824  const auto lhs_value = approx_median_materialized_buffer[lhs];
825  const auto rhs_value = approx_median_materialized_buffer[rhs];
826  ++materialized_approx_median_buffer_idx;
827  if (lhs_value == rhs_value) {
828  continue;
829  } else if (!entry_ti.get_notnull()) {
830  if (lhs_value == NULL_DOUBLE) {
831  return order_entry.nulls_first;
832  } else if (rhs_value == NULL_DOUBLE) {
833  return !order_entry.nulls_first;
834  }
835  }
836  return (lhs_value < rhs_value) != order_entry.is_desc;
837  }
838 
839  const auto lhs_v = buffer_itr_.getColumnInternal(lhs_storage->buff_,
840  fixedup_lhs,
841  order_entry.tle_no - 1,
842  lhs_storage_lookup_result);
843  const auto rhs_v = buffer_itr_.getColumnInternal(rhs_storage->buff_,
844  fixedup_rhs,
845  order_entry.tle_no - 1,
846  rhs_storage_lookup_result);
847 
848  if (UNLIKELY(isNull(entry_ti, lhs_v, float_argument_input) &&
849  isNull(entry_ti, rhs_v, float_argument_input))) {
850  continue;
851  }
852  if (UNLIKELY(isNull(entry_ti, lhs_v, float_argument_input) &&
853  !isNull(entry_ti, rhs_v, float_argument_input))) {
854  return order_entry.nulls_first;
855  }
856  if (UNLIKELY(isNull(entry_ti, rhs_v, float_argument_input) &&
857  !isNull(entry_ti, lhs_v, float_argument_input))) {
858  return !order_entry.nulls_first;
859  }
860 
861  if (LIKELY(lhs_v.isInt())) {
862  CHECK(rhs_v.isInt());
863  if (UNLIKELY(entry_ti.is_string() &&
864  entry_ti.get_compression() == kENCODING_DICT)) {
865  CHECK_EQ(4, entry_ti.get_logical_size());
866  CHECK(executor_);
867  const auto string_dict_proxy = executor_->getStringDictionaryProxy(
868  entry_ti.get_comp_param(), result_set_->row_set_mem_owner_, false);
869  auto lhs_str = string_dict_proxy->getString(lhs_v.i1);
870  auto rhs_str = string_dict_proxy->getString(rhs_v.i1);
871  if (lhs_str == rhs_str) {
872  continue;
873  }
874  return (lhs_str < rhs_str) != order_entry.is_desc;
875  }
876 
877  if (lhs_v.i1 == rhs_v.i1) {
878  continue;
879  }
880  if (entry_ti.is_fp()) {
881  if (float_argument_input) {
882  const auto lhs_dval = *reinterpret_cast<const float*>(may_alias_ptr(&lhs_v.i1));
883  const auto rhs_dval = *reinterpret_cast<const float*>(may_alias_ptr(&rhs_v.i1));
884  return (lhs_dval < rhs_dval) != order_entry.is_desc;
885  } else {
886  const auto lhs_dval =
887  *reinterpret_cast<const double*>(may_alias_ptr(&lhs_v.i1));
888  const auto rhs_dval =
889  *reinterpret_cast<const double*>(may_alias_ptr(&rhs_v.i1));
890  return (lhs_dval < rhs_dval) != order_entry.is_desc;
891  }
892  }
893  return (lhs_v.i1 < rhs_v.i1) != order_entry.is_desc;
894  } else {
895  if (lhs_v.isPair()) {
896  CHECK(rhs_v.isPair());
897  const auto lhs =
898  pair_to_double({lhs_v.i1, lhs_v.i2}, entry_ti, float_argument_input);
899  const auto rhs =
900  pair_to_double({rhs_v.i1, rhs_v.i2}, entry_ti, float_argument_input);
901  if (lhs == rhs) {
902  continue;
903  }
904  return (lhs < rhs) != order_entry.is_desc;
905  } else {
906  CHECK(lhs_v.isStr() && rhs_v.isStr());
907  const auto lhs = lhs_v.strVal();
908  const auto rhs = rhs_v.strVal();
909  if (lhs == rhs) {
910  continue;
911  }
912  return (lhs < rhs) != order_entry.is_desc;
913  }
914  }
915  }
916  return false;
917 }
#define CHECK_EQ(x, y)
Definition: Logger.h:214
#define NULL_DOUBLE
const BufferIteratorType buffer_itr_
Definition: ResultSet.h:658
const Executor * executor_
Definition: ResultSet.h:659
static bool isNull(const SQLTypeInfo &ti, const InternalTargetValue &val, const bool float_argument_input)
#define CHECK_GE(x, y)
Definition: Logger.h:219
double pair_to_double(const std::pair< int64_t, int64_t > &fp_pair, const SQLTypeInfo &ti, const bool float_argument_input)
bool takes_float_argument(const TargetInfo &target_info)
Definition: TargetInfo.h:134
const SQLTypeInfo get_compact_type(const TargetInfo &target)
const ResultSet * result_set_
Definition: ResultSet.h:656
const std::list< Analyzer::OrderEntry > & order_entries_
Definition: ResultSet.h:655
#define LIKELY(x)
Definition: likely.h:24
bool is_distinct_target(const TargetInfo &target_info)
Definition: TargetInfo.h:130
#define UNLIKELY(x)
Definition: likely.h:25
#define CHECK_LT(x, y)
Definition: Logger.h:216
#define CHECK(condition)
Definition: Logger.h:206
std::vector< std::vector< int64_t > > count_distinct_materialized_buffers_
Definition: ResultSet.h:661
const ApproxMedianBuffers approx_median_materialized_buffers_
Definition: ResultSet.h:662

+ Here is the call graph for this function:

Member Data Documentation

template<typename BUFFER_ITERATOR_TYPE>
const ApproxMedianBuffers ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::approx_median_materialized_buffers_

Definition at line 662 of file ResultSet.h.

template<typename BUFFER_ITERATOR_TYPE>
const BufferIteratorType ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::buffer_itr_

Definition at line 658 of file ResultSet.h.

template<typename BUFFER_ITERATOR_TYPE>
std::vector<std::vector<int64_t> > ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::count_distinct_materialized_buffers_

Definition at line 661 of file ResultSet.h.

template<typename BUFFER_ITERATOR_TYPE>
const Executor* ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::executor_

Definition at line 659 of file ResultSet.h.

template<typename BUFFER_ITERATOR_TYPE>
const std::list<Analyzer::OrderEntry>& ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::order_entries_

Definition at line 655 of file ResultSet.h.

template<typename BUFFER_ITERATOR_TYPE>
const PermutationView ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::permutation_

Definition at line 657 of file ResultSet.h.

template<typename BUFFER_ITERATOR_TYPE>
const ResultSet* ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::result_set_

Definition at line 656 of file ResultSet.h.

template<typename BUFFER_ITERATOR_TYPE>
const bool ResultSet::ResultSetComparator< BUFFER_ITERATOR_TYPE >::single_threaded_

Definition at line 660 of file ResultSet.h.


The documentation for this struct was generated from the following files: