OmniSciDB  ca0c39ec8f
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
ResultSet.h
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
23 #ifndef QUERYENGINE_RESULTSET_H
24 #define QUERYENGINE_RESULTSET_H
25 
26 #include "CardinalityEstimator.h"
27 #include "DataMgr/Chunk/Chunk.h"
29 #include "ResultSetStorage.h"
30 #include "Shared/quantile.h"
31 #include "TargetValue.h"
32 
33 #include <algorithm>
34 #include <atomic>
35 #include <functional>
36 #include <list>
37 #include <optional>
38 
39 /*
40  * Stores the underlying buffer and the meta-data for a result set. The buffer
41  * format reflects the main requirements for result sets. Not all queries
42  * specify a GROUP BY clause, but since it's the most important and challenging
43  * case we'll focus on it. Note that the meta-data is stored separately from
44  * the buffer and it's not transferred to GPU.
45  *
46  * 1. It has to be efficient for reduction of partial GROUP BY query results
47  * from multiple devices / cores, the cardinalities can be high. Reduction
48  * currently happens on the host.
49  * 2. No conversions should be needed when buffers are transferred from GPU to
50  * host for reduction. This implies the buffer needs to be "flat", with no
51  * pointers to chase since they have no meaning in a different address space.
52  * 3. Must be size-efficient.
53  *
54  * There are several variations of the format of a result set buffer, but the
55  * most common is a sequence of entries which represent a row in the result or
56  * an empty slot. One entry looks as follows:
57  *
58  * +-+-+-+-+-+-+-+-+-+-+-+--?--+-+-+-+-+-+-+-+-+-+-+-+-+
59  * |key_0| ... |key_N-1| padding |value_0|...|value_N-1|
60  * +-+-+-+-+-+-+-+-+-+-+-+--?--+-+-+-+-+-+-+-+-+-+-+-+-+
61  *
62  * (key_0 ... key_N-1) is a multiple component key, unique within the buffer.
63  * It stores the tuple specified by the GROUP BY clause. All components have
64  * the same width, 4 or 8 bytes. For the 4-byte components, 4-byte padding is
65  * added if the number of components is odd. Not all entries in the buffer are
66  * valid; an empty entry contains EMPTY_KEY_{64, 32} for 8-byte / 4-byte width,
67  * respectively. An empty entry is ignored by subsequent operations on the
68  * result set (reduction, iteration, sort etc).
69  *
70  * value_0 through value_N-1 are 8-byte fields which hold the columns of the
71  * result, like aggregates and projected expressions. They're reduced between
72  * multiple partial results for identical (key_0 ... key_N-1) tuples.
73  *
74  * The order of entries is decided by the type of hash used, which depends on
75  * the range of the keys. For small enough ranges, a perfect hash is used. When
76  * a perfect hash isn't feasible, open addressing (using MurmurHash) with linear
77  * probing is used instead, with a 50% fill rate.
78  */
79 
80 struct ReductionCode;
81 
82 namespace Analyzer {
83 
84 class Expr;
85 class Estimator;
86 struct OrderEntry;
87 
88 } // namespace Analyzer
89 
90 class Executor;
91 
92 class ResultSet;
93 
95  public:
96  using value_type = std::vector<TargetValue>;
97  using difference_type = std::ptrdiff_t;
98  using pointer = std::vector<TargetValue>*;
99  using reference = std::vector<TargetValue>&;
100  using iterator_category = std::input_iterator_tag;
101 
102  bool operator==(const ResultSetRowIterator& other) const {
103  return result_set_ == other.result_set_ &&
105  }
106  bool operator!=(const ResultSetRowIterator& other) const { return !(*this == other); }
107 
108  inline value_type operator*() const;
109  inline ResultSetRowIterator& operator++(void);
111  ResultSetRowIterator iter(*this);
112  ++(*this);
113  return iter;
114  }
115 
116  size_t getCurrentRowBufferIndex() const {
117  if (crt_row_buff_idx_ == 0) {
118  throw std::runtime_error("current row buffer iteration index is undefined");
119  }
120  return crt_row_buff_idx_ - 1;
121  }
122 
123  private:
131 
133  bool translate_strings,
134  bool decimal_to_double)
135  : result_set_(rs)
136  , crt_row_buff_idx_(0)
137  , global_entry_idx_(0)
139  , fetched_so_far_(0)
140  , translate_strings_(translate_strings)
141  , decimal_to_double_(decimal_to_double){};
142 
144 
145  friend class ResultSet;
146 };
147 
148 class TSerializedRows;
149 class ResultSetBuilder;
150 
151 using AppendedStorage = std::vector<std::unique_ptr<ResultSetStorage>>;
152 using PermutationIdx = uint32_t;
153 using Permutation = std::vector<PermutationIdx>;
155 using Comparator = std::function<bool(const PermutationIdx, const PermutationIdx)>;
156 
157 class ResultSet {
158  public:
160  // Can use derivatives of the builder class to construct a ResultSet
161 
162  ResultSet(const std::vector<TargetInfo>& targets,
163  const ExecutorDeviceType device_type,
165  const std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner,
166  const Catalog_Namespace::Catalog* catalog,
167  const unsigned block_size,
168  const unsigned grid_size);
169 
170  ResultSet(const std::vector<TargetInfo>& targets,
171  const std::vector<ColumnLazyFetchInfo>& lazy_fetch_info,
172  const std::vector<std::vector<const int8_t*>>& col_buffers,
173  const std::vector<std::vector<int64_t>>& frag_offsets,
174  const std::vector<int64_t>& consistent_frag_sizes,
175  const ExecutorDeviceType device_type,
176  const int device_id,
178  const std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner,
179  const Catalog_Namespace::Catalog* catalog,
180  const unsigned block_size,
181  const unsigned grid_size);
182 
183  ResultSet(const std::shared_ptr<const Analyzer::Estimator>,
184  const ExecutorDeviceType device_type,
185  const int device_id,
186  Data_Namespace::DataMgr* data_mgr);
187 
188  ResultSet(const std::string& explanation);
189 
190  ResultSet(int64_t queue_time_ms,
191  int64_t render_time_ms,
192  const std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner);
193 
194  ~ResultSet();
195 
196  std::string toString() const {
197  return typeName(this) + "(targets=" + ::toString(targets_) +
198  ", query_mem_desc=" + ::toString(query_mem_desc_) + ")";
199  }
200 
201  std::string summaryToString() const;
202 
203  inline ResultSetRowIterator rowIterator(size_t from_logical_index,
204  bool translate_strings,
205  bool decimal_to_double) const {
206  ResultSetRowIterator rowIterator(this, translate_strings, decimal_to_double);
207 
208  // move to first logical position
209  ++rowIterator;
210 
211  for (size_t index = 0; index < from_logical_index; index++) {
212  ++rowIterator;
213  }
214 
215  return rowIterator;
216  }
217 
218  inline ResultSetRowIterator rowIterator(bool translate_strings,
219  bool decimal_to_double) const {
220  return rowIterator(0, translate_strings, decimal_to_double);
221  }
222 
224 
225  const ResultSetStorage* allocateStorage() const;
226 
228  int8_t*,
229  const std::vector<int64_t>&,
230  std::shared_ptr<VarlenOutputInfo> = nullptr) const;
231 
232  const ResultSetStorage* allocateStorage(const std::vector<int64_t>&) const;
233 
234  void updateStorageEntryCount(const size_t new_entry_count) {
238  query_mem_desc_.setEntryCount(new_entry_count);
239  CHECK(storage_);
240  storage_->updateEntryCount(new_entry_count);
241  }
242 
243  std::vector<TargetValue> getNextRow(const bool translate_strings,
244  const bool decimal_to_double) const;
245 
246  size_t getCurrentRowBufferIndex() const;
247 
248  std::vector<TargetValue> getRowAt(const size_t index) const;
249 
250  TargetValue getRowAt(const size_t row_idx,
251  const size_t col_idx,
252  const bool translate_strings,
253  const bool decimal_to_double = true) const;
254 
255  // Specialized random access getter for result sets with a single column to
256  // avoid the overhead of building a std::vector<TargetValue> result with only
257  // one element. Only used by RelAlgTranslator::getInIntegerSetExpr currently.
258  OneIntegerColumnRow getOneColRow(const size_t index) const;
259 
260  std::vector<TargetValue> getRowAtNoTranslations(
261  const size_t index,
262  const std::vector<bool>& targets_to_skip = {}) const;
263 
264  bool isRowAtEmpty(const size_t index) const;
265 
266  void sort(const std::list<Analyzer::OrderEntry>& order_entries,
267  size_t top_n,
268  const Executor* executor);
269 
270  void keepFirstN(const size_t n);
271 
272  void dropFirstN(const size_t n);
273 
274  void append(ResultSet& that);
275 
276  const ResultSetStorage* getStorage() const;
277 
278  size_t colCount() const;
279 
280  SQLTypeInfo getColType(const size_t col_idx) const;
281 
311  size_t rowCount(const bool force_parallel = false) const;
312 
313  void invalidateCachedRowCount() const;
314 
315  void setCachedRowCount(const size_t row_count) const;
316 
332  bool isEmpty() const;
333 
349  size_t entryCount() const;
350 
351  size_t getBufferSizeBytes(const ExecutorDeviceType device_type) const;
352 
353  bool definitelyHasNoRows() const;
354 
355  const QueryMemoryDescriptor& getQueryMemDesc() const;
356 
357  const std::vector<TargetInfo>& getTargetInfos() const;
358 
359  const std::vector<int64_t>& getTargetInitVals() const;
360 
361  int8_t* getDeviceEstimatorBuffer() const;
362 
363  int8_t* getHostEstimatorBuffer() const;
364 
365  void syncEstimatorBuffer() const;
366 
367  size_t getNDVEstimator() const;
368 
370  // all in ms
372  int64_t render_time{0};
374  int64_t kernel_queue_time{0};
375  };
376 
377  void setQueueTime(const int64_t queue_time);
378  void setKernelQueueTime(const int64_t kernel_queue_time);
379  void addCompilationQueueTime(const int64_t compilation_queue_time);
380 
381  int64_t getQueueTime() const;
382  int64_t getRenderTime() const;
383 
384  void moveToBegin() const;
385 
386  bool isTruncated() const;
387 
388  bool isExplain() const;
389 
390  void setValidationOnlyRes();
391  bool isValidationOnlyRes() const;
392 
393  std::string getExplanation() const {
394  if (just_explain_) {
395  return explanation_;
396  }
397  return {};
398  }
399 
400  bool isGeoColOnGpu(const size_t col_idx) const;
401  int getDeviceId() const;
402 
403  // Called from the executor because in the new ResultSet we assume the 'padded' field
404  // in SlotSize already contains the padding, whereas in the executor it's computed.
405  // Once the buffer initialization moves to ResultSet we can remove this method.
407 
408  void fillOneEntry(const std::vector<int64_t>& entry) {
409  CHECK(storage_);
410  if (storage_->query_mem_desc_.didOutputColumnar()) {
411  storage_->fillOneEntryColWise(entry);
412  } else {
413  storage_->fillOneEntryRowWise(entry);
414  }
415  }
416 
417  void initializeStorage() const;
418 
419  void holdChunks(const std::list<std::shared_ptr<Chunk_NS::Chunk>>& chunks) {
420  chunks_ = chunks;
421  }
422  void holdChunkIterators(const std::shared_ptr<std::list<ChunkIter>> chunk_iters) {
423  chunk_iters_.push_back(chunk_iters);
424  }
425  void holdLiterals(std::vector<int8_t>& literal_buff) {
426  literal_buffers_.push_back(std::move(literal_buff));
427  }
428 
429  std::shared_ptr<RowSetMemoryOwner> getRowSetMemOwner() const {
430  return row_set_mem_owner_;
431  }
432 
433  const Permutation& getPermutationBuffer() const;
434  const bool isPermutationBufferEmpty() const { return permutation_.empty(); };
435 
436  void serialize(TSerializedRows& serialized_rows) const;
437 
438  static std::unique_ptr<ResultSet> unserialize(const TSerializedRows& serialized_rows,
439  const Executor*);
440 
441  size_t getLimit() const;
442 
443  // APIs for data recycler
444  ResultSetPtr copy();
445 
447  if (!permutation_.empty()) {
448  permutation_.clear();
449  }
450  }
451 
452  void initStatus() {
453  // todo(yoonmin): what else we additionally need to consider
454  // to make completely clear status of the resultset for reuse?
455  crt_row_buff_idx_ = 0;
456  fetched_so_far_ = 0;
460  drop_first_ = 0;
461  keep_first_ = 0;
462  }
463 
465  if (!chunks_.empty()) {
466  chunks_.clear();
467  }
468  if (!chunk_iters_.empty()) {
469  chunk_iters_.clear();
470  }
471  };
472 
473  const bool isEstimator() const { return !estimator_; }
474 
475  void setCached(bool val) { cached_ = val; }
476 
477  const bool isCached() const { return cached_; }
478 
479  void setExecTime(const long exec_time) { query_exec_time_ = exec_time; }
480 
481  const long getExecTime() const { return query_exec_time_; }
482 
483  void setQueryPlanHash(const QueryPlanHash query_plan) { query_plan_ = query_plan; }
484 
486 
487  std::unordered_set<size_t> getInputTableKeys() const { return input_table_keys_; }
488 
489  void setInputTableKeys(std::unordered_set<size_t>&& intput_table_keys) {
490  input_table_keys_ = std::move(intput_table_keys);
491  }
492 
493  void setTargetMetaInfo(const std::vector<TargetMetaInfo>& target_meta_info) {
494  std::copy(target_meta_info.begin(),
495  target_meta_info.end(),
496  std::back_inserter(target_meta_info_));
497  }
498 
499  std::vector<TargetMetaInfo> getTargetMetaInfo() { return target_meta_info_; }
500 
501  std::optional<bool> canUseSpeculativeTopNSort() const {
503  }
504 
506 
507  const bool hasValidBuffer() const {
508  if (storage_) {
509  return true;
510  }
511  return false;
512  }
513 
514  unsigned getBlockSize() const { return block_size_; }
515 
516  unsigned getGridSize() const { return grid_size_; }
517 
521  enum class GeoReturnType {
524  WktString,
529  };
532 
533  void copyColumnIntoBuffer(const size_t column_idx,
534  int8_t* output_buffer,
535  const size_t output_buffer_size) const;
536 
538 
539  bool didOutputColumnar() const { return this->query_mem_desc_.didOutputColumnar(); }
540 
541  bool isZeroCopyColumnarConversionPossible(size_t column_idx) const;
542  const int8_t* getColumnarBuffer(size_t column_idx) const;
543 
546  }
547 
548  const int8_t getPaddedSlotWidthBytes(const size_t slot_idx) const {
549  return query_mem_desc_.getPaddedSlotWidthBytes(slot_idx);
550  }
551 
552  // returns a bitmap of all single-slot targets, as well as its count
553  std::tuple<std::vector<bool>, size_t> getSingleSlotTargetBitmap() const;
554 
555  std::tuple<std::vector<bool>, size_t> getSupportedSingleSlotTargetBitmap() const;
556 
557  std::vector<size_t> getSlotIndicesForTargetIndices() const;
558 
559  const std::vector<ColumnLazyFetchInfo>& getLazyFetchInfo() const {
560  return lazy_fetch_info_;
561  }
562 
564  auto is_lazy = [](auto const& info) { return info.is_lazily_fetched; };
565  return std::any_of(lazy_fetch_info_.begin(), lazy_fetch_info_.end(), is_lazy);
566  }
567 
568  size_t getNumColumnsLazyFetched() const {
569  auto is_lazy = [](auto const& info) { return info.is_lazily_fetched; };
570  return std::count_if(lazy_fetch_info_.begin(), lazy_fetch_info_.end(), is_lazy);
571  }
572 
573  void setSeparateVarlenStorageValid(const bool val) {
575  }
576 
577  const std::vector<std::string> getStringDictionaryPayloadCopy(const int dict_id) const;
578 
579  const std::pair<std::vector<int32_t>, std::vector<std::string>>
580  getUniqueStringsForDictEncodedTargetCol(const size_t col_idx) const;
581 
582  StringDictionaryProxy* getStringDictionaryProxy(int const dict_id) const;
583 
584  template <typename ENTRY_TYPE, QueryDescriptionType QUERY_TYPE, bool COLUMNAR_FORMAT>
585  ENTRY_TYPE getEntryAt(const size_t row_idx,
586  const size_t target_idx,
587  const size_t slot_idx) const;
588 
589  ChunkStats getTableFunctionChunkStats(const size_t target_idx) const;
590 
591  static double calculateQuantile(quantile::TDigest* const t_digest);
592 
593  void translateDictEncodedColumns(std::vector<TargetInfo> const&,
594  size_t const start_idx);
595 
597  size_t prev_target_idx_{0};
599  size_t agg_idx_{0};
600  int8_t const* buf_ptr_{nullptr};
601  int8_t compact_sz1_;
602  };
603 
604  class CellCallback;
605  void eachCellInColumn(RowIterationState&, CellCallback const&);
606 
607  const Executor* getExecutor() const { return query_mem_desc_.getExecutor(); }
608 
609  private:
611 
612  std::vector<TargetValue> getNextRowImpl(const bool translate_strings,
613  const bool decimal_to_double) const;
614 
615  std::vector<TargetValue> getNextRowUnlocked(const bool translate_strings,
616  const bool decimal_to_double) const;
617 
618  std::vector<TargetValue> getRowAt(const size_t index,
619  const bool translate_strings,
620  const bool decimal_to_double,
621  const bool fixup_count_distinct_pointers,
622  const std::vector<bool>& targets_to_skip = {}) const;
623 
624  // NOTE: just for direct columnarization use at the moment
625  template <typename ENTRY_TYPE>
626  ENTRY_TYPE getColumnarPerfectHashEntryAt(const size_t row_idx,
627  const size_t target_idx,
628  const size_t slot_idx) const;
629 
630  template <typename ENTRY_TYPE>
631  ENTRY_TYPE getRowWisePerfectHashEntryAt(const size_t row_idx,
632  const size_t target_idx,
633  const size_t slot_idx) const;
634 
635  template <typename ENTRY_TYPE>
636  ENTRY_TYPE getRowWiseBaselineEntryAt(const size_t row_idx,
637  const size_t target_idx,
638  const size_t slot_idx) const;
639 
640  template <typename ENTRY_TYPE>
641  ENTRY_TYPE getColumnarBaselineEntryAt(const size_t row_idx,
642  const size_t target_idx,
643  const size_t slot_idx) const;
644 
645  size_t binSearchRowCount() const;
646 
647  size_t parallelRowCount() const;
648 
649  size_t advanceCursorToNextEntry() const;
650 
651  void radixSortOnGpu(const std::list<Analyzer::OrderEntry>& order_entries) const;
652 
653  void radixSortOnCpu(const std::list<Analyzer::OrderEntry>& order_entries) const;
654 
655  static bool isNull(const SQLTypeInfo& ti,
656  const InternalTargetValue& val,
657  const bool float_argument_input);
658 
660  int8_t* rowwise_target_ptr,
661  int8_t* keys_ptr,
662  const size_t entry_buff_idx,
663  const TargetInfo& target_info,
664  const size_t target_logical_idx,
665  const size_t slot_idx,
666  const bool translate_strings,
667  const bool decimal_to_double,
668  const bool fixup_count_distinct_pointers) const;
669 
670  TargetValue getTargetValueFromBufferColwise(const int8_t* col_ptr,
671  const int8_t* keys_ptr,
673  const size_t local_entry_idx,
674  const size_t global_entry_idx,
675  const TargetInfo& target_info,
676  const size_t target_logical_idx,
677  const size_t slot_idx,
678  const bool translate_strings,
679  const bool decimal_to_double) const;
680 
681  TargetValue makeTargetValue(const int8_t* ptr,
682  const int8_t compact_sz,
683  const TargetInfo& target_info,
684  const size_t target_logical_idx,
685  const bool translate_strings,
686  const bool decimal_to_double,
687  const size_t entry_buff_idx) const;
688 
689  TargetValue makeVarlenTargetValue(const int8_t* ptr1,
690  const int8_t compact_sz1,
691  const int8_t* ptr2,
692  const int8_t compact_sz2,
693  const TargetInfo& target_info,
694  const size_t target_logical_idx,
695  const bool translate_strings,
696  const size_t entry_buff_idx) const;
697 
699  int8_t* ptr1;
700  int8_t compact_sz1;
701  int8_t* ptr2;
702  int8_t compact_sz2;
703 
705  : ptr1(nullptr), compact_sz1(0), ptr2(nullptr), compact_sz2(0) {}
706  };
707  TargetValue makeGeoTargetValue(const int8_t* geo_target_ptr,
708  const size_t slot_idx,
709  const TargetInfo& target_info,
710  const size_t target_logical_idx,
711  const size_t entry_buff_idx) const;
712 
715  const size_t fixedup_entry_idx;
716  const size_t storage_idx;
717  };
718 
719  InternalTargetValue getVarlenOrderEntry(const int64_t str_ptr,
720  const size_t str_len) const;
721 
722  int64_t lazyReadInt(const int64_t ival,
723  const size_t target_logical_idx,
724  const StorageLookupResult& storage_lookup_result) const;
725 
729  std::pair<size_t, size_t> getStorageIndex(const size_t entry_idx) const;
730 
731  const std::vector<const int8_t*>& getColumnFrag(const size_t storge_idx,
732  const size_t col_logical_idx,
733  int64_t& global_idx) const;
734 
735  const VarlenOutputInfo* getVarlenOutputInfo(const size_t entry_idx) const;
736 
737  StorageLookupResult findStorage(const size_t entry_idx) const;
738 
739  struct TargetOffsets {
740  const int8_t* ptr1;
741  const size_t compact_sz1;
742  const int8_t* ptr2;
743  const size_t compact_sz2;
744  };
745 
747  RowWiseTargetAccessor(const ResultSet* result_set)
748  : result_set_(result_set)
750  , key_width_(result_set_->query_mem_desc_.getEffectiveKeyWidth())
754  }
755 
757  const int8_t* buff,
758  const size_t entry_idx,
759  const size_t target_logical_idx,
760  const StorageLookupResult& storage_lookup_result) const;
761 
763 
764  inline const int8_t* get_rowwise_ptr(const int8_t* buff,
765  const size_t entry_idx) const {
766  return buff + entry_idx * row_bytes_;
767  }
768 
769  std::vector<std::vector<TargetOffsets>> offsets_for_storage_;
770 
772 
773  // Row-wise iteration
774  const size_t row_bytes_;
775  const size_t key_width_;
777  };
778 
780  ColumnWiseTargetAccessor(const ResultSet* result_set) : result_set_(result_set) {
782  }
783 
785 
787  const int8_t* buff,
788  const size_t entry_idx,
789  const size_t target_logical_idx,
790  const StorageLookupResult& storage_lookup_result) const;
791 
792  std::vector<std::vector<TargetOffsets>> offsets_for_storage_;
793 
795  };
796 
797  using ApproxQuantileBuffers = std::vector<std::vector<double>>;
798 
799  template <typename BUFFER_ITERATOR_TYPE>
801  using BufferIteratorType = BUFFER_ITERATOR_TYPE;
802 
803  ResultSetComparator(const std::list<Analyzer::OrderEntry>& order_entries,
804  const ResultSet* result_set,
805  const PermutationView permutation,
806  const Executor* executor,
807  const bool single_threaded)
808  : order_entries_(order_entries)
809  , result_set_(result_set)
810  , permutation_(permutation)
811  , buffer_itr_(result_set)
812  , executor_(executor)
813  , single_threaded_(single_threaded)
816  }
817 
820 
821  std::vector<int64_t> materializeCountDistinctColumn(
822  const Analyzer::OrderEntry& order_entry) const;
823  ApproxQuantileBuffers::value_type materializeApproxQuantileColumn(
824  const Analyzer::OrderEntry& order_entry) const;
825 
826  bool operator()(const PermutationIdx lhs, const PermutationIdx rhs) const;
827 
828  const std::list<Analyzer::OrderEntry>& order_entries_;
832  const Executor* executor_;
833  const bool single_threaded_;
834  std::vector<std::vector<int64_t>> count_distinct_materialized_buffers_;
836  };
837 
838  Comparator createComparator(const std::list<Analyzer::OrderEntry>& order_entries,
839  const PermutationView permutation,
840  const Executor* executor,
841  const bool single_threaded) {
842  auto timer = DEBUG_TIMER(__func__);
845  order_entries, this, permutation, executor, single_threaded)](
846  const PermutationIdx lhs, const PermutationIdx rhs) {
847  return rsc(lhs, rhs);
848  };
849  } else {
851  order_entries, this, permutation, executor, single_threaded)](
852  const PermutationIdx lhs, const PermutationIdx rhs) {
853  return rsc(lhs, rhs);
854  };
855  }
856  }
857 
859  const size_t n,
860  const Comparator&);
861 
863  PermutationIdx const begin,
864  PermutationIdx const end) const;
865 
866  void parallelTop(const std::list<Analyzer::OrderEntry>& order_entries,
867  const size_t top_n,
868  const Executor* executor);
869 
870  void baselineSort(const std::list<Analyzer::OrderEntry>& order_entries,
871  const size_t top_n,
872  const Executor* executor);
873 
874  void doBaselineSort(const ExecutorDeviceType device_type,
875  const std::list<Analyzer::OrderEntry>& order_entries,
876  const size_t top_n,
877  const Executor* executor);
878 
879  bool canUseFastBaselineSort(const std::list<Analyzer::OrderEntry>& order_entries,
880  const size_t top_n);
881 
882  size_t rowCountImpl(const bool force_parallel) const;
883 
885 
886  int getGpuCount() const;
887 
888  void serializeProjection(TSerializedRows& serialized_rows) const;
889  void serializeVarlenAggColumn(int8_t* buf,
890  std::vector<std::string>& varlen_bufer) const;
891 
892  void serializeCountDistinctColumns(TSerializedRows&) const;
893 
894  void unserializeCountDistinctColumns(const TSerializedRows&);
895 
897 
898  void create_active_buffer_set(CountDistinctSet& count_distinct_active_buffer_set) const;
899 
900  int64_t getDistinctBufferRefFromBufferRowwise(int8_t* rowwise_target_ptr,
901  const TargetInfo& target_info) const;
902 
903  const std::vector<TargetInfo> targets_;
905  const int device_id_;
907  mutable std::unique_ptr<ResultSetStorage> storage_;
909  mutable size_t crt_row_buff_idx_;
910  mutable size_t fetched_so_far_;
911  size_t drop_first_;
912  size_t keep_first_;
913  std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner_;
915 
917  unsigned block_size_{0};
918  unsigned grid_size_{0};
920 
921  std::list<std::shared_ptr<Chunk_NS::Chunk>> chunks_;
922  std::vector<std::shared_ptr<std::list<ChunkIter>>> chunk_iters_;
923  // TODO(miyu): refine by using one buffer and
924  // setting offset instead of ptr in group by buffer.
925  std::vector<std::vector<int8_t>> literal_buffers_;
926  std::vector<ColumnLazyFetchInfo> lazy_fetch_info_;
927  std::vector<std::vector<std::vector<const int8_t*>>> col_buffers_;
928  std::vector<std::vector<std::vector<int64_t>>> frag_offsets_;
929  std::vector<std::vector<int64_t>> consistent_frag_sizes_;
930 
931  const std::shared_ptr<const Analyzer::Estimator> estimator_;
933  mutable int8_t* host_estimator_buffer_{nullptr};
935 
936  // only used by serialization
937  using SerializedVarlenBufferStorage = std::vector<std::string>;
938 
939  std::vector<SerializedVarlenBufferStorage> serialized_varlen_buffer_;
941  std::string explanation_;
942  const bool just_explain_;
944  mutable std::atomic<int64_t> cached_row_count_;
945  mutable std::mutex row_iteration_mutex_;
946 
947  // only used by geo
949 
950  // only used by data recycler
951  bool cached_; // indicator that this resultset is cached
952  size_t
953  query_exec_time_; // an elapsed time to process the query for this resultset (ms)
954  QueryPlanHash query_plan_; // a hashed query plan DAG of this resultset
955  std::unordered_set<size_t> input_table_keys_; // input table signatures
956  std::vector<TargetMetaInfo> target_meta_info_;
957  // if we recycle the resultset, we do not create work_unit of the query step
958  // because we may skip its child query step(s)
959  // so we try to keep whether this resultset is available to use speculative top n sort
960  // when it is inserted to the recycler, and reuse this info when recycled
961  std::optional<bool> can_use_speculative_top_n_sort;
962 
963  friend class ResultSetManager;
964  friend class ResultSetRowIterator;
965  friend class ColumnarResults;
966 };
967 
970  return {};
971  }
972 
973  if (result_set_->just_explain_) {
974  return {result_set_->explanation_};
975  }
976 
977  return result_set_->getRowAt(
979 }
980 
982  if (!result_set_->storage_ && !result_set_->just_explain_) {
983  global_entry_idx_valid_ = false;
984  } else if (result_set_->just_explain_) {
986  fetched_so_far_ = 1;
987  } else {
988  result_set_->advanceCursorToNextEntry(*this);
989  }
990  return *this;
991 }
992 
994  public:
995  ResultSet* reduce(std::vector<ResultSet*>&, const size_t executor_id);
996 
997  std::shared_ptr<ResultSet> getOwnResultSet();
998 
999  void rewriteVarlenAggregates(ResultSet*);
1000 
1001  private:
1002  std::shared_ptr<ResultSet> rs_;
1003 };
1004 
1005 class RowSortException : public std::runtime_error {
1006  public:
1007  RowSortException(const std::string& cause) : std::runtime_error(cause) {}
1008 };
1009 
1010 namespace result_set {
1011 
1012 bool can_use_parallel_algorithms(const ResultSet& rows);
1013 
1014 std::optional<size_t> first_dict_encoded_idx(std::vector<TargetInfo> const&);
1015 
1016 bool use_parallel_algorithms(const ResultSet& rows);
1017 
1018 } // namespace result_set
1019 
1020 #endif // QUERYENGINE_RESULTSET_H
void setSeparateVarlenStorageValid(const bool val)
Definition: ResultSet.h:573
void setGeoReturnType(const GeoReturnType val)
Definition: ResultSet.h:531
void serializeVarlenAggColumn(int8_t *buf, std::vector< std::string > &varlen_bufer) const
std::mutex row_iteration_mutex_
Definition: ResultSet.h:945
void setUseSpeculativeTopNSort(bool value)
Definition: ResultSet.h:505
InternalTargetValue getColumnInternal(const int8_t *buff, const size_t entry_idx, const size_t target_logical_idx, const StorageLookupResult &storage_lookup_result) const
void syncEstimatorBuffer() const
Definition: ResultSet.cpp:696
const int8_t * ptr1
Definition: ResultSet.h:740
const size_t compact_sz2
Definition: ResultSet.h:743
void holdChunks(const std::list< std::shared_ptr< Chunk_NS::Chunk >> &chunks)
Definition: ResultSet.h:419
const QueryMemoryDescriptor & getQueryMemDesc() const
Definition: ResultSet.cpp:672
void setQueryPlanHash(const QueryPlanHash query_plan)
Definition: ResultSet.h:483
robin_hood::unordered_set< int64_t > CountDistinctSet
Definition: CountDistinct.h:35
std::pair< size_t, size_t > getStorageIndex(const size_t entry_idx) const
Definition: ResultSet.cpp:914
std::shared_ptr< RowSetMemoryOwner > getRowSetMemOwner() const
Definition: ResultSet.h:429
bool isValidationOnlyRes() const
Definition: ResultSet.cpp:748
Permutation permutation_
Definition: ResultSet.h:914
bool didOutputColumnar() const
Definition: ResultSet.h:539
void setValidationOnlyRes()
Definition: ResultSet.cpp:744
PermutationView initPermutationBuffer(PermutationView permutation, PermutationIdx const begin, PermutationIdx const end) const
Definition: ResultSet.cpp:846
bool for_validation_only_
Definition: ResultSet.h:943
std::ptrdiff_t difference_type
Definition: ResultSet.h:97
void setCached(bool val)
Definition: ResultSet.h:475
ENTRY_TYPE getRowWisePerfectHashEntryAt(const size_t row_idx, const size_t target_idx, const size_t slot_idx) const
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:132
void create_active_buffer_set(CountDistinctSet &count_distinct_active_buffer_set) const
void setEntryCount(const size_t val)
double decimal_to_double(const SQLTypeInfo &otype, int64_t oval)
const ApproxQuantileBuffers approx_quantile_materialized_buffers_
Definition: ResultSet.h:835
AppendedStorage appended_storage_
Definition: ResultSet.h:908
ENTRY_TYPE getColumnarPerfectHashEntryAt(const size_t row_idx, const size_t target_idx, const size_t slot_idx) const
GeoReturnType geo_return_type_
Definition: ResultSet.h:948
ExecutorDeviceType
const BufferIteratorType buffer_itr_
Definition: ResultSet.h:831
void moveToBegin() const
Definition: ResultSet.cpp:731
Utility functions for easy access to the result set buffers.
std::shared_ptr< ResultSet > rs_
Definition: ResultSet.h:1002
Calculate approximate median and general quantiles, based on &quot;Computing Extremely Accurate Quantiles ...
const Executor * executor_
Definition: ResultSet.h:832
std::vector< std::string > SerializedVarlenBufferStorage
Definition: ResultSet.h:937
void initializeStorage() const
std::optional< bool > can_use_speculative_top_n_sort
Definition: ResultSet.h:961
QueryDescriptionType getQueryDescriptionType() const
Definition: ResultSet.h:544
ResultSetRowIterator(const ResultSet *rs, bool translate_strings, bool decimal_to_double)
Definition: ResultSet.h:132
const Catalog_Namespace::Catalog * catalog_
Definition: ResultSet.h:916
size_t getNumColumnsLazyFetched() const
Definition: ResultSet.h:568
void unserializeCountDistinctColumns(const TSerializedRows &)
ApproxQuantileBuffers materializeApproxQuantileColumns() const
Definition: ResultSet.cpp:959
std::vector< TargetValue > getNextRow(const bool translate_strings, const bool decimal_to_double) const
static bool isNull(const SQLTypeInfo &ti, const InternalTargetValue &val, const bool float_argument_input)
ResultSet(const std::vector< TargetInfo > &targets, const ExecutorDeviceType device_type, const QueryMemoryDescriptor &query_mem_desc, const std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner, const Catalog_Namespace::Catalog *catalog, const unsigned block_size, const unsigned grid_size)
Definition: ResultSet.cpp:62
QueryMemoryDescriptor query_mem_desc_
Definition: ResultSet.h:906
const std::vector< TargetInfo > & getTargetInfos() const
Definition: ResultSet.cpp:677
std::optional< size_t > first_dict_encoded_idx(std::vector< TargetInfo > const &)
Definition: ResultSet.cpp:1475
std::unique_ptr< ResultSetStorage > storage_
Definition: ResultSet.h:907
void setKernelQueueTime(const int64_t kernel_queue_time)
Definition: ResultSet.cpp:714
bool operator==(const ResultSetRowIterator &other) const
Definition: ResultSet.h:102
size_t query_exec_time_
Definition: ResultSet.h:953
std::string getExplanation() const
Definition: ResultSet.h:393
High-level representation of SQL values.
ENTRY_TYPE getEntryAt(const size_t row_idx, const size_t target_idx, const size_t slot_idx) const
size_t rowCount(const bool force_parallel=false) const
Returns the number of valid entries in the result set (i.e that will be returned from the SQL query o...
Definition: ResultSet.cpp:593
ResultSetRowIterator(const ResultSet *rs)
Definition: ResultSet.h:143
std::shared_ptr< ResultSet > ResultSetPtr
TargetValue makeGeoTargetValue(const int8_t *geo_target_ptr, const size_t slot_idx, const TargetInfo &target_info, const size_t target_logical_idx, const size_t entry_buff_idx) const
TargetValue getTargetValueFromBufferRowwise(int8_t *rowwise_target_ptr, int8_t *keys_ptr, const size_t entry_buff_idx, const TargetInfo &target_info, const size_t target_logical_idx, const size_t slot_idx, const bool translate_strings, const bool decimal_to_double, const bool fixup_count_distinct_pointers) const
size_t keep_first_
Definition: ResultSet.h:912
void keepFirstN(const size_t n)
Definition: ResultSet.cpp:52
std::vector< std::shared_ptr< std::list< ChunkIter > > > chunk_iters_
Definition: ResultSet.h:922
void addCompilationQueueTime(const int64_t compilation_queue_time)
Definition: ResultSet.cpp:718
std::vector< std::vector< double >> ApproxQuantileBuffers
Definition: ResultSet.h:797
void parallelTop(const std::list< Analyzer::OrderEntry > &order_entries, const size_t top_n, const Executor *executor)
Definition: ResultSet.cpp:866
void serialize(TSerializedRows &serialized_rows) const
std::vector< SerializedVarlenBufferStorage > serialized_varlen_buffer_
Definition: ResultSet.h:939
const size_t compact_sz1
Definition: ResultSet.h:741
int64_t lazyReadInt(const int64_t ival, const size_t target_logical_idx, const StorageLookupResult &storage_lookup_result) const
bool operator!=(const ResultSetRowIterator &other) const
Definition: ResultSet.h:106
size_t colCount() const
Definition: ResultSet.cpp:413
OneIntegerColumnRow getOneColRow(const size_t index) const
unsigned getBlockSize() const
Definition: ResultSet.h:514
TargetValue getTargetValueFromBufferColwise(const int8_t *col_ptr, const int8_t *keys_ptr, const QueryMemoryDescriptor &query_mem_desc, const size_t local_entry_idx, const size_t global_entry_idx, const TargetInfo &target_info, const size_t target_logical_idx, const size_t slot_idx, const bool translate_strings, const bool decimal_to_double) const
void setInputTableKeys(std::unordered_set< size_t > &&intput_table_keys)
Definition: ResultSet.h:489
void rewriteVarlenAggregates(ResultSet *)
size_t getLimit() const
Definition: ResultSet.cpp:1302
const bool just_explain_
Definition: ResultSet.h:942
std::vector< int64_t > materializeCountDistinctColumn(const Analyzer::OrderEntry &order_entry) const
Definition: ResultSet.cpp:972
ApproxQuantileBuffers::value_type materializeApproxQuantileColumn(const Analyzer::OrderEntry &order_entry) const
Definition: ResultSet.cpp:1018
ResultSetRowIterator rowIterator(size_t from_logical_index, bool translate_strings, bool decimal_to_double) const
Definition: ResultSet.h:203
unsigned block_size_
Definition: ResultSet.h:917
bool isTruncated() const
Definition: ResultSet.cpp:736
std::atomic< int64_t > cached_row_count_
Definition: ResultSet.h:944
const bool isPermutationBufferEmpty() const
Definition: ResultSet.h:434
size_t parallelRowCount() const
Definition: ResultSet.cpp:629
const size_t key_bytes_with_padding_
Definition: ResultSet.h:776
bool areAnyColumnsLazyFetched() const
Definition: ResultSet.h:563
const ResultSet * result_set_
Definition: ResultSet.h:771
std::vector< TargetValue > getRowAtNoTranslations(const size_t index, const std::vector< bool > &targets_to_skip={}) const
const ResultSet * result_set_
Definition: ResultSet.h:124
void radixSortOnCpu(const std::list< Analyzer::OrderEntry > &order_entries) const
Definition: ResultSet.cpp:1262
bool definitelyHasNoRows() const
Definition: ResultSet.cpp:668
ColumnWiseTargetAccessor(const ResultSet *result_set)
Definition: ResultSet.h:780
const std::vector< std::string > getStringDictionaryPayloadCopy(const int dict_id) const
Definition: ResultSet.cpp:1306
bool use_parallel_algorithms(const ResultSet &rows)
Definition: ResultSet.cpp:1482
bool isZeroCopyColumnarConversionPossible(size_t column_idx) const
Definition: ResultSet.cpp:1393
std::input_iterator_tag iterator_category
Definition: ResultSet.h:100
size_t global_entry_idx_
Definition: ResultSet.h:126
int8_t * getHostEstimatorBuffer() const
Definition: ResultSet.cpp:692
InternalTargetValue getVarlenOrderEntry(const int64_t str_ptr, const size_t str_len) const
const std::vector< TargetInfo > targets_
Definition: ResultSet.h:903
void invalidateCachedRowCount() const
Definition: ResultSet.cpp:605
std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner_
Definition: ResultSet.h:913
const Executor * getExecutor() const
Definition: ResultSet.h:607
void setExecTime(const long exec_time)
Definition: ResultSet.h:479
size_t drop_first_
Definition: ResultSet.h:911
const ResultSetStorage * allocateStorage() const
bool cached_
Definition: ResultSet.h:951
std::string toString() const
Definition: ResultSet.h:196
const int8_t * ptr2
Definition: ResultSet.h:742
std::list< std::shared_ptr< Chunk_NS::Chunk > > chunks_
Definition: ResultSet.h:921
std::unordered_set< size_t > getInputTableKeys() const
Definition: ResultSet.h:487
QueryExecutionTimings timings_
Definition: ResultSet.h:919
const ResultSet * result_set_
Definition: ResultSet.h:829
void sort(const std::list< Analyzer::OrderEntry > &order_entries, size_t top_n, const Executor *executor)
Definition: ResultSet.cpp:768
DEVICE auto copy(ARGS &&...args)
Definition: gpu_enabled.h:51
void setQueueTime(const int64_t queue_time)
Definition: ResultSet.cpp:710
void dropFirstN(const size_t n)
Definition: ResultSet.cpp:57
std::vector< std::vector< int8_t > > literal_buffers_
Definition: ResultSet.h:925
std::vector< std::vector< TargetOffsets > > offsets_for_storage_
Definition: ResultSet.h:792
bool canUseFastBaselineSort(const std::list< Analyzer::OrderEntry > &order_entries, const size_t top_n)
unsigned grid_size_
Definition: ResultSet.h:918
void setTargetMetaInfo(const std::vector< TargetMetaInfo > &target_meta_info)
Definition: ResultSet.h:493
std::unordered_set< size_t > input_table_keys_
Definition: ResultSet.h:955
const std::list< Analyzer::OrderEntry > & order_entries_
Definition: ResultSet.h:828
std::vector< TargetValue > & reference
Definition: ResultSet.h:99
std::vector< PermutationIdx > Permutation
Definition: ResultSet.h:153
std::tuple< std::vector< bool >, size_t > getSingleSlotTargetBitmap() const
Definition: ResultSet.cpp:1408
ResultSetRowIterator & operator++(void)
Definition: ResultSet.h:981
std::shared_ptr< ResultSet > getOwnResultSet()
StorageLookupResult findStorage(const size_t entry_idx) const
Definition: ResultSet.cpp:939
const long getExecTime() const
Definition: ResultSet.h:481
Comparator createComparator(const std::list< Analyzer::OrderEntry > &order_entries, const PermutationView permutation, const Executor *executor, const bool single_threaded)
Definition: ResultSet.h:838
An AbstractBuffer is a unit of data management for a data manager.
const PermutationView permutation_
Definition: ResultSet.h:830
const int8_t getPaddedSlotWidthBytes(const size_t slot_idx) const
value_type operator*() const
Definition: ResultSet.h:968
RowWiseTargetAccessor(const ResultSet *result_set)
Definition: ResultSet.h:747
ResultSetPtr copy()
Definition: ResultSet.cpp:329
void copyColumnIntoBuffer(const size_t column_idx, int8_t *output_buffer, const size_t output_buffer_size) const
RowSortException(const std::string &cause)
Definition: ResultSet.h:1007
std::function< bool(const PermutationIdx, const PermutationIdx)> Comparator
Definition: ResultSet.h:155
static double calculateQuantile(quantile::TDigest *const t_digest)
Definition: ResultSet.cpp:1008
friend ResultSetBuilder
Definition: ResultSet.h:159
const bool isEstimator() const
Definition: ResultSet.h:473
const VarlenOutputInfo * getVarlenOutputInfo(const size_t entry_idx) const
void fixupCountDistinctPointers()
void radixSortOnGpu(const std::list< Analyzer::OrderEntry > &order_entries) const
Definition: ResultSet.cpp:1222
const ResultSetStorage * getStorage() const
Definition: ResultSet.cpp:409
QueryDescriptionType getQueryDescriptionType() const
const std::pair< std::vector< int32_t >, std::vector< std::string > > getUniqueStringsForDictEncodedTargetCol(const size_t col_idx) const
Definition: ResultSet.cpp:1315
Data_Namespace::DataMgr * data_mgr_
Definition: ResultSet.h:934
Basic constructors and methods of the row set interface.
int64_t getQueueTime() const
Definition: ResultSet.cpp:722
std::optional< bool > canUseSpeculativeTopNSort() const
Definition: ResultSet.h:501
std::vector< TargetValue > getRowAt(const size_t index) const
Executor(const ExecutorId id, Data_Namespace::DataMgr *data_mgr, const size_t block_size_x, const size_t grid_size_x, const size_t max_gpu_slab_size, const std::string &debug_dir, const std::string &debug_file)
Definition: Execute.cpp:244
void fillOneEntry(const std::vector< int64_t > &entry)
Definition: ResultSet.h:408
void updateStorageEntryCount(const size_t new_entry_count)
Definition: ResultSet.h:234
uint32_t PermutationIdx
Definition: ResultSet.h:152
void serializeProjection(TSerializedRows &serialized_rows) const
ResultSetRowIterator operator++(int)
Definition: ResultSet.h:110
void invalidateResultSetChunks()
Definition: ResultSet.h:464
const std::shared_ptr< const Analyzer::Estimator > estimator_
Definition: ResultSet.h:931
SQLTypeInfo getColType(const size_t col_idx) const
Definition: ResultSet.cpp:417
GeoReturnType getGeoReturnType() const
Definition: ResultSet.h:530
void holdChunkIterators(const std::shared_ptr< std::list< ChunkIter >> chunk_iters)
Definition: ResultSet.h:422
std::tuple< std::vector< bool >, size_t > getSupportedSingleSlotTargetBitmap() const
Definition: ResultSet.cpp:1432
ExecutorDeviceType getDeviceType() const
Definition: ResultSet.cpp:250
StringDictionaryProxy * getStringDictionaryProxy(int const dict_id) const
Definition: ResultSet.cpp:426
const int8_t * getColumnarBuffer(size_t column_idx) const
Definition: ResultSet.cpp:1402
ResultSetComparator(const std::list< Analyzer::OrderEntry > &order_entries, const ResultSet *result_set, const PermutationView permutation, const Executor *executor, const bool single_threaded)
Definition: ResultSet.h:803
size_t get_row_bytes(const QueryMemoryDescriptor &query_mem_desc)
bool isExplain() const
Definition: ResultSet.cpp:740
void eachCellInColumn(RowIterationState &, CellCallback const &)
Definition: ResultSet.cpp:485
std::vector< TargetValue > value_type
Definition: ResultSet.h:96
bool isGeoColOnGpu(const size_t col_idx) const
std::vector< TargetMetaInfo > target_meta_info_
Definition: ResultSet.h:956
const QueryPlanHash getQueryPlanHash()
Definition: ResultSet.h:485
const int8_t * get_rowwise_ptr(const int8_t *buff, const size_t entry_idx) const
Definition: ResultSet.h:764
size_t getNDVEstimator() const
std::vector< std::vector< std::vector< const int8_t * > > > col_buffers_
Definition: ResultSet.h:927
bool isRowAtEmpty(const size_t index) const
const int8_t getPaddedSlotWidthBytes(const size_t slot_idx) const
Definition: ResultSet.h:548
size_t entryCount() const
Returns the number of entries the result set is allocated to hold.
size_t QueryPlanHash
static QueryMemoryDescriptor fixupQueryMemoryDescriptor(const QueryMemoryDescriptor &)
Definition: ResultSet.cpp:756
std::string typeName(const T *v)
Definition: toString.h:103
size_t rowCountImpl(const bool force_parallel) const
Definition: ResultSet.cpp:555
TargetValue makeTargetValue(const int8_t *ptr, const int8_t compact_sz, const TargetInfo &target_info, const size_t target_logical_idx, const bool translate_strings, const bool decimal_to_double, const size_t entry_buff_idx) const
void baselineSort(const std::list< Analyzer::OrderEntry > &order_entries, const size_t top_n, const Executor *executor)
size_t getCurrentRowBufferIndex() const
Definition: ResultSet.h:116
const Permutation & getPermutationBuffer() const
Definition: ResultSet.cpp:862
void append(ResultSet &that)
Definition: ResultSet.cpp:299
std::string summaryToString() const
Definition: ResultSet.cpp:218
std::string explanation_
Definition: ResultSet.h:941
std::vector< std::vector< int64_t > > consistent_frag_sizes_
Definition: ResultSet.h:929
int8_t * host_estimator_buffer_
Definition: ResultSet.h:933
friend class ResultSet
Definition: ResultSet.h:143
const ExecutorDeviceType device_type_
Definition: ResultSet.h:904
const bool hasValidBuffer() const
Definition: ResultSet.h:507
std::vector< TargetValue > getNextRowImpl(const bool translate_strings, const bool decimal_to_double) const
const bool isCached() const
Definition: ResultSet.h:477
static PermutationView topPermutation(PermutationView, const size_t n, const Comparator &)
Definition: ResultSet.cpp:1208
size_t getCurrentRowBufferIndex() const
Definition: ResultSet.cpp:291
void holdLiterals(std::vector< int8_t > &literal_buff)
Definition: ResultSet.h:425
bool g_enable_watchdog false
Definition: Execute.cpp:79
#define CHECK(condition)
Definition: Logger.h:222
#define DEBUG_TIMER(name)
Definition: Logger.h:371
int getGpuCount() const
size_t getBufferSizeBytes(const ExecutorDeviceType device_type) const
int8_t * getDeviceEstimatorBuffer() const
Definition: ResultSet.cpp:686
size_t fetched_so_far_
Definition: ResultSet.h:910
std::vector< ColumnLazyFetchInfo > lazy_fetch_info_
Definition: ResultSet.h:926
size_t crt_row_buff_idx_
Definition: ResultSet.h:909
QueryPlanHash query_plan_
Definition: ResultSet.h:954
Estimators to be used when precise cardinality isn&#39;t useful.
QueryDescriptionType
Definition: Types.h:29
int64_t getDistinctBufferRefFromBufferRowwise(int8_t *rowwise_target_ptr, const TargetInfo &target_info) const
std::vector< std::vector< std::vector< int64_t > > > frag_offsets_
Definition: ResultSet.h:928
bool operator()(const PermutationIdx lhs, const PermutationIdx rhs) const
Definition: ResultSet.cpp:1051
void doBaselineSort(const ExecutorDeviceType device_type, const std::list< Analyzer::OrderEntry > &order_entries, const size_t top_n, const Executor *executor)
bool separate_varlen_storage_valid_
Definition: ResultSet.h:940
boost::variant< ScalarTargetValue, ArrayTargetValue, GeoTargetValue, GeoTargetValuePtr > TargetValue
Definition: TargetValue.h:195
std::vector< TargetValue > getNextRowUnlocked(const bool translate_strings, const bool decimal_to_double) const
std::vector< TargetValue > * pointer
Definition: ResultSet.h:98
bool isEmpty() const
Returns a boolean signifying whether there are valid entries in the result set.
Definition: ResultSet.cpp:649
const std::vector< int64_t > & getTargetInitVals() const
Definition: ResultSet.cpp:681
std::vector< size_t > getSlotIndicesForTargetIndices() const
Definition: ResultSet.cpp:1451
unsigned getGridSize() const
Definition: ResultSet.h:516
size_t advanceCursorToNextEntry() const
constexpr double n
Definition: Utm.h:38
ResultSetRowIterator rowIterator(bool translate_strings, bool decimal_to_double) const
Definition: ResultSet.h:218
Definition: Analyzer.h:2366
BUFFER_ITERATOR_TYPE BufferIteratorType
Definition: ResultSet.h:801
ENTRY_TYPE getColumnarBaselineEntryAt(const size_t row_idx, const size_t target_idx, const size_t slot_idx) const
Data_Namespace::DataMgr * getDataManager() const
size_t crt_row_buff_idx_
Definition: ResultSet.h:125
std::vector< std::vector< int64_t > > count_distinct_materialized_buffers_
Definition: ResultSet.h:834
Data_Namespace::AbstractBuffer * device_estimator_buffer_
Definition: ResultSet.h:932
InternalTargetValue getColumnInternal(const int8_t *buff, const size_t entry_idx, const size_t target_logical_idx, const StorageLookupResult &storage_lookup_result) const
std::vector< std::vector< TargetOffsets > > offsets_for_storage_
Definition: ResultSet.h:769
std::vector< TargetMetaInfo > getTargetMetaInfo()
Definition: ResultSet.h:499
void translateDictEncodedColumns(std::vector< TargetInfo > const &, size_t const start_idx)
Definition: ResultSet.cpp:455
bool global_entry_idx_valid_
Definition: ResultSet.h:127
bool can_use_parallel_algorithms(const ResultSet &rows)
Definition: ResultSet.cpp:1463
int64_t getRenderTime() const
Definition: ResultSet.cpp:727
void setCachedRowCount(const size_t row_count) const
Definition: ResultSet.cpp:609
bool isDirectColumnarConversionPossible() const
Definition: ResultSet.cpp:1371
size_t get_key_bytes_rowwise(const QueryMemoryDescriptor &query_mem_desc)
void initStatus()
Definition: ResultSet.h:452
FORCE_INLINE HOST DEVICE T align_to_int64(T addr)
TargetValue makeVarlenTargetValue(const int8_t *ptr1, const int8_t compact_sz1, const int8_t *ptr2, const int8_t compact_sz2, const TargetInfo &target_info, const size_t target_logical_idx, const bool translate_strings, const size_t entry_buff_idx) const
void serializeCountDistinctColumns(TSerializedRows &) const
const ResultSetStorage * storage_ptr
Definition: ResultSet.h:714
ResultSet * reduce(std::vector< ResultSet * > &, const size_t executor_id)
std::vector< std::unique_ptr< ResultSetStorage >> AppendedStorage
Definition: ResultSet.h:151
const std::vector< const int8_t * > & getColumnFrag(const size_t storge_idx, const size_t col_logical_idx, int64_t &global_idx) const
const Executor * getExecutor() const
size_t binSearchRowCount() const
Definition: ResultSet.cpp:616
void clearPermutation()
Definition: ResultSet.h:446
ChunkStats getTableFunctionChunkStats(const size_t target_idx) const
const std::vector< ColumnLazyFetchInfo > & getLazyFetchInfo() const
Definition: ResultSet.h:559
ENTRY_TYPE getRowWiseBaselineEntryAt(const size_t row_idx, const size_t target_idx, const size_t slot_idx) const
int getDeviceId() const
Definition: ResultSet.cpp:752
static std::unique_ptr< ResultSet > unserialize(const TSerializedRows &serialized_rows, const Executor *)
const int device_id_
Definition: ResultSet.h:905