OmniSciDB  72c90bc290
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
ResultSet.h
Go to the documentation of this file.
1 /*
2  * Copyright 2022 HEAVY.AI, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
23 #ifndef QUERYENGINE_RESULTSET_H
24 #define QUERYENGINE_RESULTSET_H
25 
26 #include "CardinalityEstimator.h"
27 #include "DataMgr/Chunk/Chunk.h"
29 #include "ResultSetStorage.h"
30 #include "Shared/quantile.h"
31 #include "TargetValue.h"
32 
33 #include <algorithm>
34 #include <atomic>
35 #include <functional>
36 #include <list>
37 #include <optional>
38 
39 /*
40  * Stores the underlying buffer and the meta-data for a result set. The buffer
41  * format reflects the main requirements for result sets. Not all queries
42  * specify a GROUP BY clause, but since it's the most important and challenging
43  * case we'll focus on it. Note that the meta-data is stored separately from
44  * the buffer and it's not transferred to GPU.
45  *
46  * 1. It has to be efficient for reduction of partial GROUP BY query results
47  * from multiple devices / cores, the cardinalities can be high. Reduction
48  * currently happens on the host.
49  * 2. No conversions should be needed when buffers are transferred from GPU to
50  * host for reduction. This implies the buffer needs to be "flat", with no
51  * pointers to chase since they have no meaning in a different address space.
52  * 3. Must be size-efficient.
53  *
54  * There are several variations of the format of a result set buffer, but the
55  * most common is a sequence of entries which represent a row in the result or
56  * an empty slot. One entry looks as follows:
57  *
58  * +-+-+-+-+-+-+-+-+-+-+-+--?--+-+-+-+-+-+-+-+-+-+-+-+-+
59  * |key_0| ... |key_N-1| padding |value_0|...|value_N-1|
60  * +-+-+-+-+-+-+-+-+-+-+-+--?--+-+-+-+-+-+-+-+-+-+-+-+-+
61  *
62  * (key_0 ... key_N-1) is a multiple component key, unique within the buffer.
63  * It stores the tuple specified by the GROUP BY clause. All components have
64  * the same width, 4 or 8 bytes. For the 4-byte components, 4-byte padding is
65  * added if the number of components is odd. Not all entries in the buffer are
66  * valid; an empty entry contains EMPTY_KEY_{64, 32} for 8-byte / 4-byte width,
67  * respectively. An empty entry is ignored by subsequent operations on the
68  * result set (reduction, iteration, sort etc).
69  *
70  * value_0 through value_N-1 are 8-byte fields which hold the columns of the
71  * result, like aggregates and projected expressions. They're reduced between
72  * multiple partial results for identical (key_0 ... key_N-1) tuples.
73  *
74  * The order of entries is decided by the type of hash used, which depends on
75  * the range of the keys. For small enough ranges, a perfect hash is used. When
76  * a perfect hash isn't feasible, open addressing (using MurmurHash) with linear
77  * probing is used instead, with a 50% fill rate.
78  */
79 
80 struct ReductionCode;
81 
82 namespace Analyzer {
83 
84 class Expr;
85 class Estimator;
86 struct OrderEntry;
87 
88 } // namespace Analyzer
89 
90 class Executor;
91 
92 class ResultSet;
93 
95  public:
96  using value_type = std::vector<TargetValue>;
97  using difference_type = std::ptrdiff_t;
98  using pointer = std::vector<TargetValue>*;
99  using reference = std::vector<TargetValue>&;
100  using iterator_category = std::input_iterator_tag;
101 
102  bool operator==(const ResultSetRowIterator& other) const {
103  return result_set_ == other.result_set_ &&
105  }
106  bool operator!=(const ResultSetRowIterator& other) const { return !(*this == other); }
107 
108  inline value_type operator*() const;
109  inline ResultSetRowIterator& operator++(void);
111  ResultSetRowIterator iter(*this);
112  ++(*this);
113  return iter;
114  }
115 
116  size_t getCurrentRowBufferIndex() const {
117  if (crt_row_buff_idx_ == 0) {
118  throw std::runtime_error("current row buffer iteration index is undefined");
119  }
120  return crt_row_buff_idx_ - 1;
121  }
122 
123  private:
131 
133  bool translate_strings,
134  bool decimal_to_double)
135  : result_set_(rs)
136  , crt_row_buff_idx_(0)
137  , global_entry_idx_(0)
139  , fetched_so_far_(0)
140  , translate_strings_(translate_strings)
141  , decimal_to_double_(decimal_to_double){};
142 
144 
145  friend class ResultSet;
146 };
147 
148 class TSerializedRows;
149 class ResultSetBuilder;
150 
151 using AppendedStorage = std::vector<std::unique_ptr<ResultSetStorage>>;
152 using PermutationIdx = uint32_t;
153 using Permutation = std::vector<PermutationIdx>;
155 using Comparator = std::function<bool(const PermutationIdx, const PermutationIdx)>;
156 
157 class ResultSet {
158  public:
160  // Can use derivatives of the builder class to construct a ResultSet
161 
162  ResultSet(const std::vector<TargetInfo>& targets,
163  const ExecutorDeviceType device_type,
165  const std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner,
166  const unsigned block_size,
167  const unsigned grid_size);
168 
169  ResultSet(const std::vector<TargetInfo>& targets,
170  const std::vector<ColumnLazyFetchInfo>& lazy_fetch_info,
171  const std::vector<std::vector<const int8_t*>>& col_buffers,
172  const std::vector<std::vector<int64_t>>& frag_offsets,
173  const std::vector<int64_t>& consistent_frag_sizes,
174  const ExecutorDeviceType device_type,
175  const int device_id,
176  const int thread_idx,
178  const std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner,
179  const unsigned block_size,
180  const unsigned grid_size);
181 
182  ResultSet(const std::shared_ptr<const Analyzer::Estimator>,
183  const ExecutorDeviceType device_type,
184  const int device_id,
185  Data_Namespace::DataMgr* data_mgr);
186 
187  ResultSet(const std::string& explanation);
188 
189  ResultSet(int64_t queue_time_ms,
190  int64_t render_time_ms,
191  const std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner);
192 
193  ~ResultSet();
194 
195  std::string toString() const {
196  return typeName(this) + "(targets=" + ::toString(targets_) +
197  ", query_mem_desc=" + ::toString(query_mem_desc_) + ")";
198  }
199 
200  std::string summaryToString() const;
201 
202  inline ResultSetRowIterator rowIterator(size_t from_logical_index,
203  bool translate_strings,
204  bool decimal_to_double) const {
205  ResultSetRowIterator rowIterator(this, translate_strings, decimal_to_double);
206 
207  // move to first logical position
208  ++rowIterator;
209 
210  for (size_t index = 0; index < from_logical_index; index++) {
211  ++rowIterator;
212  }
213 
214  return rowIterator;
215  }
216 
217  inline ResultSetRowIterator rowIterator(bool translate_strings,
218  bool decimal_to_double) const {
219  return rowIterator(0, translate_strings, decimal_to_double);
220  }
221 
223 
224  const ResultSetStorage* allocateStorage() const;
225 
227  int8_t*,
228  const std::vector<int64_t>&,
229  std::shared_ptr<VarlenOutputInfo> = nullptr) const;
230 
231  const ResultSetStorage* allocateStorage(const std::vector<int64_t>&) const;
232 
233  void updateStorageEntryCount(const size_t new_entry_count) {
237  query_mem_desc_.setEntryCount(new_entry_count);
238  CHECK(storage_);
239  storage_->updateEntryCount(new_entry_count);
240  }
241 
242  std::vector<TargetValue> getNextRow(const bool translate_strings,
243  const bool decimal_to_double) const;
244 
245  size_t getCurrentRowBufferIndex() const;
246 
247  std::vector<TargetValue> getRowAt(const size_t index) const;
248 
249  TargetValue getRowAt(const size_t row_idx,
250  const size_t col_idx,
251  const bool translate_strings,
252  const bool decimal_to_double = true) const;
253 
254  // Specialized random access getter for result sets with a single column to
255  // avoid the overhead of building a std::vector<TargetValue> result with only
256  // one element. Only used by RelAlgTranslator::getInIntegerSetExpr currently.
257  OneIntegerColumnRow getOneColRow(const size_t index) const;
258 
259  std::vector<TargetValue> getRowAtNoTranslations(
260  const size_t index,
261  const std::vector<bool>& targets_to_skip = {}) const;
262 
263  bool isRowAtEmpty(const size_t index) const;
264 
265  void sort(const std::list<Analyzer::OrderEntry>& order_entries,
266  size_t top_n,
267  const ExecutorDeviceType device_type,
268  const Executor* executor);
269 
270  void keepFirstN(const size_t n);
271 
272  void dropFirstN(const size_t n);
273 
274  void append(ResultSet& that);
275 
276  const ResultSetStorage* getStorage() const;
277 
278  size_t colCount() const;
279 
280  SQLTypeInfo getColType(const size_t col_idx) const;
281 
311  size_t rowCount(const bool force_parallel = false) const;
312 
313  void invalidateCachedRowCount() const;
314 
315  void setCachedRowCount(const size_t row_count) const;
316 
332  bool isEmpty() const;
333 
349  size_t entryCount() const;
350 
351  size_t getBufferSizeBytes(const ExecutorDeviceType device_type) const;
352 
353  bool definitelyHasNoRows() const;
354 
355  const QueryMemoryDescriptor& getQueryMemDesc() const;
356 
357  const std::vector<TargetInfo>& getTargetInfos() const;
358 
359  const std::vector<int64_t>& getTargetInitVals() const;
360 
361  int8_t* getDeviceEstimatorBuffer() const;
362 
363  int8_t* getHostEstimatorBuffer() const;
364 
365  void syncEstimatorBuffer() const;
366 
367  size_t getNDVEstimator() const;
368 
370  // all in ms
372  int64_t render_time{0};
374  int64_t kernel_queue_time{0};
375  };
376 
377  void setQueueTime(const int64_t queue_time);
378  void setKernelQueueTime(const int64_t kernel_queue_time);
379  void addCompilationQueueTime(const int64_t compilation_queue_time);
380 
381  int64_t getQueueTime() const;
382  int64_t getRenderTime() const;
383 
384  void moveToBegin() const;
385 
386  bool isTruncated() const;
387 
388  bool isExplain() const;
389 
390  void setValidationOnlyRes();
391  bool isValidationOnlyRes() const;
392 
393  std::string getExplanation() const {
394  if (just_explain_) {
395  return explanation_;
396  }
397  return {};
398  }
399 
400  bool isGeoColOnGpu(const size_t col_idx) const;
401  int getDeviceId() const;
402  int getThreadIdx() const;
403 
404  // Materialize string from StringDictionaryProxy
405  std::string getString(SQLTypeInfo const&, int64_t const ival) const;
406 
407  // Called from the executor because in the new ResultSet we assume the 'padded' field
408  // in SlotSize already contains the padding, whereas in the executor it's computed.
409  // Once the buffer initialization moves to ResultSet we can remove this method.
411 
412  // Convert int64_t to ScalarTargetValue based on SQLTypeInfo and translate_strings.
414  bool const translate_strings,
415  int64_t const val) const;
416 
417  // Called from ResultSetComparator<>::operator().
418  bool isLessThan(SQLTypeInfo const&, int64_t const lhs, int64_t const rhs) const;
419 
420  // Required for sql_validate calls.
421  static bool isNullIval(SQLTypeInfo const&,
422  bool const translate_strings,
423  int64_t const ival);
424 
425  // Return NULL ScalarTargetValue based on SQLTypeInfo and translate_strings.
427  bool const translate_strings);
428 
429  void fillOneEntry(const std::vector<int64_t>& entry) {
430  CHECK(storage_);
431  if (storage_->query_mem_desc_.didOutputColumnar()) {
432  storage_->fillOneEntryColWise(entry);
433  } else {
434  storage_->fillOneEntryRowWise(entry);
435  }
436  }
437 
438  void initializeStorage() const;
439 
440  void holdChunks(const std::list<std::shared_ptr<Chunk_NS::Chunk>>& chunks) {
441  chunks_ = chunks;
442  }
443  void holdChunkIterators(const std::shared_ptr<std::list<ChunkIter>> chunk_iters) {
444  chunk_iters_.push_back(chunk_iters);
445  }
446  void holdLiterals(std::vector<int8_t>& literal_buff) {
447  literal_buffers_.push_back(std::move(literal_buff));
448  }
449 
450  std::shared_ptr<RowSetMemoryOwner> getRowSetMemOwner() const {
451  return row_set_mem_owner_;
452  }
453 
454  const Permutation& getPermutationBuffer() const;
455  const bool isPermutationBufferEmpty() const { return permutation_.empty(); };
456 
457  void serialize(TSerializedRows& serialized_rows) const;
458 
459  static std::unique_ptr<ResultSet> unserialize(const TSerializedRows& serialized_rows,
460  const Executor*);
461 
462  size_t getLimit() const;
463 
464  // APIs for data recycler
465  ResultSetPtr copy();
466 
468  if (!permutation_.empty()) {
469  permutation_.clear();
470  }
471  }
472 
473  void initStatus() {
474  // todo(yoonmin): what else we additionally need to consider
475  // to make completely clear status of the resultset for reuse?
476  crt_row_buff_idx_ = 0;
477  fetched_so_far_ = 0;
481  drop_first_ = 0;
482  keep_first_ = 0;
483  }
484 
486  if (!chunks_.empty()) {
487  chunks_.clear();
488  }
489  if (!chunk_iters_.empty()) {
490  chunk_iters_.clear();
491  }
492  };
493 
494  const bool isEstimator() const { return !estimator_; }
495 
496  void setCached(bool val) { cached_ = val; }
497 
498  const bool isCached() const { return cached_; }
499 
500  void setExecTime(const long exec_time) { query_exec_time_ = exec_time; }
501 
502  const long getExecTime() const { return query_exec_time_; }
503 
504  void setQueryPlanHash(const QueryPlanHash query_plan) { query_plan_ = query_plan; }
505 
507 
508  std::unordered_set<size_t> getInputTableKeys() const { return input_table_keys_; }
509 
510  void setInputTableKeys(std::unordered_set<size_t>&& intput_table_keys) {
511  input_table_keys_ = std::move(intput_table_keys);
512  }
513 
514  void setTargetMetaInfo(const std::vector<TargetMetaInfo>& target_meta_info) {
515  std::copy(target_meta_info.begin(),
516  target_meta_info.end(),
517  std::back_inserter(target_meta_info_));
518  }
519 
520  std::vector<TargetMetaInfo> getTargetMetaInfo() { return target_meta_info_; }
521 
522  std::optional<bool> canUseSpeculativeTopNSort() const {
524  }
525 
527 
528  const bool hasValidBuffer() const {
529  if (storage_) {
530  return true;
531  }
532  return false;
533  }
534 
535  unsigned getBlockSize() const { return block_size_; }
536 
537  unsigned getGridSize() const { return grid_size_; }
538 
542  enum class GeoReturnType {
545  WktString,
550  };
553 
554  void copyColumnIntoBuffer(const size_t column_idx,
555  int8_t* output_buffer,
556  const size_t output_buffer_size) const;
557 
559 
560  bool didOutputColumnar() const { return this->query_mem_desc_.didOutputColumnar(); }
561 
562  bool isZeroCopyColumnarConversionPossible(size_t column_idx) const;
563  const int8_t* getColumnarBuffer(size_t column_idx) const;
564  const size_t getColumnarBufferSize(size_t column_idx) const;
565 
568  }
569 
570  const int8_t getPaddedSlotWidthBytes(const size_t slot_idx) const {
571  return query_mem_desc_.getPaddedSlotWidthBytes(slot_idx);
572  }
573 
574  // returns a bitmap of all single-slot targets, as well as its count
575  std::tuple<std::vector<bool>, size_t> getSingleSlotTargetBitmap() const;
576 
577  std::tuple<std::vector<bool>, size_t> getSupportedSingleSlotTargetBitmap() const;
578 
579  std::vector<size_t> getSlotIndicesForTargetIndices() const;
580 
581  const std::vector<ColumnLazyFetchInfo>& getLazyFetchInfo() const {
582  return lazy_fetch_info_;
583  }
584 
586  auto is_lazy = [](auto const& info) { return info.is_lazily_fetched; };
587  return std::any_of(lazy_fetch_info_.begin(), lazy_fetch_info_.end(), is_lazy);
588  }
589 
590  size_t getNumColumnsLazyFetched() const {
591  auto is_lazy = [](auto const& info) { return info.is_lazily_fetched; };
592  return std::count_if(lazy_fetch_info_.begin(), lazy_fetch_info_.end(), is_lazy);
593  }
594 
595  void setSeparateVarlenStorageValid(const bool val) {
597  }
598 
599  const std::vector<std::string> getStringDictionaryPayloadCopy(
600  const shared::StringDictKey& dict_key) const;
601 
602  const std::pair<std::vector<int32_t>, std::vector<std::string>>
603  getUniqueStringsForDictEncodedTargetCol(const size_t col_idx) const;
604 
606  const shared::StringDictKey& dict_key) const;
607 
608  template <typename ENTRY_TYPE, QueryDescriptionType QUERY_TYPE, bool COLUMNAR_FORMAT>
609  ENTRY_TYPE getEntryAt(const size_t row_idx,
610  const size_t target_idx,
611  const size_t slot_idx) const;
612 
613  ChunkStats getTableFunctionChunkStats(const size_t target_idx) const;
614 
615  static double calculateQuantile(quantile::TDigest* const t_digest);
616 
617  void translateDictEncodedColumns(std::vector<TargetInfo> const&,
618  size_t const start_idx);
619 
621  size_t prev_target_idx_{0};
623  size_t agg_idx_{0};
624  int8_t const* buf_ptr_{nullptr};
625  int8_t compact_sz1_;
626  };
627 
628  class CellCallback;
629  void eachCellInColumn(RowIterationState&, CellCallback const&);
630 
631  const Executor* getExecutor() const { return query_mem_desc_.getExecutor(); }
632 
633  bool checkSlotUsesFlatBufferFormat(const size_t slot_idx) const {
635  }
636 
637  private:
639 
640  std::vector<TargetValue> getNextRowImpl(const bool translate_strings,
641  const bool decimal_to_double) const;
642 
643  std::vector<TargetValue> getNextRowUnlocked(const bool translate_strings,
644  const bool decimal_to_double) const;
645 
646  std::vector<TargetValue> getRowAt(const size_t index,
647  const bool translate_strings,
648  const bool decimal_to_double,
649  const bool fixup_count_distinct_pointers,
650  const std::vector<bool>& targets_to_skip = {}) const;
651 
652  // NOTE: just for direct columnarization use at the moment
653  template <typename ENTRY_TYPE>
654  ENTRY_TYPE getColumnarPerfectHashEntryAt(const size_t row_idx,
655  const size_t target_idx,
656  const size_t slot_idx) const;
657 
658  template <typename ENTRY_TYPE>
659  ENTRY_TYPE getRowWisePerfectHashEntryAt(const size_t row_idx,
660  const size_t target_idx,
661  const size_t slot_idx) const;
662 
663  template <typename ENTRY_TYPE>
664  ENTRY_TYPE getRowWiseBaselineEntryAt(const size_t row_idx,
665  const size_t target_idx,
666  const size_t slot_idx) const;
667 
668  template <typename ENTRY_TYPE>
669  ENTRY_TYPE getColumnarBaselineEntryAt(const size_t row_idx,
670  const size_t target_idx,
671  const size_t slot_idx) const;
672 
673  size_t binSearchRowCount() const;
674 
675  size_t parallelRowCount() const;
676 
677  size_t advanceCursorToNextEntry() const;
678 
679  void radixSortOnGpu(const std::list<Analyzer::OrderEntry>& order_entries) const;
680 
681  void radixSortOnCpu(const std::list<Analyzer::OrderEntry>& order_entries) const;
682 
683  static bool isNull(const SQLTypeInfo& ti,
684  const InternalTargetValue& val,
685  const bool float_argument_input);
686 
688  int8_t* rowwise_target_ptr,
689  int8_t* keys_ptr,
690  const size_t entry_buff_idx,
691  const TargetInfo& target_info,
692  const size_t target_logical_idx,
693  const size_t slot_idx,
694  const bool translate_strings,
695  const bool decimal_to_double,
696  const bool fixup_count_distinct_pointers) const;
697 
698  TargetValue getTargetValueFromBufferColwise(const int8_t* col_ptr,
699  const int8_t* keys_ptr,
701  const size_t local_entry_idx,
702  const size_t global_entry_idx,
703  const TargetInfo& target_info,
704  const size_t target_logical_idx,
705  const size_t slot_idx,
706  const bool translate_strings,
707  const bool decimal_to_double) const;
708 
709  TargetValue makeTargetValue(const int8_t* ptr,
710  const int8_t compact_sz,
711  const TargetInfo& target_info,
712  const size_t target_logical_idx,
713  const bool translate_strings,
714  const bool decimal_to_double,
715  const size_t entry_buff_idx) const;
716 
718  bool const translate_strings,
719  int64_t const ival) const;
720 
721  TargetValue makeVarlenTargetValue(const int8_t* ptr1,
722  const int8_t compact_sz1,
723  const int8_t* ptr2,
724  const int8_t compact_sz2,
725  const TargetInfo& target_info,
726  const size_t target_logical_idx,
727  const bool translate_strings,
728  const size_t entry_buff_idx) const;
729 
731  int8_t* ptr1;
732  int8_t compact_sz1;
733  int8_t* ptr2;
734  int8_t compact_sz2;
735 
737  : ptr1(nullptr), compact_sz1(0), ptr2(nullptr), compact_sz2(0) {}
738  };
739  TargetValue makeGeoTargetValue(const int8_t* geo_target_ptr,
740  const size_t slot_idx,
741  const TargetInfo& target_info,
742  const size_t target_logical_idx,
743  const size_t entry_buff_idx) const;
744 
747  const size_t fixedup_entry_idx;
748  const size_t storage_idx;
749  };
750 
751  InternalTargetValue getVarlenOrderEntry(const int64_t str_ptr,
752  const size_t str_len) const;
753 
754  int64_t lazyReadInt(const int64_t ival,
755  const size_t target_logical_idx,
756  const StorageLookupResult& storage_lookup_result) const;
757 
761  std::pair<size_t, size_t> getStorageIndex(const size_t entry_idx) const;
762 
763  const std::vector<const int8_t*>& getColumnFrag(const size_t storge_idx,
764  const size_t col_logical_idx,
765  int64_t& global_idx) const;
766 
767  const VarlenOutputInfo* getVarlenOutputInfo(const size_t entry_idx) const;
768 
769  StorageLookupResult findStorage(const size_t entry_idx) const;
770 
771  struct TargetOffsets {
772  const int8_t* ptr1;
773  const size_t compact_sz1;
774  const int8_t* ptr2;
775  const size_t compact_sz2;
776  };
777 
779  RowWiseTargetAccessor(const ResultSet* result_set)
780  : result_set_(result_set)
782  , key_width_(result_set_->query_mem_desc_.getEffectiveKeyWidth())
786  }
787 
789  const int8_t* buff,
790  const size_t entry_idx,
791  const size_t target_logical_idx,
792  const StorageLookupResult& storage_lookup_result) const;
793 
795 
796  inline const int8_t* get_rowwise_ptr(const int8_t* buff,
797  const size_t entry_idx) const {
798  return buff + entry_idx * row_bytes_;
799  }
800 
801  std::vector<std::vector<TargetOffsets>> offsets_for_storage_;
802 
804 
805  // Row-wise iteration
806  const size_t row_bytes_;
807  const size_t key_width_;
809  };
810 
812  ColumnWiseTargetAccessor(const ResultSet* result_set) : result_set_(result_set) {
814  }
815 
817 
819  const int8_t* buff,
820  const size_t entry_idx,
821  const size_t target_logical_idx,
822  const StorageLookupResult& storage_lookup_result) const;
823 
824  std::vector<std::vector<TargetOffsets>> offsets_for_storage_;
825 
827  };
828 
829  using ApproxQuantileBuffers = std::vector<std::vector<double>>;
830  using ModeBuffers = std::vector<std::vector<int64_t>>;
831 
832  template <typename BUFFER_ITERATOR_TYPE>
834  using BufferIteratorType = BUFFER_ITERATOR_TYPE;
835 
836  ResultSetComparator(const std::list<Analyzer::OrderEntry>& order_entries,
837  const ResultSet* result_set,
838  const PermutationView permutation,
839  const Executor* executor,
840  const bool single_threaded)
841  : order_entries_(order_entries)
842  , result_set_(result_set)
843  , permutation_(permutation)
844  , buffer_itr_(result_set)
845  , executor_(executor)
846  , single_threaded_(single_threaded)
850  }
851 
855 
856  std::vector<int64_t> materializeCountDistinctColumn(
857  const Analyzer::OrderEntry& order_entry) const;
858  ApproxQuantileBuffers::value_type materializeApproxQuantileColumn(
859  const Analyzer::OrderEntry& order_entry) const;
860  ModeBuffers::value_type materializeModeColumn(
861  const Analyzer::OrderEntry& order_entry) const;
862 
863  bool operator()(const PermutationIdx lhs, const PermutationIdx rhs) const;
864 
865  const std::list<Analyzer::OrderEntry>& order_entries_;
869  const Executor* executor_;
870  const bool single_threaded_;
871  std::vector<std::vector<int64_t>> count_distinct_materialized_buffers_;
874  struct ModeScatter; // Functor for setting mode_buffers_.
875  };
876 
877  Comparator createComparator(const std::list<Analyzer::OrderEntry>& order_entries,
878  const PermutationView permutation,
879  const Executor* executor,
880  const bool single_threaded) {
881  auto timer = DEBUG_TIMER(__func__);
884  order_entries, this, permutation, executor, single_threaded)](
885  const PermutationIdx lhs, const PermutationIdx rhs) {
886  return rsc(lhs, rhs);
887  };
888  } else {
890  order_entries, this, permutation, executor, single_threaded)](
891  const PermutationIdx lhs, const PermutationIdx rhs) {
892  return rsc(lhs, rhs);
893  };
894  }
895  }
896 
898  const size_t n,
899  const Comparator&);
900 
902  PermutationIdx const begin,
903  PermutationIdx const end) const;
904 
905  void parallelTop(const std::list<Analyzer::OrderEntry>& order_entries,
906  const size_t top_n,
907  const Executor* executor);
908 
909  void baselineSort(const std::list<Analyzer::OrderEntry>& order_entries,
910  const size_t top_n,
911  const ExecutorDeviceType device_type,
912  const Executor* executor);
913 
914  void doBaselineSort(const ExecutorDeviceType device_type,
915  const std::list<Analyzer::OrderEntry>& order_entries,
916  const size_t top_n,
917  const Executor* executor);
918 
919  bool canUseFastBaselineSort(const std::list<Analyzer::OrderEntry>& order_entries,
920  const size_t top_n);
921 
922  size_t rowCountImpl(const bool force_parallel) const;
923 
925 
926  int getGpuCount() const;
927 
928  void serializeProjection(TSerializedRows& serialized_rows) const;
929  void serializeVarlenAggColumn(int8_t* buf,
930  std::vector<std::string>& varlen_bufer) const;
931 
932  void serializeCountDistinctColumns(TSerializedRows&) const;
933 
934  void unserializeCountDistinctColumns(const TSerializedRows&);
935 
937 
938  void create_active_buffer_set(CountDistinctSet& count_distinct_active_buffer_set) const;
939 
940  int64_t getDistinctBufferRefFromBufferRowwise(int8_t* rowwise_target_ptr,
941  const TargetInfo& target_info) const;
942 
943  const std::vector<TargetInfo> targets_;
945  const int device_id_;
946  const int thread_idx_;
948  mutable std::unique_ptr<ResultSetStorage> storage_;
950  mutable size_t crt_row_buff_idx_;
951  mutable size_t fetched_so_far_;
952  size_t drop_first_;
953  size_t keep_first_;
954  std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner_;
956 
957  unsigned block_size_{0};
958  unsigned grid_size_{0};
960 
961  std::list<std::shared_ptr<Chunk_NS::Chunk>> chunks_;
962  std::vector<std::shared_ptr<std::list<ChunkIter>>> chunk_iters_;
963  // TODO(miyu): refine by using one buffer and
964  // setting offset instead of ptr in group by buffer.
965  std::vector<std::vector<int8_t>> literal_buffers_;
966  std::vector<ColumnLazyFetchInfo> lazy_fetch_info_;
967  std::vector<std::vector<std::vector<const int8_t*>>> col_buffers_;
968  std::vector<std::vector<std::vector<int64_t>>> frag_offsets_;
969  std::vector<std::vector<int64_t>> consistent_frag_sizes_;
970 
971  const std::shared_ptr<const Analyzer::Estimator> estimator_;
973  mutable int8_t* host_estimator_buffer_{nullptr};
975 
976  // only used by serialization
977  using SerializedVarlenBufferStorage = std::vector<std::string>;
978 
979  std::vector<SerializedVarlenBufferStorage> serialized_varlen_buffer_;
981  std::string explanation_;
982  const bool just_explain_;
984  mutable std::atomic<int64_t> cached_row_count_;
985  mutable std::mutex row_iteration_mutex_;
986 
987  // only used by geo
989 
990  // only used by data recycler
991  bool cached_; // indicator that this resultset is cached
992  size_t
993  query_exec_time_; // an elapsed time to process the query for this resultset (ms)
994  QueryPlanHash query_plan_; // a hashed query plan DAG of this resultset
995  std::unordered_set<size_t> input_table_keys_; // input table signatures
996  std::vector<TargetMetaInfo> target_meta_info_;
997  // if we recycle the resultset, we do not create work_unit of the query step
998  // because we may skip its child query step(s)
999  // so we try to keep whether this resultset is available to use speculative top n sort
1000  // when it is inserted to the recycler, and reuse this info when recycled
1001  std::optional<bool> can_use_speculative_top_n_sort;
1002 
1003  friend class ResultSetManager;
1004  friend class ResultSetRowIterator;
1005  friend class ColumnarResults;
1006 };
1007 
1009  if (!global_entry_idx_valid_) {
1010  return {};
1011  }
1012 
1013  if (result_set_->just_explain_) {
1014  return {result_set_->explanation_};
1015  }
1016 
1017  return result_set_->getRowAt(
1019 }
1020 
1022  if (!result_set_->storage_ && !result_set_->just_explain_) {
1023  global_entry_idx_valid_ = false;
1024  } else if (result_set_->just_explain_) {
1026  fetched_so_far_ = 1;
1027  } else {
1028  result_set_->advanceCursorToNextEntry(*this);
1029  }
1030  return *this;
1031 }
1032 
1034  public:
1035  ResultSet* reduce(std::vector<ResultSet*>&, const size_t executor_id);
1036 
1037  std::shared_ptr<ResultSet> getOwnResultSet();
1038 
1039  void rewriteVarlenAggregates(ResultSet*);
1040 
1041  private:
1042  std::shared_ptr<ResultSet> rs_;
1043 };
1044 
1045 class RowSortException : public std::runtime_error {
1046  public:
1047  RowSortException(const std::string& cause) : std::runtime_error(cause) {}
1048 };
1049 
1050 namespace result_set {
1051 
1052 bool can_use_parallel_algorithms(const ResultSet& rows);
1053 
1054 std::optional<size_t> first_dict_encoded_idx(std::vector<TargetInfo> const&);
1055 
1056 bool use_parallel_algorithms(const ResultSet& rows);
1057 
1058 } // namespace result_set
1059 
1060 #endif // QUERYENGINE_RESULTSET_H
void setSeparateVarlenStorageValid(const bool val)
Definition: ResultSet.h:595
void setGeoReturnType(const GeoReturnType val)
Definition: ResultSet.h:552
void serializeVarlenAggColumn(int8_t *buf, std::vector< std::string > &varlen_bufer) const
std::mutex row_iteration_mutex_
Definition: ResultSet.h:985
int getThreadIdx() const
Definition: ResultSet.cpp:762
void setUseSpeculativeTopNSort(bool value)
Definition: ResultSet.h:526
InternalTargetValue getColumnInternal(const int8_t *buff, const size_t entry_idx, const size_t target_logical_idx, const StorageLookupResult &storage_lookup_result) const
void syncEstimatorBuffer() const
Definition: ResultSet.cpp:702
const int8_t * ptr1
Definition: ResultSet.h:772
const size_t compact_sz2
Definition: ResultSet.h:775
void holdChunks(const std::list< std::shared_ptr< Chunk_NS::Chunk >> &chunks)
Definition: ResultSet.h:440
const QueryMemoryDescriptor & getQueryMemDesc() const
Definition: ResultSet.cpp:678
void setQueryPlanHash(const QueryPlanHash query_plan)
Definition: ResultSet.h:504
bool checkSlotUsesFlatBufferFormat(const size_t slot_idx) const
Definition: ResultSet.h:633
robin_hood::unordered_set< int64_t > CountDistinctSet
Definition: CountDistinct.h:35
std::pair< size_t, size_t > getStorageIndex(const size_t entry_idx) const
Definition: ResultSet.cpp:926
std::shared_ptr< RowSetMemoryOwner > getRowSetMemOwner() const
Definition: ResultSet.h:450
bool isValidationOnlyRes() const
Definition: ResultSet.cpp:754
Permutation permutation_
Definition: ResultSet.h:955
bool didOutputColumnar() const
Definition: ResultSet.h:560
void setValidationOnlyRes()
Definition: ResultSet.cpp:750
PermutationView initPermutationBuffer(PermutationView permutation, PermutationIdx const begin, PermutationIdx const end) const
Definition: ResultSet.cpp:858
bool for_validation_only_
Definition: ResultSet.h:983
std::ptrdiff_t difference_type
Definition: ResultSet.h:97
void setCached(bool val)
Definition: ResultSet.h:496
ENTRY_TYPE getRowWisePerfectHashEntryAt(const size_t row_idx, const size_t target_idx, const size_t slot_idx) const
void create_active_buffer_set(CountDistinctSet &count_distinct_active_buffer_set) const
void setEntryCount(const size_t val)
double decimal_to_double(const SQLTypeInfo &otype, int64_t oval)
static ScalarTargetValue nullScalarTargetValue(SQLTypeInfo const &, bool const translate_strings)
const ApproxQuantileBuffers approx_quantile_materialized_buffers_
Definition: ResultSet.h:872
AppendedStorage appended_storage_
Definition: ResultSet.h:949
ENTRY_TYPE getColumnarPerfectHashEntryAt(const size_t row_idx, const size_t target_idx, const size_t slot_idx) const
GeoReturnType geo_return_type_
Definition: ResultSet.h:988
const BufferIteratorType buffer_itr_
Definition: ResultSet.h:868
void moveToBegin() const
Definition: ResultSet.cpp:737
StringDictionaryProxy * getStringDictionaryProxy(const shared::StringDictKey &dict_key) const
Definition: ResultSet.cpp:429
Utility functions for easy access to the result set buffers.
void sort(const std::list< Analyzer::OrderEntry > &order_entries, size_t top_n, const ExecutorDeviceType device_type, const Executor *executor)
Definition: ResultSet.cpp:778
std::shared_ptr< ResultSet > rs_
Definition: ResultSet.h:1042
Calculate approximate median and general quantiles, based on &quot;Computing Extremely Accurate Quantiles ...
const Executor * executor_
Definition: ResultSet.h:869
std::vector< std::string > SerializedVarlenBufferStorage
Definition: ResultSet.h:977
void initializeStorage() const
std::optional< bool > can_use_speculative_top_n_sort
Definition: ResultSet.h:1001
QueryDescriptionType getQueryDescriptionType() const
Definition: ResultSet.h:566
ResultSetRowIterator(const ResultSet *rs, bool translate_strings, bool decimal_to_double)
Definition: ResultSet.h:132
size_t getNumColumnsLazyFetched() const
Definition: ResultSet.h:590
void unserializeCountDistinctColumns(const TSerializedRows &)
ApproxQuantileBuffers materializeApproxQuantileColumns() const
Definition: ResultSet.cpp:983
std::vector< TargetValue > getNextRow(const bool translate_strings, const bool decimal_to_double) const
static bool isNull(const SQLTypeInfo &ti, const InternalTargetValue &val, const bool float_argument_input)
QueryMemoryDescriptor query_mem_desc_
Definition: ResultSet.h:947
const std::vector< TargetInfo > & getTargetInfos() const
Definition: ResultSet.cpp:683
std::optional< size_t > first_dict_encoded_idx(std::vector< TargetInfo > const &)
Definition: ResultSet.cpp:1593
std::unique_ptr< ResultSetStorage > storage_
Definition: ResultSet.h:948
void setKernelQueueTime(const int64_t kernel_queue_time)
Definition: ResultSet.cpp:720
bool operator==(const ResultSetRowIterator &other) const
Definition: ResultSet.h:102
size_t query_exec_time_
Definition: ResultSet.h:993
std::string getExplanation() const
Definition: ResultSet.h:393
High-level representation of SQL values.
ENTRY_TYPE getEntryAt(const size_t row_idx, const size_t target_idx, const size_t slot_idx) const
size_t rowCount(const bool force_parallel=false) const
Returns the number of valid entries in the result set (i.e that will be returned from the SQL query o...
Definition: ResultSet.cpp:599
ResultSetRowIterator(const ResultSet *rs)
Definition: ResultSet.h:143
std::shared_ptr< ResultSet > ResultSetPtr
TargetValue makeGeoTargetValue(const int8_t *geo_target_ptr, const size_t slot_idx, const TargetInfo &target_info, const size_t target_logical_idx, const size_t entry_buff_idx) const
TargetValue getTargetValueFromBufferRowwise(int8_t *rowwise_target_ptr, int8_t *keys_ptr, const size_t entry_buff_idx, const TargetInfo &target_info, const size_t target_logical_idx, const size_t slot_idx, const bool translate_strings, const bool decimal_to_double, const bool fixup_count_distinct_pointers) const
size_t keep_first_
Definition: ResultSet.h:953
void keepFirstN(const size_t n)
Definition: ResultSet.cpp:54
std::vector< std::shared_ptr< std::list< ChunkIter > > > chunk_iters_
Definition: ResultSet.h:962
void addCompilationQueueTime(const int64_t compilation_queue_time)
Definition: ResultSet.cpp:724
std::vector< std::vector< double >> ApproxQuantileBuffers
Definition: ResultSet.h:829
void parallelTop(const std::list< Analyzer::OrderEntry > &order_entries, const size_t top_n, const Executor *executor)
Definition: ResultSet.cpp:878
void serialize(TSerializedRows &serialized_rows) const
std::vector< SerializedVarlenBufferStorage > serialized_varlen_buffer_
Definition: ResultSet.h:979
const size_t compact_sz1
Definition: ResultSet.h:773
int64_t lazyReadInt(const int64_t ival, const size_t target_logical_idx, const StorageLookupResult &storage_lookup_result) const
bool operator!=(const ResultSetRowIterator &other) const
Definition: ResultSet.h:106
size_t colCount() const
Definition: ResultSet.cpp:416
OneIntegerColumnRow getOneColRow(const size_t index) const
static bool isNullIval(SQLTypeInfo const &, bool const translate_strings, int64_t const ival)
unsigned getBlockSize() const
Definition: ResultSet.h:535
TargetValue getTargetValueFromBufferColwise(const int8_t *col_ptr, const int8_t *keys_ptr, const QueryMemoryDescriptor &query_mem_desc, const size_t local_entry_idx, const size_t global_entry_idx, const TargetInfo &target_info, const size_t target_logical_idx, const size_t slot_idx, const bool translate_strings, const bool decimal_to_double) const
void setInputTableKeys(std::unordered_set< size_t > &&intput_table_keys)
Definition: ResultSet.h:510
void rewriteVarlenAggregates(ResultSet *)
size_t getLimit() const
Definition: ResultSet.cpp:1409
const bool just_explain_
Definition: ResultSet.h:982
std::vector< int64_t > materializeCountDistinctColumn(const Analyzer::OrderEntry &order_entry) const
Definition: ResultSet.cpp:1011
ExecutorDeviceType
ApproxQuantileBuffers::value_type materializeApproxQuantileColumn(const Analyzer::OrderEntry &order_entry) const
Definition: ResultSet.cpp:1057
ResultSetRowIterator rowIterator(size_t from_logical_index, bool translate_strings, bool decimal_to_double) const
Definition: ResultSet.h:202
unsigned block_size_
Definition: ResultSet.h:957
bool isTruncated() const
Definition: ResultSet.cpp:742
std::atomic< int64_t > cached_row_count_
Definition: ResultSet.h:984
const bool isPermutationBufferEmpty() const
Definition: ResultSet.h:455
ScalarTargetValue makeStringTargetValue(SQLTypeInfo const &chosen_type, bool const translate_strings, int64_t const ival) const
size_t parallelRowCount() const
Definition: ResultSet.cpp:635
const size_t key_bytes_with_padding_
Definition: ResultSet.h:808
void baselineSort(const std::list< Analyzer::OrderEntry > &order_entries, const size_t top_n, const ExecutorDeviceType device_type, const Executor *executor)
bool areAnyColumnsLazyFetched() const
Definition: ResultSet.h:585
const ResultSet * result_set_
Definition: ResultSet.h:803
std::vector< TargetValue > getRowAtNoTranslations(const size_t index, const std::vector< bool > &targets_to_skip={}) const
const ResultSet * result_set_
Definition: ResultSet.h:124
void radixSortOnCpu(const std::list< Analyzer::OrderEntry > &order_entries) const
Definition: ResultSet.cpp:1369
bool definitelyHasNoRows() const
Definition: ResultSet.cpp:674
ColumnWiseTargetAccessor(const ResultSet *result_set)
Definition: ResultSet.h:812
bool use_parallel_algorithms(const ResultSet &rows)
Definition: ResultSet.cpp:1600
tuple rows
Definition: report.py:114
bool isZeroCopyColumnarConversionPossible(size_t column_idx) const
Definition: ResultSet.cpp:1499
std::input_iterator_tag iterator_category
Definition: ResultSet.h:100
size_t global_entry_idx_
Definition: ResultSet.h:126
int8_t * getHostEstimatorBuffer() const
Definition: ResultSet.cpp:698
InternalTargetValue getVarlenOrderEntry(const int64_t str_ptr, const size_t str_len) const
const std::vector< TargetInfo > targets_
Definition: ResultSet.h:943
void invalidateCachedRowCount() const
Definition: ResultSet.cpp:611
std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner_
Definition: ResultSet.h:954
const Executor * getExecutor() const
Definition: ResultSet.h:631
void setExecTime(const long exec_time)
Definition: ResultSet.h:500
size_t drop_first_
Definition: ResultSet.h:952
const ResultSetStorage * allocateStorage() const
bool cached_
Definition: ResultSet.h:991
std::string toString() const
Definition: ResultSet.h:195
const int8_t * ptr2
Definition: ResultSet.h:774
std::list< std::shared_ptr< Chunk_NS::Chunk > > chunks_
Definition: ResultSet.h:961
std::unordered_set< size_t > getInputTableKeys() const
Definition: ResultSet.h:508
QueryExecutionTimings timings_
Definition: ResultSet.h:959
const ResultSet * result_set_
Definition: ResultSet.h:866
DEVICE auto copy(ARGS &&...args)
Definition: gpu_enabled.h:51
void setQueueTime(const int64_t queue_time)
Definition: ResultSet.cpp:716
void dropFirstN(const size_t n)
Definition: ResultSet.cpp:59
std::vector< std::vector< int8_t > > literal_buffers_
Definition: ResultSet.h:965
std::vector< std::vector< TargetOffsets > > offsets_for_storage_
Definition: ResultSet.h:824
const size_t getColumnarBufferSize(size_t column_idx) const
Definition: ResultSet.cpp:1513
bool canUseFastBaselineSort(const std::list< Analyzer::OrderEntry > &order_entries, const size_t top_n)
unsigned grid_size_
Definition: ResultSet.h:958
void setTargetMetaInfo(const std::vector< TargetMetaInfo > &target_meta_info)
Definition: ResultSet.h:514
std::unordered_set< size_t > input_table_keys_
Definition: ResultSet.h:995
const std::list< Analyzer::OrderEntry > & order_entries_
Definition: ResultSet.h:865
std::vector< TargetValue > & reference
Definition: ResultSet.h:99
std::vector< PermutationIdx > Permutation
Definition: ResultSet.h:153
std::tuple< std::vector< bool >, size_t > getSingleSlotTargetBitmap() const
Definition: ResultSet.cpp:1525
ResultSetRowIterator & operator++(void)
Definition: ResultSet.h:1021
ScalarTargetValue convertToScalarTargetValue(SQLTypeInfo const &, bool const translate_strings, int64_t const val) const
std::shared_ptr< ResultSet > getOwnResultSet()
StorageLookupResult findStorage(const size_t entry_idx) const
Definition: ResultSet.cpp:951
const long getExecTime() const
Definition: ResultSet.h:502
Comparator createComparator(const std::list< Analyzer::OrderEntry > &order_entries, const PermutationView permutation, const Executor *executor, const bool single_threaded)
Definition: ResultSet.h:877
An AbstractBuffer is a unit of data management for a data manager.
const PermutationView permutation_
Definition: ResultSet.h:867
const int8_t getPaddedSlotWidthBytes(const size_t slot_idx) const
value_type operator*() const
Definition: ResultSet.h:1008
RowWiseTargetAccessor(const ResultSet *result_set)
Definition: ResultSet.h:779
ResultSetPtr copy()
Definition: ResultSet.cpp:333
void copyColumnIntoBuffer(const size_t column_idx, int8_t *output_buffer, const size_t output_buffer_size) const
RowSortException(const std::string &cause)
Definition: ResultSet.h:1047
std::function< bool(const PermutationIdx, const PermutationIdx)> Comparator
Definition: ResultSet.h:155
static double calculateQuantile(quantile::TDigest *const t_digest)
Definition: ResultSet.cpp:1047
friend ResultSetBuilder
Definition: ResultSet.h:159
const bool isEstimator() const
Definition: ResultSet.h:494
const VarlenOutputInfo * getVarlenOutputInfo(const size_t entry_idx) const
void fixupCountDistinctPointers()
void radixSortOnGpu(const std::list< Analyzer::OrderEntry > &order_entries) const
Definition: ResultSet.cpp:1329
const ResultSetStorage * getStorage() const
Definition: ResultSet.cpp:412
QueryDescriptionType getQueryDescriptionType() const
const std::pair< std::vector< int32_t >, std::vector< std::string > > getUniqueStringsForDictEncodedTargetCol(const size_t col_idx) const
Definition: ResultSet.cpp:1422
Data_Namespace::DataMgr * data_mgr_
Definition: ResultSet.h:974
Basic constructors and methods of the row set interface.
int64_t getQueueTime() const
Definition: ResultSet.cpp:728
std::optional< bool > canUseSpeculativeTopNSort() const
Definition: ResultSet.h:522
ModeBuffers::value_type materializeModeColumn(const Analyzer::OrderEntry &order_entry) const
Definition: ResultSet.cpp:1126
std::vector< TargetValue > getRowAt(const size_t index) const
Executor(const ExecutorId id, Data_Namespace::DataMgr *data_mgr, const size_t block_size_x, const size_t grid_size_x, const size_t max_gpu_slab_size, const std::string &debug_dir, const std::string &debug_file)
Definition: Execute.cpp:272
void fillOneEntry(const std::vector< int64_t > &entry)
Definition: ResultSet.h:429
void updateStorageEntryCount(const size_t new_entry_count)
Definition: ResultSet.h:233
uint32_t PermutationIdx
Definition: ResultSet.h:152
void serializeProjection(TSerializedRows &serialized_rows) const
ResultSetRowIterator operator++(int)
Definition: ResultSet.h:110
void invalidateResultSetChunks()
Definition: ResultSet.h:485
const std::shared_ptr< const Analyzer::Estimator > estimator_
Definition: ResultSet.h:971
SQLTypeInfo getColType(const size_t col_idx) const
Definition: ResultSet.cpp:420
GeoReturnType getGeoReturnType() const
Definition: ResultSet.h:551
void holdChunkIterators(const std::shared_ptr< std::list< ChunkIter >> chunk_iters)
Definition: ResultSet.h:443
std::tuple< std::vector< bool >, size_t > getSupportedSingleSlotTargetBitmap() const
Definition: ResultSet.cpp:1549
ExecutorDeviceType getDeviceType() const
Definition: ResultSet.cpp:254
const int8_t * getColumnarBuffer(size_t column_idx) const
Definition: ResultSet.cpp:1508
ResultSetComparator(const std::list< Analyzer::OrderEntry > &order_entries, const ResultSet *result_set, const PermutationView permutation, const Executor *executor, const bool single_threaded)
Definition: ResultSet.h:836
size_t get_row_bytes(const QueryMemoryDescriptor &query_mem_desc)
bool isExplain() const
Definition: ResultSet.cpp:746
void eachCellInColumn(RowIterationState &, CellCallback const &)
Definition: ResultSet.cpp:491
std::vector< TargetValue > value_type
Definition: ResultSet.h:96
bool isGeoColOnGpu(const size_t col_idx) const
std::vector< TargetMetaInfo > target_meta_info_
Definition: ResultSet.h:996
const QueryPlanHash getQueryPlanHash()
Definition: ResultSet.h:506
const int8_t * get_rowwise_ptr(const int8_t *buff, const size_t entry_idx) const
Definition: ResultSet.h:796
size_t getNDVEstimator() const
std::vector< std::vector< std::vector< const int8_t * > > > col_buffers_
Definition: ResultSet.h:967
bool isRowAtEmpty(const size_t index) const
const int8_t getPaddedSlotWidthBytes(const size_t slot_idx) const
Definition: ResultSet.h:570
size_t entryCount() const
Returns the number of entries the result set is allocated to hold.
const std::vector< std::string > getStringDictionaryPayloadCopy(const shared::StringDictKey &dict_key) const
Definition: ResultSet.cpp:1413
size_t QueryPlanHash
const int thread_idx_
Definition: ResultSet.h:946
static QueryMemoryDescriptor fixupQueryMemoryDescriptor(const QueryMemoryDescriptor &)
Definition: ResultSet.cpp:766
std::string typeName(const T *v)
Definition: toString.h:106
size_t rowCountImpl(const bool force_parallel) const
Definition: ResultSet.cpp:561
TargetValue makeTargetValue(const int8_t *ptr, const int8_t compact_sz, const TargetInfo &target_info, const size_t target_logical_idx, const bool translate_strings, const bool decimal_to_double, const size_t entry_buff_idx) const
size_t getCurrentRowBufferIndex() const
Definition: ResultSet.h:116
const Permutation & getPermutationBuffer() const
Definition: ResultSet.cpp:874
void append(ResultSet &that)
Definition: ResultSet.cpp:303
std::string summaryToString() const
Definition: ResultSet.cpp:222
std::string explanation_
Definition: ResultSet.h:981
std::vector< std::vector< int64_t > > consistent_frag_sizes_
Definition: ResultSet.h:969
int8_t * host_estimator_buffer_
Definition: ResultSet.h:973
friend class ResultSet
Definition: ResultSet.h:143
std::string getString(SQLTypeInfo const &, int64_t const ival) const
const ExecutorDeviceType device_type_
Definition: ResultSet.h:944
const bool hasValidBuffer() const
Definition: ResultSet.h:528
std::vector< TargetValue > getNextRowImpl(const bool translate_strings, const bool decimal_to_double) const
const bool isCached() const
Definition: ResultSet.h:498
static PermutationView topPermutation(PermutationView, const size_t n, const Comparator &)
Definition: ResultSet.cpp:1315
size_t getCurrentRowBufferIndex() const
Definition: ResultSet.cpp:295
void holdLiterals(std::vector< int8_t > &literal_buff)
Definition: ResultSet.h:446
bool g_enable_watchdog false
Definition: Execute.cpp:80
#define CHECK(condition)
Definition: Logger.h:291
#define DEBUG_TIMER(name)
Definition: Logger.h:412
int getGpuCount() const
size_t getBufferSizeBytes(const ExecutorDeviceType device_type) const
int8_t * getDeviceEstimatorBuffer() const
Definition: ResultSet.cpp:692
size_t fetched_so_far_
Definition: ResultSet.h:951
std::vector< ColumnLazyFetchInfo > lazy_fetch_info_
Definition: ResultSet.h:966
size_t crt_row_buff_idx_
Definition: ResultSet.h:950
QueryPlanHash query_plan_
Definition: ResultSet.h:994
Estimators to be used when precise cardinality isn&#39;t useful.
QueryDescriptionType
Definition: Types.h:29
int64_t getDistinctBufferRefFromBufferRowwise(int8_t *rowwise_target_ptr, const TargetInfo &target_info) const
std::vector< std::vector< std::vector< int64_t > > > frag_offsets_
Definition: ResultSet.h:968
bool operator()(const PermutationIdx lhs, const PermutationIdx rhs) const
Definition: ResultSet.cpp:1141
void doBaselineSort(const ExecutorDeviceType device_type, const std::list< Analyzer::OrderEntry > &order_entries, const size_t top_n, const Executor *executor)
bool separate_varlen_storage_valid_
Definition: ResultSet.h:980
boost::variant< ScalarTargetValue, ArrayTargetValue, GeoTargetValue, GeoTargetValuePtr > TargetValue
Definition: TargetValue.h:195
std::vector< TargetValue > getNextRowUnlocked(const bool translate_strings, const bool decimal_to_double) const
std::vector< TargetValue > * pointer
Definition: ResultSet.h:98
bool isEmpty() const
Returns a boolean signifying whether there are valid entries in the result set.
Definition: ResultSet.cpp:655
const std::vector< int64_t > & getTargetInitVals() const
Definition: ResultSet.cpp:687
std::vector< size_t > getSlotIndicesForTargetIndices() const
Definition: ResultSet.cpp:1569
unsigned getGridSize() const
Definition: ResultSet.h:537
bool any_of(std::vector< Analyzer::Expr * > const &target_exprs)
size_t advanceCursorToNextEntry() const
constexpr double n
Definition: Utm.h:38
ResultSetRowIterator rowIterator(bool translate_strings, bool decimal_to_double) const
Definition: ResultSet.h:217
Definition: Analyzer.h:2675
BUFFER_ITERATOR_TYPE BufferIteratorType
Definition: ResultSet.h:834
ENTRY_TYPE getColumnarBaselineEntryAt(const size_t row_idx, const size_t target_idx, const size_t slot_idx) const
Data_Namespace::DataMgr * getDataManager() const
size_t crt_row_buff_idx_
Definition: ResultSet.h:125
std::vector< std::vector< int64_t > > count_distinct_materialized_buffers_
Definition: ResultSet.h:871
Data_Namespace::AbstractBuffer * device_estimator_buffer_
Definition: ResultSet.h:972
InternalTargetValue getColumnInternal(const int8_t *buff, const size_t entry_idx, const size_t target_logical_idx, const StorageLookupResult &storage_lookup_result) const
bool isLessThan(SQLTypeInfo const &, int64_t const lhs, int64_t const rhs) const
std::vector< std::vector< TargetOffsets > > offsets_for_storage_
Definition: ResultSet.h:801
bool checkSlotUsesFlatBufferFormat(const size_t slot_idx) const
std::vector< TargetMetaInfo > getTargetMetaInfo()
Definition: ResultSet.h:520
void translateDictEncodedColumns(std::vector< TargetInfo > const &, size_t const start_idx)
Definition: ResultSet.cpp:459
bool global_entry_idx_valid_
Definition: ResultSet.h:127
bool can_use_parallel_algorithms(const ResultSet &rows)
Definition: ResultSet.cpp:1581
int64_t getRenderTime() const
Definition: ResultSet.cpp:733
ResultSet(const std::vector< TargetInfo > &targets, const ExecutorDeviceType device_type, const QueryMemoryDescriptor &query_mem_desc, const std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner, const unsigned block_size, const unsigned grid_size)
Definition: ResultSet.cpp:64
void setCachedRowCount(const size_t row_count) const
Definition: ResultSet.cpp:615
bool isDirectColumnarConversionPossible() const
Definition: ResultSet.cpp:1477
size_t get_key_bytes_rowwise(const QueryMemoryDescriptor &query_mem_desc)
void initStatus()
Definition: ResultSet.h:473
FORCE_INLINE HOST DEVICE T align_to_int64(T addr)
ModeBuffers materializeModeColumns() const
Definition: ResultSet.cpp:996
TargetValue makeVarlenTargetValue(const int8_t *ptr1, const int8_t compact_sz1, const int8_t *ptr2, const int8_t compact_sz2, const TargetInfo &target_info, const size_t target_logical_idx, const bool translate_strings, const size_t entry_buff_idx) const
void serializeCountDistinctColumns(TSerializedRows &) const
const ResultSetStorage * storage_ptr
Definition: ResultSet.h:746
ResultSet * reduce(std::vector< ResultSet * > &, const size_t executor_id)
const ModeBuffers mode_buffers_
Definition: ResultSet.h:873
std::vector< std::unique_ptr< ResultSetStorage >> AppendedStorage
Definition: ResultSet.h:151
const std::vector< const int8_t * > & getColumnFrag(const size_t storge_idx, const size_t col_logical_idx, int64_t &global_idx) const
const Executor * getExecutor() const
size_t binSearchRowCount() const
Definition: ResultSet.cpp:622
void clearPermutation()
Definition: ResultSet.h:467
ChunkStats getTableFunctionChunkStats(const size_t target_idx) const
const std::vector< ColumnLazyFetchInfo > & getLazyFetchInfo() const
Definition: ResultSet.h:581
ENTRY_TYPE getRowWiseBaselineEntryAt(const size_t row_idx, const size_t target_idx, const size_t slot_idx) const
int getDeviceId() const
Definition: ResultSet.cpp:758
std::vector< std::vector< int64_t >> ModeBuffers
Definition: ResultSet.h:830
boost::variant< int64_t, double, float, NullableString > ScalarTargetValue
Definition: TargetValue.h:180
static std::unique_ptr< ResultSet > unserialize(const TSerializedRows &serialized_rows, const Executor *)
const int device_id_
Definition: ResultSet.h:945