OmniSciDB  085a039ca4
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
ResultSet.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
25 #ifndef QUERYENGINE_RESULTSET_H
26 #define QUERYENGINE_RESULTSET_H
27 
28 #include "CardinalityEstimator.h"
29 #include "DataMgr/Chunk/Chunk.h"
31 #include "ResultSetStorage.h"
32 #include "Shared/quantile.h"
33 #include "TargetValue.h"
34 
35 #include <algorithm>
36 #include <atomic>
37 #include <functional>
38 #include <list>
39 #include <optional>
40 
41 /*
42  * Stores the underlying buffer and the meta-data for a result set. The buffer
43  * format reflects the main requirements for result sets. Not all queries
44  * specify a GROUP BY clause, but since it's the most important and challenging
45  * case we'll focus on it. Note that the meta-data is stored separately from
46  * the buffer and it's not transferred to GPU.
47  *
48  * 1. It has to be efficient for reduction of partial GROUP BY query results
49  * from multiple devices / cores, the cardinalities can be high. Reduction
50  * currently happens on the host.
51  * 2. No conversions should be needed when buffers are transferred from GPU to
52  * host for reduction. This implies the buffer needs to be "flat", with no
53  * pointers to chase since they have no meaning in a different address space.
54  * 3. Must be size-efficient.
55  *
56  * There are several variations of the format of a result set buffer, but the
57  * most common is a sequence of entries which represent a row in the result or
58  * an empty slot. One entry looks as follows:
59  *
60  * +-+-+-+-+-+-+-+-+-+-+-+--?--+-+-+-+-+-+-+-+-+-+-+-+-+
61  * |key_0| ... |key_N-1| padding |value_0|...|value_N-1|
62  * +-+-+-+-+-+-+-+-+-+-+-+--?--+-+-+-+-+-+-+-+-+-+-+-+-+
63  *
64  * (key_0 ... key_N-1) is a multiple component key, unique within the buffer.
65  * It stores the tuple specified by the GROUP BY clause. All components have
66  * the same width, 4 or 8 bytes. For the 4-byte components, 4-byte padding is
67  * added if the number of components is odd. Not all entries in the buffer are
68  * valid; an empty entry contains EMPTY_KEY_{64, 32} for 8-byte / 4-byte width,
69  * respectively. An empty entry is ignored by subsequent operations on the
70  * result set (reduction, iteration, sort etc).
71  *
72  * value_0 through value_N-1 are 8-byte fields which hold the columns of the
73  * result, like aggregates and projected expressions. They're reduced between
74  * multiple partial results for identical (key_0 ... key_N-1) tuples.
75  *
76  * The order of entries is decided by the type of hash used, which depends on
77  * the range of the keys. For small enough ranges, a perfect hash is used. When
78  * a perfect hash isn't feasible, open addressing (using MurmurHash) with linear
79  * probing is used instead, with a 50% fill rate.
80  */
81 
82 struct ReductionCode;
83 
84 namespace Analyzer {
85 
86 class Expr;
87 class Estimator;
88 struct OrderEntry;
89 
90 } // namespace Analyzer
91 
92 class Executor;
93 
94 class ResultSet;
95 
97  public:
98  using value_type = std::vector<TargetValue>;
99  using difference_type = std::ptrdiff_t;
100  using pointer = std::vector<TargetValue>*;
101  using reference = std::vector<TargetValue>&;
102  using iterator_category = std::input_iterator_tag;
103 
104  bool operator==(const ResultSetRowIterator& other) const {
105  return result_set_ == other.result_set_ &&
107  }
108  bool operator!=(const ResultSetRowIterator& other) const { return !(*this == other); }
109 
110  inline value_type operator*() const;
111  inline ResultSetRowIterator& operator++(void);
113  ResultSetRowIterator iter(*this);
114  ++(*this);
115  return iter;
116  }
117 
118  size_t getCurrentRowBufferIndex() const {
119  if (crt_row_buff_idx_ == 0) {
120  throw std::runtime_error("current row buffer iteration index is undefined");
121  }
122  return crt_row_buff_idx_ - 1;
123  }
124 
125  private:
133 
135  bool translate_strings,
136  bool decimal_to_double)
137  : result_set_(rs)
138  , crt_row_buff_idx_(0)
139  , global_entry_idx_(0)
141  , fetched_so_far_(0)
142  , translate_strings_(translate_strings)
143  , decimal_to_double_(decimal_to_double){};
144 
146 
147  friend class ResultSet;
148 };
149 
150 class TSerializedRows;
151 class ResultSetBuilder;
152 
153 using AppendedStorage = std::vector<std::unique_ptr<ResultSetStorage>>;
154 using PermutationIdx = uint32_t;
155 using Permutation = std::vector<PermutationIdx>;
157 using Comparator = std::function<bool(const PermutationIdx, const PermutationIdx)>;
158 
159 class ResultSet {
160  public:
162  // Can use derivatives of the builder class to construct a ResultSet
163 
164  ResultSet(const std::vector<TargetInfo>& targets,
165  const ExecutorDeviceType device_type,
167  const std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner,
168  const Catalog_Namespace::Catalog* catalog,
169  const unsigned block_size,
170  const unsigned grid_size);
171 
172  ResultSet(const std::vector<TargetInfo>& targets,
173  const std::vector<ColumnLazyFetchInfo>& lazy_fetch_info,
174  const std::vector<std::vector<const int8_t*>>& col_buffers,
175  const std::vector<std::vector<int64_t>>& frag_offsets,
176  const std::vector<int64_t>& consistent_frag_sizes,
177  const ExecutorDeviceType device_type,
178  const int device_id,
180  const std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner,
181  const Catalog_Namespace::Catalog* catalog,
182  const unsigned block_size,
183  const unsigned grid_size);
184 
185  ResultSet(const std::shared_ptr<const Analyzer::Estimator>,
186  const ExecutorDeviceType device_type,
187  const int device_id,
188  Data_Namespace::DataMgr* data_mgr);
189 
190  ResultSet(const std::string& explanation);
191 
192  ResultSet(int64_t queue_time_ms,
193  int64_t render_time_ms,
194  const std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner);
195 
196  ~ResultSet();
197 
198  std::string toString() const {
199  return typeName(this) + "(targets=" + ::toString(targets_) +
200  ", query_mem_desc=" + ::toString(query_mem_desc_) + ")";
201  }
202 
203  std::string summaryToString() const;
204 
205  inline ResultSetRowIterator rowIterator(size_t from_logical_index,
206  bool translate_strings,
207  bool decimal_to_double) const {
208  ResultSetRowIterator rowIterator(this, translate_strings, decimal_to_double);
209 
210  // move to first logical position
211  ++rowIterator;
212 
213  for (size_t index = 0; index < from_logical_index; index++) {
214  ++rowIterator;
215  }
216 
217  return rowIterator;
218  }
219 
220  inline ResultSetRowIterator rowIterator(bool translate_strings,
221  bool decimal_to_double) const {
222  return rowIterator(0, translate_strings, decimal_to_double);
223  }
224 
226 
227  const ResultSetStorage* allocateStorage() const;
228 
230  int8_t*,
231  const std::vector<int64_t>&,
232  std::shared_ptr<VarlenOutputInfo> = nullptr) const;
233 
234  const ResultSetStorage* allocateStorage(const std::vector<int64_t>&) const;
235 
236  void updateStorageEntryCount(const size_t new_entry_count) {
240  query_mem_desc_.setEntryCount(new_entry_count);
241  CHECK(storage_);
242  storage_->updateEntryCount(new_entry_count);
243  }
244 
245  std::vector<TargetValue> getNextRow(const bool translate_strings,
246  const bool decimal_to_double) const;
247 
248  size_t getCurrentRowBufferIndex() const;
249 
250  std::vector<TargetValue> getRowAt(const size_t index) const;
251 
252  TargetValue getRowAt(const size_t row_idx,
253  const size_t col_idx,
254  const bool translate_strings,
255  const bool decimal_to_double = true) const;
256 
257  // Specialized random access getter for result sets with a single column to
258  // avoid the overhead of building a std::vector<TargetValue> result with only
259  // one element. Only used by RelAlgTranslator::getInIntegerSetExpr currently.
260  OneIntegerColumnRow getOneColRow(const size_t index) const;
261 
262  std::vector<TargetValue> getRowAtNoTranslations(
263  const size_t index,
264  const std::vector<bool>& targets_to_skip = {}) const;
265 
266  bool isRowAtEmpty(const size_t index) const;
267 
268  void sort(const std::list<Analyzer::OrderEntry>& order_entries,
269  size_t top_n,
270  const Executor* executor);
271 
272  void keepFirstN(const size_t n);
273 
274  void dropFirstN(const size_t n);
275 
276  void append(ResultSet& that);
277 
278  const ResultSetStorage* getStorage() const;
279 
280  size_t colCount() const;
281 
282  SQLTypeInfo getColType(const size_t col_idx) const;
283 
313  size_t rowCount(const bool force_parallel = false) const;
314 
315  void invalidateCachedRowCount() const;
316 
317  void setCachedRowCount(const size_t row_count) const;
318 
334  bool isEmpty() const;
335 
351  size_t entryCount() const;
352 
353  size_t getBufferSizeBytes(const ExecutorDeviceType device_type) const;
354 
355  bool definitelyHasNoRows() const;
356 
357  const QueryMemoryDescriptor& getQueryMemDesc() const;
358 
359  const std::vector<TargetInfo>& getTargetInfos() const;
360 
361  const std::vector<int64_t>& getTargetInitVals() const;
362 
363  int8_t* getDeviceEstimatorBuffer() const;
364 
365  int8_t* getHostEstimatorBuffer() const;
366 
367  void syncEstimatorBuffer() const;
368 
369  size_t getNDVEstimator() const;
370 
372  // all in ms
374  int64_t render_time{0};
376  int64_t kernel_queue_time{0};
377  };
378 
379  void setQueueTime(const int64_t queue_time);
380  void setKernelQueueTime(const int64_t kernel_queue_time);
381  void addCompilationQueueTime(const int64_t compilation_queue_time);
382 
383  int64_t getQueueTime() const;
384  int64_t getRenderTime() const;
385 
386  void moveToBegin() const;
387 
388  bool isTruncated() const;
389 
390  bool isExplain() const;
391 
392  void setValidationOnlyRes();
393  bool isValidationOnlyRes() const;
394 
395  std::string getExplanation() const {
396  if (just_explain_) {
397  return explanation_;
398  }
399  return {};
400  }
401 
402  bool isGeoColOnGpu(const size_t col_idx) const;
403  int getDeviceId() const;
404 
405  // Called from the executor because in the new ResultSet we assume the 'padded' field
406  // in SlotSize already contains the padding, whereas in the executor it's computed.
407  // Once the buffer initialization moves to ResultSet we can remove this method.
409 
410  void fillOneEntry(const std::vector<int64_t>& entry) {
411  CHECK(storage_);
412  if (storage_->query_mem_desc_.didOutputColumnar()) {
413  storage_->fillOneEntryColWise(entry);
414  } else {
415  storage_->fillOneEntryRowWise(entry);
416  }
417  }
418 
419  void initializeStorage() const;
420 
421  void holdChunks(const std::list<std::shared_ptr<Chunk_NS::Chunk>>& chunks) {
422  chunks_ = chunks;
423  }
424  void holdChunkIterators(const std::shared_ptr<std::list<ChunkIter>> chunk_iters) {
425  chunk_iters_.push_back(chunk_iters);
426  }
427  void holdLiterals(std::vector<int8_t>& literal_buff) {
428  literal_buffers_.push_back(std::move(literal_buff));
429  }
430 
431  std::shared_ptr<RowSetMemoryOwner> getRowSetMemOwner() const {
432  return row_set_mem_owner_;
433  }
434 
435  const Permutation& getPermutationBuffer() const;
436  const bool isPermutationBufferEmpty() const { return permutation_.empty(); };
437 
438  void serialize(TSerializedRows& serialized_rows) const;
439 
440  static std::unique_ptr<ResultSet> unserialize(const TSerializedRows& serialized_rows,
441  const Executor*);
442 
443  size_t getLimit() const;
444 
445  // APIs for data recycler
446  ResultSetPtr copy();
447 
449  if (!permutation_.empty()) {
450  permutation_.clear();
451  }
452  }
453 
454  void initStatus() {
455  // todo(yoonmin): what else we additionally need to consider
456  // to make completely clear status of the resultset for reuse?
457  crt_row_buff_idx_ = 0;
458  fetched_so_far_ = 0;
462  drop_first_ = 0;
463  keep_first_ = 0;
464  }
465 
467  if (!chunks_.empty()) {
468  chunks_.clear();
469  }
470  if (!chunk_iters_.empty()) {
471  chunk_iters_.clear();
472  }
473  };
474 
475  const bool isEstimator() const { return !estimator_; }
476 
477  void setCached(bool val) { cached_ = val; }
478 
479  const bool isCached() const { return cached_; }
480 
481  void setExecTime(const long exec_time) { query_exec_time_ = exec_time; }
482 
483  const long getExecTime() const { return query_exec_time_; }
484 
485  void setQueryPlanHash(const QueryPlanHash query_plan) { query_plan_ = query_plan; }
486 
488 
489  std::unordered_set<size_t> getInputTableKeys() const { return input_table_keys_; }
490 
491  void setInputTableKeys(std::unordered_set<size_t>&& intput_table_keys) {
492  input_table_keys_ = std::move(intput_table_keys);
493  }
494 
495  void setTargetMetaInfo(const std::vector<TargetMetaInfo>& target_meta_info) {
496  std::copy(target_meta_info.begin(),
497  target_meta_info.end(),
498  std::back_inserter(target_meta_info_));
499  }
500 
501  std::vector<TargetMetaInfo> getTargetMetaInfo() { return target_meta_info_; }
502 
503  std::optional<bool> canUseSpeculativeTopNSort() const {
505  }
506 
508 
509  const bool hasValidBuffer() const {
510  if (storage_) {
511  return true;
512  }
513  return false;
514  }
515 
519  enum class GeoReturnType {
522  WktString,
527  };
530 
531  void copyColumnIntoBuffer(const size_t column_idx,
532  int8_t* output_buffer,
533  const size_t output_buffer_size) const;
534 
536 
537  bool didOutputColumnar() const { return this->query_mem_desc_.didOutputColumnar(); }
538 
539  bool isZeroCopyColumnarConversionPossible(size_t column_idx) const;
540  const int8_t* getColumnarBuffer(size_t column_idx) const;
541 
544  }
545 
546  const int8_t getPaddedSlotWidthBytes(const size_t slot_idx) const {
547  return query_mem_desc_.getPaddedSlotWidthBytes(slot_idx);
548  }
549 
550  // returns a bitmap of all single-slot targets, as well as its count
551  std::tuple<std::vector<bool>, size_t> getSingleSlotTargetBitmap() const;
552 
553  std::tuple<std::vector<bool>, size_t> getSupportedSingleSlotTargetBitmap() const;
554 
555  std::vector<size_t> getSlotIndicesForTargetIndices() const;
556 
557  const std::vector<ColumnLazyFetchInfo>& getLazyFetchInfo() const {
558  return lazy_fetch_info_;
559  }
560 
562  auto is_lazy = [](auto const& info) { return info.is_lazily_fetched; };
563  return std::any_of(lazy_fetch_info_.begin(), lazy_fetch_info_.end(), is_lazy);
564  }
565 
566  size_t getNumColumnsLazyFetched() const {
567  auto is_lazy = [](auto const& info) { return info.is_lazily_fetched; };
568  return std::count_if(lazy_fetch_info_.begin(), lazy_fetch_info_.end(), is_lazy);
569  }
570 
571  void setSeparateVarlenStorageValid(const bool val) {
573  }
574 
575  const std::vector<std::string> getStringDictionaryPayloadCopy(const int dict_id) const;
576 
577  const std::pair<std::vector<int32_t>, std::vector<std::string>>
578  getUniqueStringsForDictEncodedTargetCol(const size_t col_idx) const;
579 
580  StringDictionaryProxy* getStringDictionaryProxy(int const dict_id) const;
581 
582  template <typename ENTRY_TYPE, QueryDescriptionType QUERY_TYPE, bool COLUMNAR_FORMAT>
583  ENTRY_TYPE getEntryAt(const size_t row_idx,
584  const size_t target_idx,
585  const size_t slot_idx) const;
586 
587  ChunkStats getTableFunctionChunkStats(const size_t target_idx) const;
588 
589  static double calculateQuantile(quantile::TDigest* const t_digest);
590 
591  void translateDictEncodedColumns(std::vector<TargetInfo> const&,
592  size_t const start_idx);
593 
595  size_t prev_target_idx_{0};
597  size_t agg_idx_{0};
598  int8_t const* buf_ptr_{nullptr};
599  int8_t compact_sz1_;
600  };
601 
602  class CellCallback;
603  void eachCellInColumn(RowIterationState&, CellCallback const&);
604 
605  const Executor* getExecutor() const { return query_mem_desc_.getExecutor(); }
606 
607  private:
609 
610  std::vector<TargetValue> getNextRowImpl(const bool translate_strings,
611  const bool decimal_to_double) const;
612 
613  std::vector<TargetValue> getNextRowUnlocked(const bool translate_strings,
614  const bool decimal_to_double) const;
615 
616  std::vector<TargetValue> getRowAt(const size_t index,
617  const bool translate_strings,
618  const bool decimal_to_double,
619  const bool fixup_count_distinct_pointers,
620  const std::vector<bool>& targets_to_skip = {}) const;
621 
622  // NOTE: just for direct columnarization use at the moment
623  template <typename ENTRY_TYPE>
624  ENTRY_TYPE getColumnarPerfectHashEntryAt(const size_t row_idx,
625  const size_t target_idx,
626  const size_t slot_idx) const;
627 
628  template <typename ENTRY_TYPE>
629  ENTRY_TYPE getRowWisePerfectHashEntryAt(const size_t row_idx,
630  const size_t target_idx,
631  const size_t slot_idx) const;
632 
633  template <typename ENTRY_TYPE>
634  ENTRY_TYPE getRowWiseBaselineEntryAt(const size_t row_idx,
635  const size_t target_idx,
636  const size_t slot_idx) const;
637 
638  template <typename ENTRY_TYPE>
639  ENTRY_TYPE getColumnarBaselineEntryAt(const size_t row_idx,
640  const size_t target_idx,
641  const size_t slot_idx) const;
642 
643  size_t binSearchRowCount() const;
644 
645  size_t parallelRowCount() const;
646 
647  size_t advanceCursorToNextEntry() const;
648 
649  void radixSortOnGpu(const std::list<Analyzer::OrderEntry>& order_entries) const;
650 
651  void radixSortOnCpu(const std::list<Analyzer::OrderEntry>& order_entries) const;
652 
653  static bool isNull(const SQLTypeInfo& ti,
654  const InternalTargetValue& val,
655  const bool float_argument_input);
656 
658  int8_t* rowwise_target_ptr,
659  int8_t* keys_ptr,
660  const size_t entry_buff_idx,
661  const TargetInfo& target_info,
662  const size_t target_logical_idx,
663  const size_t slot_idx,
664  const bool translate_strings,
665  const bool decimal_to_double,
666  const bool fixup_count_distinct_pointers) const;
667 
668  TargetValue getTargetValueFromBufferColwise(const int8_t* col_ptr,
669  const int8_t* keys_ptr,
671  const size_t local_entry_idx,
672  const size_t global_entry_idx,
673  const TargetInfo& target_info,
674  const size_t target_logical_idx,
675  const size_t slot_idx,
676  const bool translate_strings,
677  const bool decimal_to_double) const;
678 
679  TargetValue makeTargetValue(const int8_t* ptr,
680  const int8_t compact_sz,
681  const TargetInfo& target_info,
682  const size_t target_logical_idx,
683  const bool translate_strings,
684  const bool decimal_to_double,
685  const size_t entry_buff_idx) const;
686 
687  TargetValue makeVarlenTargetValue(const int8_t* ptr1,
688  const int8_t compact_sz1,
689  const int8_t* ptr2,
690  const int8_t compact_sz2,
691  const TargetInfo& target_info,
692  const size_t target_logical_idx,
693  const bool translate_strings,
694  const size_t entry_buff_idx) const;
695 
697  int8_t* ptr1;
698  int8_t compact_sz1;
699  int8_t* ptr2;
700  int8_t compact_sz2;
701 
703  : ptr1(nullptr), compact_sz1(0), ptr2(nullptr), compact_sz2(0) {}
704  };
705  TargetValue makeGeoTargetValue(const int8_t* geo_target_ptr,
706  const size_t slot_idx,
707  const TargetInfo& target_info,
708  const size_t target_logical_idx,
709  const size_t entry_buff_idx) const;
710 
713  const size_t fixedup_entry_idx;
714  const size_t storage_idx;
715  };
716 
717  InternalTargetValue getVarlenOrderEntry(const int64_t str_ptr,
718  const size_t str_len) const;
719 
720  int64_t lazyReadInt(const int64_t ival,
721  const size_t target_logical_idx,
722  const StorageLookupResult& storage_lookup_result) const;
723 
727  std::pair<size_t, size_t> getStorageIndex(const size_t entry_idx) const;
728 
729  const std::vector<const int8_t*>& getColumnFrag(const size_t storge_idx,
730  const size_t col_logical_idx,
731  int64_t& global_idx) const;
732 
733  const VarlenOutputInfo* getVarlenOutputInfo(const size_t entry_idx) const;
734 
735  StorageLookupResult findStorage(const size_t entry_idx) const;
736 
737  struct TargetOffsets {
738  const int8_t* ptr1;
739  const size_t compact_sz1;
740  const int8_t* ptr2;
741  const size_t compact_sz2;
742  };
743 
745  RowWiseTargetAccessor(const ResultSet* result_set)
746  : result_set_(result_set)
748  , key_width_(result_set_->query_mem_desc_.getEffectiveKeyWidth())
752  }
753 
755  const int8_t* buff,
756  const size_t entry_idx,
757  const size_t target_logical_idx,
758  const StorageLookupResult& storage_lookup_result) const;
759 
761 
762  inline const int8_t* get_rowwise_ptr(const int8_t* buff,
763  const size_t entry_idx) const {
764  return buff + entry_idx * row_bytes_;
765  }
766 
767  std::vector<std::vector<TargetOffsets>> offsets_for_storage_;
768 
770 
771  // Row-wise iteration
772  const size_t row_bytes_;
773  const size_t key_width_;
775  };
776 
778  ColumnWiseTargetAccessor(const ResultSet* result_set) : result_set_(result_set) {
780  }
781 
783 
785  const int8_t* buff,
786  const size_t entry_idx,
787  const size_t target_logical_idx,
788  const StorageLookupResult& storage_lookup_result) const;
789 
790  std::vector<std::vector<TargetOffsets>> offsets_for_storage_;
791 
793  };
794 
795  using ApproxQuantileBuffers = std::vector<std::vector<double>>;
796 
797  template <typename BUFFER_ITERATOR_TYPE>
799  using BufferIteratorType = BUFFER_ITERATOR_TYPE;
800 
801  ResultSetComparator(const std::list<Analyzer::OrderEntry>& order_entries,
802  const ResultSet* result_set,
803  const PermutationView permutation,
804  const Executor* executor,
805  const bool single_threaded)
806  : order_entries_(order_entries)
807  , result_set_(result_set)
808  , permutation_(permutation)
809  , buffer_itr_(result_set)
810  , executor_(executor)
811  , single_threaded_(single_threaded)
814  }
815 
818 
819  std::vector<int64_t> materializeCountDistinctColumn(
820  const Analyzer::OrderEntry& order_entry) const;
821  ApproxQuantileBuffers::value_type materializeApproxQuantileColumn(
822  const Analyzer::OrderEntry& order_entry) const;
823 
824  bool operator()(const PermutationIdx lhs, const PermutationIdx rhs) const;
825 
826  const std::list<Analyzer::OrderEntry>& order_entries_;
830  const Executor* executor_;
831  const bool single_threaded_;
832  std::vector<std::vector<int64_t>> count_distinct_materialized_buffers_;
834  };
835 
836  Comparator createComparator(const std::list<Analyzer::OrderEntry>& order_entries,
837  const PermutationView permutation,
838  const Executor* executor,
839  const bool single_threaded) {
840  auto timer = DEBUG_TIMER(__func__);
843  order_entries, this, permutation, executor, single_threaded)](
844  const PermutationIdx lhs, const PermutationIdx rhs) {
845  return rsc(lhs, rhs);
846  };
847  } else {
849  order_entries, this, permutation, executor, single_threaded)](
850  const PermutationIdx lhs, const PermutationIdx rhs) {
851  return rsc(lhs, rhs);
852  };
853  }
854  }
855 
857  const size_t n,
858  const Comparator&);
859 
861  PermutationIdx const begin,
862  PermutationIdx const end) const;
863 
864  void parallelTop(const std::list<Analyzer::OrderEntry>& order_entries,
865  const size_t top_n,
866  const Executor* executor);
867 
868  void baselineSort(const std::list<Analyzer::OrderEntry>& order_entries,
869  const size_t top_n,
870  const Executor* executor);
871 
872  void doBaselineSort(const ExecutorDeviceType device_type,
873  const std::list<Analyzer::OrderEntry>& order_entries,
874  const size_t top_n,
875  const Executor* executor);
876 
877  bool canUseFastBaselineSort(const std::list<Analyzer::OrderEntry>& order_entries,
878  const size_t top_n);
879 
880  size_t rowCountImpl(const bool force_parallel) const;
881 
883 
884  int getGpuCount() const;
885 
886  void serializeProjection(TSerializedRows& serialized_rows) const;
887  void serializeVarlenAggColumn(int8_t* buf,
888  std::vector<std::string>& varlen_bufer) const;
889 
890  void serializeCountDistinctColumns(TSerializedRows&) const;
891 
892  void unserializeCountDistinctColumns(const TSerializedRows&);
893 
895 
896  void create_active_buffer_set(CountDistinctSet& count_distinct_active_buffer_set) const;
897 
898  int64_t getDistinctBufferRefFromBufferRowwise(int8_t* rowwise_target_ptr,
899  const TargetInfo& target_info) const;
900 
901  const std::vector<TargetInfo> targets_;
903  const int device_id_;
905  mutable std::unique_ptr<ResultSetStorage> storage_;
907  mutable size_t crt_row_buff_idx_;
908  mutable size_t fetched_so_far_;
909  size_t drop_first_;
910  size_t keep_first_;
911  std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner_;
913 
915  unsigned block_size_{0};
916  unsigned grid_size_{0};
918 
919  std::list<std::shared_ptr<Chunk_NS::Chunk>> chunks_;
920  std::vector<std::shared_ptr<std::list<ChunkIter>>> chunk_iters_;
921  // TODO(miyu): refine by using one buffer and
922  // setting offset instead of ptr in group by buffer.
923  std::vector<std::vector<int8_t>> literal_buffers_;
924  std::vector<ColumnLazyFetchInfo> lazy_fetch_info_;
925  std::vector<std::vector<std::vector<const int8_t*>>> col_buffers_;
926  std::vector<std::vector<std::vector<int64_t>>> frag_offsets_;
927  std::vector<std::vector<int64_t>> consistent_frag_sizes_;
928 
929  const std::shared_ptr<const Analyzer::Estimator> estimator_;
931  mutable int8_t* host_estimator_buffer_{nullptr};
933 
934  // only used by serialization
935  using SerializedVarlenBufferStorage = std::vector<std::string>;
936 
937  std::vector<SerializedVarlenBufferStorage> serialized_varlen_buffer_;
939  std::string explanation_;
940  const bool just_explain_;
942  mutable std::atomic<int64_t> cached_row_count_;
943  mutable std::mutex row_iteration_mutex_;
944 
945  // only used by geo
947 
948  // only used by data recycler
949  bool cached_; // indicator that this resultset is cached
950  size_t
951  query_exec_time_; // an elapsed time to process the query for this resultset (ms)
952  QueryPlanHash query_plan_; // a hashed query plan DAG of this resultset
953  std::unordered_set<size_t> input_table_keys_; // input table signatures
954  std::vector<TargetMetaInfo> target_meta_info_;
955  // if we recycle the resultset, we do not create work_unit of the query step
956  // because we may skip its child query step(s)
957  // so we try to keep whether this resultset is available to use speculative top n sort
958  // when it is inserted to the recycler, and reuse this info when recycled
959  std::optional<bool> can_use_speculative_top_n_sort;
960 
961  friend class ResultSetManager;
962  friend class ResultSetRowIterator;
963  friend class ColumnarResults;
964 };
965 
968  return {};
969  }
970 
971  if (result_set_->just_explain_) {
972  return {result_set_->explanation_};
973  }
974 
975  return result_set_->getRowAt(
977 }
978 
980  if (!result_set_->storage_ && !result_set_->just_explain_) {
981  global_entry_idx_valid_ = false;
982  } else if (result_set_->just_explain_) {
984  fetched_so_far_ = 1;
985  } else {
986  result_set_->advanceCursorToNextEntry(*this);
987  }
988  return *this;
989 }
990 
992  public:
993  ResultSet* reduce(std::vector<ResultSet*>&, const size_t executor_id);
994 
995  std::shared_ptr<ResultSet> getOwnResultSet();
996 
997  void rewriteVarlenAggregates(ResultSet*);
998 
999  private:
1000  std::shared_ptr<ResultSet> rs_;
1001 };
1002 
1003 class RowSortException : public std::runtime_error {
1004  public:
1005  RowSortException(const std::string& cause) : std::runtime_error(cause) {}
1006 };
1007 
1008 namespace result_set {
1009 
1010 bool can_use_parallel_algorithms(const ResultSet& rows);
1011 
1012 std::optional<size_t> first_dict_encoded_idx(std::vector<TargetInfo> const&);
1013 
1014 bool use_parallel_algorithms(const ResultSet& rows);
1015 
1016 } // namespace result_set
1017 
1018 #endif // QUERYENGINE_RESULTSET_H
void setSeparateVarlenStorageValid(const bool val)
Definition: ResultSet.h:571
void setGeoReturnType(const GeoReturnType val)
Definition: ResultSet.h:529
void serializeVarlenAggColumn(int8_t *buf, std::vector< std::string > &varlen_bufer) const
std::mutex row_iteration_mutex_
Definition: ResultSet.h:943
void setUseSpeculativeTopNSort(bool value)
Definition: ResultSet.h:507
InternalTargetValue getColumnInternal(const int8_t *buff, const size_t entry_idx, const size_t target_logical_idx, const StorageLookupResult &storage_lookup_result) const
void syncEstimatorBuffer() const
Definition: ResultSet.cpp:696
const int8_t * ptr1
Definition: ResultSet.h:738
const size_t compact_sz2
Definition: ResultSet.h:741
void holdChunks(const std::list< std::shared_ptr< Chunk_NS::Chunk >> &chunks)
Definition: ResultSet.h:421
const QueryMemoryDescriptor & getQueryMemDesc() const
Definition: ResultSet.cpp:672
void setQueryPlanHash(const QueryPlanHash query_plan)
Definition: ResultSet.h:485
robin_hood::unordered_set< int64_t > CountDistinctSet
Definition: CountDistinct.h:37
std::pair< size_t, size_t > getStorageIndex(const size_t entry_idx) const
Definition: ResultSet.cpp:914
std::shared_ptr< RowSetMemoryOwner > getRowSetMemOwner() const
Definition: ResultSet.h:431
bool isValidationOnlyRes() const
Definition: ResultSet.cpp:748
Permutation permutation_
Definition: ResultSet.h:912
bool didOutputColumnar() const
Definition: ResultSet.h:537
void setValidationOnlyRes()
Definition: ResultSet.cpp:744
PermutationView initPermutationBuffer(PermutationView permutation, PermutationIdx const begin, PermutationIdx const end) const
Definition: ResultSet.cpp:846
bool for_validation_only_
Definition: ResultSet.h:941
std::ptrdiff_t difference_type
Definition: ResultSet.h:99
void setCached(bool val)
Definition: ResultSet.h:477
ENTRY_TYPE getRowWisePerfectHashEntryAt(const size_t row_idx, const size_t target_idx, const size_t slot_idx) const
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:114
void create_active_buffer_set(CountDistinctSet &count_distinct_active_buffer_set) const
void setEntryCount(const size_t val)
double decimal_to_double(const SQLTypeInfo &otype, int64_t oval)
const ApproxQuantileBuffers approx_quantile_materialized_buffers_
Definition: ResultSet.h:833
AppendedStorage appended_storage_
Definition: ResultSet.h:906
ENTRY_TYPE getColumnarPerfectHashEntryAt(const size_t row_idx, const size_t target_idx, const size_t slot_idx) const
GeoReturnType geo_return_type_
Definition: ResultSet.h:946
ExecutorDeviceType
const BufferIteratorType buffer_itr_
Definition: ResultSet.h:829
void moveToBegin() const
Definition: ResultSet.cpp:731
Utility functions for easy access to the result set buffers.
std::shared_ptr< ResultSet > rs_
Definition: ResultSet.h:1000
const Executor * executor_
Definition: ResultSet.h:830
std::vector< std::string > SerializedVarlenBufferStorage
Definition: ResultSet.h:935
void initializeStorage() const
std::optional< bool > can_use_speculative_top_n_sort
Definition: ResultSet.h:959
QueryDescriptionType getQueryDescriptionType() const
Definition: ResultSet.h:542
ResultSetRowIterator(const ResultSet *rs, bool translate_strings, bool decimal_to_double)
Definition: ResultSet.h:134
const Catalog_Namespace::Catalog * catalog_
Definition: ResultSet.h:914
size_t getNumColumnsLazyFetched() const
Definition: ResultSet.h:566
void unserializeCountDistinctColumns(const TSerializedRows &)
ApproxQuantileBuffers materializeApproxQuantileColumns() const
Definition: ResultSet.cpp:959
std::vector< TargetValue > getNextRow(const bool translate_strings, const bool decimal_to_double) const
static bool isNull(const SQLTypeInfo &ti, const InternalTargetValue &val, const bool float_argument_input)
ResultSet(const std::vector< TargetInfo > &targets, const ExecutorDeviceType device_type, const QueryMemoryDescriptor &query_mem_desc, const std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner, const Catalog_Namespace::Catalog *catalog, const unsigned block_size, const unsigned grid_size)
Definition: ResultSet.cpp:62
QueryMemoryDescriptor query_mem_desc_
Definition: ResultSet.h:904
const std::vector< TargetInfo > & getTargetInfos() const
Definition: ResultSet.cpp:677
std::optional< size_t > first_dict_encoded_idx(std::vector< TargetInfo > const &)
Definition: ResultSet.cpp:1461
std::unique_ptr< ResultSetStorage > storage_
Definition: ResultSet.h:905
void setKernelQueueTime(const int64_t kernel_queue_time)
Definition: ResultSet.cpp:714
bool operator==(const ResultSetRowIterator &other) const
Definition: ResultSet.h:104
size_t query_exec_time_
Definition: ResultSet.h:951
std::string getExplanation() const
Definition: ResultSet.h:395
ENTRY_TYPE getEntryAt(const size_t row_idx, const size_t target_idx, const size_t slot_idx) const
size_t rowCount(const bool force_parallel=false) const
Returns the number of valid entries in the result set (i.e that will be returned from the SQL query o...
Definition: ResultSet.cpp:593
ResultSetRowIterator(const ResultSet *rs)
Definition: ResultSet.h:145
std::shared_ptr< ResultSet > ResultSetPtr
TargetValue makeGeoTargetValue(const int8_t *geo_target_ptr, const size_t slot_idx, const TargetInfo &target_info, const size_t target_logical_idx, const size_t entry_buff_idx) const
TargetValue getTargetValueFromBufferRowwise(int8_t *rowwise_target_ptr, int8_t *keys_ptr, const size_t entry_buff_idx, const TargetInfo &target_info, const size_t target_logical_idx, const size_t slot_idx, const bool translate_strings, const bool decimal_to_double, const bool fixup_count_distinct_pointers) const
size_t keep_first_
Definition: ResultSet.h:910
void keepFirstN(const size_t n)
Definition: ResultSet.cpp:52
std::vector< std::shared_ptr< std::list< ChunkIter > > > chunk_iters_
Definition: ResultSet.h:920
void addCompilationQueueTime(const int64_t compilation_queue_time)
Definition: ResultSet.cpp:718
std::vector< std::vector< double >> ApproxQuantileBuffers
Definition: ResultSet.h:795
void parallelTop(const std::list< Analyzer::OrderEntry > &order_entries, const size_t top_n, const Executor *executor)
Definition: ResultSet.cpp:866
void serialize(TSerializedRows &serialized_rows) const
std::vector< SerializedVarlenBufferStorage > serialized_varlen_buffer_
Definition: ResultSet.h:937
const size_t compact_sz1
Definition: ResultSet.h:739
int64_t lazyReadInt(const int64_t ival, const size_t target_logical_idx, const StorageLookupResult &storage_lookup_result) const
bool operator!=(const ResultSetRowIterator &other) const
Definition: ResultSet.h:108
size_t colCount() const
Definition: ResultSet.cpp:413
OneIntegerColumnRow getOneColRow(const size_t index) const
TargetValue getTargetValueFromBufferColwise(const int8_t *col_ptr, const int8_t *keys_ptr, const QueryMemoryDescriptor &query_mem_desc, const size_t local_entry_idx, const size_t global_entry_idx, const TargetInfo &target_info, const size_t target_logical_idx, const size_t slot_idx, const bool translate_strings, const bool decimal_to_double) const
void setInputTableKeys(std::unordered_set< size_t > &&intput_table_keys)
Definition: ResultSet.h:491
void rewriteVarlenAggregates(ResultSet *)
size_t getLimit() const
Definition: ResultSet.cpp:1302
const bool just_explain_
Definition: ResultSet.h:940
std::vector< int64_t > materializeCountDistinctColumn(const Analyzer::OrderEntry &order_entry) const
Definition: ResultSet.cpp:972
ApproxQuantileBuffers::value_type materializeApproxQuantileColumn(const Analyzer::OrderEntry &order_entry) const
Definition: ResultSet.cpp:1018
ResultSetRowIterator rowIterator(size_t from_logical_index, bool translate_strings, bool decimal_to_double) const
Definition: ResultSet.h:205
unsigned block_size_
Definition: ResultSet.h:915
bool isTruncated() const
Definition: ResultSet.cpp:736
std::atomic< int64_t > cached_row_count_
Definition: ResultSet.h:942
const bool isPermutationBufferEmpty() const
Definition: ResultSet.h:436
size_t parallelRowCount() const
Definition: ResultSet.cpp:629
const size_t key_bytes_with_padding_
Definition: ResultSet.h:774
bool areAnyColumnsLazyFetched() const
Definition: ResultSet.h:561
const ResultSet * result_set_
Definition: ResultSet.h:769
std::vector< TargetValue > getRowAtNoTranslations(const size_t index, const std::vector< bool > &targets_to_skip={}) const
const ResultSet * result_set_
Definition: ResultSet.h:126
void radixSortOnCpu(const std::list< Analyzer::OrderEntry > &order_entries) const
Definition: ResultSet.cpp:1262
bool definitelyHasNoRows() const
Definition: ResultSet.cpp:668
ColumnWiseTargetAccessor(const ResultSet *result_set)
Definition: ResultSet.h:778
const std::vector< std::string > getStringDictionaryPayloadCopy(const int dict_id) const
Definition: ResultSet.cpp:1306
bool use_parallel_algorithms(const ResultSet &rows)
Definition: ResultSet.cpp:1468
bool isZeroCopyColumnarConversionPossible(size_t column_idx) const
Definition: ResultSet.cpp:1379
std::input_iterator_tag iterator_category
Definition: ResultSet.h:102
size_t global_entry_idx_
Definition: ResultSet.h:128
int8_t * getHostEstimatorBuffer() const
Definition: ResultSet.cpp:692
InternalTargetValue getVarlenOrderEntry(const int64_t str_ptr, const size_t str_len) const
const std::vector< TargetInfo > targets_
Definition: ResultSet.h:901
void invalidateCachedRowCount() const
Definition: ResultSet.cpp:605
std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner_
Definition: ResultSet.h:911
const Executor * getExecutor() const
Definition: ResultSet.h:605
void setExecTime(const long exec_time)
Definition: ResultSet.h:481
size_t drop_first_
Definition: ResultSet.h:909
const ResultSetStorage * allocateStorage() const
bool cached_
Definition: ResultSet.h:949
std::string toString() const
Definition: ResultSet.h:198
const int8_t * ptr2
Definition: ResultSet.h:740
std::list< std::shared_ptr< Chunk_NS::Chunk > > chunks_
Definition: ResultSet.h:919
std::unordered_set< size_t > getInputTableKeys() const
Definition: ResultSet.h:489
QueryExecutionTimings timings_
Definition: ResultSet.h:917
const ResultSet * result_set_
Definition: ResultSet.h:827
void sort(const std::list< Analyzer::OrderEntry > &order_entries, size_t top_n, const Executor *executor)
Definition: ResultSet.cpp:768
DEVICE auto copy(ARGS &&...args)
Definition: gpu_enabled.h:51
void setQueueTime(const int64_t queue_time)
Definition: ResultSet.cpp:710
void dropFirstN(const size_t n)
Definition: ResultSet.cpp:57
std::vector< std::vector< int8_t > > literal_buffers_
Definition: ResultSet.h:923
std::vector< std::vector< TargetOffsets > > offsets_for_storage_
Definition: ResultSet.h:790
bool canUseFastBaselineSort(const std::list< Analyzer::OrderEntry > &order_entries, const size_t top_n)
unsigned grid_size_
Definition: ResultSet.h:916
void setTargetMetaInfo(const std::vector< TargetMetaInfo > &target_meta_info)
Definition: ResultSet.h:495
std::unordered_set< size_t > input_table_keys_
Definition: ResultSet.h:953
const std::list< Analyzer::OrderEntry > & order_entries_
Definition: ResultSet.h:826
std::vector< TargetValue > & reference
Definition: ResultSet.h:101
std::vector< PermutationIdx > Permutation
Definition: ResultSet.h:155
std::tuple< std::vector< bool >, size_t > getSingleSlotTargetBitmap() const
Definition: ResultSet.cpp:1394
ResultSetRowIterator & operator++(void)
Definition: ResultSet.h:979
std::shared_ptr< ResultSet > getOwnResultSet()
StorageLookupResult findStorage(const size_t entry_idx) const
Definition: ResultSet.cpp:939
const long getExecTime() const
Definition: ResultSet.h:483
Comparator createComparator(const std::list< Analyzer::OrderEntry > &order_entries, const PermutationView permutation, const Executor *executor, const bool single_threaded)
Definition: ResultSet.h:836
An AbstractBuffer is a unit of data management for a data manager.
const PermutationView permutation_
Definition: ResultSet.h:828
const int8_t getPaddedSlotWidthBytes(const size_t slot_idx) const
value_type operator*() const
Definition: ResultSet.h:966
RowWiseTargetAccessor(const ResultSet *result_set)
Definition: ResultSet.h:745
ResultSetPtr copy()
Definition: ResultSet.cpp:329
void copyColumnIntoBuffer(const size_t column_idx, int8_t *output_buffer, const size_t output_buffer_size) const
RowSortException(const std::string &cause)
Definition: ResultSet.h:1005
std::function< bool(const PermutationIdx, const PermutationIdx)> Comparator
Definition: ResultSet.h:157
static double calculateQuantile(quantile::TDigest *const t_digest)
Definition: ResultSet.cpp:1008
friend ResultSetBuilder
Definition: ResultSet.h:161
const bool isEstimator() const
Definition: ResultSet.h:475
const VarlenOutputInfo * getVarlenOutputInfo(const size_t entry_idx) const
void fixupCountDistinctPointers()
void radixSortOnGpu(const std::list< Analyzer::OrderEntry > &order_entries) const
Definition: ResultSet.cpp:1222
const ResultSetStorage * getStorage() const
Definition: ResultSet.cpp:409
QueryDescriptionType getQueryDescriptionType() const
const std::pair< std::vector< int32_t >, std::vector< std::string > > getUniqueStringsForDictEncodedTargetCol(const size_t col_idx) const
Definition: ResultSet.cpp:1315
Data_Namespace::DataMgr * data_mgr_
Definition: ResultSet.h:932
Basic constructors and methods of the row set interface.
int64_t getQueueTime() const
Definition: ResultSet.cpp:722
std::optional< bool > canUseSpeculativeTopNSort() const
Definition: ResultSet.h:503
std::vector< TargetValue > getRowAt(const size_t index) const
Executor(const ExecutorId id, Data_Namespace::DataMgr *data_mgr, const size_t block_size_x, const size_t grid_size_x, const size_t max_gpu_slab_size, const std::string &debug_dir, const std::string &debug_file)
Definition: Execute.cpp:260
void fillOneEntry(const std::vector< int64_t > &entry)
Definition: ResultSet.h:410
void updateStorageEntryCount(const size_t new_entry_count)
Definition: ResultSet.h:236
uint32_t PermutationIdx
Definition: ResultSet.h:154
void serializeProjection(TSerializedRows &serialized_rows) const
ResultSetRowIterator operator++(int)
Definition: ResultSet.h:112
void invalidateResultSetChunks()
Definition: ResultSet.h:466
const std::shared_ptr< const Analyzer::Estimator > estimator_
Definition: ResultSet.h:929
SQLTypeInfo getColType(const size_t col_idx) const
Definition: ResultSet.cpp:417
GeoReturnType getGeoReturnType() const
Definition: ResultSet.h:528
void holdChunkIterators(const std::shared_ptr< std::list< ChunkIter >> chunk_iters)
Definition: ResultSet.h:424
std::tuple< std::vector< bool >, size_t > getSupportedSingleSlotTargetBitmap() const
Definition: ResultSet.cpp:1418
ExecutorDeviceType getDeviceType() const
Definition: ResultSet.cpp:250
StringDictionaryProxy * getStringDictionaryProxy(int const dict_id) const
Definition: ResultSet.cpp:426
const int8_t * getColumnarBuffer(size_t column_idx) const
Definition: ResultSet.cpp:1388
ResultSetComparator(const std::list< Analyzer::OrderEntry > &order_entries, const ResultSet *result_set, const PermutationView permutation, const Executor *executor, const bool single_threaded)
Definition: ResultSet.h:801
size_t get_row_bytes(const QueryMemoryDescriptor &query_mem_desc)
bool isExplain() const
Definition: ResultSet.cpp:740
void eachCellInColumn(RowIterationState &, CellCallback const &)
Definition: ResultSet.cpp:485
std::vector< TargetValue > value_type
Definition: ResultSet.h:98
bool isGeoColOnGpu(const size_t col_idx) const
std::vector< TargetMetaInfo > target_meta_info_
Definition: ResultSet.h:954
const QueryPlanHash getQueryPlanHash()
Definition: ResultSet.h:487
const int8_t * get_rowwise_ptr(const int8_t *buff, const size_t entry_idx) const
Definition: ResultSet.h:762
size_t getNDVEstimator() const
std::vector< std::vector< std::vector< const int8_t * > > > col_buffers_
Definition: ResultSet.h:925
bool isRowAtEmpty(const size_t index) const
const int8_t getPaddedSlotWidthBytes(const size_t slot_idx) const
Definition: ResultSet.h:546
size_t entryCount() const
Returns the number of entries the result set is allocated to hold.
size_t QueryPlanHash
static QueryMemoryDescriptor fixupQueryMemoryDescriptor(const QueryMemoryDescriptor &)
Definition: ResultSet.cpp:756
std::string typeName(const T *v)
Definition: toString.h:102
size_t rowCountImpl(const bool force_parallel) const
Definition: ResultSet.cpp:555
TargetValue makeTargetValue(const int8_t *ptr, const int8_t compact_sz, const TargetInfo &target_info, const size_t target_logical_idx, const bool translate_strings, const bool decimal_to_double, const size_t entry_buff_idx) const
void baselineSort(const std::list< Analyzer::OrderEntry > &order_entries, const size_t top_n, const Executor *executor)
size_t getCurrentRowBufferIndex() const
Definition: ResultSet.h:118
const Permutation & getPermutationBuffer() const
Definition: ResultSet.cpp:862
void append(ResultSet &that)
Definition: ResultSet.cpp:299
std::string summaryToString() const
Definition: ResultSet.cpp:218
std::string explanation_
Definition: ResultSet.h:939
std::vector< std::vector< int64_t > > consistent_frag_sizes_
Definition: ResultSet.h:927
int8_t * host_estimator_buffer_
Definition: ResultSet.h:931
friend class ResultSet
Definition: ResultSet.h:145
const ExecutorDeviceType device_type_
Definition: ResultSet.h:902
const bool hasValidBuffer() const
Definition: ResultSet.h:509
std::vector< TargetValue > getNextRowImpl(const bool translate_strings, const bool decimal_to_double) const
const bool isCached() const
Definition: ResultSet.h:479
static PermutationView topPermutation(PermutationView, const size_t n, const Comparator &)
Definition: ResultSet.cpp:1208
size_t getCurrentRowBufferIndex() const
Definition: ResultSet.cpp:291
void holdLiterals(std::vector< int8_t > &literal_buff)
Definition: ResultSet.h:427
bool g_enable_watchdog false
Definition: Execute.cpp:79
#define CHECK(condition)
Definition: Logger.h:223
#define DEBUG_TIMER(name)
Definition: Logger.h:370
int getGpuCount() const
size_t getBufferSizeBytes(const ExecutorDeviceType device_type) const
int8_t * getDeviceEstimatorBuffer() const
Definition: ResultSet.cpp:686
size_t fetched_so_far_
Definition: ResultSet.h:908
std::vector< ColumnLazyFetchInfo > lazy_fetch_info_
Definition: ResultSet.h:924
size_t crt_row_buff_idx_
Definition: ResultSet.h:907
QueryPlanHash query_plan_
Definition: ResultSet.h:952
Estimators to be used when precise cardinality isn&#39;t useful.
QueryDescriptionType
Definition: Types.h:26
int64_t getDistinctBufferRefFromBufferRowwise(int8_t *rowwise_target_ptr, const TargetInfo &target_info) const
std::vector< std::vector< std::vector< int64_t > > > frag_offsets_
Definition: ResultSet.h:926
bool operator()(const PermutationIdx lhs, const PermutationIdx rhs) const
Definition: ResultSet.cpp:1051
void doBaselineSort(const ExecutorDeviceType device_type, const std::list< Analyzer::OrderEntry > &order_entries, const size_t top_n, const Executor *executor)
bool separate_varlen_storage_valid_
Definition: ResultSet.h:938
boost::variant< ScalarTargetValue, ArrayTargetValue, GeoTargetValue, GeoTargetValuePtr > TargetValue
Definition: TargetValue.h:167
std::vector< TargetValue > getNextRowUnlocked(const bool translate_strings, const bool decimal_to_double) const
std::vector< TargetValue > * pointer
Definition: ResultSet.h:100
bool isEmpty() const
Returns a boolean signifying whether there are valid entries in the result set.
Definition: ResultSet.cpp:649
const std::vector< int64_t > & getTargetInitVals() const
Definition: ResultSet.cpp:681
std::vector< size_t > getSlotIndicesForTargetIndices() const
Definition: ResultSet.cpp:1437
size_t advanceCursorToNextEntry() const
constexpr double n
Definition: Utm.h:38
ResultSetRowIterator rowIterator(bool translate_strings, bool decimal_to_double) const
Definition: ResultSet.h:220
Definition: Analyzer.h:2227
BUFFER_ITERATOR_TYPE BufferIteratorType
Definition: ResultSet.h:799
ENTRY_TYPE getColumnarBaselineEntryAt(const size_t row_idx, const size_t target_idx, const size_t slot_idx) const
Data_Namespace::DataMgr * getDataManager() const
size_t crt_row_buff_idx_
Definition: ResultSet.h:127
std::vector< std::vector< int64_t > > count_distinct_materialized_buffers_
Definition: ResultSet.h:832
Data_Namespace::AbstractBuffer * device_estimator_buffer_
Definition: ResultSet.h:930
InternalTargetValue getColumnInternal(const int8_t *buff, const size_t entry_idx, const size_t target_logical_idx, const StorageLookupResult &storage_lookup_result) const
std::vector< std::vector< TargetOffsets > > offsets_for_storage_
Definition: ResultSet.h:767
std::vector< TargetMetaInfo > getTargetMetaInfo()
Definition: ResultSet.h:501
void translateDictEncodedColumns(std::vector< TargetInfo > const &, size_t const start_idx)
Definition: ResultSet.cpp:455
bool global_entry_idx_valid_
Definition: ResultSet.h:129
bool can_use_parallel_algorithms(const ResultSet &rows)
Definition: ResultSet.cpp:1449
int64_t getRenderTime() const
Definition: ResultSet.cpp:727
void setCachedRowCount(const size_t row_count) const
Definition: ResultSet.cpp:609
bool isDirectColumnarConversionPossible() const
Definition: ResultSet.cpp:1357
size_t get_key_bytes_rowwise(const QueryMemoryDescriptor &query_mem_desc)
void initStatus()
Definition: ResultSet.h:454
FORCE_INLINE HOST DEVICE T align_to_int64(T addr)
TargetValue makeVarlenTargetValue(const int8_t *ptr1, const int8_t compact_sz1, const int8_t *ptr2, const int8_t compact_sz2, const TargetInfo &target_info, const size_t target_logical_idx, const bool translate_strings, const size_t entry_buff_idx) const
void serializeCountDistinctColumns(TSerializedRows &) const
const ResultSetStorage * storage_ptr
Definition: ResultSet.h:712
ResultSet * reduce(std::vector< ResultSet * > &, const size_t executor_id)
std::vector< std::unique_ptr< ResultSetStorage >> AppendedStorage
Definition: ResultSet.h:153
const std::vector< const int8_t * > & getColumnFrag(const size_t storge_idx, const size_t col_logical_idx, int64_t &global_idx) const
const Executor * getExecutor() const
size_t binSearchRowCount() const
Definition: ResultSet.cpp:616
void clearPermutation()
Definition: ResultSet.h:448
ChunkStats getTableFunctionChunkStats(const size_t target_idx) const
const std::vector< ColumnLazyFetchInfo > & getLazyFetchInfo() const
Definition: ResultSet.h:557
ENTRY_TYPE getRowWiseBaselineEntryAt(const size_t row_idx, const size_t target_idx, const size_t slot_idx) const
int getDeviceId() const
Definition: ResultSet.cpp:752
static std::unique_ptr< ResultSet > unserialize(const TSerializedRows &serialized_rows, const Executor *)
const int device_id_
Definition: ResultSet.h:903