OmniSciDB  1dac507f6e
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
ResultSet.h
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
25 #ifndef QUERYENGINE_RESULTSET_H
26 #define QUERYENGINE_RESULTSET_H
27 
28 #include "../Chunk/Chunk.h"
29 #include "CardinalityEstimator.h"
31 #include "TargetValue.h"
32 
33 #include <atomic>
34 #include <functional>
35 #include <list>
36 
37 /*
38  * Stores the underlying buffer and the meta-data for a result set. The buffer
39  * format reflects the main requirements for result sets. Not all queries
40  * specify a GROUP BY clause, but since it's the most important and challenging
41  * case we'll focus on it. Note that the meta-data is stored separately from
42  * the buffer and it's not transferred to GPU.
43  *
44  * 1. It has to be efficient for reduction of partial GROUP BY query results
45  * from multiple devices / cores, the cardinalities can be high. Reduction
46  * currently happens on the host.
47  * 2. No conversions should be needed when buffers are transferred from GPU to
48  * host for reduction. This implies the buffer needs to be "flat", with no
49  * pointers to chase since they have no meaning in a different address space.
50  * 3. Must be size-efficient.
51  *
52  * There are several variations of the format of a result set buffer, but the
53  * most common is a sequence of entries which represent a row in the result or
54  * an empty slot. One entry looks as follows:
55  *
56  * +-+-+-+-+-+-+-+-+-+-+-+--?--+-+-+-+-+-+-+-+-+-+-+-+-+
57  * |key_0| ... |key_N-1| padding |value_0|...|value_N-1|
58  * +-+-+-+-+-+-+-+-+-+-+-+--?--+-+-+-+-+-+-+-+-+-+-+-+-+
59  *
60  * (key_0 ... key_N-1) is a multiple component key, unique within the buffer.
61  * It stores the tuple specified by the GROUP BY clause. All components have
62  * the same width, 4 or 8 bytes. For the 4-byte components, 4-byte padding is
63  * added if the number of components is odd. Not all entries in the buffer are
64  * valid; an empty entry contains EMPTY_KEY_{64, 32} for 8-byte / 4-byte width,
65  * respectively. An empty entry is ignored by subsequent operations on the
66  * result set (reduction, iteration, sort etc).
67  *
68  * value_0 through value_N-1 are 8-byte fields which hold the columns of the
69  * result, like aggregates and projected expressions. They're reduced between
70  * multiple partial results for identical (key_0 ... key_N-1) tuples.
71  *
72  * The order of entries is decided by the type of hash used, which depends on
73  * the range of the keys. For small enough ranges, a perfect hash is used. When
74  * a perfect hash isn't feasible, open addressing (using MurmurHash) with linear
75  * probing is used instead, with a 50% fill rate.
76  */
77 
78 struct ReductionCode;
79 
81  public:
82  ResultSetStorage(const std::vector<TargetInfo>& targets,
84  int8_t* buff,
85  const bool buff_is_provided);
86 
87  void reduce(const ResultSetStorage& that,
88  const std::vector<std::string>& serialized_varlen_buffer,
89  const ReductionCode& reduction_code) const;
90 
92  const std::vector<std::string>& serialized_varlen_buffer) const;
93 
94  int8_t* getUnderlyingBuffer() const;
95 
96  template <class KeyType>
97  void moveEntriesToBuffer(int8_t* new_buff, const size_t new_entry_count) const;
98 
99  template <class KeyType>
100  void moveOneEntryToBuffer(const size_t entry_index,
101  int64_t* new_buff_i64,
102  const size_t new_entry_count,
103  const size_t key_count,
104  const size_t row_qw_count,
105  const int64_t* src_buff,
106  const size_t key_byte_width) const;
107 
108  void updateEntryCount(const size_t new_entry_count) {
109  query_mem_desc_.setEntryCount(new_entry_count);
110  }
111 
112  // Reduces results for a single row when using interleaved bin layouts
113  static bool reduceSingleRow(const int8_t* row_ptr,
114  const int8_t warp_count,
115  const bool is_columnar,
116  const bool replace_bitmap_ptr_with_bitmap_sz,
117  std::vector<int64_t>& agg_vals,
118  const QueryMemoryDescriptor& query_mem_desc,
119  const std::vector<TargetInfo>& targets,
120  const std::vector<int64_t>& agg_init_vals);
121 
122  private:
124  int8_t* this_buff,
125  const int8_t* that_buff,
126  const ResultSetStorage& that,
127  const size_t start_index,
128  const size_t end_index,
129  const std::vector<std::string>& serialized_varlen_buffer) const;
130 
131  void copyKeyColWise(const size_t entry_idx,
132  int8_t* this_buff,
133  const int8_t* that_buff) const;
134 
135  bool isEmptyEntry(const size_t entry_idx, const int8_t* buff) const;
136  bool isEmptyEntry(const size_t entry_idx) const;
137  bool isEmptyEntryColumnar(const size_t entry_idx, const int8_t* buff) const;
138 
139  void reduceOneEntryBaseline(int8_t* this_buff,
140  const int8_t* that_buff,
141  const size_t i,
142  const size_t that_entry_count,
143  const ResultSetStorage& that) const;
144 
145  void reduceOneEntrySlotsBaseline(int64_t* this_entry_slots,
146  const int64_t* that_buff,
147  const size_t that_entry_idx,
148  const size_t that_entry_count,
149  const ResultSetStorage& that) const;
150 
151  void initializeBaselineValueSlots(int64_t* this_entry_slots) const;
152 
153  void reduceOneSlotBaseline(int64_t* this_buff,
154  const size_t this_slot,
155  const int64_t* that_buff,
156  const size_t that_entry_count,
157  const size_t that_slot,
158  const TargetInfo& target_info,
159  const size_t target_logical_idx,
160  const size_t target_slot_idx,
161  const size_t init_agg_val_idx,
162  const ResultSetStorage& that) const;
163 
165  void reduceOneSlot(int8_t* this_ptr1,
166  int8_t* this_ptr2,
167  const int8_t* that_ptr1,
168  const int8_t* that_ptr2,
169  const TargetInfo& target_info,
170  const size_t target_logical_idx,
171  const size_t target_slot_idx,
172  const size_t init_agg_val_idx,
173  const ResultSetStorage& that,
174  const size_t first_slot_idx_for_target,
175  const std::vector<std::string>& serialized_varlen_buffer) const;
176 
177  void reduceOneCountDistinctSlot(int8_t* this_ptr1,
178  const int8_t* that_ptr1,
179  const size_t target_logical_idx,
180  const ResultSetStorage& that) const;
181 
182  void fillOneEntryRowWise(const std::vector<int64_t>& entry);
183 
184  void fillOneEntryColWise(const std::vector<int64_t>& entry);
185 
186  void initializeRowWise() const;
187 
188  void initializeColWise() const;
189 
190  // TODO(alex): remove the following two methods, see comment about
191  // count_distinct_sets_mapping_.
192  void addCountDistinctSetPointerMapping(const int64_t remote_ptr, const int64_t ptr);
193 
194  int64_t mappedPtr(const int64_t) const;
195 
196  const std::vector<TargetInfo> targets_;
198  int8_t* buff_;
199  const bool buff_is_provided_;
200  std::vector<int64_t> target_init_vals_;
201  // Provisional field used for multi-node until we improve the count distinct
202  // and flatten the main group by buffer and the distinct buffers in a single,
203  // contiguous buffer which we'll be able to serialize as a no-op. Used to
204  // re-route the pointers in the result set received over the wire to this
205  // machine address-space. Not efficient at all, just a placeholder!
206  std::unordered_map<int64_t, int64_t> count_distinct_sets_mapping_;
207 
208  friend class ResultSet;
209  friend class ResultSetManager;
210 };
211 
212 namespace Analyzer {
213 
214 class Expr;
215 class Estimator;
216 struct OrderEntry;
217 
218 } // namespace Analyzer
219 
220 class Executor;
221 
223  const bool is_lazily_fetched;
224  const int local_col_id;
226 };
227 
229  const int64_t value;
230  const bool valid;
231 };
232 
233 class ResultSet;
234 
236  public:
237  using value_type = std::vector<TargetValue>;
238  using difference_type = std::ptrdiff_t;
239  using pointer = std::vector<TargetValue>*;
240  using reference = std::vector<TargetValue>&;
241  using iterator_category = std::input_iterator_tag;
242 
243  bool operator==(const ResultSetRowIterator& other) const {
244  return result_set_ == other.result_set_ &&
246  }
247  bool operator!=(const ResultSetRowIterator& other) const { return !(*this == other); }
248 
249  inline value_type operator*() const;
250  inline ResultSetRowIterator& operator++(void);
252  ResultSetRowIterator iter(*this);
253  ++(*this);
254  return iter;
255  }
256 
257  size_t getCurrentRowBufferIndex() const {
258  if (crt_row_buff_idx_ == 0) {
259  throw std::runtime_error("current row buffer iteration index is undefined");
260  }
261  return crt_row_buff_idx_ - 1;
262  }
263 
264  private:
272 
274  bool translate_strings,
275  bool decimal_to_double)
276  : result_set_(rs)
277  , crt_row_buff_idx_(0)
278  , global_entry_idx_(0)
280  , fetched_so_far_(0)
281  , translate_strings_(translate_strings)
282  , decimal_to_double_(decimal_to_double){};
283 
285 
286  friend class ResultSet;
287 };
288 
289 class TSerializedRows;
290 
291 class ResultSet {
292  public:
293  ResultSet(const std::vector<TargetInfo>& targets,
294  const ExecutorDeviceType device_type,
296  const std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner,
297  const Executor* executor);
298 
299  ResultSet(const std::vector<TargetInfo>& targets,
300  const std::vector<ColumnLazyFetchInfo>& lazy_fetch_info,
301  const std::vector<std::vector<const int8_t*>>& col_buffers,
302  const std::vector<std::vector<int64_t>>& frag_offsets,
303  const std::vector<int64_t>& consistent_frag_sizes,
304  const ExecutorDeviceType device_type,
305  const int device_id,
306  const QueryMemoryDescriptor& query_mem_desc,
307  const std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner,
308  const Executor* executor);
309 
310  ResultSet(const std::shared_ptr<const Analyzer::Estimator>,
311  const ExecutorDeviceType device_type,
312  const int device_id,
313  Data_Namespace::DataMgr* data_mgr);
314 
315  ResultSet(const std::string& explanation);
316 
317  ResultSet(int64_t queue_time_ms,
318  int64_t render_time_ms,
319  const std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner);
320 
321  ~ResultSet();
322 
323  inline ResultSetRowIterator rowIterator(size_t from_logical_index,
324  bool translate_strings,
325  bool decimal_to_double) const {
326  ResultSetRowIterator rowIterator(this, translate_strings, decimal_to_double);
327 
328  // move to first logical position
329  ++rowIterator;
330 
331  for (size_t index = 0; index < from_logical_index; index++) {
332  ++rowIterator;
333  }
334 
335  return rowIterator;
336  }
337 
338  inline ResultSetRowIterator rowIterator(bool translate_strings,
339  bool decimal_to_double) const {
340  return rowIterator(0, translate_strings, decimal_to_double);
341  }
342 
344 
345  const ResultSetStorage* allocateStorage() const;
346 
347  const ResultSetStorage* allocateStorage(int8_t*, const std::vector<int64_t>&) const;
348 
349  const ResultSetStorage* allocateStorage(const std::vector<int64_t>&) const;
350 
351  void updateStorageEntryCount(const size_t new_entry_count) {
353  query_mem_desc_.setEntryCount(new_entry_count);
354  CHECK(storage_);
355  storage_->updateEntryCount(new_entry_count);
356  }
357 
358  std::vector<TargetValue> getNextRow(const bool translate_strings,
359  const bool decimal_to_double) const;
360 
361  size_t getCurrentRowBufferIndex() const;
362 
363  std::vector<TargetValue> getRowAt(const size_t index) const;
364 
365  TargetValue getRowAt(const size_t row_idx,
366  const size_t col_idx,
367  const bool translate_strings,
368  const bool decimal_to_double = true) const;
369 
370  // Specialized random access getter for result sets with a single column to
371  // avoid the overhead of building a std::vector<TargetValue> result with only
372  // one element. Only used by RelAlgTranslator::getInIntegerSetExpr currently.
373  OneIntegerColumnRow getOneColRow(const size_t index) const;
374 
375  std::vector<TargetValue> getRowAtNoTranslations(
376  const size_t index,
377  const std::vector<bool>& targets_to_skip = {}) const;
378 
379  bool isRowAtEmpty(const size_t index) const;
380 
381  void sort(const std::list<Analyzer::OrderEntry>& order_entries, const size_t top_n);
382 
383  void keepFirstN(const size_t n);
384 
385  void dropFirstN(const size_t n);
386 
387  void append(ResultSet& that);
388 
389  const ResultSetStorage* getStorage() const;
390 
391  size_t colCount() const;
392 
393  SQLTypeInfo getColType(const size_t col_idx) const;
394 
395  size_t rowCount(const bool force_parallel = false) const;
396 
397  void setCachedRowCount(const size_t row_count) const;
398 
399  size_t entryCount() const;
400 
401  size_t getBufferSizeBytes(const ExecutorDeviceType device_type) const;
402 
403  bool definitelyHasNoRows() const;
404 
405  const QueryMemoryDescriptor& getQueryMemDesc() const;
406 
407  const std::vector<TargetInfo>& getTargetInfos() const;
408 
409  const std::vector<int64_t>& getTargetInitVals() const;
410 
411  int8_t* getDeviceEstimatorBuffer() const;
412 
413  int8_t* getHostEstimatorBuffer() const;
414 
415  void syncEstimatorBuffer() const;
416 
417  size_t getNDVEstimator() const;
418 
419  void setQueueTime(const int64_t queue_time);
420 
421  int64_t getQueueTime() const;
422 
423  int64_t getRenderTime() const;
424 
425  void moveToBegin() const;
426 
427  bool isTruncated() const;
428 
429  bool isExplain() const;
430 
431  bool isGeoColOnGpu(const size_t col_idx) const;
432  int getDeviceId() const;
433 
434  // Called from the executor because in the new ResultSet we assume the 'padded' field
435  // in SlotSize already contains the padding, whereas in the executor it's computed.
436  // Once the buffer initialization moves to ResultSet we can remove this method.
438 
439  void fillOneEntry(const std::vector<int64_t>& entry) {
440  CHECK(storage_);
441  if (storage_->query_mem_desc_.didOutputColumnar()) {
442  storage_->fillOneEntryColWise(entry);
443  } else {
444  storage_->fillOneEntryRowWise(entry);
445  }
446  }
447 
448  void initializeStorage() const;
449 
450  void holdChunks(const std::list<std::shared_ptr<Chunk_NS::Chunk>>& chunks) {
451  chunks_ = chunks;
452  }
453  void holdChunkIterators(const std::shared_ptr<std::list<ChunkIter>> chunk_iters) {
454  chunk_iters_.push_back(chunk_iters);
455  }
456  void holdLiterals(std::vector<int8_t>& literal_buff) {
457  literal_buffers_.push_back(std::move(literal_buff));
458  }
459 
460  std::shared_ptr<RowSetMemoryOwner> getRowSetMemOwner() const {
461  return row_set_mem_owner_;
462  }
463 
464  const std::vector<uint32_t>& getPermutationBuffer() const;
465  const bool isPermutationBufferEmpty() const { return permutation_.empty(); };
466 
467  void serialize(TSerializedRows& serialized_rows) const;
468 
469  static std::unique_ptr<ResultSet> unserialize(const TSerializedRows& serialized_rows,
470  const Executor*);
471 
472  size_t getLimit();
473 
477  enum class GeoReturnType {
480  WktString,
485  };
488 
489  void copyColumnIntoBuffer(const size_t column_idx,
490  int8_t* output_buffer,
491  const size_t output_buffer_size) const;
492 
494 
495  bool didOutputColumnar() const { return this->query_mem_desc_.didOutputColumnar(); }
496 
499  }
500 
501  const int8_t getPaddedSlotWidthBytes(const size_t slot_idx) const {
502  return query_mem_desc_.getPaddedSlotWidthBytes(slot_idx);
503  }
504 
505  // returns a bitmap of all single-slot targets, as well as its count
506  std::tuple<std::vector<bool>, size_t> getSingleSlotTargetBitmap() const;
507 
508  std::tuple<std::vector<bool>, size_t> getSupportedSingleSlotTargetBitmap() const;
509 
510  std::vector<size_t> getSlotIndicesForTargetIndices() const;
511 
512  const std::vector<ColumnLazyFetchInfo>& getLazyFetchInfo() const {
513  return lazy_fetch_info_;
514  }
515 
516  void setSeparateVarlenStorageValid(const bool val) {
518  }
519 
520  std::shared_ptr<const std::vector<std::string>> getStringDictionaryPayloadCopy(
521  const int dict_id) const;
522 
523  template <typename ENTRY_TYPE, QueryDescriptionType QUERY_TYPE, bool COLUMNAR_FORMAT>
524  ENTRY_TYPE getEntryAt(const size_t row_idx,
525  const size_t target_idx,
526  const size_t slot_idx) const;
527 
528  private:
530 
531  std::vector<TargetValue> getNextRowImpl(const bool translate_strings,
532  const bool decimal_to_double) const;
533 
534  std::vector<TargetValue> getNextRowUnlocked(const bool translate_strings,
535  const bool decimal_to_double) const;
536 
537  std::vector<TargetValue> getRowAt(const size_t index,
538  const bool translate_strings,
539  const bool decimal_to_double,
540  const bool fixup_count_distinct_pointers,
541  const std::vector<bool>& targets_to_skip = {}) const;
542 
543  // NOTE: just for direct columnarization use at the moment
544  template <typename ENTRY_TYPE>
545  ENTRY_TYPE getColumnarPerfectHashEntryAt(const size_t row_idx,
546  const size_t target_idx,
547  const size_t slot_idx) const;
548 
549  template <typename ENTRY_TYPE>
550  ENTRY_TYPE getRowWisePerfectHashEntryAt(const size_t row_idx,
551  const size_t target_idx,
552  const size_t slot_idx) const;
553 
554  template <typename ENTRY_TYPE>
555  ENTRY_TYPE getRowWiseBaselineEntryAt(const size_t row_idx,
556  const size_t target_idx,
557  const size_t slot_idx) const;
558 
559  template <typename ENTRY_TYPE>
560  ENTRY_TYPE getColumnarBaselineEntryAt(const size_t row_idx,
561  const size_t target_idx,
562  const size_t slot_idx) const;
563 
564  size_t parallelRowCount() const;
565 
566  size_t advanceCursorToNextEntry() const;
567 
568  void radixSortOnGpu(const std::list<Analyzer::OrderEntry>& order_entries) const;
569 
570  void radixSortOnCpu(const std::list<Analyzer::OrderEntry>& order_entries) const;
571 
572  static bool isNull(const SQLTypeInfo& ti,
573  const InternalTargetValue& val,
574  const bool float_argument_input);
575 
577  int8_t* rowwise_target_ptr,
578  int8_t* keys_ptr,
579  const size_t entry_buff_idx,
580  const TargetInfo& target_info,
581  const size_t target_logical_idx,
582  const size_t slot_idx,
583  const bool translate_strings,
584  const bool decimal_to_double,
585  const bool fixup_count_distinct_pointers) const;
586 
587  TargetValue getTargetValueFromBufferColwise(const int8_t* col_ptr,
588  const int8_t* keys_ptr,
589  const QueryMemoryDescriptor& query_mem_desc,
590  const size_t local_entry_idx,
591  const size_t global_entry_idx,
592  const TargetInfo& target_info,
593  const size_t target_logical_idx,
594  const size_t slot_idx,
595  const bool translate_strings,
596  const bool decimal_to_double) const;
597 
598  TargetValue makeTargetValue(const int8_t* ptr,
599  const int8_t compact_sz,
600  const TargetInfo& target_info,
601  const size_t target_logical_idx,
602  const bool translate_strings,
603  const bool decimal_to_double,
604  const size_t entry_buff_idx) const;
605 
606  TargetValue makeVarlenTargetValue(const int8_t* ptr1,
607  const int8_t compact_sz1,
608  const int8_t* ptr2,
609  const int8_t compact_sz2,
610  const TargetInfo& target_info,
611  const size_t target_logical_idx,
612  const bool translate_strings,
613  const size_t entry_buff_idx) const;
614 
616  int8_t* ptr1;
617  int8_t compact_sz1;
618  int8_t* ptr2;
619  int8_t compact_sz2;
620 
623  };
624  TargetValue makeGeoTargetValue(const int8_t* geo_target_ptr,
625  const size_t slot_idx,
626  const TargetInfo& target_info,
627  const size_t target_logical_idx,
628  const size_t entry_buff_idx) const;
629 
632  const size_t fixedup_entry_idx;
633  const size_t storage_idx;
634  };
635 
637  const int8_t* buff,
638  const size_t entry_idx,
639  const size_t target_logical_idx,
640  const StorageLookupResult& storage_lookup_result) const;
641 
642  InternalTargetValue getVarlenOrderEntry(const int64_t str_ptr,
643  const size_t str_len) const;
644 
645  int64_t lazyReadInt(const int64_t ival,
646  const size_t target_logical_idx,
647  const StorageLookupResult& storage_lookup_result) const;
648 
652  std::pair<size_t, size_t> getStorageIndex(const size_t entry_idx) const;
653 
654  const std::vector<const int8_t*>& getColumnFrag(const size_t storge_idx,
655  const size_t col_logical_idx,
656  int64_t& global_idx) const;
657 
658  StorageLookupResult findStorage(const size_t entry_idx) const;
659 
660  struct TargetOffsets {
661  const int8_t* ptr1;
662  const size_t compact_sz1;
663  const int8_t* ptr2;
664  const size_t compact_sz2;
665  };
666 
668  RowWiseTargetAccessor(const ResultSet* result_set)
669  : result_set_(result_set)
671  , key_width_(result_set_->query_mem_desc_.getEffectiveKeyWidth())
675  }
676 
678  const int8_t* buff,
679  const size_t entry_idx,
680  const size_t target_logical_idx,
681  const StorageLookupResult& storage_lookup_result) const;
682 
684 
685  inline const int8_t* get_rowwise_ptr(const int8_t* buff,
686  const size_t entry_idx) const {
687  return buff + entry_idx * row_bytes_;
688  }
689 
690  std::vector<std::vector<TargetOffsets>> offsets_for_storage_;
691 
693 
694  // Row-wise iteration
695  const size_t row_bytes_;
696  const size_t key_width_;
698  };
699 
701  ColumnWiseTargetAccessor(const ResultSet* result_set) : result_set_(result_set) {
703  }
704 
706 
708  const int8_t* buff,
709  const size_t entry_idx,
710  const size_t target_logical_idx,
711  const StorageLookupResult& storage_lookup_result) const;
712 
713  std::vector<std::vector<TargetOffsets>> offsets_for_storage_;
714 
716  };
717 
718  template <typename BUFFER_ITERATOR_TYPE>
720  using BufferIteratorType = BUFFER_ITERATOR_TYPE;
721 
722  ResultSetComparator(const std::list<Analyzer::OrderEntry>& order_entries,
723  const bool use_heap,
724  const ResultSet* result_set)
725  : order_entries_(order_entries)
726  , use_heap_(use_heap)
727  , result_set_(result_set)
728  , buffer_itr_(result_set) {}
729 
730  bool operator()(const uint32_t lhs, const uint32_t rhs) const;
731 
732  // TODO(adb): make order_entries_ a pointer
733  const std::list<Analyzer::OrderEntry> order_entries_;
734  const bool use_heap_;
737  };
738 
739  std::function<bool(const uint32_t, const uint32_t)> createComparator(
740  const std::list<Analyzer::OrderEntry>& order_entries,
741  const bool use_heap) {
744  std::make_unique<ResultSetComparator<ColumnWiseTargetAccessor>>(
745  order_entries, use_heap, this);
746  return [this](const uint32_t lhs, const uint32_t rhs) -> bool {
747  return (*this->column_wise_comparator_)(lhs, rhs);
748  };
749  } else {
750  row_wise_comparator_ = std::make_unique<ResultSetComparator<RowWiseTargetAccessor>>(
751  order_entries, use_heap, this);
752  return [this](const uint32_t lhs, const uint32_t rhs) -> bool {
753  return (*this->row_wise_comparator_)(lhs, rhs);
754  };
755  }
756  }
757 
758  static void topPermutation(
759  std::vector<uint32_t>& to_sort,
760  const size_t n,
761  const std::function<bool(const uint32_t, const uint32_t)> compare);
762 
763  void sortPermutation(const std::function<bool(const uint32_t, const uint32_t)> compare);
764 
765  std::vector<uint32_t> initPermutationBuffer(const size_t start, const size_t step);
766 
767  void parallelTop(const std::list<Analyzer::OrderEntry>& order_entries,
768  const size_t top_n);
769 
770  void baselineSort(const std::list<Analyzer::OrderEntry>& order_entries,
771  const size_t top_n);
772 
773  void doBaselineSort(const ExecutorDeviceType device_type,
774  const std::list<Analyzer::OrderEntry>& order_entries,
775  const size_t top_n);
776 
777  bool canUseFastBaselineSort(const std::list<Analyzer::OrderEntry>& order_entries,
778  const size_t top_n);
779 
781 
782  int getGpuCount() const;
783 
784  void serializeProjection(TSerializedRows& serialized_rows) const;
785  void serializeVarlenAggColumn(int8_t* buf,
786  std::vector<std::string>& varlen_bufer) const;
787 
788  void serializeCountDistinctColumns(TSerializedRows&) const;
789 
790  void unserializeCountDistinctColumns(const TSerializedRows&);
791 
793 
794  using BufferSet = std::set<int64_t>;
795  void create_active_buffer_set(BufferSet& count_distinct_active_buffer_set) const;
796 
797  int64_t getDistinctBufferRefFromBufferRowwise(int8_t* rowwise_target_ptr,
798  const TargetInfo& target_info) const;
799 
800  const std::vector<TargetInfo> targets_;
802  const int device_id_;
804  mutable std::unique_ptr<ResultSetStorage> storage_;
805  std::vector<std::unique_ptr<ResultSetStorage>> appended_storage_;
806  mutable size_t crt_row_buff_idx_;
807  mutable size_t fetched_so_far_;
808  size_t drop_first_;
809  size_t keep_first_;
810  const std::shared_ptr<RowSetMemoryOwner> row_set_mem_owner_;
811  std::vector<uint32_t> permutation_;
812  int64_t queue_time_ms_;
814  const Executor* executor_; // TODO(alex): remove
815 
816  std::list<std::shared_ptr<Chunk_NS::Chunk>> chunks_;
817  std::vector<std::shared_ptr<std::list<ChunkIter>>> chunk_iters_;
818  // TODO(miyu): refine by using one buffer and
819  // setting offset instead of ptr in group by buffer.
820  std::vector<std::vector<int8_t>> literal_buffers_;
821  const std::vector<ColumnLazyFetchInfo> lazy_fetch_info_;
822  std::vector<std::vector<std::vector<const int8_t*>>> col_buffers_;
823  std::vector<std::vector<std::vector<int64_t>>> frag_offsets_;
824  std::vector<std::vector<int64_t>> consistent_frag_sizes_;
825 
826  const std::shared_ptr<const Analyzer::Estimator> estimator_;
828  mutable int8_t* host_estimator_buffer_;
830 
831  // only used by serialization
832  using SerializedVarlenBufferStorage = std::vector<std::string>;
833 
834  std::vector<SerializedVarlenBufferStorage> serialized_varlen_buffer_;
836  std::string explanation_;
837  const bool just_explain_;
838  mutable std::atomic<ssize_t> cached_row_count_;
839  mutable std::mutex row_iteration_mutex_;
840 
841  // only used by geo
843 
844  // comparators used for sorting (note that the actual compare function is accessed using
845  // the createComparator method)
846  std::unique_ptr<ResultSetComparator<RowWiseTargetAccessor>> row_wise_comparator_;
847  std::unique_ptr<ResultSetComparator<ColumnWiseTargetAccessor>> column_wise_comparator_;
848 
849  friend class ResultSetManager;
850  friend class ResultSetRowIterator;
851  friend class ColumnarResults;
852 };
853 
856  return {};
857  }
858 
859  if (result_set_->just_explain_) {
860  return {result_set_->explanation_};
861  }
862 
863  return result_set_->getRowAt(
865 }
866 
868  if (!result_set_->storage_ && !result_set_->just_explain_) {
869  global_entry_idx_valid_ = false;
870  } else if (result_set_->just_explain_) {
872  fetched_so_far_ = 1;
873  } else {
874  result_set_->advanceCursorToNextEntry(*this);
875  }
876  return *this;
877 }
878 
880  public:
881  ResultSet* reduce(std::vector<ResultSet*>&);
882 
883  std::shared_ptr<ResultSet> getOwnResultSet();
884 
885  void rewriteVarlenAggregates(ResultSet*);
886 
887  private:
888  std::shared_ptr<ResultSet> rs_;
889 };
890 
891 class RowSortException : public std::runtime_error {
892  public:
893  RowSortException(const std::string& cause) : std::runtime_error(cause) {}
894 };
895 
896 int64_t lazy_decode(const ColumnLazyFetchInfo& col_lazy_fetch,
897  const int8_t* byte_stream,
898  const int64_t pos);
899 
900 void fill_empty_key(void* key_ptr, const size_t key_count, const size_t key_width);
901 
902 bool can_use_parallel_algorithms(const ResultSet& rows);
903 
904 bool use_parallel_algorithms(const ResultSet& rows);
905 
906 int8_t get_width_for_slot(const size_t target_slot_idx,
907  const bool float_argument_input,
908  const QueryMemoryDescriptor& query_mem_desc);
909 
910 size_t get_byteoff_of_slot(const size_t slot_idx,
911  const QueryMemoryDescriptor& query_mem_desc);
912 
913 using GroupValueInfo = std::pair<int64_t*, bool>;
914 
916  const uint32_t groups_buffer_entry_count,
917  const int64_t* key,
918  const uint32_t key_count,
919  const size_t key_width,
920  const QueryMemoryDescriptor& query_mem_desc,
921  const int64_t* that_buff_i64,
922  const size_t that_entry_idx,
923  const size_t that_entry_count,
924  const uint32_t row_size_quad);
925 
926 #endif // QUERYENGINE_RESULTSET_H
std::pair< int64_t *, bool > GroupValueInfo
Definition: ResultSet.h:913
void setSeparateVarlenStorageValid(const bool val)
Definition: ResultSet.h:516
const std::list< Analyzer::OrderEntry > order_entries_
Definition: ResultSet.h:733
Executor(const int db_id, const size_t block_size_x, const size_t grid_size_x, const std::string &debug_dir, const std::string &debug_file,::QueryRenderer::QueryRenderManager *render_manager)
Definition: Execute.cpp:106
void setGeoReturnType(const GeoReturnType val)
Definition: ResultSet.h:487
const SQLTypeInfo type
Definition: ResultSet.h:225
void serializeVarlenAggColumn(int8_t *buf, std::vector< std::string > &varlen_bufer) const
std::mutex row_iteration_mutex_
Definition: ResultSet.h:839
InternalTargetValue getColumnInternal(const int8_t *buff, const size_t entry_idx, const size_t target_logical_idx, const StorageLookupResult &storage_lookup_result) const
void syncEstimatorBuffer() const
Definition: ResultSet.cpp:410
QueryDescriptionType
Definition: Types.h:26
const int8_t * ptr1
Definition: ResultSet.h:661
const size_t compact_sz2
Definition: ResultSet.h:664
void baselineSort(const std::list< Analyzer::OrderEntry > &order_entries, const size_t top_n)
int64_t lazy_decode(const ColumnLazyFetchInfo &col_lazy_fetch, const int8_t *byte_stream, const int64_t pos)
void holdChunks(const std::list< std::shared_ptr< Chunk_NS::Chunk >> &chunks)
Definition: ResultSet.h:450
const QueryMemoryDescriptor & getQueryMemDesc() const
Definition: ResultSet.cpp:387
std::unique_ptr< ResultSetComparator< ColumnWiseTargetAccessor > > column_wise_comparator_
Definition: ResultSet.h:847
std::pair< size_t, size_t > getStorageIndex(const size_t entry_idx) const
Definition: ResultSet.cpp:594
const int32_t groups_buffer_size return groups_buffer
std::shared_ptr< RowSetMemoryOwner > getRowSetMemOwner() const
Definition: ResultSet.h:460
bool didOutputColumnar() const
Definition: ResultSet.h:495
void moveOneEntryToBuffer(const size_t entry_index, int64_t *new_buff_i64, const size_t new_entry_count, const size_t key_count, const size_t row_qw_count, const int64_t *src_buff, const size_t key_byte_width) const
bool isEmptyEntry(const size_t entry_idx, const int8_t *buff) const
bool use_parallel_algorithms(const ResultSet &rows)
Definition: ResultSet.cpp:873
std::ptrdiff_t difference_type
Definition: ResultSet.h:238
int8_t * estimator_buffer_
Definition: ResultSet.h:827
const std::vector< TargetInfo > targets_
Definition: ResultSet.h:196
ENTRY_TYPE getRowWisePerfectHashEntryAt(const size_t row_idx, const size_t target_idx, const size_t slot_idx) const
ALWAYS_INLINE void reduceOneSlot(int8_t *this_ptr1, int8_t *this_ptr2, const int8_t *that_ptr1, const int8_t *that_ptr2, const TargetInfo &target_info, const size_t target_logical_idx, const size_t target_slot_idx, const size_t init_agg_val_idx, const ResultSetStorage &that, const size_t first_slot_idx_for_target, const std::vector< std::string > &serialized_varlen_buffer) const
double decimal_to_double(const SQLTypeInfo &otype, int64_t oval)
void setEntryCount(const size_t val)
void reduceEntriesNoCollisionsColWise(int8_t *this_buff, const int8_t *that_buff, const ResultSetStorage &that, const size_t start_index, const size_t end_index, const std::vector< std::string > &serialized_varlen_buffer) const
std::vector< std::unique_ptr< ResultSetStorage > > appended_storage_
Definition: ResultSet.h:805
std::vector< int64_t > target_init_vals_
Definition: ResultSet.h:200
void initializeColWise() const
void reduceOneEntryBaseline(int8_t *this_buff, const int8_t *that_buff, const size_t i, const size_t that_entry_count, const ResultSetStorage &that) const
ENTRY_TYPE getColumnarPerfectHashEntryAt(const size_t row_idx, const size_t target_idx, const size_t slot_idx) const
GeoReturnType geo_return_type_
Definition: ResultSet.h:842
bool isEmptyEntryColumnar(const size_t entry_idx, const int8_t *buff) const
ExecutorDeviceType
const BufferIteratorType buffer_itr_
Definition: ResultSet.h:736
void fill_empty_key(void *key_ptr, const size_t key_count, const size_t key_width)
void moveToBegin() const
Definition: ResultSet.cpp:435
Utility functions for easy access to the result set buffers.
std::shared_ptr< ResultSet > rs_
Definition: ResultSet.h:888
std::vector< std::string > SerializedVarlenBufferStorage
Definition: ResultSet.h:832
void initializeStorage() const
QueryDescriptionType getQueryDescriptionType() const
Definition: ResultSet.h:497
ResultSetRowIterator(const ResultSet *rs, bool translate_strings, bool decimal_to_double)
Definition: ResultSet.h:273
const Executor * executor_
Definition: ResultSet.h:814
size_t get_byteoff_of_slot(const size_t slot_idx, const QueryMemoryDescriptor &query_mem_desc)
void unserializeCountDistinctColumns(const TSerializedRows &)
std::vector< TargetValue > getNextRow(const bool translate_strings, const bool decimal_to_double) const
static bool isNull(const SQLTypeInfo &ti, const InternalTargetValue &val, const bool float_argument_input)
QueryMemoryDescriptor query_mem_desc_
Definition: ResultSet.h:803
const std::vector< TargetInfo > & getTargetInfos() const
Definition: ResultSet.cpp:392
void addCountDistinctSetPointerMapping(const int64_t remote_ptr, const int64_t ptr)
Definition: ResultSet.cpp:841
ResultSetStorage(const std::vector< TargetInfo > &targets, const QueryMemoryDescriptor &query_mem_desc, int8_t *buff, const bool buff_is_provided)
Definition: ResultSet.cpp:44
std::unique_ptr< ResultSetStorage > storage_
Definition: ResultSet.h:804
bool operator==(const ResultSetRowIterator &other) const
Definition: ResultSet.h:243
void initializeRowWise() const
void initializeBaselineValueSlots(int64_t *this_entry_slots) const
ENTRY_TYPE getEntryAt(const size_t row_idx, const size_t target_idx, const size_t slot_idx) const
size_t rowCount(const bool force_parallel=false) const
Definition: ResultSet.cpp:310
ResultSetRowIterator(const ResultSet *rs)
Definition: ResultSet.h:284
TargetValue makeGeoTargetValue(const int8_t *geo_target_ptr, const size_t slot_idx, const TargetInfo &target_info, const size_t target_logical_idx, const size_t entry_buff_idx) const
TargetValue getTargetValueFromBufferRowwise(int8_t *rowwise_target_ptr, int8_t *keys_ptr, const size_t entry_buff_idx, const TargetInfo &target_info, const size_t target_logical_idx, const size_t slot_idx, const bool translate_strings, const bool decimal_to_double, const bool fixup_count_distinct_pointers) const
size_t keep_first_
Definition: ResultSet.h:809
void keepFirstN(const size_t n)
Definition: ResultSet.cpp:81
std::vector< std::shared_ptr< std::list< ChunkIter > > > chunk_iters_
Definition: ResultSet.h:817
void reduceOneSlotBaseline(int64_t *this_buff, const size_t this_slot, const int64_t *that_buff, const size_t that_entry_count, const size_t that_slot, const TargetInfo &target_info, const size_t target_logical_idx, const size_t target_slot_idx, const size_t init_agg_val_idx, const ResultSetStorage &that) const
const int64_t const uint32_t groups_buffer_entry_count
void serialize(TSerializedRows &serialized_rows) const
std::vector< SerializedVarlenBufferStorage > serialized_varlen_buffer_
Definition: ResultSet.h:834
const size_t compact_sz1
Definition: ResultSet.h:662
int64_t lazyReadInt(const int64_t ival, const size_t target_logical_idx, const StorageLookupResult &storage_lookup_result) const
bool operator!=(const ResultSetRowIterator &other) const
Definition: ResultSet.h:247
size_t colCount() const
Definition: ResultSet.cpp:297
OneIntegerColumnRow getOneColRow(const size_t index) const
TargetValue getTargetValueFromBufferColwise(const int8_t *col_ptr, const int8_t *keys_ptr, const QueryMemoryDescriptor &query_mem_desc, const size_t local_entry_idx, const size_t global_entry_idx, const TargetInfo &target_info, const size_t target_logical_idx, const size_t slot_idx, const bool translate_strings, const bool decimal_to_double) const
int8_t get_width_for_slot(const size_t target_slot_idx, const bool float_argument_input, const QueryMemoryDescriptor &query_mem_desc)
friend class ResultSet
Definition: ResultSet.h:208
void rewriteVarlenAggregates(ResultSet *)
const bool just_explain_
Definition: ResultSet.h:837
ResultSetRowIterator rowIterator(size_t from_logical_index, bool translate_strings, bool decimal_to_double) const
Definition: ResultSet.h:323
bool isTruncated() const
Definition: ResultSet.cpp:440
std::vector< uint32_t > permutation_
Definition: ResultSet.h:811
size_t getLimit()
Definition: ResultSet.cpp:857
std::unordered_map< int64_t, int64_t > count_distinct_sets_mapping_
Definition: ResultSet.h:206
const int local_col_id
Definition: ResultSet.h:224
const bool isPermutationBufferEmpty() const
Definition: ResultSet.h:465
void reduce(const ResultSetStorage &that, const std::vector< std::string > &serialized_varlen_buffer, const ReductionCode &reduction_code) const
int8_t * getUnderlyingBuffer() const
Definition: ResultSet.cpp:77
ResultSet(const std::vector< TargetInfo > &targets, const ExecutorDeviceType device_type, const QueryMemoryDescriptor &query_mem_desc, const std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner, const Executor *executor)
Definition: ResultSet.cpp:91
size_t parallelRowCount() const
Definition: ResultSet.cpp:346
const size_t key_bytes_with_padding_
Definition: ResultSet.h:697
const ResultSet * result_set_
Definition: ResultSet.h:692
std::vector< uint32_t > initPermutationBuffer(const size_t start, const size_t step)
Definition: ResultSet.cpp:533
std::vector< TargetValue > getRowAtNoTranslations(const size_t index, const std::vector< bool > &targets_to_skip={}) const
const ResultSet * result_set_
Definition: ResultSet.h:265
void radixSortOnCpu(const std::list< Analyzer::OrderEntry > &order_entries) const
Definition: ResultSet.cpp:802
InternalTargetValue getColumnInternal(const int8_t *buff, const size_t entry_idx, const size_t target_logical_idx, const StorageLookupResult &storage_lookup_result) const
std::unique_ptr< ResultSetComparator< RowWiseTargetAccessor > > row_wise_comparator_
Definition: ResultSet.h:846
bool definitelyHasNoRows() const
Definition: ResultSet.cpp:383
ColumnWiseTargetAccessor(const ResultSet *result_set)
Definition: ResultSet.h:701
std::input_iterator_tag iterator_category
Definition: ResultSet.h:241
size_t global_entry_idx_
Definition: ResultSet.h:267
int8_t * getHostEstimatorBuffer() const
Definition: ResultSet.cpp:406
const bool buff_is_provided_
Definition: ResultSet.h:199
InternalTargetValue getVarlenOrderEntry(const int64_t str_ptr, const size_t str_len) const
const std::vector< TargetInfo > targets_
Definition: ResultSet.h:800
const std::shared_ptr< RowSetMemoryOwner > row_set_mem_owner_
Definition: ResultSet.h:810
void reduceOneEntrySlotsBaseline(int64_t *this_entry_slots, const int64_t *that_buff, const size_t that_entry_idx, const size_t that_entry_count, const ResultSetStorage &that) const
size_t drop_first_
Definition: ResultSet.h:808
const ResultSetStorage * allocateStorage() const
std::shared_ptr< const std::vector< std::string > > getStringDictionaryPayloadCopy(const int dict_id) const
Definition: ResultSet.cpp:861
CHECK(cgen_state)
void copyKeyColWise(const size_t entry_idx, int8_t *this_buff, const int8_t *that_buff) const
const int8_t * ptr2
Definition: ResultSet.h:663
std::list< std::shared_ptr< Chunk_NS::Chunk > > chunks_
Definition: ResultSet.h:816
void reduceOneCountDistinctSlot(int8_t *this_ptr1, const int8_t *that_ptr1, const size_t target_logical_idx, const ResultSetStorage &that) const
const ResultSet * result_set_
Definition: ResultSet.h:735
void moveEntriesToBuffer(int8_t *new_buff, const size_t new_entry_count) const
void setQueueTime(const int64_t queue_time)
Definition: ResultSet.cpp:423
void dropFirstN(const size_t n)
Definition: ResultSet.cpp:86
std::vector< std::vector< int8_t > > literal_buffers_
Definition: ResultSet.h:820
std::vector< std::vector< TargetOffsets > > offsets_for_storage_
Definition: ResultSet.h:713
void doBaselineSort(const ExecutorDeviceType device_type, const std::list< Analyzer::OrderEntry > &order_entries, const size_t top_n)
GroupValueInfo get_group_value_reduction(int64_t *groups_buffer, const uint32_t groups_buffer_entry_count, const int64_t *key, const uint32_t key_count, const size_t key_width, const QueryMemoryDescriptor &query_mem_desc, const int64_t *that_buff_i64, const size_t that_entry_idx, const size_t that_entry_count, const uint32_t row_size_quad)
bool canUseFastBaselineSort(const std::list< Analyzer::OrderEntry > &order_entries, const size_t top_n)
void updateEntryCount(const size_t new_entry_count)
Definition: ResultSet.h:108
std::vector< TargetValue > & reference
Definition: ResultSet.h:240
ResultSet * reduce(std::vector< ResultSet * > &)
std::tuple< std::vector< bool >, size_t > getSingleSlotTargetBitmap() const
Definition: ResultSet.cpp:903
ResultSetRowIterator & operator++(void)
Definition: ResultSet.h:867
const int32_t groups_buffer_size return nullptr
std::shared_ptr< ResultSet > getOwnResultSet()
StorageLookupResult findStorage(const size_t entry_idx) const
Definition: ResultSet.cpp:616
void fillOneEntryColWise(const std::vector< int64_t > &entry)
std::atomic< ssize_t > cached_row_count_
Definition: ResultSet.h:838
static void topPermutation(std::vector< uint32_t > &to_sort, const size_t n, const std::function< bool(const uint32_t, const uint32_t)> compare)
Definition: ResultSet.cpp:748
const int8_t getPaddedSlotWidthBytes(const size_t slot_idx) const
value_type operator*() const
Definition: ResultSet.h:854
const std::vector< ColumnLazyFetchInfo > lazy_fetch_info_
Definition: ResultSet.h:821
RowWiseTargetAccessor(const ResultSet *result_set)
Definition: ResultSet.h:668
void copyColumnIntoBuffer(const size_t column_idx, int8_t *output_buffer, const size_t output_buffer_size) const
RowSortException(const std::string &cause)
Definition: ResultSet.h:893
void fixupCountDistinctPointers()
void radixSortOnGpu(const std::list< Analyzer::OrderEntry > &order_entries) const
Definition: ResultSet.cpp:768
const ResultSetStorage * getStorage() const
Definition: ResultSet.cpp:293
QueryDescriptionType getQueryDescriptionType() const
Data_Namespace::DataMgr * data_mgr_
Definition: ResultSet.h:829
int64_t getQueueTime() const
Definition: ResultSet.cpp:427
std::vector< TargetValue > getRowAt(const size_t index) const
void fillOneEntry(const std::vector< int64_t > &entry)
Definition: ResultSet.h:439
const int64_t value
Definition: ResultSet.h:229
void updateStorageEntryCount(const size_t new_entry_count)
Definition: ResultSet.h:351
void serializeProjection(TSerializedRows &serialized_rows) const
ResultSetRowIterator operator++(int)
Definition: ResultSet.h:251
const std::shared_ptr< const Analyzer::Estimator > estimator_
Definition: ResultSet.h:826
SQLTypeInfo getColType(const size_t col_idx) const
Definition: ResultSet.cpp:301
GeoReturnType getGeoReturnType() const
Definition: ResultSet.h:486
void holdChunkIterators(const std::shared_ptr< std::list< ChunkIter >> chunk_iters)
Definition: ResultSet.h:453
std::tuple< std::vector< bool >, size_t > getSupportedSingleSlotTargetBitmap() const
Definition: ResultSet.cpp:927
ExecutorDeviceType getDeviceType() const
Definition: ResultSet.cpp:224
size_t get_row_bytes(const QueryMemoryDescriptor &query_mem_desc)
bool isExplain() const
Definition: ResultSet.cpp:444
int8_t * buff_
Definition: ResultSet.h:198
ResultSetComparator(const std::list< Analyzer::OrderEntry > &order_entries, const bool use_heap, const ResultSet *result_set)
Definition: ResultSet.h:722
std::vector< TargetValue > value_type
Definition: ResultSet.h:237
void sortPermutation(const std::function< bool(const uint32_t, const uint32_t)> compare)
Definition: ResultSet.cpp:763
bool isGeoColOnGpu(const size_t col_idx) const
int64_t mappedPtr(const int64_t) const
Definition: ResultSet.cpp:847
const int8_t * get_rowwise_ptr(const int8_t *buff, const size_t entry_idx) const
Definition: ResultSet.h:685
size_t getNDVEstimator() const
std::vector< std::vector< std::vector< const int8_t * > > > col_buffers_
Definition: ResultSet.h:822
bool isRowAtEmpty(const size_t index) const
const int8_t getPaddedSlotWidthBytes(const size_t slot_idx) const
Definition: ResultSet.h:501
size_t entryCount() const
const bool valid
Definition: ResultSet.h:230
static QueryMemoryDescriptor fixupQueryMemoryDescriptor(const QueryMemoryDescriptor &)
Definition: ResultSet.cpp:452
int64_t queue_time_ms_
Definition: ResultSet.h:812
TargetValue makeTargetValue(const int8_t *ptr, const int8_t compact_sz, const TargetInfo &target_info, const size_t target_logical_idx, const bool translate_strings, const bool decimal_to_double, const size_t entry_buff_idx) const
size_t getCurrentRowBufferIndex() const
Definition: ResultSet.h:257
void append(ResultSet &that)
Definition: ResultSet.cpp:263
std::string explanation_
Definition: ResultSet.h:836
const bool is_lazily_fetched
Definition: ResultSet.h:223
std::vector< std::vector< int64_t > > consistent_frag_sizes_
Definition: ResultSet.h:824
bool operator()(const uint32_t lhs, const uint32_t rhs) const
Definition: ResultSet.cpp:626
int8_t * host_estimator_buffer_
Definition: ResultSet.h:828
bool can_use_parallel_algorithms(const ResultSet &rows)
Definition: ResultSet.cpp:869
friend class ResultSet
Definition: ResultSet.h:284
const ExecutorDeviceType device_type_
Definition: ResultSet.h:801
std::vector< TargetValue > getNextRowImpl(const bool translate_strings, const bool decimal_to_double) const
size_t getCurrentRowBufferIndex() const
Definition: ResultSet.cpp:256
void holdLiterals(std::vector< int8_t > &literal_buff)
Definition: ResultSet.h:456
bool g_enable_watchdog false
Definition: Execute.cpp:71
void parallelTop(const std::list< Analyzer::OrderEntry > &order_entries, const size_t top_n)
Definition: ResultSet.cpp:555
int getGpuCount() const
size_t getBufferSizeBytes(const ExecutorDeviceType device_type) const
int8_t * getDeviceEstimatorBuffer() const
Definition: ResultSet.cpp:401
size_t fetched_so_far_
Definition: ResultSet.h:807
size_t crt_row_buff_idx_
Definition: ResultSet.h:806
Estimators to be used when precise cardinality isn&#39;t useful.
int64_t getDistinctBufferRefFromBufferRowwise(int8_t *rowwise_target_ptr, const TargetInfo &target_info) const
void fillOneEntryRowWise(const std::vector< int64_t > &entry)
std::vector< std::vector< std::vector< int64_t > > > frag_offsets_
Definition: ResultSet.h:823
bool separate_varlen_storage_valid_
Definition: ResultSet.h:835
boost::variant< ScalarTargetValue, ArrayTargetValue, GeoTargetValue, GeoTargetValuePtr > TargetValue
Definition: TargetValue.h:167
void rewriteAggregateBufferOffsets(const std::vector< std::string > &serialized_varlen_buffer) const
std::vector< TargetValue > getNextRowUnlocked(const bool translate_strings, const bool decimal_to_double) const
std::vector< TargetValue > * pointer
Definition: ResultSet.h:239
const std::vector< int64_t > & getTargetInitVals() const
Definition: ResultSet.cpp:396
std::vector< size_t > getSlotIndicesForTargetIndices() const
Definition: ResultSet.cpp:946
const std::vector< uint32_t > & getPermutationBuffer() const
Definition: ResultSet.cpp:551
size_t advanceCursorToNextEntry() const
void create_active_buffer_set(BufferSet &count_distinct_active_buffer_set) const
std::set< int64_t > BufferSet
Definition: ResultSet.h:794
std::function< bool(const uint32_t, const uint32_t)> createComparator(const std::list< Analyzer::OrderEntry > &order_entries, const bool use_heap)
Definition: ResultSet.h:739
void sort(const std::list< Analyzer::OrderEntry > &order_entries, const size_t top_n)
Definition: ResultSet.cpp:464
ResultSetRowIterator rowIterator(bool translate_strings, bool decimal_to_double) const
Definition: ResultSet.h:338
BUFFER_ITERATOR_TYPE BufferIteratorType
Definition: ResultSet.h:720
ENTRY_TYPE getColumnarBaselineEntryAt(const size_t row_idx, const size_t target_idx, const size_t slot_idx) const
Data_Namespace::DataMgr * getDataManager() const
#define ALWAYS_INLINE
size_t crt_row_buff_idx_
Definition: ResultSet.h:266
InternalTargetValue getColumnInternal(const int8_t *buff, const size_t entry_idx, const size_t target_logical_idx, const StorageLookupResult &storage_lookup_result) const
std::vector< std::vector< TargetOffsets > > offsets_for_storage_
Definition: ResultSet.h:690
static bool reduceSingleRow(const int8_t *row_ptr, const int8_t warp_count, const bool is_columnar, const bool replace_bitmap_ptr_with_bitmap_sz, std::vector< int64_t > &agg_vals, const QueryMemoryDescriptor &query_mem_desc, const std::vector< TargetInfo > &targets, const std::vector< int64_t > &agg_init_vals)
bool global_entry_idx_valid_
Definition: ResultSet.h:268
int64_t getRenderTime() const
Definition: ResultSet.cpp:431
void setCachedRowCount(const size_t row_count) const
Definition: ResultSet.cpp:341
bool isDirectColumnarConversionPossible() const
Definition: ResultSet.cpp:884
size_t get_key_bytes_rowwise(const QueryMemoryDescriptor &query_mem_desc)
FORCE_INLINE HOST DEVICE T align_to_int64(T addr)
TargetValue makeVarlenTargetValue(const int8_t *ptr1, const int8_t compact_sz1, const int8_t *ptr2, const int8_t compact_sz2, const TargetInfo &target_info, const size_t target_logical_idx, const bool translate_strings, const size_t entry_buff_idx) const
void serializeCountDistinctColumns(TSerializedRows &) const
const ResultSetStorage * storage_ptr
Definition: ResultSet.h:631
const std::vector< const int8_t * > & getColumnFrag(const size_t storge_idx, const size_t col_logical_idx, int64_t &global_idx) const
QueryMemoryDescriptor query_mem_desc_
Definition: ResultSet.h:197
const std::vector< ColumnLazyFetchInfo > & getLazyFetchInfo() const
Definition: ResultSet.h:512
int64_t render_time_ms_
Definition: ResultSet.h:813
ENTRY_TYPE getRowWiseBaselineEntryAt(const size_t row_idx, const size_t target_idx, const size_t slot_idx) const
int getDeviceId() const
Definition: ResultSet.cpp:448
static std::unique_ptr< ResultSet > unserialize(const TSerializedRows &serialized_rows, const Executor *)
const int device_id_
Definition: ResultSet.h:802