OmniSciDB  04ee39c94c
QueryFragmentDescriptor.h
Go to the documentation of this file.
1 /*
2  * Copyright 2018 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
23 #ifndef QUERYENGINE_QUERYFRAGMENTDESCRIPTOR_H
24 #define QUERYENGINE_QUERYFRAGMENTDESCRIPTOR_H
25 
26 #include <deque>
27 #include <map>
28 #include <memory>
29 #include <set>
30 #include <vector>
31 
32 #include "../CompilationOptions.h"
33 #include "Shared/Logger.h"
34 
35 namespace Fragmenter_Namespace {
36 class FragmentInfo;
37 }
38 
39 namespace Data_Namespace {
40 struct MemoryInfo;
41 }
42 
43 class Executor;
44 struct InputTableInfo;
45 struct RelAlgExecutionUnit;
46 
48  int table_id;
49  std::vector<size_t> fragment_ids;
50 };
51 
52 using FragmentsList = std::vector<FragmentsPerTable>;
53 using TableFragments = std::deque<Fragmenter_Namespace::FragmentInfo>;
54 
56  public:
58  const std::vector<InputTableInfo>& query_infos,
59  const std::vector<Data_Namespace::MemoryInfo>& gpu_mem_infos,
60  const double gpu_input_mem_limit_percent);
61 
62  static void computeAllTablesFragments(
63  std::map<int, const TableFragments*>& all_tables_fragments,
64  const RelAlgExecutionUnit& ra_exe_unit,
65  const std::vector<InputTableInfo>& query_infos);
66 
67  void buildFragmentKernelMap(const RelAlgExecutionUnit& ra_exe_unit,
68  const std::vector<uint64_t>& frag_offsets,
69  const int device_count,
70  const ExecutorDeviceType& device_type,
71  const bool enable_multifrag_kernels,
72  const bool enable_inner_join_fragment_skipping,
73  Executor* executor);
74 
75  template <typename DISPATCH_FCN>
76  void assignFragsToMultiDispatch(DISPATCH_FCN f) const {
77  for (const auto& kv : kernels_per_device_) {
78  CHECK_EQ(kv.second.size(), size_t(1));
79  const auto kernel_id = *kv.second.begin();
80  CHECK_LT(kernel_id, fragments_per_kernel_.size());
81 
82  f(kv.first, fragments_per_kernel_[kernel_id], rowid_lookup_key_);
83  }
84  }
85 
86  template <typename DISPATCH_FCN>
87  void assignFragsToKernelDispatch(DISPATCH_FCN f,
88  const RelAlgExecutionUnit& ra_exe_unit) const {
89  for (const auto& kv : kernels_per_device_) {
90  for (const auto& kernel_id : kv.second) {
91  CHECK_LT(kernel_id, fragments_per_kernel_.size());
92 
93  const auto frag_list = fragments_per_kernel_[kernel_id];
94  f(kv.first, frag_list, rowid_lookup_key_);
95 
96  if (terminateDispatchMaybe(ra_exe_unit, kernel_id)) {
97  return;
98  }
99  }
100  }
101  }
102 
104  return rowid_lookup_key_ < 0 && fragments_per_kernel_.size() > 0;
105  }
106 
107  protected:
108  size_t outer_fragments_size_ = 0;
109  int64_t rowid_lookup_key_ = -1;
110 
111  std::map<int, const TableFragments*> selected_tables_fragments_;
112 
113  std::vector<FragmentsList> fragments_per_kernel_;
114  std::map<int, std::set<size_t>> kernels_per_device_;
115  std::vector<size_t> outer_fragment_tuple_sizes_;
116 
118  std::map<size_t, size_t> tuple_count_per_device_;
119  std::map<size_t, size_t> available_gpu_mem_bytes_;
120 
121  void buildFragmentPerKernelMap(const RelAlgExecutionUnit& ra_exe_unit,
122  const std::vector<uint64_t>& frag_offsets,
123  const int device_count,
124  const ExecutorDeviceType& device_type,
125  Executor* executor);
126 
127  void buildMultifragKernelMap(const RelAlgExecutionUnit& ra_exe_unit,
128  const std::vector<uint64_t>& frag_offsets,
129  const int device_count,
130  const ExecutorDeviceType& device_type,
131  const bool enable_inner_join_fragment_skipping,
132  Executor* executor);
133 
134  const size_t getOuterFragmentTupleSize(const size_t frag_index) const {
135  if (frag_index < outer_fragment_tuple_sizes_.size()) {
136  return outer_fragment_tuple_sizes_[frag_index];
137  } else {
138  return 0;
139  }
140  }
141 
142  bool terminateDispatchMaybe(const RelAlgExecutionUnit& ra_exe_unit,
143  const size_t kernel_id) const;
144 
145  void checkDeviceMemoryUsage(const Fragmenter_Namespace::FragmentInfo& fragment,
146  const int device_id,
147  const size_t num_cols);
148 };
149 
150 #endif // QUERYENGINE_QUERYFRAGMENTDESCRIPTOR_H
std::map< int, const TableFragments * > selected_tables_fragments_
#define CHECK_EQ(x, y)
Definition: Logger.h:195
std::map< size_t, size_t > tuple_count_per_device_
ExecutorDeviceType
void assignFragsToMultiDispatch(DISPATCH_FCN f) const
std::vector< size_t > outer_fragment_tuple_sizes_
std::vector< FragmentsPerTable > FragmentsList
std::map< int, std::set< size_t > > kernels_per_device_
std::vector< FragmentsList > fragments_per_kernel_
Used by Fragmenter classes to store info about each fragment - the fragment id and number of tuples(r...
Definition: Fragmenter.h:79
std::deque< Fragmenter_Namespace::FragmentInfo > TableFragments
#define CHECK_LT(x, y)
Definition: Logger.h:197
const size_t getOuterFragmentTupleSize(const size_t frag_index) const
void assignFragsToKernelDispatch(DISPATCH_FCN f, const RelAlgExecutionUnit &ra_exe_unit) const
std::vector< size_t > fragment_ids
std::map< size_t, size_t > available_gpu_mem_bytes_