OmniSciDB  eee9fa949c
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
RelAlgExecutor.cpp
Go to the documentation of this file.
1 /*
2  * Copyright 2017 MapD Technologies, Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "RelAlgExecutor.h"
19 #include "RelAlgTranslator.h"
20 
22 #include "CardinalityEstimator.h"
23 #include "ColumnFetcher.h"
24 #include "EquiJoinCondition.h"
25 #include "ErrorHandling.h"
26 #include "ExpressionRewrite.h"
27 #include "FromTableReordering.h"
28 #include "InputMetadata.h"
29 #include "JoinFilterPushDown.h"
31 #include "RangeTableIndexVisitor.h"
32 #include "RexVisitor.h"
34 #include "UsedColumnsVisitor.h"
35 #include "WindowContext.h"
36 
37 #include "../Parser/ParserNode.h"
38 #include "../Shared/measure.h"
39 
40 #include <algorithm>
41 #include <numeric>
42 
44 extern bool g_enable_bump_allocator;
45 namespace {
46 
47 bool node_is_aggregate(const RelAlgNode* ra) {
48  const auto compound = dynamic_cast<const RelCompound*>(ra);
49  const auto aggregate = dynamic_cast<const RelAggregate*>(ra);
50  return ((compound && compound->isAggregate()) || aggregate);
51 }
52 
53 std::unordered_set<PhysicalInput> get_physical_inputs(
54  const Catalog_Namespace::Catalog& cat,
55  const RelAlgNode* ra) {
56  auto phys_inputs = get_physical_inputs(ra);
57  std::unordered_set<PhysicalInput> phys_inputs2;
58  for (auto& phi : phys_inputs) {
59  phys_inputs2.insert(
60  PhysicalInput{cat.getColumnIdBySpi(phi.table_id, phi.col_id), phi.table_id});
61  }
62  return phys_inputs2;
63 }
64 
65 } // namespace
66 
68  const ExecutionOptions& eo,
69  RenderInfo* render_info) {
71  auto timer = DEBUG_TIMER(__func__);
73  try {
74  return executeRelAlgQueryNoRetry(co, eo, render_info);
75  } catch (const QueryMustRunOnCpu&) {
76  if (!g_allow_cpu_retry) {
77  throw;
78  }
79  }
80  LOG(INFO) << "Query unable to run in GPU mode, retrying on CPU";
82  co.hoist_literals_,
83  co.opt_level_,
85  co.explain_type_,
87  if (render_info) {
88  render_info->setForceNonInSituData();
89  }
90  return executeRelAlgQueryNoRetry(co_cpu, eo, render_info);
91 }
92 
94  const ExecutionOptions& eo,
95  RenderInfo* render_info) {
97 
98  query_dag_->resetQueryExecutionState();
99  const auto& ra = query_dag_->getRootNode();
100 
101  // capture the lock acquistion time
102  auto clock_begin = timer_start();
103  std::lock_guard<std::mutex> lock(executor_->execute_mutex_);
104  int64_t queue_time_ms = timer_stop(clock_begin);
106  executor_->resetInterrupt();
107  }
108  ScopeGuard row_set_holder = [this] { cleanupPostExecution(); };
109  const auto phys_inputs = get_physical_inputs(cat_, &ra);
110  const auto phys_table_ids = get_physical_table_inputs(&ra);
111  executor_->setCatalog(&cat_);
112  executor_->setupCaching(phys_inputs, phys_table_ids);
113 
114  ScopeGuard restore_metainfo_cache = [this] { executor_->clearMetaInfoCache(); };
115  auto ed_seq = RaExecutionSequence(&ra);
116 
117  if (render_info) {
118  // set render to be non-insitu in certain situations.
120  ed_seq.size() > 1) {
121  // old logic
122  // disallow if more than one ED
123  render_info->setInSituDataIfUnset(false);
124  }
125  }
126 
127  if (eo.find_push_down_candidates) {
128  // this extra logic is mainly due to current limitations on multi-step queries
129  // and/or subqueries.
131  ed_seq, co, eo, render_info, queue_time_ms);
132  }
133 
134  // Dispatch the subqueries first
135  for (auto subquery : getSubqueries()) {
136  const auto subquery_ra = subquery->getRelAlg();
137  CHECK(subquery_ra);
138  if (subquery_ra->hasContextData()) {
139  continue;
140  }
141  // Execute the subquery and cache the result.
142  RelAlgExecutor ra_executor(executor_, cat_, query_state_);
143  RaExecutionSequence subquery_seq(subquery_ra);
144  auto result = ra_executor.executeRelAlgSeq(subquery_seq, co, eo, nullptr, 0);
145  subquery->setExecutionResult(std::make_shared<ExecutionResult>(result));
146  }
147  return executeRelAlgSeq(ed_seq, co, eo, render_info, queue_time_ms);
148 }
149 
151  AggregatedColRange agg_col_range_cache;
152  const auto phys_inputs = get_physical_inputs(cat_, &getRootRelAlgNode());
153  return executor_->computeColRangesCache(phys_inputs);
154 }
155 
157  const auto phys_inputs = get_physical_inputs(cat_, &getRootRelAlgNode());
158  return executor_->computeStringDictionaryGenerations(phys_inputs);
159 }
160 
162  const auto phys_table_ids = get_physical_table_inputs(&getRootRelAlgNode());
163  return executor_->computeTableGenerations(phys_table_ids);
164 }
165 
166 Executor* RelAlgExecutor::getExecutor() const {
167  return executor_;
168 }
169 
171  CHECK(executor_);
172  executor_->row_set_mem_owner_ = nullptr;
173  executor_->lit_str_dict_proxy_ = nullptr;
174 }
175 
176 namespace {
177 
178 inline void check_sort_node_source_constraint(const RelSort* sort) {
179  CHECK_EQ(size_t(1), sort->inputCount());
180  const auto source = sort->getInput(0);
181  if (dynamic_cast<const RelSort*>(source)) {
182  throw std::runtime_error("Sort node not supported as input to another sort");
183  }
184 }
185 
186 } // namespace
187 
189  const RaExecutionSequence& seq,
190  const size_t step_idx,
191  const CompilationOptions& co,
192  const ExecutionOptions& eo,
193  RenderInfo* render_info) {
194  INJECT_TIMER(executeRelAlgQueryStep);
195  auto exe_desc_ptr = seq.getDescriptor(step_idx);
196  CHECK(exe_desc_ptr);
197  const auto sort = dynamic_cast<const RelSort*>(exe_desc_ptr->getBody());
198 
199  size_t shard_count{0};
200  auto merge_type = [&shard_count](const RelAlgNode* body) -> MergeType {
201  return node_is_aggregate(body) && !shard_count ? MergeType::Reduce : MergeType::Union;
202  };
203 
204  if (sort) {
206  const auto source_work_unit = createSortInputWorkUnit(sort, eo.just_explain);
208  source_work_unit.exe_unit, *executor_->getCatalog());
209  if (!shard_count) {
210  // No point in sorting on the leaf, only execute the input to the sort node.
211  CHECK_EQ(size_t(1), sort->inputCount());
212  const auto source = sort->getInput(0);
213  if (sort->collationCount() || node_is_aggregate(source)) {
214  auto temp_seq = RaExecutionSequence(std::make_unique<RaExecutionDesc>(source));
215  CHECK_EQ(temp_seq.size(), size_t(1));
216  // Use subseq to avoid clearing existing temporary tables
217  return {executeRelAlgSubSeq(temp_seq, std::make_pair(0, 1), co, eo, nullptr, 0),
218  merge_type(source),
219  source->getId(),
220  false};
221  }
222  }
223  }
224  return {executeRelAlgSubSeq(seq,
225  std::make_pair(step_idx, step_idx + 1),
226  co,
227  eo,
228  render_info,
230  merge_type(exe_desc_ptr->getBody()),
231  exe_desc_ptr->getBody()->getId(),
232  false};
233 }
234 
236  const AggregatedColRange& agg_col_range,
237  const StringDictionaryGenerations& string_dictionary_generations,
238  const TableGenerations& table_generations) {
239  // capture the lock acquistion time
240  auto clock_begin = timer_start();
242  executor_->resetInterrupt();
243  }
244  queue_time_ms_ = timer_stop(clock_begin);
245  executor_->row_set_mem_owner_ = std::make_shared<RowSetMemoryOwner>();
246  executor_->table_generations_ = table_generations;
247  executor_->agg_col_range_cache_ = agg_col_range;
248  executor_->string_dictionary_generations_ = string_dictionary_generations;
249 }
250 
252  const CompilationOptions& co,
253  const ExecutionOptions& eo,
254  RenderInfo* render_info,
255  const int64_t queue_time_ms,
256  const bool with_existing_temp_tables) {
258  if (!with_existing_temp_tables) {
259  decltype(temporary_tables_)().swap(temporary_tables_);
260  }
261  decltype(target_exprs_owned_)().swap(target_exprs_owned_);
262  executor_->catalog_ = &cat_;
263  executor_->temporary_tables_ = &temporary_tables_;
264 
265  time(&now_);
266  CHECK(!seq.empty());
267  const auto exec_desc_count = eo.just_explain ? size_t(1) : seq.size();
268 
269  for (size_t i = 0; i < exec_desc_count; i++) {
270  // only render on the last step
271  executeRelAlgStep(seq,
272  i,
273  co,
274  eo,
275  (i == exec_desc_count - 1) ? render_info : nullptr,
276  queue_time_ms);
277  }
278 
279  return seq.getDescriptor(exec_desc_count - 1)->getResult();
280 }
281 
283  const RaExecutionSequence& seq,
284  const std::pair<size_t, size_t> interval,
285  const CompilationOptions& co,
286  const ExecutionOptions& eo,
287  RenderInfo* render_info,
288  const int64_t queue_time_ms) {
290  executor_->catalog_ = &cat_;
291  executor_->temporary_tables_ = &temporary_tables_;
292 
293  time(&now_);
294  CHECK(!eo.just_explain);
295 
296  for (size_t i = interval.first; i < interval.second; i++) {
297  // only render on the last step
298  executeRelAlgStep(seq,
299  i,
300  co,
301  eo,
302  (i == interval.second - 1) ? render_info : nullptr,
303  queue_time_ms);
304  }
305 
306  return seq.getDescriptor(interval.second - 1)->getResult();
307 }
308 
310  const size_t step_idx,
311  const CompilationOptions& co,
312  const ExecutionOptions& eo,
313  RenderInfo* render_info,
314  const int64_t queue_time_ms) {
317  auto exec_desc_ptr = seq.getDescriptor(step_idx);
318  CHECK(exec_desc_ptr);
319  auto& exec_desc = *exec_desc_ptr;
320  const auto body = exec_desc.getBody();
321  if (body->isNop()) {
322  handleNop(exec_desc);
323  return;
324  }
325  const ExecutionOptions eo_work_unit{
327  eo.allow_multifrag,
328  eo.just_explain,
329  eo.allow_loop_joins,
330  eo.with_watchdog && (step_idx == 0 || dynamic_cast<const RelProject*>(body)),
331  eo.jit_debug,
332  eo.just_validate,
338 
339  const auto compound = dynamic_cast<const RelCompound*>(body);
340  if (compound) {
341  if (compound->isDeleteViaSelect()) {
342  executeDeleteViaCompound(compound, co, eo_work_unit, render_info, queue_time_ms);
343  } else if (compound->isUpdateViaSelect()) {
344  executeUpdateViaCompound(compound, co, eo_work_unit, render_info, queue_time_ms);
345  } else {
346  exec_desc.setResult(
347  executeCompound(compound, co, eo_work_unit, render_info, queue_time_ms));
348  if (exec_desc.getResult().isFilterPushDownEnabled()) {
349  return;
350  }
351  addTemporaryTable(-compound->getId(), exec_desc.getResult().getDataPtr());
352  }
353  return;
354  }
355  const auto project = dynamic_cast<const RelProject*>(body);
356  if (project) {
357  if (project->isDeleteViaSelect()) {
358  executeDeleteViaProject(project, co, eo_work_unit, render_info, queue_time_ms);
359  } else if (project->isUpdateViaSelect()) {
360  executeUpdateViaProject(project, co, eo_work_unit, render_info, queue_time_ms);
361  } else {
362  ssize_t prev_count = -1;
363  // Disabling the intermediate count optimization in distributed, as the previous
364  // execution descriptor will likely not hold the aggregated result.
365  if (g_skip_intermediate_count && step_idx > 0 && !g_cluster) {
366  auto prev_exec_desc = seq.getDescriptor(step_idx - 1);
367  CHECK(prev_exec_desc);
368  if (dynamic_cast<const RelCompound*>(prev_exec_desc->getBody())) {
369  const auto& prev_exe_result = prev_exec_desc->getResult();
370  const auto prev_result = prev_exe_result.getRows();
371  if (prev_result) {
372  prev_count = static_cast<ssize_t>(prev_result->rowCount());
373  }
374  }
375  }
376  exec_desc.setResult(executeProject(
377  project, co, eo_work_unit, render_info, queue_time_ms, prev_count));
378  if (exec_desc.getResult().isFilterPushDownEnabled()) {
379  return;
380  }
381  addTemporaryTable(-project->getId(), exec_desc.getResult().getDataPtr());
382  }
383  return;
384  }
385  const auto aggregate = dynamic_cast<const RelAggregate*>(body);
386  if (aggregate) {
387  exec_desc.setResult(
388  executeAggregate(aggregate, co, eo_work_unit, render_info, queue_time_ms));
389  addTemporaryTable(-aggregate->getId(), exec_desc.getResult().getDataPtr());
390  return;
391  }
392  const auto filter = dynamic_cast<const RelFilter*>(body);
393  if (filter) {
394  exec_desc.setResult(
395  executeFilter(filter, co, eo_work_unit, render_info, queue_time_ms));
396  addTemporaryTable(-filter->getId(), exec_desc.getResult().getDataPtr());
397  return;
398  }
399  const auto sort = dynamic_cast<const RelSort*>(body);
400  if (sort) {
401  exec_desc.setResult(executeSort(sort, co, eo_work_unit, render_info, queue_time_ms));
402  if (exec_desc.getResult().isFilterPushDownEnabled()) {
403  return;
404  }
405  addTemporaryTable(-sort->getId(), exec_desc.getResult().getDataPtr());
406  return;
407  }
408  const auto logical_values = dynamic_cast<const RelLogicalValues*>(body);
409  if (logical_values) {
410  exec_desc.setResult(executeLogicalValues(logical_values, eo_work_unit));
411  addTemporaryTable(-logical_values->getId(), exec_desc.getResult().getDataPtr());
412  return;
413  }
414  const auto modify = dynamic_cast<const RelModify*>(body);
415  if (modify) {
416  exec_desc.setResult(executeModify(modify, eo_work_unit));
417  return;
418  }
419  const auto table_func = dynamic_cast<const RelTableFunction*>(body);
420  if (table_func) {
421  exec_desc.setResult(
422  executeTableFunction(table_func, co, eo_work_unit, queue_time_ms));
423  addTemporaryTable(-table_func->getId(), exec_desc.getResult().getDataPtr());
424  return;
425  }
426  CHECK(false);
427 }
428 
430  // just set the result of the previous node as the result of no op
431  auto body = ed.getBody();
432  CHECK(dynamic_cast<const RelAggregate*>(body));
433  CHECK_EQ(size_t(1), body->inputCount());
434  const auto input = body->getInput(0);
435  body->setOutputMetainfo(input->getOutputMetainfo());
436  const auto it = temporary_tables_.find(-input->getId());
437  CHECK(it != temporary_tables_.end());
438  // set up temp table as it could be used by the outer query or next step
439  addTemporaryTable(-body->getId(), it->second);
440 
441  ed.setResult({it->second, input->getOutputMetainfo()});
442 }
443 
444 namespace {
445 
446 class RexUsedInputsVisitor : public RexVisitor<std::unordered_set<const RexInput*>> {
447  public:
449 
450  const std::vector<std::shared_ptr<RexInput>>& get_inputs_owned() const {
451  return synthesized_physical_inputs_owned;
452  }
453 
454  std::unordered_set<const RexInput*> visitInput(
455  const RexInput* rex_input) const override {
456  const auto input_ra = rex_input->getSourceNode();
457  CHECK(input_ra);
458  const auto scan_ra = dynamic_cast<const RelScan*>(input_ra);
459  if (scan_ra) {
460  const auto td = scan_ra->getTableDescriptor();
461  if (td) {
462  const auto col_id = rex_input->getIndex();
463  const auto cd = cat_.getMetadataForColumnBySpi(td->tableId, col_id + 1);
464  if (cd && cd->columnType.get_physical_cols() > 0) {
465  CHECK(IS_GEO(cd->columnType.get_type()));
466  std::unordered_set<const RexInput*> synthesized_physical_inputs;
467  for (auto i = 0; i < cd->columnType.get_physical_cols(); i++) {
468  auto physical_input =
469  new RexInput(scan_ra, SPIMAP_GEO_PHYSICAL_INPUT(col_id, i));
470  synthesized_physical_inputs_owned.emplace_back(physical_input);
471  synthesized_physical_inputs.insert(physical_input);
472  }
473  return synthesized_physical_inputs;
474  }
475  }
476  }
477  return {rex_input};
478  }
479 
480  protected:
481  std::unordered_set<const RexInput*> aggregateResult(
482  const std::unordered_set<const RexInput*>& aggregate,
483  const std::unordered_set<const RexInput*>& next_result) const override {
484  auto result = aggregate;
485  result.insert(next_result.begin(), next_result.end());
486  return result;
487  }
488 
489  private:
490  mutable std::vector<std::shared_ptr<RexInput>> synthesized_physical_inputs_owned;
492 };
493 
494 const RelAlgNode* get_data_sink(const RelAlgNode* ra_node) {
495  if (auto join = dynamic_cast<const RelJoin*>(ra_node)) {
496  CHECK_EQ(size_t(2), join->inputCount());
497  return join;
498  }
499  CHECK_EQ(size_t(1), ra_node->inputCount());
500  auto only_src = ra_node->getInput(0);
501  const bool is_join = dynamic_cast<const RelJoin*>(only_src) ||
502  dynamic_cast<const RelLeftDeepInnerJoin*>(only_src);
503  return is_join ? only_src : ra_node;
504 }
505 
506 std::pair<std::unordered_set<const RexInput*>, std::vector<std::shared_ptr<RexInput>>>
508  RexUsedInputsVisitor visitor(cat);
509  const auto filter_expr = compound->getFilterExpr();
510  std::unordered_set<const RexInput*> used_inputs =
511  filter_expr ? visitor.visit(filter_expr) : std::unordered_set<const RexInput*>{};
512  const auto sources_size = compound->getScalarSourcesSize();
513  for (size_t i = 0; i < sources_size; ++i) {
514  const auto source_inputs = visitor.visit(compound->getScalarSource(i));
515  used_inputs.insert(source_inputs.begin(), source_inputs.end());
516  }
517  std::vector<std::shared_ptr<RexInput>> used_inputs_owned(visitor.get_inputs_owned());
518  return std::make_pair(used_inputs, used_inputs_owned);
519 }
520 
521 std::pair<std::unordered_set<const RexInput*>, std::vector<std::shared_ptr<RexInput>>>
523  CHECK_EQ(size_t(1), aggregate->inputCount());
524  std::unordered_set<const RexInput*> used_inputs;
525  std::vector<std::shared_ptr<RexInput>> used_inputs_owned;
526  const auto source = aggregate->getInput(0);
527  const auto& in_metainfo = source->getOutputMetainfo();
528  const auto group_count = aggregate->getGroupByCount();
529  CHECK_GE(in_metainfo.size(), group_count);
530  for (size_t i = 0; i < group_count; ++i) {
531  auto synthesized_used_input = new RexInput(source, i);
532  used_inputs_owned.emplace_back(synthesized_used_input);
533  used_inputs.insert(synthesized_used_input);
534  }
535  for (const auto& agg_expr : aggregate->getAggExprs()) {
536  for (size_t i = 0; i < agg_expr->size(); ++i) {
537  const auto operand_idx = agg_expr->getOperand(i);
538  CHECK_GE(in_metainfo.size(), static_cast<size_t>(operand_idx));
539  auto synthesized_used_input = new RexInput(source, operand_idx);
540  used_inputs_owned.emplace_back(synthesized_used_input);
541  used_inputs.insert(synthesized_used_input);
542  }
543  }
544  return std::make_pair(used_inputs, used_inputs_owned);
545 }
546 
547 std::pair<std::unordered_set<const RexInput*>, std::vector<std::shared_ptr<RexInput>>>
549  RexUsedInputsVisitor visitor(cat);
550  std::unordered_set<const RexInput*> used_inputs;
551  for (size_t i = 0; i < project->size(); ++i) {
552  const auto proj_inputs = visitor.visit(project->getProjectAt(i));
553  used_inputs.insert(proj_inputs.begin(), proj_inputs.end());
554  }
555  std::vector<std::shared_ptr<RexInput>> used_inputs_owned(visitor.get_inputs_owned());
556  return std::make_pair(used_inputs, used_inputs_owned);
557 }
558 
559 std::pair<std::unordered_set<const RexInput*>, std::vector<std::shared_ptr<RexInput>>>
561  const Catalog_Namespace::Catalog& cat) {
562  RexUsedInputsVisitor visitor(cat);
563  std::unordered_set<const RexInput*> used_inputs;
564  for (size_t i = 0; i < table_func->getTableFuncInputsSize(); ++i) {
565  const auto table_func_inputs = visitor.visit(table_func->getTableFuncInputAt(i));
566  used_inputs.insert(table_func_inputs.begin(), table_func_inputs.end());
567  }
568  std::vector<std::shared_ptr<RexInput>> used_inputs_owned(visitor.get_inputs_owned());
569  return std::make_pair(used_inputs, used_inputs_owned);
570 }
571 
572 std::pair<std::unordered_set<const RexInput*>, std::vector<std::shared_ptr<RexInput>>>
574  std::unordered_set<const RexInput*> used_inputs;
575  std::vector<std::shared_ptr<RexInput>> used_inputs_owned;
576  const auto data_sink_node = get_data_sink(filter);
577  for (size_t nest_level = 0; nest_level < data_sink_node->inputCount(); ++nest_level) {
578  const auto source = data_sink_node->getInput(nest_level);
579  const auto scan_source = dynamic_cast<const RelScan*>(source);
580  if (scan_source) {
581  CHECK(source->getOutputMetainfo().empty());
582  for (size_t i = 0; i < scan_source->size(); ++i) {
583  auto synthesized_used_input = new RexInput(scan_source, i);
584  used_inputs_owned.emplace_back(synthesized_used_input);
585  used_inputs.insert(synthesized_used_input);
586  }
587  } else {
588  const auto& partial_in_metadata = source->getOutputMetainfo();
589  for (size_t i = 0; i < partial_in_metadata.size(); ++i) {
590  auto synthesized_used_input = new RexInput(source, i);
591  used_inputs_owned.emplace_back(synthesized_used_input);
592  used_inputs.insert(synthesized_used_input);
593  }
594  }
595  }
596  return std::make_pair(used_inputs, used_inputs_owned);
597 }
598 
599 int table_id_from_ra(const RelAlgNode* ra_node) {
600  const auto scan_ra = dynamic_cast<const RelScan*>(ra_node);
601  if (scan_ra) {
602  const auto td = scan_ra->getTableDescriptor();
603  CHECK(td);
604  return td->tableId;
605  }
606  return -ra_node->getId();
607 }
608 
609 std::unordered_map<const RelAlgNode*, int> get_input_nest_levels(
610  const RelAlgNode* ra_node,
611  const std::vector<size_t>& input_permutation) {
612  const auto data_sink_node = get_data_sink(ra_node);
613  std::unordered_map<const RelAlgNode*, int> input_to_nest_level;
614  for (size_t input_idx = 0; input_idx < data_sink_node->inputCount(); ++input_idx) {
615  const auto input_node_idx =
616  input_permutation.empty() ? input_idx : input_permutation[input_idx];
617  const auto input_ra = data_sink_node->getInput(input_node_idx);
618  const auto it_ok = input_to_nest_level.emplace(input_ra, input_idx);
619  CHECK(it_ok.second);
620  LOG_IF(INFO, !input_permutation.empty())
621  << "Assigned input " << input_ra->toString() << " to nest level " << input_idx;
622  }
623  return input_to_nest_level;
624 }
625 
626 std::pair<std::unordered_set<const RexInput*>, std::vector<std::shared_ptr<RexInput>>>
628  const Catalog_Namespace::Catalog& cat) {
629  const auto data_sink_node = get_data_sink(ra_node);
630  if (auto join = dynamic_cast<const RelJoin*>(data_sink_node)) {
631  CHECK_EQ(join->inputCount(), 2u);
632  const auto condition = join->getCondition();
633  RexUsedInputsVisitor visitor(cat);
634  auto condition_inputs = visitor.visit(condition);
635  std::vector<std::shared_ptr<RexInput>> condition_inputs_owned(
636  visitor.get_inputs_owned());
637  return std::make_pair(condition_inputs, condition_inputs_owned);
638  }
639 
640  if (auto left_deep_join = dynamic_cast<const RelLeftDeepInnerJoin*>(data_sink_node)) {
641  CHECK_GE(left_deep_join->inputCount(), 2u);
642  const auto condition = left_deep_join->getInnerCondition();
643  RexUsedInputsVisitor visitor(cat);
644  auto result = visitor.visit(condition);
645  for (size_t nesting_level = 1; nesting_level <= left_deep_join->inputCount() - 1;
646  ++nesting_level) {
647  const auto outer_condition = left_deep_join->getOuterCondition(nesting_level);
648  if (outer_condition) {
649  const auto outer_result = visitor.visit(outer_condition);
650  result.insert(outer_result.begin(), outer_result.end());
651  }
652  }
653  std::vector<std::shared_ptr<RexInput>> used_inputs_owned(visitor.get_inputs_owned());
654  return std::make_pair(result, used_inputs_owned);
655  }
656 
657  CHECK_EQ(ra_node->inputCount(), 1u);
658  return std::make_pair(std::unordered_set<const RexInput*>{},
659  std::vector<std::shared_ptr<RexInput>>{});
660 }
661 
662 std::vector<const RelAlgNode*> get_non_join_sequence(const RelAlgNode* ra) {
663  std::vector<const RelAlgNode*> seq;
664  for (auto join = dynamic_cast<const RelJoin*>(ra); join;
665  join = static_cast<const RelJoin*>(join->getInput(0))) {
666  CHECK_EQ(size_t(2), join->inputCount());
667  seq.emplace_back(join->getInput(1));
668  auto lhs = join->getInput(0);
669  if (!dynamic_cast<const RelJoin*>(lhs)) {
670  seq.emplace_back(lhs);
671  break;
672  }
673  }
674  std::reverse(seq.begin(), seq.end());
675  return seq;
676 }
677 
679  std::vector<InputDescriptor>& input_descs,
680  const Catalog_Namespace::Catalog& cat,
681  std::unordered_set<std::shared_ptr<const InputColDescriptor>>& input_col_descs_unique,
682  const RelAlgNode* ra_node,
683  const std::unordered_set<const RexInput*>& source_used_inputs,
684  const std::unordered_map<const RelAlgNode*, int>& input_to_nest_level) {
685  std::unordered_set<InputDescriptor> input_descs_unique(input_descs.begin(),
686  input_descs.end());
687  const auto non_join_src_seq = get_non_join_sequence(get_data_sink(ra_node));
688  std::unordered_map<const RelAlgNode*, int> non_join_to_nest_level;
689  for (const auto node : non_join_src_seq) {
690  non_join_to_nest_level.insert(std::make_pair(node, non_join_to_nest_level.size()));
691  }
692  for (const auto used_input : source_used_inputs) {
693  const auto input_ra = used_input->getSourceNode();
694  const int table_id = table_id_from_ra(input_ra);
695  const auto col_id = used_input->getIndex();
696  auto it = input_to_nest_level.find(input_ra);
697  if (it == input_to_nest_level.end()) {
698  throw std::runtime_error("Bushy joins not supported");
699  }
700  const int input_desc = it->second;
701  input_col_descs_unique.insert(std::make_shared<const InputColDescriptor>(
702  dynamic_cast<const RelScan*>(input_ra)
703  ? cat.getColumnIdBySpi(table_id, col_id + 1)
704  : col_id,
705  table_id,
706  input_desc));
707  }
708 }
709 
710 template <class RA>
711 std::pair<std::vector<InputDescriptor>,
712  std::list<std::shared_ptr<const InputColDescriptor>>>
713 get_input_desc_impl(const RA* ra_node,
714  const std::unordered_set<const RexInput*>& used_inputs,
715  const std::unordered_map<const RelAlgNode*, int>& input_to_nest_level,
716  const std::vector<size_t>& input_permutation,
717  const Catalog_Namespace::Catalog& cat) {
718  std::vector<InputDescriptor> input_descs;
719  const auto data_sink_node = get_data_sink(ra_node);
720  for (size_t input_idx = 0; input_idx < data_sink_node->inputCount(); ++input_idx) {
721  const auto input_node_idx =
722  input_permutation.empty() ? input_idx : input_permutation[input_idx];
723  const auto input_ra = data_sink_node->getInput(input_node_idx);
724  const int table_id = table_id_from_ra(input_ra);
725  input_descs.emplace_back(table_id, input_idx);
726  }
727  std::sort(input_descs.begin(),
728  input_descs.end(),
729  [](const InputDescriptor& lhs, const InputDescriptor& rhs) {
730  return lhs.getNestLevel() < rhs.getNestLevel();
731  });
732  std::unordered_set<std::shared_ptr<const InputColDescriptor>> input_col_descs_unique;
733  collect_used_input_desc(input_descs,
734  cat,
735  input_col_descs_unique,
736  ra_node,
737  used_inputs,
738  input_to_nest_level);
739  std::unordered_set<const RexInput*> join_source_used_inputs;
740  std::vector<std::shared_ptr<RexInput>> join_source_used_inputs_owned;
741  std::tie(join_source_used_inputs, join_source_used_inputs_owned) =
742  get_join_source_used_inputs(ra_node, cat);
743  collect_used_input_desc(input_descs,
744  cat,
745  input_col_descs_unique,
746  ra_node,
747  join_source_used_inputs,
748  input_to_nest_level);
749  std::vector<std::shared_ptr<const InputColDescriptor>> input_col_descs(
750  input_col_descs_unique.begin(), input_col_descs_unique.end());
751 
752  std::sort(
753  input_col_descs.begin(),
754  input_col_descs.end(),
755  [](std::shared_ptr<const InputColDescriptor> const& lhs,
756  std::shared_ptr<const InputColDescriptor> const& rhs) {
757  if (lhs->getScanDesc().getNestLevel() == rhs->getScanDesc().getNestLevel()) {
758  return lhs->getColId() < rhs->getColId();
759  }
760  return lhs->getScanDesc().getNestLevel() < rhs->getScanDesc().getNestLevel();
761  });
762  return {input_descs,
763  std::list<std::shared_ptr<const InputColDescriptor>>(input_col_descs.begin(),
764  input_col_descs.end())};
765 }
766 
767 template <class RA>
768 std::tuple<std::vector<InputDescriptor>,
769  std::list<std::shared_ptr<const InputColDescriptor>>,
770  std::vector<std::shared_ptr<RexInput>>>
771 get_input_desc(const RA* ra_node,
772  const std::unordered_map<const RelAlgNode*, int>& input_to_nest_level,
773  const std::vector<size_t>& input_permutation,
774  const Catalog_Namespace::Catalog& cat) {
775  std::unordered_set<const RexInput*> used_inputs;
776  std::vector<std::shared_ptr<RexInput>> used_inputs_owned;
777  std::tie(used_inputs, used_inputs_owned) = get_used_inputs(ra_node, cat);
778  auto input_desc_pair = get_input_desc_impl(
779  ra_node, used_inputs, input_to_nest_level, input_permutation, cat);
780  return std::make_tuple(
781  input_desc_pair.first, input_desc_pair.second, used_inputs_owned);
782 }
783 
784 size_t get_scalar_sources_size(const RelCompound* compound) {
785  return compound->getScalarSourcesSize();
786 }
787 
788 size_t get_scalar_sources_size(const RelProject* project) {
789  return project->size();
790 }
791 
792 size_t get_scalar_sources_size(const RelTableFunction* table_func) {
793  return table_func->getTableFuncInputsSize();
794 }
795 
796 const RexScalar* scalar_at(const size_t i, const RelCompound* compound) {
797  return compound->getScalarSource(i);
798 }
799 
800 const RexScalar* scalar_at(const size_t i, const RelProject* project) {
801  return project->getProjectAt(i);
802 }
803 
804 const RexScalar* scalar_at(const size_t i, const RelTableFunction* table_func) {
805  return table_func->getTableFuncInputAt(i);
806 }
807 
808 std::shared_ptr<Analyzer::Expr> set_transient_dict(
809  const std::shared_ptr<Analyzer::Expr> expr) {
810  const auto& ti = expr->get_type_info();
811  if (!ti.is_string() || ti.get_compression() != kENCODING_NONE) {
812  return expr;
813  }
814  auto transient_dict_ti = ti;
815  transient_dict_ti.set_compression(kENCODING_DICT);
816  transient_dict_ti.set_comp_param(TRANSIENT_DICT_ID);
817  transient_dict_ti.set_fixed_size();
818  return expr->add_cast(transient_dict_ti);
819 }
820 
822  std::vector<std::shared_ptr<Analyzer::Expr>>& scalar_sources,
823  const std::shared_ptr<Analyzer::Expr>& expr) {
824  try {
825  scalar_sources.push_back(set_transient_dict(fold_expr(expr.get())));
826  } catch (...) {
827  scalar_sources.push_back(fold_expr(expr.get()));
828  }
829 }
830 
831 template <class RA>
832 std::vector<std::shared_ptr<Analyzer::Expr>> translate_scalar_sources(
833  const RA* ra_node,
834  const RelAlgTranslator& translator) {
835  std::vector<std::shared_ptr<Analyzer::Expr>> scalar_sources;
836  for (size_t i = 0; i < get_scalar_sources_size(ra_node); ++i) {
837  const auto scalar_rex = scalar_at(i, ra_node);
838  if (dynamic_cast<const RexRef*>(scalar_rex)) {
839  // RexRef are synthetic scalars we append at the end of the real ones
840  // for the sake of taking memory ownership, no real work needed here.
841  continue;
842  }
843 
844  const auto scalar_expr =
845  rewrite_array_elements(translator.translateScalarRex(scalar_rex).get());
846  const auto rewritten_expr = rewrite_expr(scalar_expr.get());
847  set_transient_dict_maybe(scalar_sources, rewritten_expr);
848  }
849 
850  return scalar_sources;
851 }
852 
853 template <class RA>
854 std::vector<std::shared_ptr<Analyzer::Expr>> translate_scalar_sources_for_update(
855  const RA* ra_node,
856  const RelAlgTranslator& translator,
857  int32_t tableId,
858  const Catalog_Namespace::Catalog& cat,
859  const ColumnNameList& colNames,
860  size_t starting_projection_column_idx) {
861  std::vector<std::shared_ptr<Analyzer::Expr>> scalar_sources;
862  for (size_t i = 0; i < get_scalar_sources_size(ra_node); ++i) {
863  const auto scalar_rex = scalar_at(i, ra_node);
864  if (dynamic_cast<const RexRef*>(scalar_rex)) {
865  // RexRef are synthetic scalars we append at the end of the real ones
866  // for the sake of taking memory ownership, no real work needed here.
867  continue;
868  }
869 
870  std::shared_ptr<Analyzer::Expr> translated_expr;
871  if (i >= starting_projection_column_idx && i < get_scalar_sources_size(ra_node) - 1) {
872  translated_expr = cast_to_column_type(translator.translateScalarRex(scalar_rex),
873  tableId,
874  cat,
875  colNames[i - starting_projection_column_idx]);
876  } else {
877  translated_expr = translator.translateScalarRex(scalar_rex);
878  }
879  const auto scalar_expr = rewrite_array_elements(translated_expr.get());
880  const auto rewritten_expr = rewrite_expr(scalar_expr.get());
881  set_transient_dict_maybe(scalar_sources, rewritten_expr);
882  }
883 
884  return scalar_sources;
885 }
886 
887 std::list<std::shared_ptr<Analyzer::Expr>> translate_groupby_exprs(
888  const RelCompound* compound,
889  const std::vector<std::shared_ptr<Analyzer::Expr>>& scalar_sources) {
890  if (!compound->isAggregate()) {
891  return {nullptr};
892  }
893  std::list<std::shared_ptr<Analyzer::Expr>> groupby_exprs;
894  for (size_t group_idx = 0; group_idx < compound->getGroupByCount(); ++group_idx) {
895  groupby_exprs.push_back(set_transient_dict(scalar_sources[group_idx]));
896  }
897  return groupby_exprs;
898 }
899 
900 std::list<std::shared_ptr<Analyzer::Expr>> translate_groupby_exprs(
901  const RelAggregate* aggregate,
902  const std::vector<std::shared_ptr<Analyzer::Expr>>& scalar_sources) {
903  std::list<std::shared_ptr<Analyzer::Expr>> groupby_exprs;
904  for (size_t group_idx = 0; group_idx < aggregate->getGroupByCount(); ++group_idx) {
905  groupby_exprs.push_back(set_transient_dict(scalar_sources[group_idx]));
906  }
907  return groupby_exprs;
908 }
909 
911  const RelAlgTranslator& translator) {
912  const auto filter_rex = compound->getFilterExpr();
913  const auto filter_expr =
914  filter_rex ? translator.translateScalarRex(filter_rex) : nullptr;
915  return filter_expr ? qual_to_conjunctive_form(fold_expr(filter_expr.get()))
917 }
918 
919 std::vector<Analyzer::Expr*> translate_targets(
920  std::vector<std::shared_ptr<Analyzer::Expr>>& target_exprs_owned,
921  const std::vector<std::shared_ptr<Analyzer::Expr>>& scalar_sources,
922  const std::list<std::shared_ptr<Analyzer::Expr>>& groupby_exprs,
923  const RelCompound* compound,
924  const RelAlgTranslator& translator) {
925  std::vector<Analyzer::Expr*> target_exprs;
926  for (size_t i = 0; i < compound->size(); ++i) {
927  const auto target_rex = compound->getTargetExpr(i);
928  const auto target_rex_agg = dynamic_cast<const RexAgg*>(target_rex);
929  std::shared_ptr<Analyzer::Expr> target_expr;
930  if (target_rex_agg) {
931  target_expr =
932  RelAlgTranslator::translateAggregateRex(target_rex_agg, scalar_sources);
933  } else {
934  const auto target_rex_scalar = dynamic_cast<const RexScalar*>(target_rex);
935  const auto target_rex_ref = dynamic_cast<const RexRef*>(target_rex_scalar);
936  if (target_rex_ref) {
937  const auto ref_idx = target_rex_ref->getIndex();
938  CHECK_GE(ref_idx, size_t(1));
939  CHECK_LE(ref_idx, groupby_exprs.size());
940  const auto groupby_expr = *std::next(groupby_exprs.begin(), ref_idx - 1);
941  target_expr = var_ref(groupby_expr.get(), Analyzer::Var::kGROUPBY, ref_idx);
942  } else {
943  target_expr = translator.translateScalarRex(target_rex_scalar);
944  auto rewritten_expr = rewrite_expr(target_expr.get());
945  target_expr = fold_expr(rewritten_expr.get());
946  try {
947  target_expr = set_transient_dict(target_expr);
948  } catch (...) {
949  // noop
950  }
951  }
952  }
953  CHECK(target_expr);
954  target_exprs_owned.push_back(target_expr);
955  target_exprs.push_back(target_expr.get());
956  }
957  return target_exprs;
958 }
959 
960 std::vector<Analyzer::Expr*> translate_targets(
961  std::vector<std::shared_ptr<Analyzer::Expr>>& target_exprs_owned,
962  const std::vector<std::shared_ptr<Analyzer::Expr>>& scalar_sources,
963  const std::list<std::shared_ptr<Analyzer::Expr>>& groupby_exprs,
964  const RelAggregate* aggregate,
965  const RelAlgTranslator& translator) {
966  std::vector<Analyzer::Expr*> target_exprs;
967  size_t group_key_idx = 0;
968  for (const auto& groupby_expr : groupby_exprs) {
969  auto target_expr =
970  var_ref(groupby_expr.get(), Analyzer::Var::kGROUPBY, group_key_idx++);
971  target_exprs_owned.push_back(target_expr);
972  target_exprs.push_back(target_expr.get());
973  }
974 
975  for (const auto& target_rex_agg : aggregate->getAggExprs()) {
976  auto target_expr =
977  RelAlgTranslator::translateAggregateRex(target_rex_agg.get(), scalar_sources);
978  CHECK(target_expr);
979  target_expr = fold_expr(target_expr.get());
980  target_exprs_owned.push_back(target_expr);
981  target_exprs.push_back(target_expr.get());
982  }
983  return target_exprs;
984 }
985 
987  const auto agg_expr = dynamic_cast<const Analyzer::AggExpr*>(expr);
988  return agg_expr && agg_expr->get_is_distinct();
989 }
990 
991 bool is_agg(const Analyzer::Expr* expr) {
992  const auto agg_expr = dynamic_cast<const Analyzer::AggExpr*>(expr);
993  if (agg_expr && agg_expr->get_contains_agg()) {
994  auto agg_type = agg_expr->get_aggtype();
995  if (agg_type == SQLAgg::kMIN || agg_type == SQLAgg::kMAX ||
996  agg_type == SQLAgg::kSUM || agg_type == SQLAgg::kAVG) {
997  return true;
998  }
999  }
1000  return false;
1001 }
1002 
1004  if (is_count_distinct(&expr)) {
1005  return SQLTypeInfo(kBIGINT, false);
1006  } else if (is_agg(&expr)) {
1008  }
1009  return get_logical_type_info(expr.get_type_info());
1010 }
1011 
1012 template <class RA>
1013 std::vector<TargetMetaInfo> get_targets_meta(
1014  const RA* ra_node,
1015  const std::vector<Analyzer::Expr*>& target_exprs) {
1016  std::vector<TargetMetaInfo> targets_meta;
1017  for (size_t i = 0; i < ra_node->size(); ++i) {
1018  CHECK(target_exprs[i]);
1019  // TODO(alex): remove the count distinct type fixup.
1020  targets_meta.emplace_back(ra_node->getFieldName(i),
1021  get_logical_type_for_expr(*target_exprs[i]),
1022  target_exprs[i]->get_type_info());
1023  }
1024  return targets_meta;
1025 }
1026 
1027 } // namespace
1028 
1030  const CompilationOptions& co,
1031  const ExecutionOptions& eo,
1032  RenderInfo* render_info,
1033  const int64_t queue_time_ms) {
1034  if (!compound->validateTargetColumns(
1036  throw std::runtime_error(
1037  "Unsupported update operation encountered. (None-encoded string column updates "
1038  "are not supported.)");
1039  }
1040 
1041  const auto work_unit = createCompoundWorkUnit(
1042  compound, {{}, SortAlgorithm::Default, 0, 0}, eo.just_explain);
1043  const auto table_infos = get_table_infos(work_unit.exe_unit, executor_);
1044  CompilationOptions co_project = co;
1046 
1047  try {
1049 
1050  UpdateTransactionParameters update_params(compound->getModifiedTableDescriptor(),
1051  compound->getTargetColumns(),
1052  compound->getOutputMetainfo(),
1053  compound->isVarlenUpdateRequired());
1054  auto update_callback = yieldUpdateCallback(update_params);
1055  executor_->executeUpdate(work_unit.exe_unit,
1056  table_infos,
1057  co_project,
1058  eo,
1059  cat_,
1060  executor_->row_set_mem_owner_,
1061  update_callback,
1062  compound->isAggregate());
1063  update_params.finalizeTransaction();
1064  } catch (...) {
1065  LOG(INFO) << "Update operation failed.";
1066  throw;
1067  }
1068 }
1069 
1071  const CompilationOptions& co,
1072  const ExecutionOptions& eo,
1073  RenderInfo* render_info,
1074  const int64_t queue_time_ms) {
1075  if (!project->validateTargetColumns(
1077  throw std::runtime_error(
1078  "Unsupported update operation encountered. (None-encoded string column updates "
1079  "are not supported.)");
1080  }
1081 
1082  auto work_unit =
1083  createProjectWorkUnit(project, {{}, SortAlgorithm::Default, 0, 0}, eo.just_explain);
1084  const auto table_infos = get_table_infos(work_unit.exe_unit, executor_);
1085  CompilationOptions co_project = co;
1087 
1088  if (project->isSimple()) {
1089  CHECK_EQ(size_t(1), project->inputCount());
1090  const auto input_ra = project->getInput(0);
1091  if (dynamic_cast<const RelSort*>(input_ra)) {
1092  const auto& input_table =
1093  get_temporary_table(&temporary_tables_, -input_ra->getId());
1094  CHECK(input_table);
1095  work_unit.exe_unit.scan_limit = input_table->rowCount();
1096  }
1097  }
1098 
1099  try {
1101 
1102  UpdateTransactionParameters update_params(project->getModifiedTableDescriptor(),
1103  project->getTargetColumns(),
1104  project->getOutputMetainfo(),
1105  project->isVarlenUpdateRequired());
1106  auto update_callback = yieldUpdateCallback(update_params);
1107  executor_->executeUpdate(work_unit.exe_unit,
1108  table_infos,
1109  co_project,
1110  eo,
1111  cat_,
1112  executor_->row_set_mem_owner_,
1113  update_callback);
1114  update_params.finalizeTransaction();
1115  } catch (...) {
1116  LOG(INFO) << "Update operation failed.";
1117  throw;
1118  }
1119 }
1120 
1122  const CompilationOptions& co,
1123  const ExecutionOptions& eo,
1124  RenderInfo* render_info,
1125  const int64_t queue_time_ms) {
1126  auto* table_descriptor = compound->getModifiedTableDescriptor();
1127  if (!table_descriptor->hasDeletedCol) {
1128  throw std::runtime_error(
1129  "DELETE only supported on tables with the vacuum attribute set to 'delayed'");
1130  }
1131  if (table_is_temporary(table_descriptor)) {
1132  throw std::runtime_error("DELETE not yet supported on temporary tables.");
1133  }
1134 
1135  const auto work_unit = createCompoundWorkUnit(
1136  compound, {{}, SortAlgorithm::Default, 0, 0}, eo.just_explain);
1137  const auto table_infos = get_table_infos(work_unit.exe_unit, executor_);
1138  CompilationOptions co_project = co;
1140 
1141  try {
1143 
1144  DeleteTransactionParameters delete_params;
1145  auto delete_callback = yieldDeleteCallback(delete_params);
1146 
1147  executor_->executeUpdate(work_unit.exe_unit,
1148  table_infos,
1149  co_project,
1150  eo,
1151  cat_,
1152  executor_->row_set_mem_owner_,
1153  delete_callback,
1154  compound->isAggregate());
1155  delete_params.finalizeTransaction();
1156  } catch (...) {
1157  LOG(INFO) << "Delete operation failed.";
1158  throw;
1159  }
1160 }
1161 
1163  const CompilationOptions& co,
1164  const ExecutionOptions& eo,
1165  RenderInfo* render_info,
1166  const int64_t queue_time_ms) {
1167  auto* table_descriptor = project->getModifiedTableDescriptor();
1168  if (!table_descriptor->hasDeletedCol) {
1169  throw std::runtime_error(
1170  "DELETE only supported on tables with the vacuum attribute set to 'delayed'");
1171  }
1172  if (table_is_temporary(table_descriptor)) {
1173  throw std::runtime_error("DELETE not yet supported on temporary tables.");
1174  }
1175 
1176  auto work_unit =
1177  createProjectWorkUnit(project, {{}, SortAlgorithm::Default, 0, 0}, eo.just_explain);
1178  const auto table_infos = get_table_infos(work_unit.exe_unit, executor_);
1179  CompilationOptions co_project = co;
1181 
1182  if (project->isSimple()) {
1183  CHECK_EQ(size_t(1), project->inputCount());
1184  const auto input_ra = project->getInput(0);
1185  if (dynamic_cast<const RelSort*>(input_ra)) {
1186  const auto& input_table =
1187  get_temporary_table(&temporary_tables_, -input_ra->getId());
1188  CHECK(input_table);
1189  work_unit.exe_unit.scan_limit = input_table->rowCount();
1190  }
1191  }
1192 
1193  try {
1195 
1196  DeleteTransactionParameters delete_params;
1197  auto delete_callback = yieldDeleteCallback(delete_params);
1198 
1199  executor_->executeUpdate(work_unit.exe_unit,
1200  table_infos,
1201  co_project,
1202  eo,
1203  cat_,
1204  executor_->row_set_mem_owner_,
1205  delete_callback);
1206  delete_params.finalizeTransaction();
1207  } catch (...) {
1208  LOG(INFO) << "Delete operation failed.";
1209  throw;
1210  }
1211 }
1212 
1214  const CompilationOptions& co,
1215  const ExecutionOptions& eo,
1216  RenderInfo* render_info,
1217  const int64_t queue_time_ms) {
1218  const auto work_unit = createCompoundWorkUnit(
1219  compound, {{}, SortAlgorithm::Default, 0, 0}, eo.just_explain);
1220  CompilationOptions co_compound = co;
1221  return executeWorkUnit(work_unit,
1222  compound->getOutputMetainfo(),
1223  compound->isAggregate(),
1224  co_compound,
1225  eo,
1226  render_info,
1227  queue_time_ms);
1228 }
1229 
1231  const CompilationOptions& co,
1232  const ExecutionOptions& eo,
1233  RenderInfo* render_info,
1234  const int64_t queue_time_ms) {
1235  const auto work_unit = createAggregateWorkUnit(
1236  aggregate, {{}, SortAlgorithm::Default, 0, 0}, eo.just_explain);
1237  return executeWorkUnit(work_unit,
1238  aggregate->getOutputMetainfo(),
1239  true,
1240  co,
1241  eo,
1242  render_info,
1243  queue_time_ms);
1244 }
1245 
1246 namespace {
1247 
1248 // Returns true iff the execution unit contains window functions.
1250  return std::any_of(ra_exe_unit.target_exprs.begin(),
1251  ra_exe_unit.target_exprs.end(),
1252  [](const Analyzer::Expr* expr) {
1253  return dynamic_cast<const Analyzer::WindowFunction*>(expr);
1254  });
1255 }
1256 
1257 } // namespace
1258 
1260  const CompilationOptions& co,
1261  const ExecutionOptions& eo,
1262  RenderInfo* render_info,
1263  const int64_t queue_time_ms,
1264  const ssize_t previous_count) {
1265  auto work_unit =
1266  createProjectWorkUnit(project, {{}, SortAlgorithm::Default, 0, 0}, eo.just_explain);
1267  CompilationOptions co_project = co;
1268  if (project->isSimple()) {
1269  CHECK_EQ(size_t(1), project->inputCount());
1270  const auto input_ra = project->getInput(0);
1271  if (dynamic_cast<const RelSort*>(input_ra)) {
1272  co_project.device_type_ = ExecutorDeviceType::CPU;
1273  const auto& input_table =
1274  get_temporary_table(&temporary_tables_, -input_ra->getId());
1275  CHECK(input_table);
1276  work_unit.exe_unit.scan_limit =
1277  std::min(input_table->getLimit(), input_table->rowCount());
1278  }
1279  }
1280  return executeWorkUnit(work_unit,
1281  project->getOutputMetainfo(),
1282  false,
1283  co_project,
1284  eo,
1285  render_info,
1286  queue_time_ms,
1287  previous_count);
1288 }
1289 
1291  const CompilationOptions& co_in,
1292  const ExecutionOptions& eo,
1293  const int64_t queue_time_ms) {
1295 
1296  auto co = co_in;
1297 
1298  if (g_cluster) {
1299  throw std::runtime_error("Table functions not supported in distributed mode yet");
1300  }
1301  if (!g_enable_table_functions) {
1302  throw std::runtime_error("Table function support is disabled");
1303  }
1304 
1305  auto table_func_work_unit = createTableFunctionWorkUnit(table_func, eo.just_explain);
1306  const auto body = table_func_work_unit.body;
1307  CHECK(body);
1308 
1309  const auto table_infos =
1310  get_table_infos(table_func_work_unit.exe_unit.input_descs, executor_);
1311 
1312  ExecutionResult result{std::make_shared<ResultSet>(std::vector<TargetInfo>{},
1313  co.device_type_,
1315  nullptr,
1316  executor_),
1317  {}};
1318 
1319  try {
1320  result = {executor_->executeTableFunction(
1321  table_func_work_unit.exe_unit, table_infos, co, eo, cat_),
1322  body->getOutputMetainfo()};
1323  } catch (const QueryExecutionError& e) {
1326  throw std::runtime_error("Table function ran out of memory during execution");
1327  }
1328  result.setQueueTime(queue_time_ms);
1329  return result;
1330 }
1331 
1332 namespace {
1333 
1334 // Creates a new expression which has the range table index set to 1. This is needed to
1335 // reuse the hash join construction helpers to generate a hash table for the window
1336 // function partition: create an equals expression with left and right sides identical
1337 // except for the range table index.
1338 std::shared_ptr<Analyzer::Expr> transform_to_inner(const Analyzer::Expr* expr) {
1339  const auto tuple = dynamic_cast<const Analyzer::ExpressionTuple*>(expr);
1340  if (tuple) {
1341  std::vector<std::shared_ptr<Analyzer::Expr>> transformed_tuple;
1342  for (const auto& element : tuple->getTuple()) {
1343  transformed_tuple.push_back(transform_to_inner(element.get()));
1344  }
1345  return makeExpr<Analyzer::ExpressionTuple>(transformed_tuple);
1346  }
1347  const auto col = dynamic_cast<const Analyzer::ColumnVar*>(expr);
1348  if (!col) {
1349  throw std::runtime_error("Only columns supported in the window partition for now");
1350  }
1351  return makeExpr<Analyzer::ColumnVar>(
1352  col->get_type_info(), col->get_table_id(), col->get_column_id(), 1);
1353 }
1354 
1355 } // namespace
1356 
1358  const CompilationOptions& co,
1359  const ExecutionOptions& eo,
1360  ColumnCacheMap& column_cache_map,
1361  const int64_t queue_time_ms) {
1362  auto query_infos = get_table_infos(ra_exe_unit.input_descs, executor_);
1363  CHECK_EQ(query_infos.size(), size_t(1));
1364  if (query_infos.front().info.fragments.size() != 1) {
1365  throw std::runtime_error(
1366  "Only single fragment tables supported for window functions for now");
1367  }
1368  query_infos.push_back(query_infos.front());
1369  auto window_project_node_context = WindowProjectNodeContext::create();
1370  for (size_t target_index = 0; target_index < ra_exe_unit.target_exprs.size();
1371  ++target_index) {
1372  const auto& target_expr = ra_exe_unit.target_exprs[target_index];
1373  const auto window_func = dynamic_cast<const Analyzer::WindowFunction*>(target_expr);
1374  if (!window_func) {
1375  continue;
1376  }
1377  // Always use baseline layout hash tables for now, make the expression a tuple.
1378  const auto& partition_keys = window_func->getPartitionKeys();
1379  std::shared_ptr<Analyzer::Expr> partition_key_tuple;
1380  if (partition_keys.size() > 1) {
1381  partition_key_tuple = makeExpr<Analyzer::ExpressionTuple>(partition_keys);
1382  } else {
1383  if (partition_keys.empty()) {
1384  throw std::runtime_error(
1385  "Empty window function partitions are not supported yet");
1386  }
1387  CHECK_EQ(partition_keys.size(), size_t(1));
1388  partition_key_tuple = partition_keys.front();
1389  }
1390  // Creates a tautology equality with the partition expression on both sides.
1391  const auto partition_key_cond =
1392  makeExpr<Analyzer::BinOper>(kBOOLEAN,
1393  kBW_EQ,
1394  kONE,
1395  partition_key_tuple,
1396  transform_to_inner(partition_key_tuple.get()));
1397  auto context = createWindowFunctionContext(
1398  window_func, partition_key_cond, ra_exe_unit, query_infos, co, column_cache_map);
1399  context->compute();
1400  window_project_node_context->addWindowFunctionContext(std::move(context),
1401  target_index);
1402  }
1403 }
1404 
1405 std::unique_ptr<WindowFunctionContext> RelAlgExecutor::createWindowFunctionContext(
1406  const Analyzer::WindowFunction* window_func,
1407  const std::shared_ptr<Analyzer::BinOper>& partition_key_cond,
1408  const RelAlgExecutionUnit& ra_exe_unit,
1409  const std::vector<InputTableInfo>& query_infos,
1410  const CompilationOptions& co,
1411  ColumnCacheMap& column_cache_map) {
1412  const auto memory_level = co.device_type_ == ExecutorDeviceType::GPU
1415  const auto join_table_or_err =
1416  executor_->buildHashTableForQualifier(partition_key_cond,
1417  query_infos,
1418  memory_level,
1420  column_cache_map);
1421  if (!join_table_or_err.fail_reason.empty()) {
1422  throw std::runtime_error(join_table_or_err.fail_reason);
1423  }
1424  CHECK(join_table_or_err.hash_table->getHashType() ==
1426  const auto& order_keys = window_func->getOrderKeys();
1427  std::vector<std::shared_ptr<Chunk_NS::Chunk>> chunks_owner;
1428  const size_t elem_count = query_infos.front().info.fragments.front().getNumTuples();
1429  auto context = std::make_unique<WindowFunctionContext>(
1430  window_func, join_table_or_err.hash_table, elem_count, co.device_type_);
1431  for (const auto& order_key : order_keys) {
1432  const auto order_col =
1433  std::dynamic_pointer_cast<const Analyzer::ColumnVar>(order_key);
1434  if (!order_col) {
1435  throw std::runtime_error("Only order by columns supported for now");
1436  }
1437  const int8_t* column;
1438  size_t join_col_elem_count;
1439  std::tie(column, join_col_elem_count) =
1441  *order_col,
1442  query_infos.front().info.fragments.front(),
1443  memory_level,
1444  0,
1445  chunks_owner,
1446  column_cache_map);
1447  CHECK_EQ(join_col_elem_count, elem_count);
1448  context->addOrderColumn(column, order_col.get(), chunks_owner);
1449  }
1450  return context;
1451 }
1452 
1454  const CompilationOptions& co,
1455  const ExecutionOptions& eo,
1456  RenderInfo* render_info,
1457  const int64_t queue_time_ms) {
1458  const auto work_unit =
1459  createFilterWorkUnit(filter, {{}, SortAlgorithm::Default, 0, 0}, eo.just_explain);
1460  return executeWorkUnit(
1461  work_unit, filter->getOutputMetainfo(), false, co, eo, render_info, queue_time_ms);
1462 }
1463 
1465  const ExecutionOptions& eo) {
1466  if (eo.just_explain) {
1467  throw std::runtime_error("EXPLAIN not supported for ModifyTable");
1468  }
1469 
1470  auto rs = std::make_shared<ResultSet>(TargetInfoList{},
1473  executor_->getRowSetMemoryOwner(),
1474  executor_);
1475 
1476  std::vector<TargetMetaInfo> empty_targets;
1477  return {rs, empty_targets};
1478 }
1479 
1481  const RelLogicalValues* logical_values,
1482  const ExecutionOptions& eo) {
1483  if (eo.just_explain) {
1484  throw std::runtime_error("EXPLAIN not supported for LogicalValues");
1485  }
1487  1,
1489  /*is_table_function=*/false);
1490 
1491  const auto& tuple_type = logical_values->getTupleType();
1492  for (size_t i = 0; i < tuple_type.size(); ++i) {
1493  query_mem_desc.addColSlotInfo({std::make_tuple(8, 8)});
1494  }
1495  logical_values->setOutputMetainfo(tuple_type);
1496  std::vector<std::unique_ptr<Analyzer::ColumnVar>> owned_column_expressions;
1497  std::vector<Analyzer::Expr*> target_expressions;
1498  for (const auto& tuple_component : tuple_type) {
1499  const auto column_var =
1500  new Analyzer::ColumnVar(tuple_component.get_type_info(), 0, 0, 0);
1501  target_expressions.push_back(column_var);
1502  owned_column_expressions.emplace_back(column_var);
1503  }
1504  std::vector<TargetInfo> target_infos;
1505  for (const auto& tuple_type_component : tuple_type) {
1506  target_infos.emplace_back(TargetInfo{false,
1507  kCOUNT,
1508  tuple_type_component.get_type_info(),
1509  SQLTypeInfo(kNULLT, false),
1510  false,
1511  false});
1512  }
1513  auto rs = std::make_shared<ResultSet>(target_infos,
1516  executor_->getRowSetMemoryOwner(),
1517  executor_);
1518  return {rs, tuple_type};
1519 }
1520 
1521 namespace {
1522 
1523 // TODO(alex): Once we're fully migrated to the relational algebra model, change
1524 // the executor interface to use the collation directly and remove this conversion.
1525 std::list<Analyzer::OrderEntry> get_order_entries(const RelSort* sort) {
1526  std::list<Analyzer::OrderEntry> result;
1527  for (size_t i = 0; i < sort->collationCount(); ++i) {
1528  const auto sort_field = sort->getCollation(i);
1529  result.emplace_back(sort_field.getField() + 1,
1530  sort_field.getSortDir() == SortDirection::Descending,
1531  sort_field.getNullsPosition() == NullSortedPosition::First);
1532  }
1533  return result;
1534 }
1535 
1536 size_t get_scan_limit(const RelAlgNode* ra, const size_t limit) {
1537  const auto aggregate = dynamic_cast<const RelAggregate*>(ra);
1538  if (aggregate) {
1539  return 0;
1540  }
1541  const auto compound = dynamic_cast<const RelCompound*>(ra);
1542  return (compound && compound->isAggregate()) ? 0 : limit;
1543 }
1544 
1545 bool first_oe_is_desc(const std::list<Analyzer::OrderEntry>& order_entries) {
1546  return !order_entries.empty() && order_entries.front().is_desc;
1547 }
1548 
1549 } // namespace
1550 
1552  const CompilationOptions& co,
1553  const ExecutionOptions& eo,
1554  RenderInfo* render_info,
1555  const int64_t queue_time_ms) {
1557  const auto source = sort->getInput(0);
1558  const bool is_aggregate = node_is_aggregate(source);
1559  auto it = leaf_results_.find(sort->getId());
1560  if (it != leaf_results_.end()) {
1561  // Add any transient string literals to the sdp on the agg
1562  const auto source_work_unit = createSortInputWorkUnit(sort, eo.just_explain);
1564  source_work_unit.exe_unit, executor_, executor_->row_set_mem_owner_);
1565 
1566  // Handle push-down for LIMIT for multi-node
1567  auto& aggregated_result = it->second;
1568  auto& result_rows = aggregated_result.rs;
1569  const size_t limit = sort->getLimit();
1570  const size_t offset = sort->getOffset();
1571  const auto order_entries = get_order_entries(sort);
1572  if (limit || offset) {
1573  if (!order_entries.empty()) {
1574  result_rows->sort(order_entries, limit + offset);
1575  }
1576  result_rows->dropFirstN(offset);
1577  if (limit) {
1578  result_rows->keepFirstN(limit);
1579  }
1580  }
1581  ExecutionResult result(result_rows, aggregated_result.targets_meta);
1582  sort->setOutputMetainfo(aggregated_result.targets_meta);
1583 
1584  return result;
1585  }
1586  while (true) {
1587  std::list<std::shared_ptr<Analyzer::Expr>> groupby_exprs;
1588  bool is_desc{false};
1589  try {
1590  const auto source_work_unit = createSortInputWorkUnit(sort, eo.just_explain);
1591  is_desc = first_oe_is_desc(source_work_unit.exe_unit.sort_info.order_entries);
1592  ExecutionOptions eo_copy = {
1594  eo.allow_multifrag,
1595  eo.just_explain,
1596  eo.allow_loop_joins,
1597  eo.with_watchdog,
1598  eo.jit_debug,
1599  eo.just_validate || sort->isEmptyResult(),
1605  };
1606 
1607  groupby_exprs = source_work_unit.exe_unit.groupby_exprs;
1608  auto source_result = executeWorkUnit(source_work_unit,
1609  source->getOutputMetainfo(),
1610  is_aggregate,
1611  co,
1612  eo_copy,
1613  render_info,
1614  queue_time_ms);
1615  if (render_info && render_info->isPotentialInSituRender()) {
1616  return source_result;
1617  }
1618  if (source_result.isFilterPushDownEnabled()) {
1619  return source_result;
1620  }
1621  auto rows_to_sort = source_result.getRows();
1622  if (eo.just_explain) {
1623  return {rows_to_sort, {}};
1624  }
1625  const size_t limit = sort->getLimit();
1626  const size_t offset = sort->getOffset();
1627  if (sort->collationCount() != 0 && !rows_to_sort->definitelyHasNoRows() &&
1628  !use_speculative_top_n(source_work_unit.exe_unit,
1629  rows_to_sort->getQueryMemDesc())) {
1630  rows_to_sort->sort(source_work_unit.exe_unit.sort_info.order_entries,
1631  limit + offset);
1632  }
1633  if (limit || offset) {
1634  if (g_cluster && sort->collationCount() == 0) {
1635  if (offset >= rows_to_sort->rowCount()) {
1636  rows_to_sort->dropFirstN(offset);
1637  } else {
1638  rows_to_sort->keepFirstN(limit + offset);
1639  }
1640  } else {
1641  rows_to_sort->dropFirstN(offset);
1642  if (limit) {
1643  rows_to_sort->keepFirstN(limit);
1644  }
1645  }
1646  }
1647  return {rows_to_sort, source_result.getTargetsMeta()};
1648  } catch (const SpeculativeTopNFailed&) {
1649  CHECK_EQ(size_t(1), groupby_exprs.size());
1650  speculative_topn_blacklist_.add(groupby_exprs.front(), is_desc);
1651  }
1652  }
1653  CHECK(false);
1654  return {std::make_shared<ResultSet>(std::vector<TargetInfo>{},
1655  co.device_type_,
1657  nullptr,
1658  executor_),
1659  {}};
1660 }
1661 
1663  const RelSort* sort,
1664  const bool just_explain) {
1665  const auto source = sort->getInput(0);
1666  const size_t limit = sort->getLimit();
1667  const size_t offset = sort->getOffset();
1668  const size_t scan_limit = sort->collationCount() ? 0 : get_scan_limit(source, limit);
1669  const size_t scan_total_limit =
1670  scan_limit ? get_scan_limit(source, scan_limit + offset) : 0;
1671  size_t max_groups_buffer_entry_guess{
1672  scan_total_limit ? scan_total_limit : max_groups_buffer_entry_default_guess};
1674  const auto order_entries = get_order_entries(sort);
1675  SortInfo sort_info{order_entries, sort_algorithm, limit, offset};
1676  auto source_work_unit = createWorkUnit(source, sort_info, just_explain);
1677  const auto& source_exe_unit = source_work_unit.exe_unit;
1678  if (source_exe_unit.groupby_exprs.size() == 1) {
1679  if (!source_exe_unit.groupby_exprs.front()) {
1680  sort_algorithm = SortAlgorithm::StreamingTopN;
1681  } else {
1682  if (speculative_topn_blacklist_.contains(source_exe_unit.groupby_exprs.front(),
1683  first_oe_is_desc(order_entries))) {
1684  sort_algorithm = SortAlgorithm::Default;
1685  }
1686  }
1687  }
1688 
1689  sort->setOutputMetainfo(source->getOutputMetainfo());
1690  // NB: the `body` field of the returned `WorkUnit` needs to be the `source` node,
1691  // not the `sort`. The aggregator needs the pre-sorted result from leaves.
1692  return {RelAlgExecutionUnit{source_exe_unit.input_descs,
1693  std::move(source_exe_unit.input_col_descs),
1694  source_exe_unit.simple_quals,
1695  source_exe_unit.quals,
1696  source_exe_unit.join_quals,
1697  source_exe_unit.groupby_exprs,
1698  source_exe_unit.target_exprs,
1699  nullptr,
1700  {sort_info.order_entries, sort_algorithm, limit, offset},
1701  scan_total_limit,
1702  source_exe_unit.query_features,
1703  source_exe_unit.use_bump_allocator,
1704  source_exe_unit.query_state},
1705  source,
1706  max_groups_buffer_entry_guess,
1707  std::move(source_work_unit.query_rewriter),
1708  source_work_unit.input_permutation,
1709  source_work_unit.left_deep_join_input_sizes};
1710 }
1711 
1712 namespace {
1713 
1720 size_t groups_approx_upper_bound(const std::vector<InputTableInfo>& table_infos) {
1721  CHECK(!table_infos.empty());
1722  const auto& first_table = table_infos.front();
1723  size_t max_num_groups = first_table.info.getNumTuplesUpperBound();
1724  for (const auto& table_info : table_infos) {
1725  if (table_info.info.getNumTuplesUpperBound() > max_num_groups) {
1726  max_num_groups = table_info.info.getNumTuplesUpperBound();
1727  }
1728  }
1729  return std::max(max_num_groups, size_t(1));
1730 }
1731 
1739  for (const auto target_expr : ra_exe_unit.target_exprs) {
1740  if (dynamic_cast<const Analyzer::AggExpr*>(target_expr)) {
1741  return false;
1742  }
1743  }
1744  if (ra_exe_unit.groupby_exprs.size() == 1 && !ra_exe_unit.groupby_exprs.front() &&
1745  (!ra_exe_unit.scan_limit || ra_exe_unit.scan_limit > Executor::high_scan_limit)) {
1746  return true;
1747  }
1748  return false;
1749 }
1750 
1751 inline bool exe_unit_has_quals(const RelAlgExecutionUnit ra_exe_unit) {
1752  return !(ra_exe_unit.quals.empty() && ra_exe_unit.join_quals.empty() &&
1753  ra_exe_unit.simple_quals.empty());
1754 }
1755 
1757  const RelAlgExecutionUnit& ra_exe_unit_in,
1758  const std::vector<InputTableInfo>& table_infos,
1759  const Executor* executor,
1760  const ExecutorDeviceType device_type_in,
1761  std::vector<std::shared_ptr<Analyzer::Expr>>& target_exprs_owned) {
1762  RelAlgExecutionUnit ra_exe_unit = ra_exe_unit_in;
1763  for (size_t i = 0; i < ra_exe_unit.target_exprs.size(); ++i) {
1764  const auto target_expr = ra_exe_unit.target_exprs[i];
1765  const auto agg_info = get_target_info(target_expr, g_bigint_count);
1766  if (agg_info.agg_kind != kAPPROX_COUNT_DISTINCT) {
1767  continue;
1768  }
1769  CHECK(dynamic_cast<const Analyzer::AggExpr*>(target_expr));
1770  const auto arg = static_cast<Analyzer::AggExpr*>(target_expr)->get_own_arg();
1771  CHECK(arg);
1772  const auto& arg_ti = arg->get_type_info();
1773  // Avoid calling getExpressionRange for variable length types (string and array),
1774  // it'd trigger an assertion since that API expects to be called only for types
1775  // for which the notion of range is well-defined. A bit of a kludge, but the
1776  // logic to reject these types anyway is at lower levels in the stack and not
1777  // really worth pulling into a separate function for now.
1778  if (!(arg_ti.is_number() || arg_ti.is_boolean() || arg_ti.is_time() ||
1779  (arg_ti.is_string() && arg_ti.get_compression() == kENCODING_DICT))) {
1780  continue;
1781  }
1782  const auto arg_range = getExpressionRange(arg.get(), table_infos, executor);
1783  if (arg_range.getType() != ExpressionRangeType::Integer) {
1784  continue;
1785  }
1786  // When running distributed, the threshold for using the precise implementation
1787  // must be consistent across all leaves, otherwise we could have a mix of precise
1788  // and approximate bitmaps and we cannot aggregate them.
1789  const auto device_type = g_cluster ? ExecutorDeviceType::GPU : device_type_in;
1790  const auto bitmap_sz_bits = arg_range.getIntMax() - arg_range.getIntMin() + 1;
1791  const auto sub_bitmap_count =
1792  get_count_distinct_sub_bitmap_count(bitmap_sz_bits, ra_exe_unit, device_type);
1793  int64_t approx_bitmap_sz_bits{0};
1794  const auto error_rate =
1795  static_cast<Analyzer::AggExpr*>(target_expr)->get_error_rate();
1796  if (error_rate) {
1797  CHECK(error_rate->get_type_info().get_type() == kINT);
1798  CHECK_GE(error_rate->get_constval().intval, 1);
1799  approx_bitmap_sz_bits = hll_size_for_rate(error_rate->get_constval().intval);
1800  } else {
1801  approx_bitmap_sz_bits = g_hll_precision_bits;
1802  }
1803  CountDistinctDescriptor approx_count_distinct_desc{CountDistinctImplType::Bitmap,
1804  arg_range.getIntMin(),
1805  approx_bitmap_sz_bits,
1806  true,
1807  device_type,
1808  sub_bitmap_count};
1809  CountDistinctDescriptor precise_count_distinct_desc{CountDistinctImplType::Bitmap,
1810  arg_range.getIntMin(),
1811  bitmap_sz_bits,
1812  false,
1813  device_type,
1814  sub_bitmap_count};
1815  if (approx_count_distinct_desc.bitmapPaddedSizeBytes() >=
1816  precise_count_distinct_desc.bitmapPaddedSizeBytes()) {
1817  auto precise_count_distinct = makeExpr<Analyzer::AggExpr>(
1818  get_agg_type(kCOUNT, arg.get()), kCOUNT, arg, true, nullptr);
1819  target_exprs_owned.push_back(precise_count_distinct);
1820  ra_exe_unit.target_exprs[i] = precise_count_distinct.get();
1821  }
1822  }
1823  return ra_exe_unit;
1824 }
1825 
1827  const std::vector<Analyzer::Expr*>& work_unit_target_exprs,
1828  const std::vector<TargetMetaInfo>& targets_meta) {
1829  CHECK_EQ(work_unit_target_exprs.size(), targets_meta.size());
1830  render_info.targets.clear();
1831  for (size_t i = 0; i < targets_meta.size(); ++i) {
1832  render_info.targets.emplace_back(std::make_shared<Analyzer::TargetEntry>(
1833  targets_meta[i].get_resname(),
1834  work_unit_target_exprs[i]->get_shared_ptr(),
1835  false));
1836  }
1837 }
1838 
1839 inline bool can_use_bump_allocator(const RelAlgExecutionUnit& ra_exe_unit,
1840  const CompilationOptions& co,
1841  const ExecutionOptions& eo) {
1843  !eo.output_columnar_hint && ra_exe_unit.sort_info.order_entries.empty();
1844 }
1845 
1846 } // namespace
1847 
1849  const RelAlgExecutor::WorkUnit& work_unit,
1850  const std::vector<TargetMetaInfo>& targets_meta,
1851  const bool is_agg,
1852  const CompilationOptions& co_in,
1853  const ExecutionOptions& eo,
1854  RenderInfo* render_info,
1855  const int64_t queue_time_ms,
1856  const ssize_t previous_count) {
1858  auto timer = DEBUG_TIMER(__func__);
1859 
1860  auto co = co_in;
1861  ColumnCacheMap column_cache;
1862  if (is_window_execution_unit(work_unit.exe_unit)) {
1863  if (g_cluster) {
1864  throw std::runtime_error(
1865  "Window functions support not supported in distributed mode");
1866  }
1868  throw std::runtime_error("Window functions support is disabled");
1869  }
1870  co.device_type_ = ExecutorDeviceType::CPU;
1871  computeWindow(work_unit.exe_unit, co, eo, column_cache, queue_time_ms);
1872  }
1873  if (!eo.just_explain && eo.find_push_down_candidates) {
1874  // find potential candidates:
1875  auto selected_filters = selectFiltersToBePushedDown(work_unit, co, eo);
1876  if (!selected_filters.empty() || eo.just_calcite_explain) {
1877  return ExecutionResult(selected_filters, eo.find_push_down_candidates);
1878  }
1879  }
1880  const auto body = work_unit.body;
1881  CHECK(body);
1882  auto it = leaf_results_.find(body->getId());
1883  if (it != leaf_results_.end()) {
1885  work_unit.exe_unit, executor_, executor_->row_set_mem_owner_);
1886  auto& aggregated_result = it->second;
1887  auto& result_rows = aggregated_result.rs;
1888  ExecutionResult result(result_rows, aggregated_result.targets_meta);
1889  body->setOutputMetainfo(aggregated_result.targets_meta);
1890  if (render_info) {
1891  build_render_targets(*render_info, work_unit.exe_unit.target_exprs, targets_meta);
1892  }
1893  return result;
1894  }
1895  const auto table_infos = get_table_infos(work_unit.exe_unit, executor_);
1896 
1898  work_unit.exe_unit, table_infos, executor_, co.device_type_, target_exprs_owned_);
1899  auto max_groups_buffer_entry_guess = work_unit.max_groups_buffer_entry_guess;
1900  if (is_window_execution_unit(ra_exe_unit)) {
1901  CHECK_EQ(table_infos.size(), size_t(1));
1902  CHECK_EQ(table_infos.front().info.fragments.size(), size_t(1));
1903  max_groups_buffer_entry_guess =
1904  table_infos.front().info.fragments.front().getNumTuples();
1905  ra_exe_unit.scan_limit = max_groups_buffer_entry_guess;
1906  } else if (compute_output_buffer_size(ra_exe_unit) && !isRowidLookup(work_unit)) {
1907  if (previous_count > 0 && !exe_unit_has_quals(ra_exe_unit)) {
1908  ra_exe_unit.scan_limit = static_cast<size_t>(previous_count);
1909  } else {
1910  // TODO(adb): enable bump allocator path for render queries
1911  if (can_use_bump_allocator(ra_exe_unit, co, eo) && !render_info) {
1912  ra_exe_unit.scan_limit = 0;
1913  ra_exe_unit.use_bump_allocator = true;
1914  } else if (!eo.just_explain) {
1915  const auto filter_count_all = getFilteredCountAll(work_unit, true, co, eo);
1916  if (filter_count_all >= 0) {
1917  ra_exe_unit.scan_limit = std::max(filter_count_all, ssize_t(1));
1918  }
1919  }
1920  }
1921  }
1922 
1923  ExecutionResult result{std::make_shared<ResultSet>(std::vector<TargetInfo>{},
1924  co.device_type_,
1926  nullptr,
1927  executor_),
1928  {}};
1929 
1930  auto execute_and_handle_errors =
1931  [&](const auto max_groups_buffer_entry_guess_in,
1932  const bool has_cardinality_estimation) -> ExecutionResult {
1933  // Note that the groups buffer entry guess may be modified during query execution.
1934  // Create a local copy so we can track those changes if we need to attempt a retry due
1935  // to OOM
1936  auto local_groups_buffer_entry_guess = max_groups_buffer_entry_guess_in;
1937  try {
1938  return {executor_->executeWorkUnit(local_groups_buffer_entry_guess,
1939  is_agg,
1940  table_infos,
1941  ra_exe_unit,
1942  co,
1943  eo,
1944  cat_,
1945  executor_->row_set_mem_owner_,
1946  render_info,
1947  has_cardinality_estimation,
1948  column_cache),
1949  targets_meta};
1950  } catch (const QueryExecutionError& e) {
1952  return handleOutOfMemoryRetry(
1953  {ra_exe_unit, work_unit.body, local_groups_buffer_entry_guess},
1954  targets_meta,
1955  is_agg,
1956  co,
1957  eo,
1958  render_info,
1960  queue_time_ms);
1961  }
1962  };
1963 
1964  try {
1965  result = execute_and_handle_errors(
1966  max_groups_buffer_entry_guess,
1968  } catch (const CardinalityEstimationRequired&) {
1969  const auto estimated_groups_buffer_entry_guess =
1970  2 * std::min(groups_approx_upper_bound(table_infos),
1971  getNDVEstimation(work_unit, is_agg, co, eo));
1972  CHECK_GT(estimated_groups_buffer_entry_guess, size_t(0));
1973  result = execute_and_handle_errors(estimated_groups_buffer_entry_guess, true);
1974  }
1975 
1976  result.setQueueTime(queue_time_ms);
1977  if (render_info) {
1978  build_render_targets(*render_info, work_unit.exe_unit.target_exprs, targets_meta);
1979  if (render_info->isPotentialInSituRender()) {
1980  // return an empty result (with the same queue time, and zero render time)
1981  return {std::make_shared<ResultSet>(
1982  queue_time_ms,
1983  0,
1984  executor_->row_set_mem_owner_
1985  ? executor_->row_set_mem_owner_->cloneStrDictDataOnly()
1986  : nullptr),
1987  {}};
1988  }
1989  }
1990  return result;
1991 }
1992 
1994  const bool is_agg,
1995  const CompilationOptions& co,
1996  const ExecutionOptions& eo) {
1997  const auto count =
1998  makeExpr<Analyzer::AggExpr>(SQLTypeInfo(g_bigint_count ? kBIGINT : kINT, false),
1999  kCOUNT,
2000  nullptr,
2001  false,
2002  nullptr);
2003  const auto count_all_exe_unit =
2004  create_count_all_execution_unit(work_unit.exe_unit, count);
2005  size_t one{1};
2006  ResultSetPtr count_all_result;
2007  try {
2008  ColumnCacheMap column_cache;
2009  count_all_result =
2010  executor_->executeWorkUnit(one,
2011  is_agg,
2012  get_table_infos(work_unit.exe_unit, executor_),
2013  count_all_exe_unit,
2014  co,
2015  eo,
2016  cat_,
2017  executor_->row_set_mem_owner_,
2018  nullptr,
2019  false,
2020  column_cache);
2021  } catch (const std::exception& e) {
2022  LOG(WARNING) << "Failed to run pre-flight filtered count with error " << e.what();
2023  return -1;
2024  }
2025  const auto count_row = count_all_result->getNextRow(false, false);
2026  CHECK_EQ(size_t(1), count_row.size());
2027  const auto& count_tv = count_row.front();
2028  const auto count_scalar_tv = boost::get<ScalarTargetValue>(&count_tv);
2029  CHECK(count_scalar_tv);
2030  const auto count_ptr = boost::get<int64_t>(count_scalar_tv);
2031  CHECK(count_ptr);
2032  CHECK_GE(*count_ptr, 0);
2033  auto count_upper_bound = static_cast<size_t>(*count_ptr);
2034  return std::max(count_upper_bound, size_t(1));
2035 }
2036 
2037 bool RelAlgExecutor::isRowidLookup(const WorkUnit& work_unit) {
2038  const auto& ra_exe_unit = work_unit.exe_unit;
2039  if (ra_exe_unit.input_descs.size() != 1) {
2040  return false;
2041  }
2042  const auto& table_desc = ra_exe_unit.input_descs.front();
2043  if (table_desc.getSourceType() != InputSourceType::TABLE) {
2044  return false;
2045  }
2046  const int table_id = table_desc.getTableId();
2047  for (const auto simple_qual : ra_exe_unit.simple_quals) {
2048  const auto comp_expr =
2049  std::dynamic_pointer_cast<const Analyzer::BinOper>(simple_qual);
2050  if (!comp_expr || comp_expr->get_optype() != kEQ) {
2051  return false;
2052  }
2053  const auto lhs = comp_expr->get_left_operand();
2054  const auto lhs_col = dynamic_cast<const Analyzer::ColumnVar*>(lhs);
2055  if (!lhs_col || !lhs_col->get_table_id() || lhs_col->get_rte_idx()) {
2056  return false;
2057  }
2058  const auto rhs = comp_expr->get_right_operand();
2059  const auto rhs_const = dynamic_cast<const Analyzer::Constant*>(rhs);
2060  if (!rhs_const) {
2061  return false;
2062  }
2063  auto cd = get_column_descriptor(lhs_col->get_column_id(), table_id, cat_);
2064  if (cd->isVirtualCol) {
2065  CHECK_EQ("rowid", cd->columnName);
2066  return true;
2067  }
2068  }
2069  return false;
2070 }
2071 
2073  const RelAlgExecutor::WorkUnit& work_unit,
2074  const std::vector<TargetMetaInfo>& targets_meta,
2075  const bool is_agg,
2076  const CompilationOptions& co,
2077  const ExecutionOptions& eo,
2078  RenderInfo* render_info,
2079  const bool was_multifrag_kernel_launch,
2080  const int64_t queue_time_ms) {
2081  // Disable the bump allocator
2082  // Note that this will have basically the same affect as using the bump allocator for
2083  // the kernel per fragment path. Need to unify the max_groups_buffer_entry_guess = 0
2084  // path and the bump allocator path for kernel per fragment execution.
2085  auto ra_exe_unit_in = work_unit.exe_unit;
2086  ra_exe_unit_in.use_bump_allocator = false;
2087 
2088  auto result = ExecutionResult{std::make_shared<ResultSet>(std::vector<TargetInfo>{},
2089  co.device_type_,
2091  nullptr,
2092  executor_),
2093  {}};
2094 
2095  const auto table_infos = get_table_infos(ra_exe_unit_in, executor_);
2096  auto max_groups_buffer_entry_guess = work_unit.max_groups_buffer_entry_guess;
2097  ExecutionOptions eo_no_multifrag{eo.output_columnar_hint,
2098  false,
2099  false,
2100  eo.allow_loop_joins,
2101  eo.with_watchdog,
2102  eo.jit_debug,
2103  false,
2106  false,
2107  false,
2109 
2110  if (was_multifrag_kernel_launch) {
2111  try {
2112  // Attempt to retry using the kernel per fragment path. The smaller input size
2113  // required may allow the entire kernel to execute in GPU memory.
2114  LOG(WARNING) << "Multifrag query ran out of memory, retrying with multifragment "
2115  "kernels disabled.";
2116  const auto ra_exe_unit = decide_approx_count_distinct_implementation(
2117  ra_exe_unit_in, table_infos, executor_, co.device_type_, target_exprs_owned_);
2118  ColumnCacheMap column_cache;
2119  result = {executor_->executeWorkUnit(max_groups_buffer_entry_guess,
2120  is_agg,
2121  table_infos,
2122  ra_exe_unit,
2123  co,
2124  eo_no_multifrag,
2125  cat_,
2126  executor_->row_set_mem_owner_,
2127  nullptr,
2128  true,
2129  column_cache),
2130  targets_meta};
2131  result.setQueueTime(queue_time_ms);
2132  } catch (const QueryExecutionError& e) {
2134  LOG(WARNING) << "Kernel per fragment query ran out of memory, retrying on CPU.";
2135  }
2136  }
2137 
2138  if (render_info) {
2139  render_info->setForceNonInSituData();
2140  }
2141 
2143  co.hoist_literals_,
2144  co.opt_level_,
2146 
2147  // Only reset the group buffer entry guess if we ran out of slots, which
2148  // suggests a
2149  // highly pathological input which prevented a good estimation of distinct tuple
2150  // count. For projection queries, this will force a per-fragment scan limit, which is
2151  // compatible with the CPU path
2152  VLOG(1) << "Resetting max groups buffer entry guess.";
2153  max_groups_buffer_entry_guess = 0;
2154 
2155  int iteration_ctr = -1;
2156  while (true) {
2157  iteration_ctr++;
2159  ra_exe_unit_in, table_infos, executor_, co_cpu.device_type_, target_exprs_owned_);
2160  ColumnCacheMap column_cache;
2161  try {
2162  result = {executor_->executeWorkUnit(max_groups_buffer_entry_guess,
2163  is_agg,
2164  table_infos,
2165  ra_exe_unit,
2166  co_cpu,
2167  eo_no_multifrag,
2168  cat_,
2169  executor_->row_set_mem_owner_,
2170  nullptr,
2171  true,
2172  column_cache),
2173  targets_meta};
2174  } catch (const QueryExecutionError& e) {
2175  // Ran out of slots
2176  if (e.getErrorCode() < 0) {
2177  // Even the conservative guess failed; it should only happen when we group
2178  // by a huge cardinality array. Maybe we should throw an exception instead?
2179  // Such a heavy query is entirely capable of exhausting all the host memory.
2180  CHECK(max_groups_buffer_entry_guess);
2181  // Only allow two iterations of increasingly large entry guesses up to a maximum
2182  // of 512MB per column per kernel
2183  if (g_enable_watchdog || iteration_ctr > 1) {
2184  throw std::runtime_error("Query ran out of output slots in the result");
2185  }
2186  max_groups_buffer_entry_guess *= 2;
2187  LOG(WARNING) << "Query ran out of slots in the output buffer, retrying with max "
2188  "groups buffer entry "
2189  "guess equal to "
2190  << max_groups_buffer_entry_guess;
2191  } else {
2193  }
2194  continue;
2195  }
2196  result.setQueueTime(queue_time_ms);
2197  return result;
2198  }
2199  return result;
2200 }
2201 
2203  LOG(ERROR) << "Query execution failed with error "
2204  << getErrorMessageFromCode(error_code);
2205  if (error_code == Executor::ERR_SPECULATIVE_TOP_OOM) {
2206  throw SpeculativeTopNFailed();
2207  }
2208  if (error_code == Executor::ERR_OUT_OF_GPU_MEM) {
2209  // We ran out of GPU memory, this doesn't count as an error if the query is
2210  // allowed to continue on CPU because retry on CPU is explicitly allowed through
2211  // --allow-cpu-retry.
2212  LOG(INFO) << "Query ran out of GPU memory, attempting punt to CPU";
2213  if (!g_allow_cpu_retry) {
2214  throw std::runtime_error(
2215  "Query ran out of GPU memory, unable to automatically retry on CPU");
2216  }
2217  return;
2218  }
2219  throw std::runtime_error(getErrorMessageFromCode(error_code));
2220 }
2221 
2223  if (error_code < 0) {
2224  return "Ran out of slots in the query output buffer";
2225  }
2226  switch (error_code) {
2228  return "Division by zero";
2230  return "Query couldn't keep the entire working set of columns in GPU memory";
2232  return "Self joins not supported yet";
2234  return "Not enough host memory to execute the query";
2236  return "Overflow or underflow";
2238  return "Query execution has exceeded the time limit";
2240  return "Query execution has been interrupted";
2242  return "Columnar conversion not supported for variable length types";
2244  return "Too many literals in the query";
2246  return "NONE ENCODED String types are not supported as input result set.";
2248  return "Not enough OpenGL memory to render the query results";
2250  return "Streaming-Top-N not supported in Render Query";
2252  return "Multiple distinct values encountered";
2253  }
2254  return "Other error: code " + std::to_string(error_code);
2255 }
2256 
2258  const SortInfo& sort_info,
2259  const bool just_explain) {
2260  const auto compound = dynamic_cast<const RelCompound*>(node);
2261  if (compound) {
2262  return createCompoundWorkUnit(compound, sort_info, just_explain);
2263  }
2264  const auto project = dynamic_cast<const RelProject*>(node);
2265  if (project) {
2266  return createProjectWorkUnit(project, sort_info, just_explain);
2267  }
2268  const auto aggregate = dynamic_cast<const RelAggregate*>(node);
2269  if (aggregate) {
2270  return createAggregateWorkUnit(aggregate, sort_info, just_explain);
2271  }
2272  const auto filter = dynamic_cast<const RelFilter*>(node);
2273  CHECK(filter);
2274  return createFilterWorkUnit(filter, sort_info, just_explain);
2275 }
2276 
2277 namespace {
2278 
2280  auto sink = get_data_sink(ra);
2281  if (auto join = dynamic_cast<const RelJoin*>(sink)) {
2282  return join->getJoinType();
2283  }
2284  if (dynamic_cast<const RelLeftDeepInnerJoin*>(sink)) {
2285  return JoinType::INNER;
2286  }
2287 
2288  return JoinType::INVALID;
2289 }
2290 
2291 std::unique_ptr<const RexOperator> get_bitwise_equals(const RexScalar* scalar) {
2292  const auto condition = dynamic_cast<const RexOperator*>(scalar);
2293  if (!condition || condition->getOperator() != kOR || condition->size() != 2) {
2294  return nullptr;
2295  }
2296  const auto equi_join_condition =
2297  dynamic_cast<const RexOperator*>(condition->getOperand(0));
2298  if (!equi_join_condition || equi_join_condition->getOperator() != kEQ) {
2299  return nullptr;
2300  }
2301  const auto both_are_null_condition =
2302  dynamic_cast<const RexOperator*>(condition->getOperand(1));
2303  if (!both_are_null_condition || both_are_null_condition->getOperator() != kAND ||
2304  both_are_null_condition->size() != 2) {
2305  return nullptr;
2306  }
2307  const auto lhs_is_null =
2308  dynamic_cast<const RexOperator*>(both_are_null_condition->getOperand(0));
2309  const auto rhs_is_null =
2310  dynamic_cast<const RexOperator*>(both_are_null_condition->getOperand(1));
2311  if (!lhs_is_null || !rhs_is_null || lhs_is_null->getOperator() != kISNULL ||
2312  rhs_is_null->getOperator() != kISNULL) {
2313  return nullptr;
2314  }
2315  CHECK_EQ(size_t(1), lhs_is_null->size());
2316  CHECK_EQ(size_t(1), rhs_is_null->size());
2317  CHECK_EQ(size_t(2), equi_join_condition->size());
2318  const auto eq_lhs = dynamic_cast<const RexInput*>(equi_join_condition->getOperand(0));
2319  const auto eq_rhs = dynamic_cast<const RexInput*>(equi_join_condition->getOperand(1));
2320  const auto is_null_lhs = dynamic_cast<const RexInput*>(lhs_is_null->getOperand(0));
2321  const auto is_null_rhs = dynamic_cast<const RexInput*>(rhs_is_null->getOperand(0));
2322  if (!eq_lhs || !eq_rhs || !is_null_lhs || !is_null_rhs) {
2323  return nullptr;
2324  }
2325  std::vector<std::unique_ptr<const RexScalar>> eq_operands;
2326  if (*eq_lhs == *is_null_lhs && *eq_rhs == *is_null_rhs) {
2327  RexDeepCopyVisitor deep_copy_visitor;
2328  auto lhs_op_copy = deep_copy_visitor.visit(equi_join_condition->getOperand(0));
2329  auto rhs_op_copy = deep_copy_visitor.visit(equi_join_condition->getOperand(1));
2330  eq_operands.emplace_back(lhs_op_copy.release());
2331  eq_operands.emplace_back(rhs_op_copy.release());
2332  return boost::make_unique<const RexOperator>(
2333  kBW_EQ, eq_operands, equi_join_condition->getType());
2334  }
2335  return nullptr;
2336 }
2337 
2338 std::unique_ptr<const RexOperator> get_bitwise_equals_conjunction(
2339  const RexScalar* scalar) {
2340  const auto condition = dynamic_cast<const RexOperator*>(scalar);
2341  if (condition && condition->getOperator() == kAND) {
2342  CHECK_GE(condition->size(), size_t(2));
2343  auto acc = get_bitwise_equals(condition->getOperand(0));
2344  if (!acc) {
2345  return nullptr;
2346  }
2347  for (size_t i = 1; i < condition->size(); ++i) {
2348  std::vector<std::unique_ptr<const RexScalar>> and_operands;
2349  and_operands.emplace_back(std::move(acc));
2350  and_operands.emplace_back(get_bitwise_equals_conjunction(condition->getOperand(i)));
2351  acc =
2352  boost::make_unique<const RexOperator>(kAND, and_operands, condition->getType());
2353  }
2354  return acc;
2355  }
2356  return get_bitwise_equals(scalar);
2357 }
2358 
2359 std::vector<JoinType> left_deep_join_types(const RelLeftDeepInnerJoin* left_deep_join) {
2360  CHECK_GE(left_deep_join->inputCount(), size_t(2));
2361  std::vector<JoinType> join_types(left_deep_join->inputCount() - 1, JoinType::INNER);
2362  for (size_t nesting_level = 1; nesting_level <= left_deep_join->inputCount() - 1;
2363  ++nesting_level) {
2364  if (left_deep_join->getOuterCondition(nesting_level)) {
2365  join_types[nesting_level - 1] = JoinType::LEFT;
2366  }
2367  }
2368  return join_types;
2369 }
2370 
2371 template <class RA>
2372 std::vector<size_t> do_table_reordering(
2373  std::vector<InputDescriptor>& input_descs,
2374  std::list<std::shared_ptr<const InputColDescriptor>>& input_col_descs,
2375  const JoinQualsPerNestingLevel& left_deep_join_quals,
2376  std::unordered_map<const RelAlgNode*, int>& input_to_nest_level,
2377  const RA* node,
2378  const std::vector<InputTableInfo>& query_infos,
2379  const Executor* executor) {
2380  if (g_cluster) {
2381  // Disable table reordering in distributed mode. The aggregator does not have enough
2382  // information to break ties
2383  return {};
2384  }
2385  const auto& cat = *executor->getCatalog();
2386  for (const auto& table_info : query_infos) {
2387  if (table_info.table_id < 0) {
2388  continue;
2389  }
2390  const auto td = cat.getMetadataForTable(table_info.table_id);
2391  CHECK(td);
2392  if (table_is_replicated(td)) {
2393  return {};
2394  }
2395  }
2396  const auto input_permutation =
2397  get_node_input_permutation(left_deep_join_quals, query_infos, executor);
2398  input_to_nest_level = get_input_nest_levels(node, input_permutation);
2399  std::tie(input_descs, input_col_descs, std::ignore) =
2400  get_input_desc(node, input_to_nest_level, input_permutation, cat);
2401  return input_permutation;
2402 }
2403 
2405  const RelLeftDeepInnerJoin* left_deep_join) {
2406  std::vector<size_t> input_sizes;
2407  for (size_t i = 0; i < left_deep_join->inputCount(); ++i) {
2408  const auto inputs = get_node_output(left_deep_join->getInput(i));
2409  input_sizes.push_back(inputs.size());
2410  }
2411  return input_sizes;
2412 }
2413 
2414 std::list<std::shared_ptr<Analyzer::Expr>> rewrite_quals(
2415  const std::list<std::shared_ptr<Analyzer::Expr>>& quals) {
2416  std::list<std::shared_ptr<Analyzer::Expr>> rewritten_quals;
2417  for (const auto& qual : quals) {
2418  const auto rewritten_qual = rewrite_expr(qual.get());
2419  rewritten_quals.push_back(rewritten_qual ? rewritten_qual : qual);
2420  }
2421  return rewritten_quals;
2422 }
2423 
2424 } // namespace
2425 
2427  const RelCompound* compound,
2428  const SortInfo& sort_info,
2429  const bool just_explain) {
2430  std::vector<InputDescriptor> input_descs;
2431  std::list<std::shared_ptr<const InputColDescriptor>> input_col_descs;
2432  auto input_to_nest_level = get_input_nest_levels(compound, {});
2433  std::tie(input_descs, input_col_descs, std::ignore) =
2434  get_input_desc(compound, input_to_nest_level, {}, cat_);
2435  const auto query_infos = get_table_infos(input_descs, executor_);
2436  CHECK_EQ(size_t(1), compound->inputCount());
2437  const auto left_deep_join =
2438  dynamic_cast<const RelLeftDeepInnerJoin*>(compound->getInput(0));
2439  JoinQualsPerNestingLevel left_deep_join_quals;
2440  const auto join_types = left_deep_join ? left_deep_join_types(left_deep_join)
2441  : std::vector<JoinType>{get_join_type(compound)};
2442  std::vector<size_t> input_permutation;
2443  std::vector<size_t> left_deep_join_input_sizes;
2444  if (left_deep_join) {
2445  left_deep_join_input_sizes = get_left_deep_join_input_sizes(left_deep_join);
2446  left_deep_join_quals = translateLeftDeepJoinFilter(
2447  left_deep_join, input_descs, input_to_nest_level, just_explain);
2449  std::find(join_types.begin(), join_types.end(), JoinType::LEFT) ==
2450  join_types.end()) {
2451  input_permutation = do_table_reordering(input_descs,
2452  input_col_descs,
2453  left_deep_join_quals,
2454  input_to_nest_level,
2455  compound,
2456  query_infos,
2457  executor_);
2458  input_to_nest_level = get_input_nest_levels(compound, input_permutation);
2459  std::tie(input_descs, input_col_descs, std::ignore) =
2460  get_input_desc(compound, input_to_nest_level, input_permutation, cat_);
2461  left_deep_join_quals = translateLeftDeepJoinFilter(
2462  left_deep_join, input_descs, input_to_nest_level, just_explain);
2463  }
2464  }
2465  QueryFeatureDescriptor query_features;
2466  RelAlgTranslator translator(cat_,
2467  executor_,
2468  input_to_nest_level,
2469  join_types,
2470  now_,
2471  just_explain,
2472  query_features);
2473  const auto scalar_sources = translate_scalar_sources(compound, translator);
2474  const auto groupby_exprs = translate_groupby_exprs(compound, scalar_sources);
2475  const auto quals_cf = translate_quals(compound, translator);
2476  const auto target_exprs = translate_targets(
2477  target_exprs_owned_, scalar_sources, groupby_exprs, compound, translator);
2478  CHECK_EQ(compound->size(), target_exprs.size());
2479  const RelAlgExecutionUnit exe_unit = {input_descs,
2480  input_col_descs,
2481  quals_cf.simple_quals,
2482  rewrite_quals(quals_cf.quals),
2483  left_deep_join_quals,
2484  groupby_exprs,
2485  target_exprs,
2486  nullptr,
2487  sort_info,
2488  0,
2489  query_features,
2490  false,
2491  query_state_};
2492  auto query_rewriter = std::make_unique<QueryRewriter>(query_infos, executor_);
2493  const auto rewritten_exe_unit = query_rewriter->rewrite(exe_unit);
2494  const auto targets_meta = get_targets_meta(compound, rewritten_exe_unit.target_exprs);
2495  compound->setOutputMetainfo(targets_meta);
2496  return {rewritten_exe_unit,
2497  compound,
2499  std::move(query_rewriter),
2500  input_permutation,
2501  left_deep_join_input_sizes};
2502 }
2503 
2504 namespace {
2505 
2506 std::vector<const RexScalar*> rex_to_conjunctive_form(const RexScalar* qual_expr) {
2507  CHECK(qual_expr);
2508  const auto bin_oper = dynamic_cast<const RexOperator*>(qual_expr);
2509  if (!bin_oper || bin_oper->getOperator() != kAND) {
2510  return {qual_expr};
2511  }
2512  CHECK_GE(bin_oper->size(), size_t(2));
2513  auto lhs_cf = rex_to_conjunctive_form(bin_oper->getOperand(0));
2514  for (size_t i = 1; i < bin_oper->size(); ++i) {
2515  const auto rhs_cf = rex_to_conjunctive_form(bin_oper->getOperand(i));
2516  lhs_cf.insert(lhs_cf.end(), rhs_cf.begin(), rhs_cf.end());
2517  }
2518  return lhs_cf;
2519 }
2520 
2521 std::shared_ptr<Analyzer::Expr> build_logical_expression(
2522  const std::vector<std::shared_ptr<Analyzer::Expr>>& factors,
2523  const SQLOps sql_op) {
2524  CHECK(!factors.empty());
2525  auto acc = factors.front();
2526  for (size_t i = 1; i < factors.size(); ++i) {
2527  acc = Parser::OperExpr::normalize(sql_op, kONE, acc, factors[i]);
2528  }
2529  return acc;
2530 }
2531 
2532 template <class QualsList>
2533 bool list_contains_expression(const QualsList& haystack,
2534  const std::shared_ptr<Analyzer::Expr>& needle) {
2535  for (const auto& qual : haystack) {
2536  if (*qual == *needle) {
2537  return true;
2538  }
2539  }
2540  return false;
2541 }
2542 
2543 // Transform `(p AND q) OR (p AND r)` to `p AND (q OR r)`. Avoids redundant
2544 // evaluations of `p` and allows use of the original form in joins if `p`
2545 // can be used for hash joins.
2546 std::shared_ptr<Analyzer::Expr> reverse_logical_distribution(
2547  const std::shared_ptr<Analyzer::Expr>& expr) {
2548  const auto expr_terms = qual_to_disjunctive_form(expr);
2549  CHECK_GE(expr_terms.size(), size_t(1));
2550  const auto& first_term = expr_terms.front();
2551  const auto first_term_factors = qual_to_conjunctive_form(first_term);
2552  std::vector<std::shared_ptr<Analyzer::Expr>> common_factors;
2553  // First, collect the conjunctive components common to all the disjunctive components.
2554  // Don't do it for simple qualifiers, we only care about expensive or join qualifiers.
2555  for (const auto& first_term_factor : first_term_factors.quals) {
2556  bool is_common =
2557  expr_terms.size() > 1; // Only report common factors for disjunction.
2558  for (size_t i = 1; i < expr_terms.size(); ++i) {
2559  const auto crt_term_factors = qual_to_conjunctive_form(expr_terms[i]);
2560  if (!list_contains_expression(crt_term_factors.quals, first_term_factor)) {
2561  is_common = false;
2562  break;
2563  }
2564  }
2565  if (is_common) {
2566  common_factors.push_back(first_term_factor);
2567  }
2568  }
2569  if (common_factors.empty()) {
2570  return expr;
2571  }
2572  // Now that the common expressions are known, collect the remaining expressions.
2573  std::vector<std::shared_ptr<Analyzer::Expr>> remaining_terms;
2574  for (const auto& term : expr_terms) {
2575  const auto term_cf = qual_to_conjunctive_form(term);
2576  std::vector<std::shared_ptr<Analyzer::Expr>> remaining_quals(
2577  term_cf.simple_quals.begin(), term_cf.simple_quals.end());
2578  for (const auto& qual : term_cf.quals) {
2579  if (!list_contains_expression(common_factors, qual)) {
2580  remaining_quals.push_back(qual);
2581  }
2582  }
2583  if (!remaining_quals.empty()) {
2584  remaining_terms.push_back(build_logical_expression(remaining_quals, kAND));
2585  }
2586  }
2587  // Reconstruct the expression with the transformation applied.
2588  const auto common_expr = build_logical_expression(common_factors, kAND);
2589  if (remaining_terms.empty()) {
2590  return common_expr;
2591  }
2592  const auto remaining_expr = build_logical_expression(remaining_terms, kOR);
2593  return Parser::OperExpr::normalize(kAND, kONE, common_expr, remaining_expr);
2594 }
2595 
2596 } // namespace
2597 
2598 std::list<std::shared_ptr<Analyzer::Expr>> RelAlgExecutor::makeJoinQuals(
2599  const RexScalar* join_condition,
2600  const std::vector<JoinType>& join_types,
2601  const std::unordered_map<const RelAlgNode*, int>& input_to_nest_level,
2602  const bool just_explain) const {
2603  QueryFeatureDescriptor query_features;
2604  RelAlgTranslator translator(cat_,
2605  executor_,
2606  input_to_nest_level,
2607  join_types,
2608  now_,
2609  just_explain,
2610  query_features);
2611  const auto rex_condition_cf = rex_to_conjunctive_form(join_condition);
2612  std::list<std::shared_ptr<Analyzer::Expr>> join_condition_quals;
2613  for (const auto rex_condition_component : rex_condition_cf) {
2614  const auto bw_equals = get_bitwise_equals_conjunction(rex_condition_component);
2615  const auto join_condition =
2617  bw_equals ? bw_equals.get() : rex_condition_component));
2618  auto join_condition_cf = qual_to_conjunctive_form(join_condition);
2619  join_condition_quals.insert(join_condition_quals.end(),
2620  join_condition_cf.quals.begin(),
2621  join_condition_cf.quals.end());
2622  join_condition_quals.insert(join_condition_quals.end(),
2623  join_condition_cf.simple_quals.begin(),
2624  join_condition_cf.simple_quals.end());
2625  }
2626  return combine_equi_join_conditions(join_condition_quals);
2627 }
2628 
2629 // Translate left deep join filter and separate the conjunctive form qualifiers
2630 // per nesting level. The code generated for hash table lookups on each level
2631 // must dominate its uses in deeper nesting levels.
2633  const RelLeftDeepInnerJoin* join,
2634  const std::vector<InputDescriptor>& input_descs,
2635  const std::unordered_map<const RelAlgNode*, int>& input_to_nest_level,
2636  const bool just_explain) {
2637  const auto join_types = left_deep_join_types(join);
2638  const auto join_condition_quals = makeJoinQuals(
2639  join->getInnerCondition(), join_types, input_to_nest_level, just_explain);
2640  MaxRangeTableIndexVisitor rte_idx_visitor;
2641  JoinQualsPerNestingLevel result(input_descs.size() - 1);
2642  std::unordered_set<std::shared_ptr<Analyzer::Expr>> visited_quals;
2643  for (size_t rte_idx = 1; rte_idx < input_descs.size(); ++rte_idx) {
2644  const auto outer_condition = join->getOuterCondition(rte_idx);
2645  if (outer_condition) {
2646  result[rte_idx - 1].quals =
2647  makeJoinQuals(outer_condition, join_types, input_to_nest_level, just_explain);
2648  CHECK_LE(rte_idx, join_types.size());
2649  CHECK(join_types[rte_idx - 1] == JoinType::LEFT);
2650  result[rte_idx - 1].type = JoinType::LEFT;
2651  continue;
2652  }
2653  for (const auto qual : join_condition_quals) {
2654  if (visited_quals.count(qual)) {
2655  continue;
2656  }
2657  const auto qual_rte_idx = rte_idx_visitor.visit(qual.get());
2658  if (static_cast<size_t>(qual_rte_idx) <= rte_idx) {
2659  const auto it_ok = visited_quals.emplace(qual);
2660  CHECK(it_ok.second);
2661  result[rte_idx - 1].quals.push_back(qual);
2662  }
2663  }
2664  CHECK_LE(rte_idx, join_types.size());
2665  CHECK(join_types[rte_idx - 1] == JoinType::INNER);
2666  result[rte_idx - 1].type = JoinType::INNER;
2667  }
2668  return result;
2669 }
2670 
2671 namespace {
2672 
2673 std::vector<std::shared_ptr<Analyzer::Expr>> synthesize_inputs(
2674  const RelAlgNode* ra_node,
2675  const size_t nest_level,
2676  const std::vector<TargetMetaInfo>& in_metainfo,
2677  const std::unordered_map<const RelAlgNode*, int>& input_to_nest_level) {
2678  CHECK_LE(size_t(1), ra_node->inputCount());
2679  CHECK_GE(size_t(2), ra_node->inputCount());
2680  const auto input = ra_node->getInput(nest_level);
2681  const auto it_rte_idx = input_to_nest_level.find(input);
2682  CHECK(it_rte_idx != input_to_nest_level.end());
2683  const int rte_idx = it_rte_idx->second;
2684  const int table_id = table_id_from_ra(input);
2685  std::vector<std::shared_ptr<Analyzer::Expr>> inputs;
2686  const auto scan_ra = dynamic_cast<const RelScan*>(input);
2687  int input_idx = 0;
2688  for (const auto& input_meta : in_metainfo) {
2689  inputs.push_back(
2690  std::make_shared<Analyzer::ColumnVar>(input_meta.get_type_info(),
2691  table_id,
2692  scan_ra ? input_idx + 1 : input_idx,
2693  rte_idx));
2694  ++input_idx;
2695  }
2696  return inputs;
2697 }
2698 
2699 } // namespace
2700 
2702  const RelAggregate* aggregate,
2703  const SortInfo& sort_info,
2704  const bool just_explain) {
2705  std::vector<InputDescriptor> input_descs;
2706  std::list<std::shared_ptr<const InputColDescriptor>> input_col_descs;
2707  std::vector<std::shared_ptr<RexInput>> used_inputs_owned;
2708  const auto input_to_nest_level = get_input_nest_levels(aggregate, {});
2709  std::tie(input_descs, input_col_descs, used_inputs_owned) =
2710  get_input_desc(aggregate, input_to_nest_level, {}, cat_);
2711  const auto join_type = get_join_type(aggregate);
2712  QueryFeatureDescriptor query_features;
2713  RelAlgTranslator translator(cat_,
2714  executor_,
2715  input_to_nest_level,
2716  {join_type},
2717  now_,
2718  just_explain,
2719  query_features);
2720  CHECK_EQ(size_t(1), aggregate->inputCount());
2721  const auto source = aggregate->getInput(0);
2722  const auto& in_metainfo = source->getOutputMetainfo();
2723  const auto scalar_sources =
2724  synthesize_inputs(aggregate, size_t(0), in_metainfo, input_to_nest_level);
2725  const auto groupby_exprs = translate_groupby_exprs(aggregate, scalar_sources);
2726  const auto target_exprs = translate_targets(
2727  target_exprs_owned_, scalar_sources, groupby_exprs, aggregate, translator);
2728  const auto targets_meta = get_targets_meta(aggregate, target_exprs);
2729  aggregate->setOutputMetainfo(targets_meta);
2730  return {RelAlgExecutionUnit{input_descs,
2731  input_col_descs,
2732  {},
2733  {},
2734  {},
2735  groupby_exprs,
2736  target_exprs,
2737  nullptr,
2738  sort_info,
2739  0,
2740  query_features,
2741  false,
2742  query_state_},
2743  aggregate,
2745  nullptr};
2746 }
2747 
2749  const SortInfo& sort_info,
2750  const bool just_explain) {
2751  std::vector<InputDescriptor> input_descs;
2752  std::list<std::shared_ptr<const InputColDescriptor>> input_col_descs;
2753  auto input_to_nest_level = get_input_nest_levels(project, {});
2754  std::tie(input_descs, input_col_descs, std::ignore) =
2755  get_input_desc(project, input_to_nest_level, {}, cat_);
2756  const auto query_infos = get_table_infos(input_descs, executor_);
2757 
2758  const auto left_deep_join =
2759  dynamic_cast<const RelLeftDeepInnerJoin*>(project->getInput(0));
2760  JoinQualsPerNestingLevel left_deep_join_quals;
2761  const auto join_types = left_deep_join ? left_deep_join_types(left_deep_join)
2762  : std::vector<JoinType>{get_join_type(project)};
2763  std::vector<size_t> input_permutation;
2764  std::vector<size_t> left_deep_join_input_sizes;
2765  if (left_deep_join) {
2766  left_deep_join_input_sizes = get_left_deep_join_input_sizes(left_deep_join);
2767  const auto query_infos = get_table_infos(input_descs, executor_);
2768  left_deep_join_quals = translateLeftDeepJoinFilter(
2769  left_deep_join, input_descs, input_to_nest_level, just_explain);
2771  input_permutation = do_table_reordering(input_descs,
2772  input_col_descs,
2773  left_deep_join_quals,
2774  input_to_nest_level,
2775  project,
2776  query_infos,
2777  executor_);
2778  input_to_nest_level = get_input_nest_levels(project, input_permutation);
2779  std::tie(input_descs, input_col_descs, std::ignore) =
2780  get_input_desc(project, input_to_nest_level, input_permutation, cat_);
2781  left_deep_join_quals = translateLeftDeepJoinFilter(
2782  left_deep_join, input_descs, input_to_nest_level, just_explain);
2783  }
2784  }
2785 
2786  QueryFeatureDescriptor query_features;
2787  RelAlgTranslator translator(cat_,
2788  executor_,
2789  input_to_nest_level,
2790  join_types,
2791  now_,
2792  just_explain,
2793  query_features);
2794  const auto target_exprs_owned = translate_scalar_sources(project, translator);
2795  target_exprs_owned_.insert(
2796  target_exprs_owned_.end(), target_exprs_owned.begin(), target_exprs_owned.end());
2797  const auto target_exprs = get_exprs_not_owned(target_exprs_owned);
2798  const RelAlgExecutionUnit exe_unit = {input_descs,
2799  input_col_descs,
2800  {},
2801  {},
2802  left_deep_join_quals,
2803  {nullptr},
2804  target_exprs,
2805  nullptr,
2806  sort_info,
2807  0,
2808  query_features,
2809  false,
2810  query_state_};
2811  auto query_rewriter = std::make_unique<QueryRewriter>(query_infos, executor_);
2812  const auto rewritten_exe_unit = query_rewriter->rewrite(exe_unit);
2813  const auto targets_meta = get_targets_meta(project, rewritten_exe_unit.target_exprs);
2814  project->setOutputMetainfo(targets_meta);
2815  return {rewritten_exe_unit,
2816  project,
2818  std::move(query_rewriter),
2819  input_permutation,
2820  left_deep_join_input_sizes};
2821 }
2822 
2824  const RelTableFunction* table_func,
2825  const bool just_explain) {
2826  std::vector<InputDescriptor> input_descs;
2827  std::list<std::shared_ptr<const InputColDescriptor>> input_col_descs;
2828  auto input_to_nest_level = get_input_nest_levels(table_func, {});
2829  std::tie(input_descs, input_col_descs, std::ignore) =
2830  get_input_desc(table_func, input_to_nest_level, {}, cat_);
2831  const auto query_infos = get_table_infos(input_descs, executor_);
2832  CHECK_EQ(size_t(1), table_func->inputCount());
2833 
2834  QueryFeatureDescriptor query_features; // TODO(adb): remove/make optional
2835  RelAlgTranslator translator(
2836  cat_, executor_, input_to_nest_level, {}, now_, just_explain, query_features);
2837  const auto input_exprs_owned = translate_scalar_sources(table_func, translator);
2838  target_exprs_owned_.insert(
2839  target_exprs_owned_.end(), input_exprs_owned.begin(), input_exprs_owned.end());
2840  const auto input_exprs = get_exprs_not_owned(input_exprs_owned);
2841 
2842  std::vector<Analyzer::ColumnVar*> input_col_exprs;
2843  for (auto input_expr : input_exprs) {
2844  if (auto col_var = dynamic_cast<Analyzer::ColumnVar*>(input_expr)) {
2845  input_col_exprs.push_back(col_var);
2846  }
2847  }
2848  CHECK_EQ(input_col_exprs.size(), table_func->getColInputsSize());
2849 
2850  const auto& table_function_impl =
2852 
2853  std::vector<Analyzer::Expr*> table_func_outputs;
2854  for (size_t i = 0; i < table_function_impl.getOutputsSize(); i++) {
2855  const auto ti = table_function_impl.getOutputSQLType(i);
2856  target_exprs_owned_.push_back(std::make_shared<Analyzer::ColumnVar>(ti, 0, i, -1));
2857  table_func_outputs.push_back(target_exprs_owned_.back().get());
2858  }
2859 
2860  std::optional<size_t> output_row_multiplier;
2861  if (table_function_impl.hasUserSpecifiedOutputMultiplier()) {
2862  const auto parameter_index = table_function_impl.getOutputRowParameter();
2863  CHECK_GT(parameter_index, size_t(0));
2864  const auto parameter_expr = table_func->getTableFuncInputAt(parameter_index - 1);
2865  const auto parameter_expr_literal = dynamic_cast<const RexLiteral*>(parameter_expr);
2866  if (!parameter_expr_literal) {
2867  throw std::runtime_error(
2868  "Provided output buffer multiplier parameter is not a literal. Only literal "
2869  "values are supported with output buffer multiplier configured table "
2870  "functions.");
2871  }
2872  int64_t literal_val = parameter_expr_literal->getVal<int64_t>();
2873  if (literal_val < 0) {
2874  throw std::runtime_error("Provided output row multiplier " +
2875  std::to_string(literal_val) +
2876  " is not valid for table functions.");
2877  }
2878  output_row_multiplier = static_cast<size_t>(literal_val);
2879  }
2880 
2881  const TableFunctionExecutionUnit exe_unit = {
2882  input_descs,
2883  input_col_descs,
2884  input_exprs, // table function inputs
2885  input_col_exprs, // table function column inputs (duplicates w/ above)
2886  table_func_outputs, // table function projected exprs
2887  output_row_multiplier, // output buffer multiplier
2888  table_func->getFunctionName()};
2889  const auto targets_meta = get_targets_meta(table_func, exe_unit.target_exprs);
2890  table_func->setOutputMetainfo(targets_meta);
2891  return {exe_unit, table_func};
2892 }
2893 
2894 namespace {
2895 
2896 std::pair<std::vector<TargetMetaInfo>, std::vector<std::shared_ptr<Analyzer::Expr>>>
2898  const RelAlgTranslator& translator,
2899  const std::vector<std::shared_ptr<RexInput>>& inputs_owned,
2900  const std::unordered_map<const RelAlgNode*, int>& input_to_nest_level) {
2901  std::vector<TargetMetaInfo> in_metainfo;
2902  std::vector<std::shared_ptr<Analyzer::Expr>> exprs_owned;
2903  const auto data_sink_node = get_data_sink(filter);
2904  auto input_it = inputs_owned.begin();
2905  for (size_t nest_level = 0; nest_level < data_sink_node->inputCount(); ++nest_level) {
2906  const auto source = data_sink_node->getInput(nest_level);
2907  const auto scan_source = dynamic_cast<const RelScan*>(source);
2908  if (scan_source) {
2909  CHECK(source->getOutputMetainfo().empty());
2910  std::vector<std::shared_ptr<Analyzer::Expr>> scalar_sources_owned;
2911  for (size_t i = 0; i < scan_source->size(); ++i, ++input_it) {
2912  scalar_sources_owned.push_back(translator.translateScalarRex(input_it->get()));
2913  }
2914  const auto source_metadata =
2915  get_targets_meta(scan_source, get_exprs_not_owned(scalar_sources_owned));
2916  in_metainfo.insert(
2917  in_metainfo.end(), source_metadata.begin(), source_metadata.end());
2918  exprs_owned.insert(
2919  exprs_owned.end(), scalar_sources_owned.begin(), scalar_sources_owned.end());
2920  } else {
2921  const auto& source_metadata = source->getOutputMetainfo();
2922  input_it += source_metadata.size();
2923  in_metainfo.insert(
2924  in_metainfo.end(), source_metadata.begin(), source_metadata.end());
2925  const auto scalar_sources_owned = synthesize_inputs(
2926  data_sink_node, nest_level, source_metadata, input_to_nest_level);
2927  exprs_owned.insert(
2928  exprs_owned.end(), scalar_sources_owned.begin(), scalar_sources_owned.end());
2929  }
2930  }
2931  return std::make_pair(in_metainfo, exprs_owned);
2932 }
2933 
2934 } // namespace
2935 
2937  const SortInfo& sort_info,
2938  const bool just_explain) {
2939  CHECK_EQ(size_t(1), filter->inputCount());
2940  std::vector<InputDescriptor> input_descs;
2941  std::list<std::shared_ptr<const InputColDescriptor>> input_col_descs;
2942  std::vector<TargetMetaInfo> in_metainfo;
2943  std::vector<std::shared_ptr<RexInput>> used_inputs_owned;
2944  std::vector<std::shared_ptr<Analyzer::Expr>> target_exprs_owned;
2945 
2946  const auto input_to_nest_level = get_input_nest_levels(filter, {});
2947  std::tie(input_descs, input_col_descs, used_inputs_owned) =
2948  get_input_desc(filter, input_to_nest_level, {}, cat_);
2949  const auto join_type = get_join_type(filter);
2950  QueryFeatureDescriptor query_features;
2951  RelAlgTranslator translator(cat_,
2952  executor_,
2953  input_to_nest_level,
2954  {join_type},
2955  now_,
2956  just_explain,
2957  query_features);
2958  std::tie(in_metainfo, target_exprs_owned) =
2959  get_inputs_meta(filter, translator, used_inputs_owned, input_to_nest_level);
2960  const auto filter_expr = translator.translateScalarRex(filter->getCondition());
2961  const auto qual = fold_expr(filter_expr.get());
2962  target_exprs_owned_.insert(
2963  target_exprs_owned_.end(), target_exprs_owned.begin(), target_exprs_owned.end());
2964  const auto target_exprs = get_exprs_not_owned(target_exprs_owned);
2965  filter->setOutputMetainfo(in_metainfo);
2966  const auto rewritten_qual = rewrite_expr(qual.get());
2967  return {{input_descs,
2968  input_col_descs,
2969  {},
2970  {rewritten_qual ? rewritten_qual : qual},
2971  {},
2972  {nullptr},
2973  target_exprs,
2974  nullptr,
2975  sort_info,
2976  0},
2977  filter,
2979  nullptr};
2980 }
2981 
const size_t getGroupByCount() const
SQLTypeInfo getOutputSQLType(const size_t idx) const
bool is_agg(const Analyzer::Expr *expr)
Analyzer::ExpressionPtr rewrite_array_elements(Analyzer::Expr const *expr)
std::vector< Analyzer::Expr * > target_exprs
SortField getCollation(const size_t i) const
int64_t queue_time_ms_
#define CHECK_EQ(x, y)
Definition: Logger.h:205
void collect_used_input_desc(std::vector< InputDescriptor > &input_descs, const Catalog_Namespace::Catalog &cat, std::unordered_set< std::shared_ptr< const InputColDescriptor >> &input_col_descs_unique, const RelAlgNode *ra_node, const std::unordered_set< const RexInput * > &source_used_inputs, const std::unordered_map< const RelAlgNode *, int > &input_to_nest_level)
size_t getOffset() const
ExecutionResult executeAggregate(const RelAggregate *aggregate, const CompilationOptions &co, const ExecutionOptions &eo, RenderInfo *render_info, const int64_t queue_time_ms)
RaExecutionDesc * getDescriptor(size_t idx) const
ExecutionResult executeProject(const RelProject *, const CompilationOptions &, const ExecutionOptions &, RenderInfo *, const int64_t queue_time_ms, const ssize_t previous_count)
std::unordered_map< const RelAlgNode *, int > get_input_nest_levels(const RelAlgNode *ra_node, const std::vector< size_t > &input_permutation)
std::vector< JoinType > left_deep_join_types(const RelLeftDeepInnerJoin *left_deep_join)
JoinType
Definition: sqldefs.h:107
int32_t getErrorCode() const
Definition: ErrorHandling.h:55
TableFunctionWorkUnit createTableFunctionWorkUnit(const RelTableFunction *table_func, const bool just_explain)
bool can_use_bump_allocator(const RelAlgExecutionUnit &ra_exe_unit, const CompilationOptions &co, const ExecutionOptions &eo)
bool contains(const std::shared_ptr< Analyzer::Expr > expr, const bool desc) const
const Rex * getTargetExpr(const size_t i) const
AggregatedColRange computeColRangesCache()
static const int32_t ERR_INTERRUPTED
Definition: Execute.h:980
std::vector< size_t > do_table_reordering(std::vector< InputDescriptor > &input_descs, std::list< std::shared_ptr< const InputColDescriptor >> &input_col_descs, const JoinQualsPerNestingLevel &left_deep_join_quals, std::unordered_map< const RelAlgNode *, int > &input_to_nest_level, const RA *node, const std::vector< InputTableInfo > &query_infos, const Executor *executor)
class for a per-database catalog. also includes metadata for the current database and the current use...
Definition: Catalog.h:81
WorkUnit createProjectWorkUnit(const RelProject *, const SortInfo &, const bool just_explain)
size_t size() const override
int hll_size_for_rate(const int err_percent)
Definition: HyperLogLog.h:115
bool g_enable_bump_allocator
Definition: Execute.cpp:99
ExecutionResult executeRelAlgSeq(const RaExecutionSequence &seq, const CompilationOptions &co, const ExecutionOptions &eo, RenderInfo *render_info, const int64_t queue_time_ms, const bool with_existing_temp_tables=false)
const RexScalar * getFilterExpr() const
bool g_cluster
std::shared_ptr< Analyzer::Expr > set_transient_dict(const std::shared_ptr< Analyzer::Expr > expr)
std::vector< std::shared_ptr< Analyzer::Expr > > synthesize_inputs(const RelAlgNode *ra_node, const size_t nest_level, const std::vector< TargetMetaInfo > &in_metainfo, const std::unordered_map< const RelAlgNode *, int > &input_to_nest_level)
WorkUnit createAggregateWorkUnit(const RelAggregate *, const SortInfo &, const bool just_explain)
const RelAlgNode * body
ExecutorDeviceType
WorkUnit createWorkUnit(const RelAlgNode *, const SortInfo &, const bool just_explain)
void setForceNonInSituData()
Definition: RenderInfo.cpp:42
std::vector< Analyzer::Expr * > translate_targets(std::vector< std::shared_ptr< Analyzer::Expr >> &target_exprs_owned, const std::vector< std::shared_ptr< Analyzer::Expr >> &scalar_sources, const std::list< std::shared_ptr< Analyzer::Expr >> &groupby_exprs, const RelCompound *compound, const RelAlgTranslator &translator)
size_t getIndex() const
#define SPIMAP_GEO_PHYSICAL_INPUT(c, i)
Definition: Catalog.h:70
size_t get_scan_limit(const RelAlgNode *ra, const size_t limit)
std::unique_ptr< const RexOperator > get_bitwise_equals_conjunction(const RexScalar *scalar)
RexUsedInputsVisitor(const Catalog_Namespace::Catalog &cat)
const RexScalar * getOuterCondition(const size_t nesting_level) const
TableGenerations computeTableGenerations()
SQLTypeInfo get_nullable_logical_type_info(const SQLTypeInfo &type_info)
Definition: sqltypes.h:884
std::shared_ptr< Analyzer::Expr > translateScalarRex(const RexScalar *rex) const
#define LOG(tag)
Definition: Logger.h:188
TargetInfo get_target_info(const PointerType target_expr, const bool bigint_count)
Definition: TargetInfo.h:66
size_t size() const override
static SpeculativeTopNBlacklist speculative_topn_blacklist_
size_t get_scalar_sources_size(const RelCompound *compound)
RelAlgExecutionUnit exe_unit
std::pair< std::vector< TargetMetaInfo >, std::vector< std::shared_ptr< Analyzer::Expr > > > get_inputs_meta(const RelFilter *filter, const RelAlgTranslator &translator, const std::vector< std::shared_ptr< RexInput >> &inputs_owned, const std::unordered_map< const RelAlgNode *, int > &input_to_nest_level)
std::vector< std::shared_ptr< Analyzer::TargetEntry > > targets
Definition: RenderInfo.h:37
SQLOps
Definition: sqldefs.h:29
TemporaryTables temporary_tables_
static const size_t max_groups_buffer_entry_default_guess
const std::list< Analyzer::OrderEntry > order_entries
ExecutionResult executeRelAlgQueryWithFilterPushDown(const RaExecutionSequence &seq, const CompilationOptions &co, const ExecutionOptions &eo, RenderInfo *render_info, const int64_t queue_time_ms)
bool setInSituDataIfUnset(const bool is_in_situ_data)
Definition: RenderInfo.cpp:95
const RexScalar * getCondition() const
const ExecutorOptLevel opt_level_
std::string join(T const &container, std::string const &delim)
const std::vector< TargetMetaInfo > getTupleType() const
std::vector< std::shared_ptr< Analyzer::Expr > > translate_scalar_sources_for_update(const RA *ra_node, const RelAlgTranslator &translator, int32_t tableId, const Catalog_Namespace::Catalog &cat, const ColumnNameList &colNames, size_t starting_projection_column_idx)
Definition: sqldefs.h:38
std::shared_ptr< Analyzer::Var > var_ref(const Analyzer::Expr *expr, const Analyzer::Var::WhichRow which_row, const int varno)
Definition: Analyzer.h:1589
bool use_speculative_top_n(const RelAlgExecutionUnit &ra_exe_unit, const QueryMemoryDescriptor &query_mem_desc)
#define CHECK_GE(x, y)
Definition: Logger.h:210
std::vector< PushedDownFilterInfo > selectFiltersToBePushedDown(const RelAlgExecutor::WorkUnit &work_unit, const CompilationOptions &co, const ExecutionOptions &eo)
static const TableFunction & get(const std::string &name)
SQLTypeInfo get_logical_type_info(const SQLTypeInfo &type_info)
Definition: sqltypes.h:869
TypeR::rep timer_stop(Type clock_begin)
Definition: measure.h:46
Definition: sqldefs.h:30
ExecutionResult executeModify(const RelModify *modify, const ExecutionOptions &eo)
void addTemporaryTable(const int table_id, const ResultSetPtr &result)
void check_sort_node_source_constraint(const RelSort *sort)
std::unordered_map< unsigned, AggregatedResult > leaf_results_
SQLTypeInfo get_agg_type(const SQLAgg agg_kind, const Analyzer::Expr *arg_expr)
std::vector< TargetInfo > TargetInfoList
std::vector< JoinCondition > JoinQualsPerNestingLevel
std::shared_ptr< ResultSet > ResultSetPtr
WorkUnit createCompoundWorkUnit(const RelCompound *, const SortInfo &, const bool just_explain)
static const int32_t ERR_TOO_MANY_LITERALS
Definition: Execute.h:982
ExecutionResult executeLogicalValues(const RelLogicalValues *, const ExecutionOptions &)
bool g_enable_dynamic_watchdog
Definition: Execute.cpp:72
const std::list< std::shared_ptr< Analyzer::Expr > > groupby_exprs
std::vector< Analyzer::Expr * > get_exprs_not_owned(const std::vector< std::shared_ptr< Analyzer::Expr >> &exprs)
Definition: Execute.h:213
Analyzer::ExpressionPtr rewrite_expr(const Analyzer::Expr *expr)
static void invalidateCaches()
int g_hll_precision_bits
ExecutionResult executeFilter(const RelFilter *, const CompilationOptions &, const ExecutionOptions &, RenderInfo *, const int64_t queue_time_ms)
const std::vector< std::shared_ptr< RexSubQuery > > & getSubqueries() const noexcept
size_t getNDVEstimation(const WorkUnit &work_unit, const bool is_agg, const CompilationOptions &co, const ExecutionOptions &eo)
QualsConjunctiveForm qual_to_conjunctive_form(const std::shared_ptr< Analyzer::Expr > qual_expr)
const std::vector< InputDescriptor > input_descs
const std::vector< std::shared_ptr< Analyzer::Expr > > & getOrderKeys() const
Definition: Analyzer.h:1404
#define CHECK_GT(x, y)
Definition: Logger.h:209
bool is_window_execution_unit(const RelAlgExecutionUnit &ra_exe_unit)
std::vector< const RexScalar * > rex_to_conjunctive_form(const RexScalar *qual_expr)
bool is_count_distinct(const Analyzer::Expr *expr)
std::shared_ptr< Analyzer::Expr > transform_to_inner(const Analyzer::Expr *expr)
std::string to_string(char const *&&v)
void handleNop(RaExecutionDesc &ed)
#define LOG_IF(severity, condition)
Definition: Logger.h:287
void computeWindow(const RelAlgExecutionUnit &ra_exe_unit, const CompilationOptions &co, const ExecutionOptions &eo, ColumnCacheMap &column_cache_map, const int64_t queue_time_ms)
bool g_enable_watchdog
bool disallow_in_situ_only_if_final_ED_is_aggregate
Definition: RenderInfo.h:41
std::vector< node_t > get_node_input_permutation(const JoinQualsPerNestingLevel &left_deep_join_quals, const std::vector< InputTableInfo > &table_infos, const Executor *executor)
static const size_t high_scan_limit
Definition: Execute.h:409
bool list_contains_expression(const QualsList &haystack, const std::shared_ptr< Analyzer::Expr > &needle)
size_t get_count_distinct_sub_bitmap_count(const size_t bitmap_sz_bits, const RelAlgExecutionUnit &ra_exe_unit, const ExecutorDeviceType device_type)
static const int32_t ERR_STRING_CONST_IN_RESULTSET
Definition: Execute.h:983
Definition: sqldefs.h:73
size_t getColInputsSize() const
virtual T visit(const RexScalar *rex_scalar) const
Definition: RexVisitor.h:27
const size_t getScalarSourcesSize() const
static const int32_t ERR_STREAMING_TOP_N_NOT_SUPPORTED_IN_RENDER_QUERY
Definition: Execute.h:984
static const int32_t ERR_COLUMNAR_CONVERSION_NOT_SUPPORTED
Definition: Execute.h:981
const ResultSetPtr & get_temporary_table(const TemporaryTables *temporary_tables, const int table_id)
Definition: Execute.h:178
RelAlgExecutionUnit create_count_all_execution_unit(const RelAlgExecutionUnit &ra_exe_unit, std::shared_ptr< Analyzer::Expr > replacement_target)
std::vector< std::shared_ptr< Analyzer::Expr > > target_exprs_owned_
unsigned getIndex() const
static const int32_t ERR_DIV_BY_ZERO
Definition: Execute.h:971
const bool allow_multifrag
const bool find_push_down_candidates
bool g_from_table_reordering
Definition: Execute.cpp:77
CHECK(cgen_state)
const bool just_validate
unsigned getId() const
const SortInfo sort_info
ColumnNameList const & getTargetColumns() const
#define INJECT_TIMER(DESC)
Definition: measure.h:91
const bool with_dynamic_watchdog
static const int32_t ERR_OUT_OF_RENDER_MEM
Definition: Execute.h:975
std::list< std::shared_ptr< Analyzer::Expr > > translate_groupby_exprs(const RelCompound *compound, const std::vector< std::shared_ptr< Analyzer::Expr >> &scalar_sources)
const JoinQualsPerNestingLevel join_quals
SQLTypeInfo get_logical_type_for_expr(const Analyzer::Expr &expr)
std::vector< std::shared_ptr< RexInput > > synthesized_physical_inputs_owned
std::pair< std::vector< InputDescriptor >, std::list< std::shared_ptr< const InputColDescriptor > > > get_input_desc_impl(const RA *ra_node, const std::unordered_set< const RexInput * > &used_inputs, const std::unordered_map< const RelAlgNode *, int > &input_to_nest_level, const std::vector< size_t > &input_permutation, const Catalog_Namespace::Catalog &cat)
const int8_t const int64_t const uint64_t const int32_t const int64_t int64_t uint32_t const int64_t int32_t * error_code
SortAlgorithm
const double gpu_input_mem_limit_percent
MergeType
A container for relational algebra descriptors defining the execution order for a relational algebra ...
const Catalog_Namespace::Catalog & cat_
size_t g_big_group_threshold
Definition: Execute.cpp:92
static const int32_t ERR_OVERFLOW_OR_UNDERFLOW
Definition: Execute.h:977
const std::shared_ptr< ResultSet > & getRows() const
static const int32_t ERR_OUT_OF_TIME
Definition: Execute.h:979
ExecutionResult handleOutOfMemoryRetry(const RelAlgExecutor::WorkUnit &work_unit, const std::vector< TargetMetaInfo > &targets_meta, const bool is_agg, const CompilationOptions &co, const ExecutionOptions &eo, RenderInfo *render_info, const bool was_multifrag_kernel_launch, const int64_t queue_time_ms)
bool g_bigint_count
size_t groups_approx_upper_bound(const std::vector< InputTableInfo > &table_infos)
const RelAlgNode * getInput(const size_t idx) const
Definition: sqldefs.h:37
Definition: sqldefs.h:75
std::shared_ptr< Analyzer::Expr > build_logical_expression(const std::vector< std::shared_ptr< Analyzer::Expr >> &factors, const SQLOps sql_op)
ColumnValidationFunction yieldColumnValidator(TableDescriptorType const *table_descriptor)
bool table_is_temporary(const TableDescriptor *td)
const bool register_intel_jit_listener_
static WindowProjectNodeContext * create()
static const int32_t ERR_UNSUPPORTED_SELF_JOIN
Definition: Execute.h:974
bool isSimple() const
ExecutionResult executeRelAlgSubSeq(const RaExecutionSequence &seq, const std::pair< size_t, size_t > interval, const CompilationOptions &co, const ExecutionOptions &eo, RenderInfo *render_info, const int64_t queue_time_ms)
const bool output_columnar_hint
bool get_is_distinct() const
Definition: Analyzer.h:1047
static const int32_t ERR_SINGLE_VALUE_FOUND_MULTIPLE_VALUES
Definition: Execute.h:985
ExpressionRange getExpressionRange(const Analyzer::BinOper *expr, const std::vector< InputTableInfo > &query_infos, const Executor *, boost::optional< std::list< std::shared_ptr< Analyzer::Expr >>> simple_quals)
void build_render_targets(RenderInfo &render_info, const std::vector< Analyzer::Expr * > &work_unit_target_exprs, const std::vector< TargetMetaInfo > &targets_meta)
JoinType get_join_type(const RelAlgNode *ra)
void executeRelAlgStep(const RaExecutionSequence &seq, const size_t step_idx, const CompilationOptions &, const ExecutionOptions &, RenderInfo *, const int64_t queue_time_ms)
static std::pair< const int8_t *, size_t > getOneColumnFragment(Executor *executor, const Analyzer::ColumnVar &hash_col, const Fragmenter_Namespace::FragmentInfo &fragment, const Data_Namespace::MemoryLevel effective_mem_lvl, const int device_id, std::vector< std::shared_ptr< Chunk_NS::Chunk >> &chunks_owner, ColumnCacheMap &column_cache)
static const int32_t ERR_OUT_OF_GPU_MEM
Definition: Execute.h:972
size_t getTableFuncInputsSize() const
SQLTypeInfoCore< ArrayContextTypeSizer, ExecutorTypePackaging, DateTimeFacilities > SQLTypeInfo
Definition: sqltypes.h:852
std::shared_ptr< Analyzer::Expr > reverse_logical_distribution(const std::shared_ptr< Analyzer::Expr > &expr)
const SQLTypeInfo & get_type_info() const
Definition: Analyzer.h:78
ExecutorDeviceType device_type_
Executor * getExecutor() const
static std::shared_ptr< Analyzer::Expr > normalize(const SQLOps optype, const SQLQualifier qual, std::shared_ptr< Analyzer::Expr > left_expr, std::shared_ptr< Analyzer::Expr > right_expr)
Definition: ParserNode.cpp:261
const std::vector< std::unique_ptr< const RexAgg > > & getAggExprs() const
UpdateCallback yieldUpdateCallback(UpdateTransactionParameters &update_parameters)
bool node_is_aggregate(const RelAlgNode *ra)
std::vector< TargetMetaInfo > get_targets_meta(const RA *ra_node, const std::vector< Analyzer::Expr * > &target_exprs)
bool g_enable_window_functions
Definition: Execute.cpp:93
bool isEmptyResult() const
ExecutionResult executeTableFunction(const RelTableFunction *, const CompilationOptions &, const ExecutionOptions &, const int64_t queue_time_ms)
UpdateCallback yieldDeleteCallback(DeleteTransactionParameters &delete_parameters)
ExecutionResult executeRelAlgQueryNoRetry(const CompilationOptions &co, const ExecutionOptions &eo, RenderInfo *render_info)
const RexScalar * getProjectAt(const size_t idx) const
const ExecutorExplainType explain_type_
ssize_t getFilteredCountAll(const WorkUnit &work_unit, const bool is_agg, const CompilationOptions &co, const ExecutionOptions &eo)
Definition: sqldefs.h:69
#define TRANSIENT_DICT_ID
Definition: sqltypes.h:189
const bool just_calcite_explain
std::vector< std::shared_ptr< Analyzer::Expr > > translate_scalar_sources(const RA *ra_node, const RelAlgTranslator &translator)
std::pair< std::unordered_set< const RexInput * >, std::vector< std::shared_ptr< RexInput > > > get_used_inputs(const RelCompound *compound, const Catalog_Namespace::Catalog &cat)
#define CHECK_LE(x, y)
Definition: Logger.h:208
std::shared_ptr< const query_state::QueryState > query_state_
bool table_is_replicated(const TableDescriptor *td)
std::unordered_set< const RexInput * > visitInput(const RexInput *rex_input) const override
std::pair< std::unordered_set< const RexInput * >, std::vector< std::shared_ptr< RexInput > > > get_join_source_used_inputs(const RelAlgNode *ra_node, const Catalog_Namespace::Catalog &cat)
void executeDeleteViaCompound(const RelCompound *compound, const CompilationOptions &co, const ExecutionOptions &eo, RenderInfo *render_info, const int64_t queue_time_ms)
Definition: sqldefs.h:76
const size_t getGroupByCount() const
std::unordered_set< const RexInput * > aggregateResult(const std::unordered_set< const RexInput * > &aggregate, const std::unordered_set< const RexInput * > &next_result) const override
const RelAlgNode & getRootRelAlgNode() const
size_t collationCount() const
const RelAlgNode * getSourceNode() const
bool isAggregate() const
size_t getLimit() const
void executeUpdateViaProject(const RelProject *, const CompilationOptions &, const ExecutionOptions &, RenderInfo *, const int64_t queue_time_ms)
ExecutionResult executeSort(const RelSort *, const CompilationOptions &, const ExecutionOptions &, RenderInfo *, const int64_t queue_time_ms)
bool isRowidLookup(const WorkUnit &work_unit)
bool exe_unit_has_quals(const RelAlgExecutionUnit ra_exe_unit)
int table_id_from_ra(const RelAlgNode *ra_node)
bool g_enable_table_functions
Definition: Execute.cpp:94
static void handlePersistentError(const int32_t error_code)
const RexScalar * getTableFuncInputAt(const size_t idx) const
void executeDeleteViaProject(const RelProject *, const CompilationOptions &, const ExecutionOptions &, RenderInfo *, const int64_t queue_time_ms)
bool compute_output_buffer_size(const RelAlgExecutionUnit &ra_exe_unit)
SQLAgg get_aggtype() const
Definition: Analyzer.h:1044
const size_t max_groups_buffer_entry_guess
std::string getFunctionName() const
std::list< std::shared_ptr< Analyzer::Expr > > quals
const bool allow_loop_joins
bool wasMultifragKernelLaunch() const
Definition: ErrorHandling.h:57
bool g_skip_intermediate_count
static const int32_t ERR_SPECULATIVE_TOP_OOM
Definition: Execute.h:978
ExecutionResult executeWorkUnit(const WorkUnit &work_unit, const std::vector< TargetMetaInfo > &targets_meta, const bool is_agg, const CompilationOptions &co_in, const ExecutionOptions &eo, RenderInfo *, const int64_t queue_time_ms, const ssize_t previous_count=-1)
StringDictionaryGenerations computeStringDictionaryGenerations()
void executeUpdateViaCompound(const RelCompound *compound, const CompilationOptions &co, const ExecutionOptions &eo, RenderInfo *render_info, const int64_t queue_time_ms)
const RexScalar * getInnerCondition() const
bool isPotentialInSituRender() const
Definition: RenderInfo.cpp:61
std::vector< InputTableInfo > get_table_infos(const std::vector< InputDescriptor > &input_descs, Executor *executor)
#define DEBUG_TIMER(name)
Definition: Logger.h:308
std::vector< std::shared_ptr< Analyzer::Expr > > qual_to_disjunctive_form(const std::shared_ptr< Analyzer::Expr > &qual_expr)
void setOutputMetainfo(const std::vector< TargetMetaInfo > &targets_metainfo) const
Definition: sqldefs.h:31
const std::vector< std::shared_ptr< RexInput > > & get_inputs_owned() const
ExecutionResult executeCompound(const RelCompound *, const CompilationOptions &, const ExecutionOptions &, RenderInfo *, const int64_t queue_time_ms)
std::unique_ptr< RelAlgDagBuilder > query_dag_
const RelAlgNode * getBody() const
std::list< Analyzer::OrderEntry > get_order_entries(const RelSort *sort)
RANodeOutput get_node_output(const RelAlgNode *ra_node)
std::unique_ptr< const RexOperator > get_bitwise_equals(const RexScalar *scalar)
Estimators to be used when precise cardinality isn&#39;t useful.
bool g_allow_cpu_retry
Definition: Execute.cpp:74
static std::string getErrorMessageFromCode(const int32_t error_code)
std::unordered_map< int, std::unordered_map< int, std::shared_ptr< const ColumnarResults > > > ColumnCacheMap
WorkUnit createSortInputWorkUnit(const RelSort *, const bool just_explain)
QualsConjunctiveForm translate_quals(const RelCompound *compound, const RelAlgTranslator &translator)
void add(const std::shared_ptr< Analyzer::Expr > expr, const bool desc)
const Expr * get_left_operand() const
Definition: Analyzer.h:436
void cleanupPostExecution()
std::unique_ptr< WindowFunctionContext > createWindowFunctionContext(const Analyzer::WindowFunction *window_func, const std::shared_ptr< Analyzer::BinOper > &partition_key_cond, const RelAlgExecutionUnit &ra_exe_unit, const std::vector< InputTableInfo > &query_infos, const CompilationOptions &co, ColumnCacheMap &column_cache_map)
std::vector< std::string > ColumnNameList
std::list< std::shared_ptr< Analyzer::Expr > > makeJoinQuals(const RexScalar *join_condition, const std::vector< JoinType > &join_types, const std::unordered_map< const RelAlgNode *, int > &input_to_nest_level, const bool just_explain) const
const RexScalar * scalar_at(const size_t i, const RelCompound *compound)
Definition: sqltypes.h:48
std::vector< Analyzer::Expr * > target_exprs
std::unordered_set< PhysicalInput > get_physical_inputs(const RelAlgNode *ra)
std::tuple< std::vector< InputDescriptor >, std::list< std::shared_ptr< const InputColDescriptor > >, std::vector< std::shared_ptr< RexInput > > > get_input_desc(const RA *ra_node, const std::unordered_map< const RelAlgNode *, int > &input_to_nest_level, const std::vector< size_t > &input_permutation, const Catalog_Namespace::Catalog &cat)
const size_t inputCount() const
void addColSlotInfo(const std::vector< std::tuple< int8_t, int8_t >> &slots_for_col)
auto const isVarlenUpdateRequired() const
const bool with_dynamic_watchdog_
FirstStepExecutionResult executeRelAlgQuerySingleStep(const RaExecutionSequence &seq, const size_t step_idx, const CompilationOptions &co, const ExecutionOptions &eo, RenderInfo *render_info)
RelAlgExecutionUnit decide_approx_count_distinct_implementation(const RelAlgExecutionUnit &ra_exe_unit_in, const std::vector< InputTableInfo > &table_infos, const Executor *executor, const ExecutorDeviceType device_type_in, std::vector< std::shared_ptr< Analyzer::Expr >> &target_exprs_owned)
Definition: sqldefs.h:74
const std::vector< std::shared_ptr< Analyzer::Expr > > & getPartitionKeys() const
Definition: Analyzer.h:1400
const RelAlgNode * get_data_sink(const RelAlgNode *ra_node)
const unsigned dynamic_watchdog_time_limit
const std::vector< TargetMetaInfo > & getOutputMetainfo() const
static const int32_t ERR_OUT_OF_CPU_MEM
Definition: Execute.h:976
const TableDescriptor * getTableDescriptor() const
Definition: sqldefs.h:72
std::vector< const RelAlgNode * > get_non_join_sequence(const RelAlgNode *ra)
Executor * executor_
ExecutionResult executeRelAlgQuery(const CompilationOptions &co, const ExecutionOptions &eo, RenderInfo *render_info)
int getNestLevel() const
std::list< std::shared_ptr< Analyzer::Expr > > combine_equi_join_conditions(const std::list< std::shared_ptr< Analyzer::Expr >> &join_quals)
#define IS_GEO(T)
Definition: sqltypes.h:167
std::unordered_set< int > get_physical_table_inputs(const RelAlgNode *ra)
WorkUnit createFilterWorkUnit(const RelFilter *, const SortInfo &, const bool just_explain)
#define VLOG(n)
Definition: Logger.h:291
Type timer_start()
Definition: measure.h:40
std::list< std::shared_ptr< Analyzer::Expr > > simple_quals
std::shared_ptr< Analyzer::Expr > fold_expr(const Analyzer::Expr *expr)
const int getColumnIdBySpi(const int tableId, const size_t spi) const
Definition: Catalog.cpp:1459
const bool with_watchdog
JoinQualsPerNestingLevel translateLeftDeepJoinFilter(const RelLeftDeepInnerJoin *join, const std::vector< InputDescriptor > &input_descs, const std::unordered_map< const RelAlgNode *, int > &input_to_nest_level, const bool just_explain)
const ColumnDescriptor * get_column_descriptor(const int col_id, const int table_id, const Catalog_Namespace::Catalog &cat)
Definition: Execute.h:141
TableDescriptor const * getModifiedTableDescriptor() const
static size_t shard_count_for_top_groups(const RelAlgExecutionUnit &ra_exe_unit, const Catalog_Namespace::Catalog &catalog)
static std::shared_ptr< Analyzer::Expr > translateAggregateRex(const RexAgg *rex, const std::vector< std::shared_ptr< Analyzer::Expr >> &scalar_sources)
bool first_oe_is_desc(const std::list< Analyzer::OrderEntry > &order_entries)
void prepareLeafExecution(const AggregatedColRange &agg_col_range, const StringDictionaryGenerations &string_dictionary_generations, const TableGenerations &table_generations)
const RexScalar * getScalarSource(const size_t i) const
std::vector< size_t > get_left_deep_join_input_sizes(const RelLeftDeepInnerJoin *left_deep_join)
bool validateTargetColumns(VALIDATION_FUNCTOR validator) const
void set_transient_dict_maybe(std::vector< std::shared_ptr< Analyzer::Expr >> &scalar_sources, const std::shared_ptr< Analyzer::Expr > &expr)
std::list< std::shared_ptr< Analyzer::Expr > > rewrite_quals(const std::list< std::shared_ptr< Analyzer::Expr >> &quals)